From 91a38b6d4ecf21826385da49e919d84ef8b162ee Mon Sep 17 00:00:00 2001
From: Thomas Veerman <thomas@minix3.org>
Date: Mon, 2 Apr 2012 15:20:05 +0000
Subject: [PATCH] VFS: fix dead lock

When running out of worker threads to handle device replies a dead
lock resolver thread is used. However, it was only used for FS
endpoints; it is now used for "system processes" (drivers and FS
endpoints). Also, drivers were marked as system process when they
were not "forced" to map (i.e., mapping was done before endpoint was
alive).
---
 servers/vfs/dmap.c  | 13 ++++++------
 servers/vfs/main.c  | 49 +++++++++++++++++++++++++--------------------
 servers/vfs/proto.h |  1 -
 3 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/servers/vfs/dmap.c b/servers/vfs/dmap.c
index 54235e29d..ed83553e2 100644
--- a/servers/vfs/dmap.c
+++ b/servers/vfs/dmap.c
@@ -87,9 +87,10 @@ int flags;			/* device flags */
  * NONE, we're supposed to unmap it.
  */
 
-  int slot;
+  int slot, s;
   size_t len;
   struct dmap *dp;
+  struct fproc *rfp;
 
   /* Get pointer to device entry in the dmap table. */
   if (major < 0 || major >= NR_DEVICES) return(ENODEV);
@@ -105,12 +106,12 @@ int flags;			/* device flags */
   }
 
   /* Check process number of new driver if it was alive before mapping */
-  if (! (flags & DRV_FORCED)) {
-	struct fproc *rfp;
-
-	if (isokendpt(proc_nr_e, &slot) != OK)
+  s = isokendpt(proc_nr_e, &slot);
+  if (s != OK) {
+	/* This is not a problem only when we force this driver mapping */
+	if (! (flags & DRV_FORCED))
 		return(EINVAL);
-
+  } else {
 	rfp = &fproc[slot];
 	rfp->fp_flags |= FP_SYS_PROC;	/* Process is a driver */
   }
diff --git a/servers/vfs/main.c b/servers/vfs/main.c
index 97883d3f1..863691890 100644
--- a/servers/vfs/main.c
+++ b/servers/vfs/main.c
@@ -147,31 +147,36 @@ static void handle_work(void *(*func)(void *arg))
 
   proc_e = m_in.m_source;
 
-  if ((vmp = find_vmnt(proc_e)) != NULL) {
-	/* A call back or dev result from an FS endpoint */
-
-	/* Set call back flag. We assume that an FS does only one call back
-	 * at a time */
-	vmp->m_flags |= VMNT_CALLBACK;
-
-	/* When an FS point has to make a call back in order to mount, force
-	 * its device to a "none device" so block reads/writes will be handled
-	 * by ROOT_FS_E.
-	 */
-	if (vmp->m_flags & VMNT_MOUNTING)
-		vmp->m_flags |= VMNT_FORCEROOTBSF;
-
+  if (fp->fp_flags & FP_SYS_PROC) {
 	if (worker_available() == 0) {
-		/* No worker threads available to handle call */
-		if (deadlock_resolving) {
-			/* Already trying to resolve a deadlock, can't
-			 * handle more, sorry */
-			vmp->m_flags &= ~VMNT_CALLBACK;
-			reply(proc_e, EAGAIN);
+		if (!deadlock_resolving) {
+			if ((vmp = find_vmnt(proc_e)) != NULL) {
+				/* A call back or dev result from an FS
+				 * endpoint. Set call back flag. Can do only
+				 * one call back at a time.
+				 */
+				if (vmp->m_flags & VMNT_CALLBACK) {
+					reply(proc_e, EAGAIN);
+					return;
+				}
+				vmp->m_flags |= VMNT_CALLBACK;
+
+				/* When an FS endpoint has to make a call back
+				 * in order to mount, force its device to a
+				 * "none device" so block reads/writes will be
+				 * handled by ROOT_FS_E.
+				 */
+				if (vmp->m_flags & VMNT_MOUNTING)
+					vmp->m_flags |= VMNT_FORCEROOTBSF;
+			}
+			deadlock_resolving = 1;
+			dl_worker_start(func);
 			return;
 		}
-		deadlock_resolving = 1;
-		dl_worker_start(func);
+		/* Already trying to resolve a deadlock, can't
+		 * handle more, sorry */
+
+		reply(proc_e, EAGAIN);
 		return;
 	}
   }
diff --git a/servers/vfs/proto.h b/servers/vfs/proto.h
index 3689c6c29..6cde7927c 100644
--- a/servers/vfs/proto.h
+++ b/servers/vfs/proto.h
@@ -62,7 +62,6 @@ struct dmap *get_dmap(endpoint_t proc_e);
 int do_mapdriver(void);
 int map_service(struct rprocpub *rpub);
 void dmap_unmap_by_endpt(int proc_nr);
-struct dmap *get_dmap(endpoint_t proc_e);
 int map_driver(const char *label, int major, endpoint_t proc_nr, int
 	dev_style, int flags);
 int map_service(struct rprocpub *rpub);
-- 
2.44.0