improve thread termination handling

The reason the DirWatcher had to wait for program termination was because
it used withINotify, so when it finished, its watcher threads were killed.
But since I have two DirWatcher threads now, that was not good, and could
perhaps explain the MVar problem I saw yesterday. In any case, fixed this
part of the code by making the DirWatcher return a handle that can be used
to stop it, and now the main Assistant thread is the only one calling
waitForTermination.

4 files changed, 33 insertions, 11 deletions

diff --git a/Assistant.hs b/Assistant.hs
index b61270613..2a11741b4 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -73,6 +73,7 @@ import Assistant.Threads.Merger
 import Assistant.Threads.SanityChecker
 import qualified Utility.Daemon
 import Utility.LogFile
+import Utility.ThreadScheduler
 
 import Control.Concurrent
 
@@ -99,8 +100,8 @@ startDaemon assistant foreground
 				_ <- forkIO $ mergeThread st
 				_ <- forkIO $ daemonStatusThread st dstatus
 				_ <- forkIO $ sanityCheckerThread st dstatus changechan
-				-- Does not return.
-				watchThread st dstatus changechan
+				_ <- forkIO $ watchThread st dstatus changechan
+				waitForTermination
 
 stopDaemon :: Annex ()
 stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs
index de172e8da..5d24d1862 100644
--- a/Assistant/Threads/Merger.hs
+++ b/Assistant/Threads/Merger.hs
@@ -35,7 +35,7 @@ mergeThread st = do
 		{ addHook = hook onAdd
 		, errHook = hook onErr
 		}
-	watchDir dir (const False) hooks id
+	void $ watchDir dir (const False) hooks id
 
 type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO ()
 
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 1b6ec15f1..e250f4b4a 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -46,7 +46,7 @@ needLsof = error $ unlines
 	]
 
 watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
-watchThread st dstatus changechan = watchDir "." ignored hooks startup
+watchThread st dstatus changechan = void $ watchDir "." ignored hooks startup
 	where
 		startup = statupScan st dstatus
 		hook a = Just $ runHandler st dstatus changechan a
diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs
index 11ce7baef..93c3ecb02 100644
--- a/Utility/DirWatcher.hs
+++ b/Utility/DirWatcher.hs
@@ -17,7 +17,6 @@ import Utility.Types.DirWatcher
 #if WITH_INOTIFY
 import qualified Utility.INotify as INotify
 import qualified System.INotify as INotify
-import Utility.ThreadScheduler
 #endif
 #if WITH_KQUEUE
 import qualified Utility.Kqueue as Kqueue
@@ -72,19 +71,41 @@ closingTracked = undefined
 #endif
 #endif
 
+/* Starts a watcher thread. The runStartup action is passed a scanner action
+ * to run, that will return once the initial directory scan is complete.
+ * Once runStartup returns, the watcher thread continues running,
+ * and processing events. Returns a DirWatcherHandle that can be used
+ * to shutdown later.  */
 #if WITH_INOTIFY
-watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
-watchDir dir prune hooks runstartup = INotify.withINotify $ \i -> do
+type DirWatcherHandle = INotify.INotify
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle
+watchDir dir prune hooks runstartup = do
+	i <- INotify.initINotify
 	runstartup $ INotify.watchDir i dir prune hooks
-	waitForTermination -- Let the inotify thread run.
+	return i
 #else
 #if WITH_KQUEUE
-watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO ()
+type DirWatcherHandle = ThreadID
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle
 watchDir dir ignored hooks runstartup = do
 	kq <- runstartup $ Kqueue.initKqueue dir ignored
-	Kqueue.runHooks kq hooks
+	forkIO $ Kqueue.runHooks kq hooks
 #else
-watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
+type DirWatcherHandle = ()
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle
 watchDir = undefined
 #endif
 #endif
+
+#if WITH_INOTIFY
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = INotify.killINotify
+#else
+#if WITH_KQUEUE
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = killThread
+#else
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = undefined
+#endif
+#endif