From bc51387e6dd426f46f9ab0ef23e6e3eefe7a4417 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 7 Apr 2011 13:59:31 -0400
Subject: Periodically flush git command queue, to avoid boating memory usage
 too much.

Since the queue is flushed in between subcommand actions being run,
there should be no issues with actions that expect to queue up some stuff
and have it run after they do other stuff. So I didn't have to audit for
such assumptions.
---
 GitQueue.hs | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

(limited to 'GitQueue.hs')

diff --git a/GitQueue.hs b/GitQueue.hs
index dfe2976da..480027fa0 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -10,7 +10,8 @@ module GitQueue (
 	empty,
 	add,
 	size,
-	run
+	full,
+	flush
 ) where
 
 import qualified Data.Map as M
@@ -32,9 +33,21 @@ data Action = Action {
 {- A queue of actions to perform (in any order) on a git repository,
  - with lists of files to perform them on. This allows coalescing 
  - similar git commands. -}
-data Queue = Queue Integer (M.Map Action [FilePath])
+data Queue = Queue Int (M.Map Action [FilePath])
 	deriving (Show, Eq)
 
+{- A recommended maximum size for the queue, after which it should be
+ - run.
+ -
+ - 10240 is semi-arbitrary. If we assume git filenames are between 10 and
+ - 255 characters long, then the queue will build up between 100kb and
+ - 2550kb long commands. The max command line length on linux is somewhere
+ - above 20k, so this is a fairly good balance -- the queue will buffer
+ - only a few megabytes of stuff and a minimal number of commands will be
+ - run by xargs. -}
+maxSize :: Int
+maxSize = 10240
+
 {- Constructor for empty queue. -}
 empty :: Queue
 empty = Queue 0 M.empty
@@ -47,14 +60,18 @@ add (Queue n m) subcommand params file = Queue (n + 1) m'
 		m' = M.insertWith' (++) action [file] m
 
 {- Number of items in a queue. -}
-size :: Queue -> Integer
+size :: Queue -> Int
 size (Queue n _) = n
 
+{- Is a queue large enough that it should be flushed? -}
+full :: Queue -> Bool
+full (Queue n _) = n > maxSize
+
 {- Runs a queue on a git repository. -}
-run :: Git.Repo -> Queue -> IO ()
-run repo (Queue _ m) = do
+flush :: Git.Repo -> Queue -> IO Queue
+flush repo (Queue _ m) = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
-	return ()
+	return empty
 
 {- Runs an Action on a list of files in a git repository.
  -
-- 
cgit v1.2.3