added basic run statistics to the end of each run's log

1 files changed, 93 insertions, 3 deletions

diff --git a/headless/src/main/java/com/galois/fiveui/HeadlessRunner.java b/headless/src/main/java/com/galois/fiveui/HeadlessRunner.java
index b5f98ff..93e0ba7 100644
--- a/headless/src/main/java/com/galois/fiveui/HeadlessRunner.java
+++ b/headless/src/main/java/com/galois/fiveui/HeadlessRunner.java
@@ -22,6 +22,11 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.galois.fiveui.Result;
 import com.google.common.collect.ImmutableList;
@@ -52,15 +57,17 @@ import org.apache.log4j.BasicConfigurator;
 public class HeadlessRunner {
 
 	private static Logger logger = Logger.getLogger("com.galois.fiveui.HeadlessRunner");
+	private static Map<String, int[]> passFailMap = new HashMap<String, int[]>();
 	
     /**
      * @param args list of headless run description filenames
      * @throws IOException
      * @throws URISyntaxException
+     * @throws ParseException 
      */
     @SuppressWarnings("static-access")
 	public static void main(final String[] args)
-    	throws IOException, URISyntaxException {
+    	throws IOException, URISyntaxException, ParseException {
 
     	// Setup command line options
     	Options options = new Options();
@@ -87,7 +94,7 @@ public class HeadlessRunner {
         // Display help if requested
         if (cmd.hasOption("h")) {
         	HelpFormatter formatter = new HelpFormatter();
-        	formatter.printHelp("headless", options);
+        	formatter.printHelp("headless <input file 1> [<input file 2> ...]", options);
         	System.exit(1);
         }	
         
@@ -122,17 +129,100 @@ public class HeadlessRunner {
     	 
         // Process input files
         for (String in: cmd.getArgs()) {
-            logger.debug("parsing headless run description: " + in);
          	HeadlessRunDescription descr = HeadlessRunDescription.parse(in);
         	logger.debug("invoking headless run...");
         	BatchRunner runner = new BatchRunner();
             ImmutableList<Result> results = runner.runHeadless(descr);
             logger.debug("runHeadless returned " + results.size() + " results");
+            updateStats(results);
+            // write results to the output stream as we go
             for (Result result : results) {
                 outStream.println(result.toString());
             }
             outStream.flush();
         }
+        
+        reportStats();
         outStream.close();
     }
+
+    /**
+     * Update the global frequency map we're using to keep track of unique URLs,
+     * passes, and fails.
+     */
+    private static void updateStats(List<Result> results) throws ParseException {  
+    	// compute statistics on results
+        String url;
+        int pass, fail;
+        int[] passFailList;
+        Pattern numberPassedPattern = Pattern.compile("passed ([0-9]+) tests");
+        Matcher matcher;
+      
+        for (Result result : results) {
+        	pass = fail = 0;
+        	url = result.getURL();
+        	if (result.getType() == ResType.Pass) {
+        		// now we have to parse out how many tests passed
+        		matcher = numberPassedPattern.matcher(result.getDesc());
+        		if (matcher.find()) {
+	        		try {
+	        			pass = Integer.parseInt(matcher.group(1));
+	        		} catch (Exception e) {
+	        			e.printStackTrace();
+	        			logger.error("error parsing pass rate from \"" + result.getDesc() + "\"");
+	        		}
+        		}
+        	} else if (result.getType() == ResType.Error) {
+        		// each error result corresponds to one test
+        		fail = 1;
+        	}
+        	
+        	if (passFailMap.containsKey(url)) {
+        		passFailList = passFailMap.get(url);
+        	} else {
+        		passFailList = new int[] { 0, 0};
+        		passFailMap.put(url, passFailList);
+        	}
+    		passFailList[0] += pass;
+    		passFailList[1] += fail;
+    		//passFailMap.put(url, passFailList);
+        }
+    }
+    
+    /**
+     * Report statistics on the headless run to the log. Note, "pass"
+     * means the URL passed all tests in the ruleset, but "fail" can be
+     * reported for the same test on multiple offenders in the page.
+     */
+    private static void reportStats() {
+    	// report statistics on results
+        int uniqueURLs = passFailMap.size();
+        String stats = "\n\nRESULT STATISTICS\n\n"; // update as we go
+        int pass, fail;
+        int[] passFailList;
+        
+        stats += "Unique URLs: " + uniqueURLs + "\n";
+        for (String key: passFailMap.keySet()) {
+        	passFailList = passFailMap.get(key);
+        	pass = passFailList[0];
+        	fail = passFailList[1];
+        	stats += String.format("URL: %s, pass: %3d, fail %3d\n",
+        		(key.length()>40 ? key.substring(0,37)+"..." : key + sp(40-key.length())),
+        		pass, fail, pass+fail);
+        }
+        for (String s: stats.split("\n"))
+        	logger.info(s);
+    }
+    
+    /**
+     * return a string of n spaces
+     */
+    private static String sp(int n) {
+    	String s = "";
+    	for (int i=0; i<n; i++)
+    		s += " ";
+    	return s;
+    }
+    
 }
+