incorporated multiple web drivers (chrome, ff) and improved result reporting

7 files changed, 165 insertions, 100 deletions

diff --git a/contexts/batch/src/main/java/com/galois/fiveui/BatchExecutor.java b/contexts/batch/src/main/java/com/galois/fiveui/BatchExecutor.java
index dc1c135..3d97055 100644
--- a/contexts/batch/src/main/java/com/galois/fiveui/BatchExecutor.java
+++ b/contexts/batch/src/main/java/com/galois/fiveui/BatchExecutor.java
@@ -17,99 +17,57 @@
  */
 package com.galois.fiveui;
 
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
 
 import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
 import org.openqa.selenium.firefox.FirefoxDriver;
 
-import com.google.common.base.Predicate;
-import com.google.common.collect.Collections2;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableList.Builder;
-import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
 
 /**
  * @author creswick
  * 
  */
 public class BatchExecutor {
-//
-//    private static final Predicate<String> isRuleSet = new Predicate<String>() {
-//        public boolean apply(final String input) {
-//            return input.endsWith(".json");
-//        }
-//    };
-//
-//    private static final Predicate<String> isUrl = new Predicate<String>() {
-//        public boolean apply(final String input) {
-//            return input.startsWith("http");
-//        }
-//    };
+
+    private static final ImmutableList<WebDriver> DRIVERS = ImmutableList.of(
+            Drivers.buildFFDriver(), Drivers.buildChromeDriver());
 
     /**
      * @param args
      * @throws IOException
-     * @throws URISyntaxException 
+     * @throws URISyntaxException
      */
-    public static void main(final String[] args) throws IOException, URISyntaxException {
-        String runDescFile = args[0];
-        RunDescription descr = RunDescription.deserialize(runDescFile);
+    public static void main(final String[] args)
+            throws IOException, URISyntaxException {
+        
+        for (int i = 0; i < args.length; i++) {
+            String runDescFile = args[i];
+            RunDescription descr = RunDescription.parse(runDescFile);
 
-        WebDriver driver = new FirefoxDriver();
-        ImmutableList<Result> results;
-        try {
-            BatchRunner runner = new BatchRunner(driver);
+            for (WebDriver driver : DRIVERS) {
+                try {
+                    ImmutableList<Result> results = invokeTest(descr, driver);
 
-            results = runner.runTests(descr.getTests());
-            
-            for (Result result : results) {
-                System.out.println(result);
+                    for (Result result : results) {
+                        System.out.println(result);
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                } finally {
+                    driver.quit();
+                }
             }
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            driver.close();
         }
     }
-//
-//    private static ImmutableList<String> readRuleSets(Config conf) {
-//        Builder<String> contentBuilder = ImmutableList.builder();
-//        for (String fileName : conf.getRuleSets()) {
-//            try {
-//                contentBuilder.add(Utils.readFile(fileName));
-//            } catch (IOException e) {
-//                System.err.println("could not read file: " + fileName);
-//                e.printStackTrace();
-//            }
-//        }
-//
-//        return contentBuilder.build();
-//    }
-//
-//    private static Config parseArgs(final ArrayList<String> args) {
-//
-//        Collection<String> ruleSets = Collections2.filter(args, isRuleSet);
-//        Collection<String> rawUrls = Collections2.filter(args, isUrl);
-//
-//        Builder<URI> b = ImmutableList.builder();
-//        for (String str : rawUrls) {
-//            try {
-//                b.add(new URI(str));
-//            } catch (URISyntaxException e) {
-//                System.err.println("Warning: Could not parse: " + str
-//                        + " into URI: \n" + e);
-//                System.err.println("         Discarding " + str + " as uri.");
-//            }
-//        }
-//        return new Config(ImmutableList.copyOf(ruleSets), b.build());
-//    }
+    
+    private static ImmutableList<Result> invokeTest(RunDescription descr,
+            WebDriver driver) throws IOException {
+        BatchRunner runner = new BatchRunner(driver);
+
+        return runner.runTests(descr.getTests());
+
+    }
 }
diff --git a/contexts/batch/src/main/java/com/galois/fiveui/BatchRunner.java b/contexts/batch/src/main/java/com/galois/fiveui/BatchRunner.java
index 7ac5baa..a6996ab 100644
--- a/contexts/batch/src/main/java/com/galois/fiveui/BatchRunner.java
+++ b/contexts/batch/src/main/java/com/galois/fiveui/BatchRunner.java
@@ -67,31 +67,34 @@ public class BatchRunner {
     public ImmutableList<Result> runTest(final RuleTest test) {
         RuleSet rule = test.getRule();
         
-        ImmutableList<Result> results;
+        ImmutableList<Result> rawResults;
+        Builder<Result> builder = ImmutableList.builder();
         try {
             _driver.get(test.getUri().toString());
-            results = runRule(rule);
+            rawResults = runRule(rule);
             
             List<ResType> oracle = Lists.newArrayList(test.getOracle());
-            for (Result result : results) {
-                // TODO build up Results object *here*
+            for (Result result : rawResults) {
+                Result res;
                 if ( oracle.remove(result.getType()) ) {
-                    System.out.println("Success: expected result type: "+result.getType());
+                    res = Result.pass(_driver,
+                            test.getRuleId() + ": Got expected result: "+result.getType());
                 } else {
-                    System.out.println("Failure: expected result type: "+result.getType());
+                    res = Result.error(_driver,
+                            test.getRuleId() + ": Unexpected Result: "+result);
                 }
+                builder.add(res);
             }
-//            System.out.println(results);
         } catch (Exception e) {
             String errStr = "Could not run rule: " + rule.getName() + "\n";
             errStr += e.toString();
-            results = ImmutableList.of(
-                    Result.exception("Could not run rule: "+errStr));
+            rawResults = ImmutableList.of(
+                    Result.exception(_driver, "Could not run rule: "+errStr));
             
             e.printStackTrace();
         }
         
-        return results;
+        return builder.build();
     }
 
     private ImmutableList<Result> runRule(final RuleSet ruleSet) throws IOException {
@@ -111,7 +114,7 @@ public class BatchRunner {
         if (res.getClass() == String.class) {
             // we received an error via the expected mechanisms:
             System.err.println("Exception running rule: " + res);
-            builder.add(Result.exception((String) res));
+            builder.add(Result.exception(_driver, (String) res));
             return builder.build();
         } else {
 
@@ -120,18 +123,18 @@ public class BatchRunner {
                 List<Map<String, Map<String, String>>> results = (List) res;
 
                 if (0 == results.size()) {
-                    builder.add(Result.pass("passed"));
+                    builder.add(Result.pass(_driver, "passed"));
                 }
 
                 for (Map<String, Map<String, String>> r : results) {
                     Map<String, String> problem = r.get("payload");
                     
-                    builder.add(Result.error(problem.get("descr")));
+                    builder.add(Result.error(_driver, problem.get("descr")));
                 }
 
             } catch (ClassCastException e) {
                 // An unexpected error happened:
-                builder.add(Result.exception("Unexpected object returned: "
+                builder.add(Result.exception(_driver, "Unexpected object returned: "
                         + res));
                 e.printStackTrace();
             }
diff --git a/contexts/batch/src/main/java/com/galois/fiveui/Drivers.java b/contexts/batch/src/main/java/com/galois/fiveui/Drivers.java
new file mode 100644
index 0000000..60d2b3b
--- /dev/null
+++ b/contexts/batch/src/main/java/com/galois/fiveui/Drivers.java
@@ -0,0 +1,96 @@
+/**
+ * Module : Drivers.java Copyright : (c) 2011-2012, Galois, Inc.
+ * 
+ * Maintainer : Stability : Provisional Portability: Portable
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.galois.fiveui;
+
+import java.io.File;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.openqa.selenium.firefox.FirefoxDriver;
+import org.openqa.selenium.firefox.FirefoxProfile;
+
+/**
+ * @author creswick
+ * 
+ */
+public class Drivers {
+    private static final String CD_BINARY_NAME = "chromedriver";
+    private static final String CD_BASE_PATH = mkPath("..", "..", "tools",
+            "seleniumChromeDrivers");
+
+    private static final String CHROME_PROFILE = mkPath("..", "..", "profiles",
+            "chrome");
+
+    private static final String FF_PROFILE = mkPath("..", "..", "profiles",
+            "firefox");
+    
+    public static WebDriver buildFFDriver() {
+        // Extracted into a method so we can set up profiles
+
+        File profileDir = new File(FF_PROFILE);
+        FirefoxProfile profile = new FirefoxProfile(profileDir);
+        FirefoxDriver driver = new FirefoxDriver(profile);
+
+        return driver;
+    }
+
+    public static WebDriver buildChromeDriver() {
+        // set the chrome driver path:
+        String chromeDriverPth =
+                mkPath(CD_BASE_PATH, osNameArch(), CD_BINARY_NAME);
+        System.setProperty("webdriver.chrome.driver", chromeDriverPth);
+
+        ChromeOptions options = new ChromeOptions();
+        options.addArguments("--user-data-dir="+CHROME_PROFILE);
+
+        return new ChromeDriver(options);
+    }
+
+    private static String mkPath(String... components) {
+        StringBuilder path = new StringBuilder();
+        int remaining = components.length;
+        for (String c : components) {
+            path.append(c);
+            remaining--;
+            if (remaining != 0) {
+                path.append(File.separator);
+            }
+        }
+
+        return path.toString();
+    }
+
+    /**
+     * Determine the name of the directory that the chromedriver is in, based on
+     * os.name and os.arch.
+     * 
+     * @return The name of the directory containing 'chromedriver'
+     */
+    private static String osNameArch() {
+        String rawOsName = System.getProperty("os.name").toLowerCase();
+        String osName = rawOsName.substring(0, 3);
+        boolean is64bit = System.getProperty("os.arch").indexOf("64") >= 0;
+
+        if (osName.equals("lin")) {
+            osName += is64bit ? "64" : "32";
+        }
+        return osName;
+    }
+
+}
diff --git a/contexts/batch/src/main/java/com/galois/fiveui/Result.java b/contexts/batch/src/main/java/com/galois/fiveui/Result.java
index 91f889b..2f80aca 100644
--- a/contexts/batch/src/main/java/com/galois/fiveui/Result.java
+++ b/contexts/batch/src/main/java/com/galois/fiveui/Result.java
@@ -17,31 +17,35 @@
  */
 package com.galois.fiveui;
 
+import org.openqa.selenium.WebDriver;
+
 public class Result {
 
-    public static Result exception(String res) {
-        return new Result(ResType.Exception, res);
+    public static Result exception(WebDriver driver, String res) {
+        return new Result(ResType.Exception, driver, res);
     }
     
-    public static Result pass(String res) {
-        return new Result(ResType.Pass, res);
+    public static Result pass(WebDriver driver, String res) {
+        return new Result(ResType.Pass, driver, res);
     }
     
-    public static Result error(String res) {
-        return new Result(ResType.Error, res);
+    public static Result error(WebDriver driver, String res) {
+        return new Result(ResType.Error, driver, res);
     }
 
-    public static Result warning(String res) {
-        return new Result(ResType.Warning, res);
+    public static Result warning(WebDriver driver, String res) {
+        return new Result(ResType.Warning, driver, res);
     }
     
     private ResType _type;
     private String _desc;
+    private WebDriver _driver;
     
-    private Result(ResType type, String desc) {
+    private Result(ResType type, WebDriver driver, String desc) {
         super();
         _type = type;
         _desc = desc;
+        _driver = driver;
     }
 
     public ResType getType() {
@@ -51,9 +55,13 @@ public class Result {
     public String getDesc() {
         return _desc;
     }
+    
+    public WebDriver getDriver() {
+        return _driver;
+    }
 
     @Override
     public String toString() {
-        return getType() + ": " + getDesc();
+        return getType() + " - " + _driver + ": " + getDesc();
     }
 }
diff --git a/contexts/batch/src/main/java/com/galois/fiveui/RunDescription.java b/contexts/batch/src/main/java/com/galois/fiveui/RunDescription.java
index 7b34a61..9139460 100644
--- a/contexts/batch/src/main/java/com/galois/fiveui/RunDescription.java
+++ b/contexts/batch/src/main/java/com/galois/fiveui/RunDescription.java
@@ -61,7 +61,7 @@ public class RunDescription {
      * @return A populated RunDescription object.
      * @throws FileNotFoundException if runDescFile can't be found.
      */
-    public static RunDescription deserialize(String runDescFile) 
+    public static RunDescription parse(String runDescFile) 
             throws FileNotFoundException {
 
         GsonBuilder gsonBuilder = new GsonBuilder();
diff --git a/contexts/batch/src/test/java/com/galois/fiveui/RunDescriptionTest.java b/contexts/batch/src/test/java/com/galois/fiveui/RunDescriptionTest.java
index 46cfb72..d6f1e76 100644
--- a/contexts/batch/src/test/java/com/galois/fiveui/RunDescriptionTest.java
+++ b/contexts/batch/src/test/java/com/galois/fiveui/RunDescriptionTest.java
@@ -45,7 +45,7 @@ public class RunDescriptionTest {
                         new ArrayList<RunDescription.URIMap>(), rsOracle); 
         
         
-        RunDescription actual = RunDescription.deserialize(jsonFileName);
+        RunDescription actual = RunDescription.parse(jsonFileName);
         assertObjEqual("Object deserialized incorrectly.", oracle, actual);
     }
 
@@ -69,7 +69,7 @@ public class RunDescriptionTest {
                         new ArrayList<RunDescription.URIMap>(), rsOracle); 
         
         
-        RunDescription actual = RunDescription.deserialize(jsonFileName);
+        RunDescription actual = RunDescription.parse(jsonFileName);
         assertObjEqual("Object deserialized incorrectly.", oracle, actual);
     }
 
diff --git a/contexts/batch/src/test/resources/sample.json b/contexts/batch/src/test/resources/runDescriptions/headingSample.json
index 0104d1c..2659148 100644
--- a/contexts/batch/src/test/resources/sample.json
+++ b/contexts/batch/src/test/resources/runDescriptions/headingSample.json
@@ -1,5 +1,5 @@
 {
-  'ruleSet': '../../../../../exampleData/ruleSets/headingGuidelines.json',
+  'ruleSet': '../../../../../../exampleData/ruleSets/headingGuidelines.json',
   'tests': [ { 'url': 'http://localhost:8000/exampleData/basic/headings.html',
                'oracle': [ { 'ruleId': 1
                            , 'results': ['Error', 'Error']