Add a new action for generating reconciled R classes for Robolectric.

This includes some refactoring:
* Move the symbol deserialization our of the merger and into the ParsedAndroidData (probably move again.)
* Change the FailedFutureAggregator generics to work more callables

RELNOTES: None
PiperOrigin-RevId: 156863698

8 files changed, 267 insertions, 66 deletions

diff --git a/src/tools/android/java/com/google/devtools/build/android/AndroidDataMerger.java b/src/tools/android/java/com/google/devtools/build/android/AndroidDataMerger.java
index b9a372d32c..44967168b9 100644
--- a/src/tools/android/java/com/google/devtools/build/android/AndroidDataMerger.java
+++ b/src/tools/android/java/com/google/devtools/build/android/AndroidDataMerger.java
@@ -17,12 +17,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.devtools.build.android.AndroidResourceMerger.MergingException;
-import com.google.devtools.build.android.ParsedAndroidData.Builder;
-import com.google.devtools.build.android.ParsedAndroidData.ParsedAndroidDataBuildingPathWalker;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
@@ -32,7 +29,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
@@ -42,44 +38,6 @@ class AndroidDataMerger {
 
   private static final Logger logger = Logger.getLogger(AndroidDataMerger.class.getCanonicalName());
 
-  private final class ParseDependencyDataTask implements Callable<Boolean> {
-
-    private final SerializedAndroidData dependency;
-
-    private final Builder targetBuilder;
-
-    private ParseDependencyDataTask(SerializedAndroidData dependency, Builder targetBuilder) {
-      this.dependency = dependency;
-      this.targetBuilder = targetBuilder;
-    }
-
-    @Override
-    public Boolean call() throws Exception {
-      final Builder parsedDataBuilder = ParsedAndroidData.Builder.newBuilder();
-      try {
-        dependency.deserialize(deserializer, parsedDataBuilder.consumers());
-      } catch (DeserializationException e) {
-        if (!e.isLegacy()) {
-          throw MergingException.wrapException(e);
-        }
-        logger.fine(
-            String.format(
-                "\u001B[31mDEPRECATION:\u001B[0m Legacy resources used for %s",
-                dependency.getLabel()));
-        // Legacy android resources -- treat them as direct dependencies.
-        dependency.walk(ParsedAndroidDataBuildingPathWalker.create(parsedDataBuilder));
-      }
-      // The builder isn't threadsafe, so synchronize the copyTo call.
-      synchronized (targetBuilder) {
-        // All the resources are sorted before writing, so they can be aggregated in
-        // whatever order here.
-        parsedDataBuilder.copyTo(targetBuilder);
-      }
-      // Had to return something?
-      return Boolean.TRUE;
-    }
-  }
-
   /** Interface for comparing paths. */
   interface SourceChecker {
     boolean checkEquality(DataSource one, DataSource two) throws IOException;
@@ -184,27 +142,12 @@ class AndroidDataMerger {
       throws MergingException {
     Stopwatch timer = Stopwatch.createStarted();
     try {
-      final ParsedAndroidData.Builder directBuilder = ParsedAndroidData.Builder.newBuilder();
-      final ParsedAndroidData.Builder transitiveBuilder = ParsedAndroidData.Builder.newBuilder();
-      final List<ListenableFuture<Boolean>> tasks = new ArrayList<>();
-      for (final SerializedAndroidData dependency : direct) {
-        tasks.add(executorService.submit(new ParseDependencyDataTask(dependency, directBuilder)));
-      }
-      for (final SerializedAndroidData dependency : transitive) {
-        tasks.add(
-            executorService.submit(new ParseDependencyDataTask(dependency, transitiveBuilder)));
-      }
-      // Wait for all the parsing to complete.
-      FailedFutureAggregator<MergingException> aggregator =
-          FailedFutureAggregator.createForMergingExceptionWithMessage(
-              "Failure(s) during dependency parsing");
-      aggregator.aggregateAndMaybeThrow(tasks);
       logger.fine(
           String.format("Merged dependencies read in %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
       timer.reset().start();
       return doMerge(
-          transitiveBuilder.build(),
-          directBuilder.build(),
+          ParsedAndroidData.loadedFrom(transitive, executorService, deserializer),
+          ParsedAndroidData.loadedFrom(direct, executorService, deserializer),
           primary,
           primaryManifest,
           allowPrimaryOverrideAll);
diff --git a/src/tools/android/java/com/google/devtools/build/android/AndroidResourceClassWriter.java b/src/tools/android/java/com/google/devtools/build/android/AndroidResourceClassWriter.java
index b64f9774f6..ec7a3286c9 100644
--- a/src/tools/android/java/com/google/devtools/build/android/AndroidResourceClassWriter.java
+++ b/src/tools/android/java/com/google/devtools/build/android/AndroidResourceClassWriter.java
@@ -44,7 +44,7 @@ public class AndroidResourceClassWriter implements Flushable, AndroidResourceSym
   public static AndroidResourceClassWriter of(
       AndroidFrameworkAttrIdProvider androidIdProvider, Path outputBasePath, String packageName) {
     return new AndroidResourceClassWriter(
-        new PlaceholderIdFieldInitializerBuilder(androidIdProvider), outputBasePath, packageName);
+        PlaceholderIdFieldInitializerBuilder.from(androidIdProvider), outputBasePath, packageName);
   }
   
   private final Path outputBasePath;
diff --git a/src/tools/android/java/com/google/devtools/build/android/AndroidResourceMerger.java b/src/tools/android/java/com/google/devtools/build/android/AndroidResourceMerger.java
index 7101033c89..8fc6075382 100644
--- a/src/tools/android/java/com/google/devtools/build/android/AndroidResourceMerger.java
+++ b/src/tools/android/java/com/google/devtools/build/android/AndroidResourceMerger.java
@@ -70,8 +70,6 @@ public class AndroidResourceMerger {
     final ListeningExecutorService executorService =
         MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(15));
     try (Closeable closeable = ExecutorServiceCloser.createWith(executorService)) {
-      AndroidDataMerger merger =
-          AndroidDataMerger.createWithPathDeduplictor(executorService, deserializer);
       UnwrittenMergedAndroidData merged =
           mergeData(
               executorService,
diff --git a/src/tools/android/java/com/google/devtools/build/android/FailedFutureAggregator.java b/src/tools/android/java/com/google/devtools/build/android/FailedFutureAggregator.java
index da61920803..06f49036e2 100644
--- a/src/tools/android/java/com/google/devtools/build/android/FailedFutureAggregator.java
+++ b/src/tools/android/java/com/google/devtools/build/android/FailedFutureAggregator.java
@@ -56,11 +56,12 @@ class FailedFutureAggregator<T extends Throwable> {
     this.exceptionFactory = exceptionFactory;
   }
 
-  /** Iterates throw a list of futures, throwing an Exception if any have failed. */
-  public void aggregateAndMaybeThrow(List<ListenableFuture<Boolean>> tasks) throws T {
+  /** Iterates a list of futures, throwing an Exception if any have failed. */
+  public <V> void aggregateAndMaybeThrow(List<? extends ListenableFuture<? extends V>> tasks)
+      throws T {
     // Retrieve all the exceptions and wrap them in an IOException.
     T exception = null;
-    for (ListenableFuture<Boolean> task : tasks) {
+    for (ListenableFuture<?> task : tasks) {
       try {
         task.get();
       } catch (ExecutionException | InterruptedException e) {
diff --git a/src/tools/android/java/com/google/devtools/build/android/GenerateRobolectricResourceSymbolsAction.java b/src/tools/android/java/com/google/devtools/build/android/GenerateRobolectricResourceSymbolsAction.java
new file mode 100644
index 0000000000..e95b646d6d
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/GenerateRobolectricResourceSymbolsAction.java
@@ -0,0 +1,176 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android;
+
+import com.android.builder.dependency.SymbolFileProvider;
+import com.android.resources.ResourceType;
+import com.google.common.base.Optional;
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.devtools.build.android.AndroidResourceProcessor.AaptConfigOptions;
+import com.google.devtools.build.android.Converters.DependencyAndroidDataListConverter;
+import com.google.devtools.build.android.Converters.PathConverter;
+import com.google.devtools.build.android.resources.RClassGenerator;
+import com.google.devtools.build.android.resources.ResourceSymbols;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import java.io.Closeable;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This action generates consistant ids R.class files for use in robolectric tests.
+ */
+public class GenerateRobolectricResourceSymbolsAction {
+
+  private static final Logger logger =
+      Logger.getLogger(GenerateRobolectricResourceSymbolsAction.class.getName());
+
+  private static final class WriteLibraryRClass implements Callable<Boolean> {
+    private final Entry<String, ListenableFuture<ResourceSymbols>> librarySymbolEntry;
+    private final RClassGenerator generator;
+
+    private WriteLibraryRClass(
+        Entry<String, ListenableFuture<ResourceSymbols>> librarySymbolEntry,
+        RClassGenerator generator) {
+      this.librarySymbolEntry = librarySymbolEntry;
+      this.generator = generator;
+    }
+
+    @Override
+    public Boolean call() throws Exception {
+      generator.write(
+          librarySymbolEntry.getKey(), librarySymbolEntry.getValue().get().asInitializers());
+      return true;
+    }
+  }
+
+  /** Flag specifications for this action. */
+  public static final class Options extends OptionsBase {
+
+    @Option(
+      name = "data",
+      defaultValue = "",
+      converter = DependencyAndroidDataListConverter.class,
+      category = "input",
+      help =
+          "Data dependencies. The expected format is "
+              + DependencyAndroidData.EXPECTED_FORMAT
+              + "[&...]"
+    )
+    public List<DependencyAndroidData> data;
+
+    @Option(
+      name = "classJarOutput",
+      defaultValue = "null",
+      converter = PathConverter.class,
+      category = "output",
+      help = "Path for the generated java class jar."
+    )
+    public Path classJarOutput;
+  }
+
+  public static void main(String[] args) throws Exception {
+
+    final Stopwatch timer = Stopwatch.createStarted();
+    OptionsParser optionsParser =
+        OptionsParser.newOptionsParser(Options.class, AaptConfigOptions.class);
+    optionsParser.enableParamsFileSupport(FileSystems.getDefault());
+    optionsParser.parseAndExitUponError(args);
+    AaptConfigOptions aaptConfigOptions = optionsParser.getOptions(AaptConfigOptions.class);
+    Options options = optionsParser.getOptions(Options.class);
+
+    try (ScopedTemporaryDirectory scopedTmp =
+        new ScopedTemporaryDirectory("robolectric_resources_tmp")) {
+      Path tmp = scopedTmp.getPath();
+      Path generatedSources = tmp.resolve("generated_resources");
+      // The reported availableProcessors may be higher than the actual resources
+      // (on a shared system). On the other hand, a lot of the work is I/O, so it's not completely
+      // CPU bound. As a compromise, divide by 2 the reported availableProcessors.
+      int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2);
+      ListeningExecutorService executorService =
+          MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads));
+      try (Closeable closeable = ExecutorServiceCloser.createWith(executorService)) {
+
+        logger.fine(String.format("Setup finished at %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
+
+        final PlaceholderIdFieldInitializerBuilder robolectricIds =
+            PlaceholderIdFieldInitializerBuilder.from(aaptConfigOptions.androidJar);
+        ParsedAndroidData.loadedFrom(
+                options.data, executorService, AndroidDataDeserializer.create())
+            .writeResourcesTo(
+                new AndroidResourceSymbolSink() {
+
+                  @Override
+                  public void acceptSimpleResource(ResourceType type, String name) {
+                    robolectricIds.addSimpleResource(type, name);
+                  }
+
+                  @Override
+                  public void acceptPublicResource(
+                      ResourceType type, String name, Optional<Integer> value) {
+                    robolectricIds.addPublicResource(type, name, value);
+                  }
+
+                  @Override
+                  public void acceptStyleableResource(
+                      FullyQualifiedName key, Map<FullyQualifiedName, Boolean> attrs) {
+                    robolectricIds.addStyleableResource(key, attrs);
+                  }
+                });
+
+        final RClassGenerator generator =
+            RClassGenerator.with(generatedSources, robolectricIds.build(), false);
+
+        List<SymbolFileProvider> libraries = new ArrayList<>();
+        for (DependencyAndroidData dataDep : options.data) {
+          SymbolFileProvider library = dataDep.asSymbolFileProvider();
+          libraries.add(library);
+        }
+        List<ListenableFuture<Boolean>> writeSymbolsTask = new ArrayList<>();
+        for (final Entry<String, ListenableFuture<ResourceSymbols>> librarySymbolEntry :
+            ResourceSymbols.loadFrom(libraries, executorService, null).entries()) {
+          writeSymbolsTask.add(
+              executorService.submit(new WriteLibraryRClass(librarySymbolEntry, generator)));
+        }
+        FailedFutureAggregator.forIOExceptionsWithMessage("Errors writing symbols.")
+            .aggregateAndMaybeThrow(writeSymbolsTask);
+      }
+
+      logger.fine(String.format("Merging finished at %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
+
+      AndroidResourceOutputs.createClassJar(generatedSources, options.classJarOutput);
+      System.out.println(options.classJarOutput);
+      logger.fine(
+          String.format("Create classJar finished at %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
+
+    } catch (Exception e) {
+      logger.log(Level.SEVERE, "Unexpected", e);
+      throw e;
+    }
+    logger.fine(String.format("Resources merged in %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
+  }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ParsedAndroidData.java b/src/tools/android/java/com/google/devtools/build/android/ParsedAndroidData.java
index 36eeabe9bd..cd3401e1db 100644
--- a/src/tools/android/java/com/google/devtools/build/android/ParsedAndroidData.java
+++ b/src/tools/android/java/com/google/devtools/build/android/ParsedAndroidData.java
@@ -24,6 +24,8 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Sets.SetView;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.devtools.build.android.AndroidResourceMerger.MergingException;
 import com.google.devtools.build.android.xml.StyleableXmlResourceValue;
 import java.io.IOException;
@@ -42,6 +44,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.logging.Logger;
 import javax.annotation.concurrent.Immutable;
 import javax.annotation.concurrent.NotThreadSafe;
@@ -394,6 +397,70 @@ public class ParsedAndroidData {
     return pathWalker.createParsedAndroidData();
   }
 
+  private static final class ParseDependencyDataTask implements Callable<Void> {
+
+    private final SerializedAndroidData dependency;
+
+    private final Builder targetBuilder;
+
+    private final AndroidDataDeserializer deserializer;
+
+    private ParseDependencyDataTask(
+        AndroidDataDeserializer deserializer,
+        SerializedAndroidData dependency,
+        Builder targetBuilder) {
+      this.deserializer = deserializer;
+      this.dependency = dependency;
+      this.targetBuilder = targetBuilder;
+    }
+
+    @Override
+    public Void call() throws Exception {
+      final Builder parsedDataBuilder = ParsedAndroidData.Builder.newBuilder();
+      try {
+        dependency.deserialize(deserializer, parsedDataBuilder.consumers());
+      } catch (DeserializationException e) {
+        if (!e.isLegacy()) {
+          throw MergingException.wrapException(e);
+        }
+        logger.fine(
+            String.format(
+                "\u001B[31mDEPRECATION:\u001B[0m Legacy resources used for %s",
+                dependency.getLabel()));
+        // Legacy android resources -- treat them as direct dependencies.
+        dependency.walk(ParsedAndroidDataBuildingPathWalker.create(parsedDataBuilder));
+      }
+      // The builder isn't threadsafe, so synchronize the copyTo call.
+      synchronized (targetBuilder) {
+        // All the resources are sorted before writing, so they can be aggregated in
+        // whatever order here.
+        parsedDataBuilder.copyTo(targetBuilder);
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Deserializes data and merges them into a single {@link ParsedAndroidData}.
+   *
+   * @throws MergingException for deserialization errors.
+   */
+  public static ParsedAndroidData loadedFrom(
+      List<? extends SerializedAndroidData> data,
+      ListeningExecutorService executorService,
+      AndroidDataDeserializer deserializer) {
+    List<ListenableFuture<Void>> tasks = new ArrayList<>();
+    final Builder target = Builder.newBuilder();
+    for (SerializedAndroidData serialized : data) {
+      tasks.add(
+          executorService.submit(new ParseDependencyDataTask(deserializer, serialized, target)));
+    }
+    FailedFutureAggregator.createForMergingExceptionWithMessage(
+            "Failure(s) during dependency parsing")
+        .aggregateAndMaybeThrow(tasks);
+    return target.build();
+  }
+
   private final ImmutableSet<MergeConflict> conflicts;
   private final ImmutableMap<DataKey, DataResource> overwritingResources;
   private final ImmutableMap<DataKey, DataResource> combiningResources;
diff --git a/src/tools/android/java/com/google/devtools/build/android/PlaceholderIdFieldInitializerBuilder.java b/src/tools/android/java/com/google/devtools/build/android/PlaceholderIdFieldInitializerBuilder.java
index 189ea76d8e..0065f8fc15 100644
--- a/src/tools/android/java/com/google/devtools/build/android/PlaceholderIdFieldInitializerBuilder.java
+++ b/src/tools/android/java/com/google/devtools/build/android/PlaceholderIdFieldInitializerBuilder.java
@@ -26,6 +26,7 @@ import com.google.devtools.build.android.resources.FieldInitializer;
 import com.google.devtools.build.android.resources.FieldInitializers;
 import com.google.devtools.build.android.resources.IntArrayFieldInitializer;
 import com.google.devtools.build.android.resources.IntFieldInitializer;
+import java.nio.file.Path;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.HashMap;
@@ -158,6 +159,15 @@ class PlaceholderIdFieldInitializerBuilder {
     return resourceName.replace('.', '_');
   }
 
+  public static PlaceholderIdFieldInitializerBuilder from(
+      AndroidFrameworkAttrIdProvider androidIdProvider) {
+    return new PlaceholderIdFieldInitializerBuilder(androidIdProvider);
+  }
+
+  public static PlaceholderIdFieldInitializerBuilder from(Path androidJar) {
+    return from(new AndroidFrameworkAttrIdJar(androidJar));
+  }
+
   private final AndroidFrameworkAttrIdProvider androidIdProvider;
 
   private final Map<ResourceType, Set<String>> innerClasses = new EnumMap<>(ResourceType.class);
@@ -167,7 +177,7 @@ class PlaceholderIdFieldInitializerBuilder {
 
   private final Map<String, Map<String, Boolean>> styleableAttrs = new HashMap<>();
 
-  public PlaceholderIdFieldInitializerBuilder(AndroidFrameworkAttrIdProvider androidIdProvider) {
+  private PlaceholderIdFieldInitializerBuilder(AndroidFrameworkAttrIdProvider androidIdProvider) {
     this.androidIdProvider = androidIdProvider;
   }
 
diff --git a/src/tools/android/java/com/google/devtools/build/android/ResourceProcessorBusyBox.java b/src/tools/android/java/com/google/devtools/build/android/ResourceProcessorBusyBox.java
index d9a3204465..59f0aebdd4 100644
--- a/src/tools/android/java/com/google/devtools/build/android/ResourceProcessorBusyBox.java
+++ b/src/tools/android/java/com/google/devtools/build/android/ResourceProcessorBusyBox.java
@@ -71,6 +71,12 @@ public class ResourceProcessorBusyBox {
         LibraryRClassGeneratorAction.main(args);
       }
     },
+    GENERATE_ROBOLECTRIC_R() {
+      @Override
+      void call(String[] args) throws Exception {
+        GenerateRobolectricResourceSymbolsAction.main(args);
+      }
+    },
     PARSE() {
       @Override
       void call(String[] args) throws Exception {