Make external repository implementations not re-fetch things on server restart.

This is accomplished by saving a proto of the repository rule in the output tree, then comparing it to that of the previous version. This makes HTTP_DOWNLOAD_CHECKER somewhat superfluous because it only matters if the external repository directory is modified manually.

Local repository implementations are not included, mainly because the symlinking is cheap (maybe they should be for reasons of symmetry?)

--
MOS_MIGRATED_REVID=108706396

3 files changed, 433 insertions, 0 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index e547541985..fdb1029b73 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -510,12 +510,14 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:vfs",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/common/options",
+        "//src/main/protobuf:build_proto",
         "//src/main/protobuf:extra_actions_base_proto",
         "//third_party:guava",
         "//third_party:guava-testlib",
         "//third_party:jsr305",
         "//third_party:junit4",
         "//third_party:mockito",
+        "//third_party:protobuf",
         "//third_party:truth",
     ],
 )
@@ -550,6 +552,7 @@ java_test(
         "//third_party:guava-testlib",
         "//third_party:jsr305",
         "//third_party:junit4",
+        "//third_party:protobuf",
         "//third_party:truth",
     ],
 )
diff --git a/src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java b/src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java
new file mode 100644
index 0000000000..fa98a9de12
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java
@@ -0,0 +1,282 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
+import com.google.devtools.build.lib.packages.util.PackageSerializationTestCase;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.Attribute.Discriminator;
+import com.google.devtools.build.lib.syntax.GlobCriteria;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.ExtensionRegistryLite;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Unit tests for package serialization and deserialization.
+ */
+public class PackageSerializationTest extends PackageSerializationTestCase {
+  @Override
+  protected List<EnvironmentExtension> getPackageEnvironmentExtensions() {
+    return ImmutableList.of();
+  }
+
+  public void testLocationsOmitted() throws Exception {
+    Package pkg = scratchPackage("bacon",
+        "sh_library(name='bacon',",
+        "           srcs=['bacon.sh'])");
+    // By default we keep full location.
+    Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
+    Rule rule2 = pkg2.getRule("bacon");
+
+    assertEmpty(rule2.getLocation());
+    assertEmpty(rule2.getAttributeLocation("name"));
+    assertEmpty(rule2.getAttributeLocation("srcs"));
+  }
+
+  public void testGlobInformationKept() throws Exception {
+    scratch.file("ham/head.sh");
+    scratch.file("ham/tbone.sh");
+    Package pkg = scratchPackage("ham",
+        "sh_library(name='ham',",
+        "           srcs=glob(['*.sh'], exclude=['tbo*.sh']))");
+    Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
+    Rule ham = pkg2.getRule("ham");
+    GlobList<?> globs = Rule.getGlobInfo(RawAttributeMapper.of(ham)
+        .get("srcs", BuildType.LABEL_LIST));
+    assertThat(globs).containsExactly(Label.parseAbsolute("//ham:head.sh"));
+    List<GlobCriteria> criteria = globs.getCriteria();
+    assertThat(criteria).hasSize(1);
+    assertThat(criteria.get(0).getIncludePatterns()).containsExactly("*.sh");
+    assertThat(criteria.get(0).getExcludePatterns()).containsExactly("tbo*.sh");
+    assertTrue(criteria.get(0).isGlob());
+  }
+
+  public void testCanConcatenateGlobs() throws Exception {
+    scratch.file("serrano/sinew.sh");
+    scratch.file("serrano/muscle.sh");
+    scratch.file("serrano/bone.sh");
+    Package pkg = scratchPackage("serrano",
+        "sh_library(name='serrano',",
+        "           srcs=glob(['s*.sh']) + ['muscle.sh'] + glob(['b*.sh']))");
+    Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
+    Rule ham = pkg2.getRule("serrano");
+    GlobList<?> globs = Rule.getGlobInfo(RawAttributeMapper.of(ham)
+        .get("srcs", BuildType.LABEL_LIST));
+    assertThat(globs).containsExactly(
+        Label.parseAbsolute("//serrano:sinew.sh"),
+        Label.parseAbsolute("//serrano:muscle.sh"),
+        Label.parseAbsolute("//serrano:bone.sh"));
+    List<GlobCriteria> criteria = globs.getCriteria();
+    assertThat(criteria).hasSize(3);
+    assertThat(criteria.get(0).getIncludePatterns()).containsExactly("s*.sh");
+    assertTrue(criteria.get(0).isGlob());
+    assertFalse(criteria.get(1).isGlob());
+    assertThat(criteria.get(2).getIncludePatterns()).containsExactly("b*.sh");
+    assertTrue(criteria.get(2).isGlob());
+  }
+
+  public void testEvents() throws Exception {
+    Package pkg = scratchPackage("j", "sh_library(name='s', srcs='s.sh')");
+    Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
+    assertThat(pkg2.getEvents()).hasSize(1);
+    Event ev = pkg2.getEvents().get(0);
+    assertThat(ev.getMessage()).contains("expected value of type 'list(label)'");
+  }
+
+  public void testPermanentErrorBitIsKept() throws Exception {
+    Package pkg = scratchPackage("g", "sh_library(name='g', srcs=g.sh)");
+    Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
+    assertTrue(pkg2.containsErrors());
+  }
+
+  public void testBasicSerialization() throws Exception {
+    checkSerialization(scratchPackage("bacon",
+        "sh_library(name='bacon',",
+        "           srcs=['bacon.sh'],",
+        "           data=glob(['*.txt']))"));
+  }
+
+  public void testIntList() throws Exception {
+    Build.Attribute.Builder attrPb = Build.Attribute.newBuilder();
+    attrPb.setName("has_int_list").setType(Discriminator.INTEGER_LIST);
+    attrPb.addIntListValue(1).addIntListValue(2).addIntListValue(3).addIntListValue(4);
+    assertThat(PackageDeserializer.deserializeAttributeValue(
+        com.google.devtools.build.lib.syntax.Type.INTEGER_LIST, attrPb.build()))
+        .isEqualTo(ImmutableList.of(1, 2, 3, 4));
+  }
+
+  public void testExternalPackageLabel() throws Exception {
+    PackageIdentifier packageId = PackageIdentifier.create("@foo", new PathFragment("p"));
+    Package pkg = scratchPackage(packageId, "filegroup(name = 'rumple', srcs = ['dark_one'])");
+    assertEquals(packageId, pkg.getPackageIdentifier());
+    Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
+    assertEquals(pkg.getPackageIdentifier(), pkg2.getPackageIdentifier());
+  }
+
+  public void testConfigurableAttributes() throws Exception {
+    Package pkg = scratchPackage("pkg",
+        "sh_library(",
+        "    name = 'bread',",
+        "    srcs = select({",
+        "        ':one': ['rye.sh'],",
+        "        ':two': ['wheat.sh'],",
+        "    }))");
+    Package deserialized = deserializer.deserialize(codedInputFromPackage(pkg));
+    AttributeMap attrs = RawAttributeMapper.of(deserialized.getRule("bread"));
+    // We expect package serialization to "flatten" configurable attributes, e.g. the deserialized
+    // rule should look like "srcs = ['rye.sh', 'wheat.sh']" (without the select). Eventually
+    // we'll want to preserve the original structure, but that requires syntactic changes to the
+    // proto which we'll need to ensure the depserver understands.
+    assertThat(attrs.get("srcs", BuildType.LABEL_LIST)).containsExactly(
+        Label.parseAbsolute("//pkg:rye.sh"), Label.parseAbsolute("//pkg:wheat.sh"));
+  }
+
+  public void testConfigurableDictionaryAttribute() throws Exception {
+    // Although we expect package serialization to "flatten" configurable attributes, as described
+    // above, package deserialization should not crash if multiple configuration entries for a
+    // map valued attribute have keys in common.
+    checkSerialization(scratchPackage("pkg",
+        "cc_library(",
+        "    name = 'bread',",
+        "    srcs = ['PROTECTED/rye.cc'],",
+        "    abi_deps = select({",
+        "        ':one': {'duplicated_key': [':value1']},",
+        "        ':two': {'duplicated_key': [':value2']},",
+        "    }))"));
+  }
+
+  public void testRuleAttributesWithNullValuesAreIncludedInSerializedRepresentation()
+      throws Exception {
+    Package pkg = scratchPackage("lettuce",
+        "sh_library(name='lettuce',",
+        "           srcs=['romaine.sh'])");
+
+    // Manually parse stream, we're interested in wire representation.
+    CodedInputStream codedIn = codedInputFromPackage(pkg);
+    readPackage(codedIn);
+    Multimap<Build.Target.Discriminator, Build.Target> targets = readTargets(codedIn);
+
+    // Check that we encoded "deprecation" in the rule proto output even though it had no value.
+    Build.Rule pbRule =
+        Iterables.getOnlyElement(targets.get(Build.Target.Discriminator.RULE)).getRule();
+    boolean foundEmptyAttribute = false;
+    for (Build.Attribute attribute : pbRule.getAttributeList()) {
+      if (attribute.getName().equals("deprecation")) {
+        assertFalse(attribute.hasStringValue());
+        foundEmptyAttribute = true;
+      }
+    }
+    assertTrue(foundEmptyAttribute);
+  }
+
+  public void testTargetsIndividuallySerializedAndDeserialized() throws Exception {
+    scratchPackage("empty",
+        "sh_library(name='noop')");
+
+    Package pkg = scratchPackage("food",
+        "sh_library(name='pork',",
+        "           srcs=['pig.sh'])",
+        "sh_library(name='salmon',",
+        "           srcs=['river.sh'])",
+        "package_group(name='self',",
+        "           packages=['//food'])");
+
+    // Manually parse stream, we're interested in wire representation.
+    CodedInputStream codedIn = codedInputFromPackage(pkg);
+    readPackage(codedIn);
+
+    Multimap<Build.Target.Discriminator, Build.Target> targets = readTargets(codedIn);
+    assertThat(targets).hasSize(6);
+    assertThat(targets.get(Build.Target.Discriminator.SOURCE_FILE)).hasSize(3);
+    assertThat(targets.get(Build.Target.Discriminator.RULE)).hasSize(2);
+    assertThat(targets.get(Build.Target.Discriminator.PACKAGE_GROUP)).hasSize(1);
+
+    // Make sure we see the same thing when deserializing all the way.
+    Package deserialized = deserializer.deserialize(codedInputFromPackage(pkg));
+    assertThat(deserialized.getTargets(InputFile.class)).hasSize(3);
+    assertThat(deserialized.getTargets(Rule.class)).hasSize(2);
+    assertThat(deserialized.getTargets(PackageGroup.class)).hasSize(1);
+  }
+
+  public void testMassivePackageDeserializesFine() throws Exception {
+    // Create a package definition which exports 2^16 files with name lengths 2^10 each, which
+    // should result in 2^26 (64MB) of targets. With overhead this should push us comfortably
+    // over the 64MB default protocol buffer deserialization limit.
+    StringBuilder sb = new StringBuilder();
+    sb.append("exports_files([");
+    String srcName = Strings.repeat("x", 1 << 10);
+    for (int i = 0; i < (1 << 16); i++) {
+      sb.append("'").append(srcName).append(i).append("',");
+    }
+    sb.append("'last'])");
+
+    Package pkg = scratchPackage("meat", sb.toString());
+
+    // Check that we created our package correctly. There should be 2^16 + 1 dummy targets from
+    // our exports_files, plus the BUILD file.
+    assertThat(pkg.containsErrors()).isFalse();
+    assertThat(pkg.getTargets()).hasSize((1 << 16) + 2);
+
+    checkSerialization(pkg);
+  }
+
+  private static Multimap<Build.Target.Discriminator, Build.Target> readTargets(
+      CodedInputStream codedIn) throws IOException {
+    Multimap<Build.Target.Discriminator, Build.Target> targets = ArrayListMultimap.create();
+    Build.TargetOrTerminator tot;
+    while (!(tot = readNext(codedIn)).getIsTerminator()) {
+      Build.Target pbTarget = tot.getTarget();
+      targets.put(pbTarget.getType(), pbTarget);
+    }
+    return targets;
+  }
+
+  private static Build.Package readPackage(CodedInputStream codedIn) throws IOException {
+    Build.Package.Builder builder = Build.Package.newBuilder();
+    codedIn.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
+    return builder.build();
+  }
+
+  private static Build.TargetOrTerminator readNext(CodedInputStream codedIn) throws IOException {
+    Build.TargetOrTerminator.Builder builder = Build.TargetOrTerminator.newBuilder();
+    codedIn.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
+    return builder.build();
+  }
+
+  private void assertEmpty(Location location) {
+    assertEquals(0, location.getStartOffset());
+    assertEquals(0, location.getEndOffset());
+    assertEquals(new PathFragment("/dev/null"), location.getPath());
+    assertEquals(0, location.getStartLineAndColumn().getLine());
+    assertEquals(0, location.getStartLineAndColumn().getColumn());
+    assertEquals(0, location.getEndLineAndColumn().getLine());
+    assertEquals(0, location.getEndLineAndColumn().getColumn());
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java b/src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java
new file mode 100644
index 0000000000..8217e8feef
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java
@@ -0,0 +1,148 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages.util;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageDeserializer;
+import com.google.devtools.build.lib.packages.PackageDeserializer.AttributesToDeserialize;
+import com.google.devtools.build.lib.packages.PackageDeserializer.PackageDeserializationEnvironment;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
+import com.google.devtools.build.lib.packages.PackageSerializer;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build;
+import com.google.devtools.build.lib.testutil.FoundationTestCase;
+import com.google.devtools.build.lib.testutil.TestConstants;
+import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Provides test infrastructure for package serialization tests. */
+public abstract class PackageSerializationTestCase extends FoundationTestCase {
+
+  private CachingPackageLocator packageLocator;
+  private Map<PackageIdentifier, Path> buildFileMap;
+  private PackageFactory packageFactory;
+  private RuleClassProvider ruleClassProvider;
+
+  protected PackageSerializer serializer;
+  protected PackageDeserializer deserializer;
+
+  protected abstract List<EnvironmentExtension> getPackageEnvironmentExtensions();
+
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+
+    reporter = new Reporter();
+    buildFileMap = new HashMap<>();
+
+    packageLocator = new CachingPackageLocator() {
+      @Override
+      public Path getBuildFileForPackage(PackageIdentifier packageName) {
+        return buildFileMap.get(packageName);
+      }
+    };
+
+    ruleClassProvider = TestRuleClassProvider.getRuleClassProvider();
+    packageFactory = new PackageFactory(ruleClassProvider, getPackageEnvironmentExtensions());
+    serializer = new PackageSerializer();
+    deserializer = new PackageDeserializer(createDeserializationEnvironment());
+
+  }
+
+  protected PackageDeserializationEnvironment createDeserializationEnvironment() {
+    return new TestPackageDeserializationEnvironment();
+  }
+
+  private void registerBuildFile(PackageIdentifier packageName, Path path) {
+    buildFileMap.put(packageName, path);
+  }
+
+  protected Package scratchPackage(String name, String... lines) throws Exception {
+    return scratchPackage(PackageIdentifier.createInDefaultRepo(name), lines);
+  }
+
+  protected Package scratchPackage(PackageIdentifier packageId, String... lines)
+      throws Exception {
+    Path buildFile = scratch.file("" + packageId.getPathFragment() + "/BUILD", lines);
+    registerBuildFile(packageId, buildFile);
+    Package.Builder externalPkg =
+        Package.newExternalPackageBuilder(buildFile.getRelative("WORKSPACE"), "TESTING");
+    externalPkg.setWorkspaceName(TestConstants.WORKSPACE_NAME);
+    return packageFactory.createPackageForTesting(
+        packageId, externalPkg.build(), buildFile, packageLocator, reporter);
+  }
+
+  protected Package checkSerialization(Package pkg) throws Exception {
+    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+    CodedOutputStream codedOut = CodedOutputStream.newInstance(bytesOut);
+    serializer.serialize(pkg, codedOut);
+    codedOut.flush();
+
+    Package deserializedPkg = deserializer.deserialize(
+        CodedInputStream.newInstance(bytesOut.toByteArray()));
+
+    // Check equality of an arbitrary sample of properties.
+    assertEquals(pkg.getName(), deserializedPkg.getName());
+    assertEquals(pkg.getPackageIdentifier(), deserializedPkg.getPackageIdentifier());
+    assertEquals(pkg.getBuildFileLabel(), deserializedPkg.getBuildFileLabel());
+    assertEquals(pkg.getFilename(), deserializedPkg.getFilename());
+    assertEquals(pkg.toString(), deserializedPkg.toString());
+    // Not all implementations of Target implement equals, so just check sizes match up.
+    assertThat(deserializedPkg.getTargets()).hasSize(pkg.getTargets().size());
+    return deserializedPkg;
+  }
+
+  private class TestPackageDeserializationEnvironment implements PackageDeserializationEnvironment {
+
+    @Override
+    public Path getPath(String buildFilePath) {
+      return scratch.getFileSystem().getPath(buildFilePath);
+    }
+
+    @Override
+    public RuleClass getRuleClass(Build.Rule rulePb, Location ruleLocation) {
+      return ruleClassProvider.getRuleClassMap().get(rulePb.getRuleClass());
+    }
+
+    @Override
+    public AttributesToDeserialize attributesToDeserialize() {
+      return PackageDeserializer.DESERIALIZE_ALL_ATTRS;
+    }
+  }
+
+  protected CodedInputStream codedInputFromPackage(Package pkg) throws IOException {
+    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+    CodedOutputStream codedOut = CodedOutputStream.newInstance(bytesOut);
+    serializer.serialize(pkg, codedOut);
+    codedOut.flush();
+    return CodedInputStream.newInstance(bytesOut.toByteArray());
+  }
+}