aboutsummaryrefslogtreecommitdiffhomepage
path: root/src
diff options
context:
space:
mode:
authorGravatar Michajlo Matijkiw <michajlo@google.com>2015-11-30 17:42:26 +0000
committerGravatar Kristina Chodorow <kchodorow@google.com>2015-11-30 18:30:49 +0000
commit2d699e9f5570cd0a96e712bd8ab89fe36b08ad6d (patch)
treed45923d3f980843bf9dca9c74e342149286a16e1 /src
parent27760a6efefc64f9577ed1add120b60399203be3 (diff)
Simplify rule fingerprinting in RepositoryFunction
-- MOS_MIGRATED_REVID=108986856
Diffstat (limited to 'src')
-rw-r--r--src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java834
-rw-r--r--src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java329
-rw-r--r--src/main/java/com/google/devtools/build/lib/packages/RuleSerializer.java37
-rw-r--r--src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java6
-rw-r--r--src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java282
-rw-r--r--src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java148
6 files changed, 40 insertions, 1596 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java b/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java
deleted file mode 100644
index f393a78f77..0000000000
--- a/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java
+++ /dev/null
@@ -1,834 +0,0 @@
-// Copyright 2015 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.packages;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableList.Builder;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Interners;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
-import com.google.devtools.build.lib.cmdline.Label;
-import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier;
-import com.google.devtools.build.lib.events.Event;
-import com.google.devtools.build.lib.events.EventHandler;
-import com.google.devtools.build.lib.events.Location;
-import com.google.devtools.build.lib.events.NullEventHandler;
-import com.google.devtools.build.lib.events.StoredEventHandler;
-import com.google.devtools.build.lib.packages.License.DistributionType;
-import com.google.devtools.build.lib.packages.License.LicenseParsingException;
-import com.google.devtools.build.lib.packages.Package.Builder.GeneratedLabelConflict;
-import com.google.devtools.build.lib.packages.Package.NameConflictException;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build.StringDictUnaryEntry;
-import com.google.devtools.build.lib.syntax.GlobCriteria;
-import com.google.devtools.build.lib.syntax.GlobList;
-import com.google.devtools.build.lib.syntax.Type;
-import com.google.devtools.build.lib.syntax.Type.ConversionException;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.ExtensionRegistryLite;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-/**
- * Functionality to deserialize loaded packages.
- */
-public class PackageDeserializer {
-
- private static final Logger LOG = Logger.getLogger(PackageDeserializer.class.getName());
-
- /**
- * Provides the deserializer with tools it needs to build a package from its serialized form.
- */
- public interface PackageDeserializationEnvironment {
-
- /** Converts the serialized package's path string into a {@link Path} object. */
- Path getPath(String buildFilePath);
-
- /** Returns a {@link RuleClass} object for the serialized rule. */
- RuleClass getRuleClass(Build.Rule rulePb, Location ruleLocation)
- throws PackageDeserializationException;
-
- /** Description of what rule attributes of each rule should be deserialized. */
- AttributesToDeserialize attributesToDeserialize();
- }
-
- /**
- * A class that defines what attributes to keep after deserialization. Note that all attributes of
- * type label are kept in order to navigate between dependencies.
- *
- * <p>If {@code addSyntheticAttributeHash} is {@code true}, a synthetic attribute is added to each
- * Rule that contains a stable hash of the entire serialized rule for the sake of permitting
- * equality comparisons that respect the attributes that were dropped according to {@code
- * attributesToKeep}.
- */
- public static class AttributesToDeserialize {
-
- private final boolean addSyntheticAttributeHash;
- private final Predicate<String> shouldKeepAttributeWithName;
-
- public AttributesToDeserialize(boolean addSyntheticAttributeHash,
- Predicate<String> shouldKeepAttributeWithName) {
- this.addSyntheticAttributeHash = addSyntheticAttributeHash;
- this.shouldKeepAttributeWithName = shouldKeepAttributeWithName;
- }
-
- public boolean includeAttribute(String attr) {
- return shouldKeepAttributeWithName.apply(attr);
- }
- }
-
- public static final AttributesToDeserialize DESERIALIZE_ALL_ATTRS =
- new AttributesToDeserialize(false, Predicates.<String>alwaysTrue());
-
- // Workaround for Java serialization making it tough to pass in a deserialization environment
- // manually.
- // volatile is needed to ensure that the objects are published safely.
- // TODO(bazel-team): Subclass ObjectOutputStream to pass this through instead.
- public static volatile PackageDeserializationEnvironment defaultPackageDeserializationEnvironment;
-
- // Cache label deserialization across all instances- PackgeDeserializers need to be created on
- // demand due to initialiation constraints wrt the setting of static members.
- private static final Interner<Label> LABEL_INTERNER = Interners.newWeakInterner();
-
- /** Class encapsulating state for a single package deserialization. */
- private static class DeserializationContext {
- private final Package.Builder packageBuilder;
-
- private DeserializationContext(Package.Builder packageBuilder) {
- this.packageBuilder = packageBuilder;
- }
- }
-
- private final PackageDeserializationEnvironment packageDeserializationEnvironment;
-
- /**
- * Creates a {@link PackageDeserializer} using {@link #defaultPackageDeserializationEnvironment}.
- */
- public PackageDeserializer() {
- this.packageDeserializationEnvironment = defaultPackageDeserializationEnvironment;
- }
-
- public PackageDeserializer(PackageDeserializationEnvironment packageDeserializationEnvironment) {
- this.packageDeserializationEnvironment =
- Preconditions.checkNotNull(packageDeserializationEnvironment);
- }
-
- private static ParsedAttributeValue deserializeAttribute(
- Type<?> expectedType, Build.Attribute attrPb) throws PackageDeserializationException {
- Object value = deserializeAttributeValue(expectedType, attrPb);
- return new ParsedAttributeValue(
- attrPb.hasExplicitlySpecified() && attrPb.getExplicitlySpecified(), value
- );
- }
-
- private static void deserializeInputFile(DeserializationContext context,
- Build.SourceFile sourceFile)
- throws PackageDeserializationException {
- InputFile inputFile;
- try {
- inputFile = context.packageBuilder.createInputFile(
- sourceFile.getName(), EmptyLocation.INSTANCE);
- } catch (GeneratedLabelConflict e) {
- throw new PackageDeserializationException(e);
- }
-
- if (!sourceFile.getVisibilityLabelList().isEmpty() || sourceFile.hasLicense()) {
- context.packageBuilder.setVisibilityAndLicense(inputFile,
- PackageFactory.getVisibility(deserializeLabels(sourceFile.getVisibilityLabelList())),
- deserializeLicense(sourceFile.getLicense()));
- }
- }
-
- private static void deserializePackageGroup(DeserializationContext context,
- Build.PackageGroup packageGroupPb) throws PackageDeserializationException {
- List<String> specifications = new ArrayList<>();
- for (String containedPackage : packageGroupPb.getContainedPackageList()) {
- specifications.add("//" + containedPackage);
- }
-
- try {
- context.packageBuilder.addPackageGroup(
- packageGroupPb.getName(),
- specifications,
- deserializeLabels(packageGroupPb.getIncludedPackageGroupList()),
- NullEventHandler.INSTANCE, // TODO(bazel-team): Handle errors properly
- EmptyLocation.INSTANCE);
- } catch (LabelSyntaxException | Package.NameConflictException e) {
- throw new PackageDeserializationException(e);
- }
- }
-
- private void deserializeRule(
- PackageIdentifier packageId, DeserializationContext context, Build.Rule rulePb)
- throws PackageDeserializationException, InterruptedException {
- Location ruleLocation = EmptyLocation.INSTANCE;
- RuleClass ruleClass = packageDeserializationEnvironment.getRuleClass(rulePb, ruleLocation);
- Map<String, ParsedAttributeValue> attributeValues = new HashMap<>();
- AttributesToDeserialize attrToDeserialize =
- packageDeserializationEnvironment.attributesToDeserialize();
-
- Hasher hasher = Hashing.md5().newHasher();
- for (Build.Attribute attrPb : rulePb.getAttributeList()) {
- Type<?> type = ruleClass.getAttributeByName(attrPb.getName()).getType();
- attributeValues.put(attrPb.getName(), deserializeAttribute(type, attrPb));
- if (attrToDeserialize.addSyntheticAttributeHash) {
- // TODO(bazel-team): This might give false positives because of explicit vs implicit.
- hasher.putBytes(attrPb.toByteArray());
- }
- }
- AttributeContainerWithoutLocation attributeContainer =
- new AttributeContainerWithoutLocation(ruleClass, hasher.hash().asBytes());
-
- try {
- Label ruleLabel = LABEL_INTERNER.intern(Label.create(packageId, rulePb.getName()));
- Rule rule = createRuleWithParsedAttributeValues(ruleClass,
- ruleLabel, context.packageBuilder, ruleLocation, attributeValues,
- NullEventHandler.INSTANCE, attributeContainer);
- context.packageBuilder.addRule(rule);
-
- // Remove the attribute after it is added to package in order to pass the validations
- // and be able to compute all the outputs.
- if (attrToDeserialize != DESERIALIZE_ALL_ATTRS) {
- for (String attrName : attributeValues.keySet()) {
- Attribute attribute = ruleClass.getAttributeByName(attrName);
- if (!(attrToDeserialize.shouldKeepAttributeWithName.apply(attrName)
- || BuildType.isLabelType(attribute.getType()))) {
- attributeContainer.clearIfNotLabel(attrName);
- }
- }
- }
-
- Preconditions.checkState(!rule.containsErrors());
- } catch (NameConflictException | LabelSyntaxException e) {
- throw new PackageDeserializationException(e);
- }
-
- }
-
- /** "Empty" location implementation, all methods should return non-null, but empty, values. */
- private static class EmptyLocation extends Location {
- private static final EmptyLocation INSTANCE = new EmptyLocation();
-
- private static final PathFragment DEV_NULL = new PathFragment("/dev/null");
- private static final LineAndColumn EMPTY_LINE_AND_COLUMN = new LineAndColumn(0, 0);
-
- private EmptyLocation() {
- super(0, 0);
- }
-
- @Override
- public PathFragment getPath() {
- return DEV_NULL;
- }
-
- @Override
- public LineAndColumn getStartLineAndColumn() {
- return EMPTY_LINE_AND_COLUMN;
- }
-
- @Override
- public LineAndColumn getEndLineAndColumn() {
- return EMPTY_LINE_AND_COLUMN;
- }
-
- @Override
- public int hashCode() {
- return 42;
- }
-
- @Override
- public boolean equals(Object other) {
- return other instanceof EmptyLocation;
- }
- }
-
- /**
- * Exception thrown when something goes wrong during package deserialization.
- */
- public static class PackageDeserializationException extends Exception {
- private PackageDeserializationException(String message) {
- super(message);
- }
-
- private PackageDeserializationException(String message, Exception reason) {
- super(message, reason);
- }
-
- private PackageDeserializationException(Exception reason) {
- super(reason);
- }
- }
-
- private static Label deserializeLabel(String labelName) throws PackageDeserializationException {
- try {
- return LABEL_INTERNER.intern(Label.parseAbsolute(labelName));
- } catch (LabelSyntaxException e) {
- throw new PackageDeserializationException(
- "Invalid label '" + labelName + "':" + e.getMessage(), e);
- }
- }
-
- private static List<Label> deserializeLabels(List<String> labelNames)
- throws PackageDeserializationException {
- ImmutableList.Builder<Label> result = ImmutableList.builder();
- for (String labelName : labelNames) {
- result.add(deserializeLabel(labelName));
- }
-
- return result.build();
- }
-
- private static License deserializeLicense(Build.License licensePb)
- throws PackageDeserializationException {
- List<String> licenseStrings = new ArrayList<>();
- licenseStrings.addAll(licensePb.getLicenseTypeList());
- for (String exception : licensePb.getExceptionList()) {
- licenseStrings.add("exception=" + exception);
- }
-
- try {
- return License.parseLicense(licenseStrings);
- } catch (LicenseParsingException e) {
- throw new PackageDeserializationException(e);
- }
- }
-
- private static Set<DistributionType> deserializeDistribs(List<String> distributions)
- throws PackageDeserializationException {
- try {
- return License.parseDistributions(distributions);
- } catch (LicenseParsingException e) {
- throw new PackageDeserializationException(e);
- }
- }
-
- private static TriState deserializeTriStateValue(String value)
- throws PackageDeserializationException {
- if (value.equals("yes")) {
- return TriState.YES;
- } else if (value.equals("no")) {
- return TriState.NO;
- } else if (value.equals("auto")) {
- return TriState.AUTO;
- } else {
- throw new PackageDeserializationException(
- String.format("Invalid tristate value: '%s'", value));
- }
- }
-
- private static List<FilesetEntry> deserializeFilesetEntries(
- List<Build.FilesetEntry> filesetPbs)
- throws PackageDeserializationException {
- ImmutableList.Builder<FilesetEntry> result = ImmutableList.builder();
- for (Build.FilesetEntry filesetPb : filesetPbs) {
- Label srcLabel = deserializeLabel(filesetPb.getSource());
- List<Label> files =
- filesetPb.getFilesPresent() ? deserializeLabels(filesetPb.getFileList()) : null;
- List<String> excludes =
- filesetPb.getExcludeList().isEmpty()
- ? null : ImmutableList.copyOf(filesetPb.getExcludeList());
- String destDir = filesetPb.getDestinationDirectory();
- FilesetEntry.SymlinkBehavior symlinkBehavior =
- pbToSymlinkBehavior(filesetPb.getSymlinkBehavior());
- String stripPrefix = filesetPb.hasStripPrefix() ? filesetPb.getStripPrefix() : null;
-
- result.add(
- new FilesetEntry(srcLabel, files, excludes, destDir, symlinkBehavior, stripPrefix));
- }
-
- return result.build();
- }
-
- /**
- * Deserialize a package from its representation as a protocol message. The inverse of
- * {@link PackageSerializer#serialize}.
- * @throws IOException
- * @throws InterruptedException
- */
- private void deserializeInternal(PackageIdentifier packageId, Build.Package packagePb,
- StoredEventHandler eventHandler, Package.Builder builder, CodedInputStream codedIn)
- throws PackageDeserializationException, IOException, InterruptedException {
- Path buildFile = packageDeserializationEnvironment.getPath(packagePb.getBuildFilePath());
- Preconditions.checkNotNull(buildFile);
- DeserializationContext context = new DeserializationContext(builder);
- builder.setFilename(buildFile);
-
- if (packagePb.hasDefaultVisibilitySet() && packagePb.getDefaultVisibilitySet()) {
- builder.setDefaultVisibility(
- PackageFactory.getVisibility(
- deserializeLabels(packagePb.getDefaultVisibilityLabelList())));
- }
-
- // It's important to do this after setting the default visibility, since that implicitly sets
- // this bit to true
- builder.setDefaultVisibilitySet(packagePb.getDefaultVisibilitySet());
- if (packagePb.hasDefaultTestonly()) {
- builder.setDefaultTestonly(packagePb.getDefaultTestonly());
- }
- if (packagePb.hasDefaultDeprecation()) {
- builder.setDefaultDeprecation(packagePb.getDefaultDeprecation());
- }
-
- builder.setDefaultCopts(packagePb.getDefaultCoptList());
- if (packagePb.hasDefaultHdrsCheck()) {
- builder.setDefaultHdrsCheck(packagePb.getDefaultHdrsCheck());
- }
- if (packagePb.hasDefaultLicense()) {
- builder.setDefaultLicense(deserializeLicense(packagePb.getDefaultLicense()));
- }
- builder.setDefaultDistribs(deserializeDistribs(packagePb.getDefaultDistribList()));
-
- for (String subinclude : packagePb.getSubincludeLabelList()) {
- Label label = deserializeLabel(subinclude);
- builder.addSubinclude(label, null);
- }
-
- ImmutableList.Builder<Label> skylarkFileDependencies = ImmutableList.builder();
- for (String skylarkFile : packagePb.getSkylarkLabelList()) {
- skylarkFileDependencies.add(deserializeLabel(skylarkFile));
- }
- builder.setSkylarkFileDependencies(skylarkFileDependencies.build());
-
- MakeEnvironment.Builder makeEnvBuilder = new MakeEnvironment.Builder();
- for (Build.MakeVar makeVar : packagePb.getMakeVariableList()) {
- for (Build.MakeVarBinding binding : makeVar.getBindingList()) {
- makeEnvBuilder.update(
- makeVar.getName(), binding.getValue(), binding.getPlatformSetRegexp());
- }
- }
- builder.setMakeEnv(makeEnvBuilder);
-
- for (Build.Event event : packagePb.getEventList()) {
- deserializeEvent(eventHandler, event);
- }
-
- if (packagePb.hasContainsErrors() && packagePb.getContainsErrors()) {
- builder.setContainsErrors();
- }
-
- builder.setWorkspaceName(packagePb.getWorkspaceName());
-
- deserializeTargets(packageId, codedIn, context);
- }
-
- private void deserializeTargets(
- PackageIdentifier packageId, CodedInputStream codedIn, DeserializationContext context)
- throws IOException, PackageDeserializationException, InterruptedException {
- Build.TargetOrTerminator tot;
- while (!(tot = readTargetOrTerminator(codedIn)).getIsTerminator()) {
- Build.Target target = tot.getTarget();
- switch (target.getType()) {
- case SOURCE_FILE:
- deserializeInputFile(context, target.getSourceFile());
- break;
- case PACKAGE_GROUP:
- deserializePackageGroup(context, target.getPackageGroup());
- break;
- case RULE:
- deserializeRule(packageId, context, target.getRule());
- break;
- default:
- throw new IllegalStateException("Unexpected Target type: " + target.getType());
- }
- }
- }
-
- private static Build.TargetOrTerminator readTargetOrTerminator(CodedInputStream codedIn)
- throws IOException {
- Build.TargetOrTerminator.Builder builder = Build.TargetOrTerminator.newBuilder();
- codedIn.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
- return builder.build();
- }
-
- /**
- * Deserializes a {@link Package} from {@code in}. The inverse of
- * {@link PackageSerializer#serialize}.
- *
- * <p>Expects {@code codedIn} to contain a single
- * {@link com.google.devtools.build.lib.query2.proto.proto2api.Build.Package} message followed
- * by a series of
- * {@link com.google.devtools.build.lib.query2.proto.proto2api.Build.TargetOrTerminator}
- * messages encoding the associated targets.
- *
- * @param codedIn stream to read from
- * @return a new {@link Package} as read from {@code in}
- * @throws PackageDeserializationException on failures deserializing the input
- * @throws IOException on failures reading from {@code in}
- * @throws InterruptedException
- */
- public Package deserialize(CodedInputStream codedIn)
- throws PackageDeserializationException, IOException, InterruptedException {
- try {
- return deserializeInternal(codedIn);
- } catch (PackageDeserializationException | RuntimeException e) {
- LOG.log(Level.WARNING, "Failed to deserialize Package object", e);
- throw e;
- }
- }
-
- private Package deserializeInternal(CodedInputStream codedIn)
- throws PackageDeserializationException, IOException, InterruptedException {
- // Read the initial Package message so we have the data to initialize the builder. We will read
- // the Targets in individually later.
- Build.Package packagePb = readPackageProto(codedIn);
- PackageIdentifier packageId;
- try {
- packageId = PackageIdentifier.create(
- packagePb.getRepository(), new PathFragment(packagePb.getName()));
- } catch (LabelSyntaxException e) {
- throw new PackageDeserializationException(e);
- }
-
- Package.Builder builder = new Package.Builder(packageId, null);
- StoredEventHandler eventHandler = new StoredEventHandler();
- deserializeInternal(packageId, packagePb, eventHandler, builder, codedIn);
- builder.addEvents(eventHandler.getEvents());
- return builder.build();
- }
-
- private static Build.Package readPackageProto(CodedInputStream codedIn) throws IOException {
- Build.Package.Builder builder = Build.Package.newBuilder();
- codedIn.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
- return builder.build();
- }
-
- private static void deserializeEvent(StoredEventHandler eventHandler, Build.Event event) {
- String message = event.getMessage();
- switch (event.getKind()) {
- case ERROR: eventHandler.handle(Event.error(message)); break;
- case WARNING: eventHandler.handle(Event.warn(message)); break;
- case INFO: eventHandler.handle(Event.info(message)); break;
- case PROGRESS: eventHandler.handle(Event.progress(message)); break;
- default: break; // Ignore
- }
- }
-
- private static List<?> deserializeGlobs(List<?> matches,
- Build.Attribute attrPb) {
- if (attrPb.getGlobCriteriaCount() == 0) {
- return matches;
- }
-
- Builder<GlobCriteria> criteriaBuilder = ImmutableList.builder();
- for (Build.GlobCriteria criteriaPb : attrPb.getGlobCriteriaList()) {
- if (criteriaPb.hasGlob() && criteriaPb.getGlob()) {
- criteriaBuilder.add(GlobCriteria.fromGlobCall(
- ImmutableList.copyOf(criteriaPb.getIncludeList()),
- ImmutableList.copyOf(criteriaPb.getExcludeList())));
- } else {
- criteriaBuilder.add(
- GlobCriteria.fromList(ImmutableList.copyOf(criteriaPb.getIncludeList())));
- }
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"}) GlobList<?> result =
- new GlobList(criteriaBuilder.build(), matches);
- return result;
- }
-
- // TODO(bazel-team): Verify that these put sane values in the attribute
- @VisibleForTesting
- static Object deserializeAttributeValue(Type<?> expectedType,
- Build.Attribute attrPb)
- throws PackageDeserializationException {
- switch (attrPb.getType()) {
- case INTEGER:
- return attrPb.hasIntValue() ? attrPb.getIntValue() : null;
-
- case STRING:
- if (!attrPb.hasStringValue()) {
- return null;
- } else if (expectedType == BuildType.NODEP_LABEL) {
- return deserializeLabel(attrPb.getStringValue());
- } else {
- return attrPb.getStringValue();
- }
-
- case LABEL:
- case OUTPUT:
- return attrPb.hasStringValue() ? deserializeLabel(attrPb.getStringValue()) : null;
-
- case STRING_LIST:
- if (expectedType == BuildType.NODEP_LABEL_LIST) {
- return deserializeGlobs(deserializeLabels(attrPb.getStringListValueList()), attrPb);
- } else {
- return deserializeGlobs(ImmutableList.copyOf(attrPb.getStringListValueList()), attrPb);
- }
-
- case LABEL_LIST:
- case OUTPUT_LIST:
- return deserializeGlobs(deserializeLabels(attrPb.getStringListValueList()), attrPb);
-
- case DISTRIBUTION_SET:
- return deserializeDistribs(attrPb.getStringListValueList());
-
- case LICENSE:
- return attrPb.hasLicense() ? deserializeLicense(attrPb.getLicense()) : null;
-
- case STRING_DICT: {
- // Building an immutable map will fail if the builder was given duplicate keys. These entry
- // lists may contain duplicate keys if the serialized map value was configured (e.g. via
- // the select function) and the different configuration values had keys in common. This is
- // because serialization flattens configurable map-valued attributes.
- //
- // As long as serialization does this flattening, to avoid failure during deserialization,
- // we dedupe entries in the list by their keys.
- // TODO(bazel-team): Serialize and deserialize configured values with fidelity (without
- // flattening them).
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- HashSet<String> keysSeenSoFar = Sets.newHashSet();
- for (Build.StringDictEntry entry : attrPb.getStringDictValueList()) {
- String key = entry.getKey();
- if (keysSeenSoFar.add(key)) {
- builder.put(key, entry.getValue());
- }
- }
- return builder.build();
- }
-
- case STRING_DICT_UNARY: {
- // See STRING_DICT case's comment about why this dedupes entries by their keys.
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- HashSet<String> keysSeenSoFar = Sets.newHashSet();
- for (StringDictUnaryEntry entry : attrPb.getStringDictUnaryValueList()) {
- String key = entry.getKey();
- if (keysSeenSoFar.add(key)) {
- builder.put(key, entry.getValue());
- }
- }
- return builder.build();
- }
-
- case FILESET_ENTRY_LIST:
- return deserializeFilesetEntries(attrPb.getFilesetListValueList());
-
- case LABEL_LIST_DICT: {
- // See STRING_DICT case's comment about why this dedupes entries by their keys.
- ImmutableMap.Builder<String, List<Label>> builder = ImmutableMap.builder();
- HashSet<String> keysSeenSoFar = Sets.newHashSet();
- for (Build.LabelListDictEntry entry : attrPb.getLabelListDictValueList()) {
- String key = entry.getKey();
- if (keysSeenSoFar.add(key)) {
- builder.put(key, deserializeLabels(entry.getValueList()));
- }
- }
- return builder.build();
- }
-
- case STRING_LIST_DICT: {
- // See STRING_DICT case's comment about why this dedupes entries by their keys.
- ImmutableMap.Builder<String, List<String>> builder = ImmutableMap.builder();
- HashSet<String> keysSeenSoFar = Sets.newHashSet();
- for (Build.StringListDictEntry entry : attrPb.getStringListDictValueList()) {
- String key = entry.getKey();
- if (keysSeenSoFar.add(key)) {
- builder.put(key, ImmutableList.copyOf(entry.getValueList()));
- }
- }
- return builder.build();
- }
-
- case BOOLEAN:
- return attrPb.hasBooleanValue() ? attrPb.getBooleanValue() : null;
-
- case TRISTATE:
- return attrPb.hasStringValue() ? deserializeTriStateValue(attrPb.getStringValue()) : null;
-
- case INTEGER_LIST:
- return ImmutableList.copyOf(attrPb.getIntListValueList());
-
- default:
- throw new PackageDeserializationException("Invalid discriminator: " + attrPb.getType());
- }
- }
-
- private static FilesetEntry.SymlinkBehavior pbToSymlinkBehavior(
- Build.FilesetEntry.SymlinkBehavior symlinkBehavior) {
- switch (symlinkBehavior) {
- case COPY:
- return FilesetEntry.SymlinkBehavior.COPY;
- case DEREFERENCE:
- return FilesetEntry.SymlinkBehavior.DEREFERENCE;
- default:
- throw new IllegalStateException();
- }
- }
-
- /**
- * An special {@code AttributeContainer} implementation that does not keep
- * the location and can contain a hashcode of the target attributes.
- */
- public static class AttributeContainerWithoutLocation extends AttributeContainer {
-
- @Nullable
- private final byte[] syntheticAttrHash;
-
- private AttributeContainerWithoutLocation(RuleClass ruleClass,
- @Nullable byte[] syntheticAttrHash) {
- super(ruleClass, null);
- this.syntheticAttrHash = syntheticAttrHash;
- }
-
- @Override
- public Location getAttributeLocation(String attrName) {
- return EmptyLocation.INSTANCE;
- }
-
- @Override
- void setAttributeLocation(int attrIndex, Location location) {
- throw new UnsupportedOperationException("Setting location not supported");
- }
-
- @Override
- void setAttributeLocation(Attribute attribute, Location location) {
- throw new UnsupportedOperationException("Setting location not supported");
- }
-
- @Nullable
- public byte[] getSyntheticAttrHash() {
- return syntheticAttrHash;
- }
-
- private void clearIfNotLabel(String attr) {
- setAttributeValueByName(attr, null);
- }
- }
-
- /**
- * Creates a rule with the attribute values that are already parsed.
- *
- * <p><b>WARNING:</b> This assumes that the attribute values here have the right type and
- * bypasses some sanity checks. If they are of the wrong type, everything will come down burning.
- */
- @SuppressWarnings("unchecked")
- private static Rule createRuleWithParsedAttributeValues(RuleClass ruleClass, Label label,
- Package.Builder pkgBuilder, Location ruleLocation,
- Map<String, ParsedAttributeValue> attributeValues, EventHandler eventHandler,
- AttributeContainer attributeContainer)
- throws LabelSyntaxException, InterruptedException {
- Rule rule = pkgBuilder.newRuleWithLabelAndAttrContainer(label, ruleClass, ruleLocation,
- attributeContainer);
- rule.checkValidityPredicate(eventHandler);
-
- for (Attribute attribute : rule.getRuleClassObject().getAttributes()) {
- ParsedAttributeValue value = attributeValues.get(attribute.getName());
- if (attribute.isMandatory()) {
- Preconditions.checkState(value != null);
- }
-
- if (value == null) {
- continue;
- }
-
- rule.setAttributeValue(attribute, value.value, value.explicitlySpecified);
- ruleClass.checkAllowedValues(rule, attribute, eventHandler);
-
- if (attribute.getName().equals("visibility")) {
- // TODO(bazel-team): Verify that this cast works
- rule.setVisibility(PackageFactory.getVisibility((List<Label>) value.value));
- }
- }
-
- rule.populateOutputFiles(eventHandler, pkgBuilder);
- Preconditions.checkState(!rule.containsErrors());
- return rule;
- }
-
- /**
- * Implemetation of a SkylarkAspectClass deserialized with a package.
- */
- public static class DeserializedSkylarkAspectClass extends SkylarkAspectClass {
-
- private final Label extenesionFileLabel;
- private final String exportedName;
- private final AspectDefinition definition;
-
- private DeserializedSkylarkAspectClass(Build.SkylarkAspect aspect)
- throws PackageDeserializationException {
- this.extenesionFileLabel = deserializeLabel(aspect.getExtensionFileLabel());
- this.exportedName = aspect.getExportedName();
- AspectDefinition.Builder builder = new AspectDefinition.Builder(getName());
-
- for (Build.Attribute attributePb : aspect.getAttributeList()) {
- Type<?> type =
- ProtoUtils.getTypeFromDiscriminator(
- attributePb.getType(), Optional.<Boolean>absent(), "", attributePb.getName());
- ParsedAttributeValue parsedAttributeValue = deserializeAttribute(type, attributePb);
- Attribute.Builder<?> attribute = Attribute.attr(attributePb.getName(), type);
- try {
- attribute.defaultValue(parsedAttributeValue.value);
- } catch (ConversionException e) {
- throw new PackageDeserializationException(
- "Wrong type of a value for attribute " + attributePb.getName(), e);
- }
- builder.add(attribute);
- }
-
- this.definition = builder.build();
- }
-
- @Override
- public Label getExtensionLabel() {
- return extenesionFileLabel;
- }
-
- @Override
- public String getExportedName() {
- return exportedName;
- }
-
- @Override
- public AspectDefinition getDefinition(AspectParameters aspectParameters) {
- return definition;
- }
- }
-
- public SkylarkAspectClass createDeserializedSkylarkAspectClass(Build.SkylarkAspect aspect)
- throws PackageDeserializationException {
- return new DeserializedSkylarkAspectClass(aspect);
- }
-
-
- private static class ParsedAttributeValue {
- private final boolean explicitlySpecified;
- private final Object value;
-
- private ParsedAttributeValue(boolean explicitlySpecified, Object value) {
- this.explicitlySpecified = explicitlySpecified;
- this.value = value;
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java b/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java
deleted file mode 100644
index 5fd518814f..0000000000
--- a/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java
+++ /dev/null
@@ -1,329 +0,0 @@
-// Copyright 2015 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.packages;
-
-import com.google.common.base.Verify;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Sets;
-import com.google.devtools.build.lib.cmdline.Label;
-import com.google.devtools.build.lib.events.Event;
-import com.google.devtools.build.lib.packages.License.DistributionType;
-import com.google.devtools.build.lib.packages.MakeEnvironment.Binding;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build.AttributeAspect;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build.SkylarkAspect;
-import com.google.protobuf.CodedOutputStream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-/** Functionality to serialize loaded packages. */
-public class PackageSerializer {
-
- public PackageSerializer() {}
-
- /**
- * Serialize a package to {@code out}. The inverse of {@link PackageDeserializer#deserialize}.
- *
- * <p>Writes pkg as a single
- * {@link com.google.devtools.build.lib.query2.proto.proto2api.Build.Package} protocol buffer
- * message followed by a series of
- * {@link com.google.devtools.build.lib.query2.proto.proto2api.Build.TargetOrTerminator} messages
- * encoding the targets.
- *
- * @param pkg the {@link Package} to be serialized
- * @param codedOut the stream to pkg's serialized representation to
- * @throws IOException on failure writing to {@code out}
- */
- public void serialize(Package pkg, CodedOutputStream codedOut) throws IOException {
- Build.Package.Builder builder = Build.Package.newBuilder();
- builder.setName(pkg.getName());
- builder.setRepository(pkg.getPackageIdentifier().getRepository().getName());
- builder.setBuildFilePath(pkg.getFilename().getPathString());
- // The extra bit is needed to handle the corner case when the default visibility is [], i.e.
- // zero labels.
- builder.setDefaultVisibilitySet(pkg.isDefaultVisibilitySet());
- if (pkg.isDefaultVisibilitySet()) {
- for (Label visibilityLabel : pkg.getDefaultVisibility().getDeclaredLabels()) {
- builder.addDefaultVisibilityLabel(visibilityLabel.toString());
- }
- }
-
- builder.setDefaultTestonly(pkg.getDefaultTestOnly());
- if (pkg.getDefaultDeprecation() != null) {
- builder.setDefaultDeprecation(pkg.getDefaultDeprecation());
- }
-
- for (String defaultCopt : pkg.getDefaultCopts()) {
- builder.addDefaultCopt(defaultCopt);
- }
-
- if (pkg.isDefaultHdrsCheckSet()) {
- builder.setDefaultHdrsCheck(pkg.getDefaultHdrsCheck());
- }
-
- builder.setDefaultLicense(serializeLicense(pkg.getDefaultLicense()));
-
- for (DistributionType distributionType : pkg.getDefaultDistribs()) {
- builder.addDefaultDistrib(distributionType.toString());
- }
-
- for (String feature : pkg.getFeatures()) {
- builder.addDefaultSetting(feature);
- }
-
- for (Label subincludeLabel : pkg.getSubincludeLabels()) {
- builder.addSubincludeLabel(subincludeLabel.toString());
- }
-
- for (Label skylarkLabel : pkg.getSkylarkFileDependencies()) {
- builder.addSkylarkLabel(skylarkLabel.toString());
- }
-
- for (Build.MakeVar makeVar :
- serializeMakeEnvironment(pkg.getMakeEnvironment())) {
- builder.addMakeVariable(makeVar);
- }
-
- for (Event event : pkg.getEvents()) {
- builder.addEvent(serializeEvent(event));
- }
-
- builder.setContainsErrors(pkg.containsErrors());
-
- builder.setWorkspaceName(pkg.getWorkspaceName());
-
- codedOut.writeMessageNoTag(builder.build());
-
- // Targets are emitted separately as individual protocol buffers as to prevent overwhelming
- // protocol buffer deserialization size limits.
- emitTargets(pkg.getTargets(), codedOut);
- }
-
- private Build.Target serializeInputFile(InputFile inputFile) {
- Build.SourceFile.Builder builder = Build.SourceFile.newBuilder();
- builder.setName(inputFile.getLabel().getName());
- if (inputFile.isVisibilitySpecified()) {
- for (Label visibilityLabel : inputFile.getVisibility().getDeclaredLabels()) {
- builder.addVisibilityLabel(visibilityLabel.toString());
- }
- }
- if (inputFile.isLicenseSpecified()) {
- builder.setLicense(serializeLicense(inputFile.getLicense()));
- }
-
- return Build.Target.newBuilder()
- .setType(Build.Target.Discriminator.SOURCE_FILE)
- .setSourceFile(builder.build())
- .build();
- }
-
- private Build.Target serializePackageGroup(PackageGroup packageGroup) {
- Build.PackageGroup.Builder builder = Build.PackageGroup.newBuilder();
-
- builder.setName(packageGroup.getLabel().getName());
-
- for (PackageSpecification packageSpecification : packageGroup.getPackageSpecifications()) {
- builder.addContainedPackage(packageSpecification.toString());
- }
-
- for (Label include : packageGroup.getIncludes()) {
- builder.addIncludedPackageGroup(include.toString());
- }
-
- return Build.Target.newBuilder()
- .setType(Build.Target.Discriminator.PACKAGE_GROUP)
- .setPackageGroup(builder.build())
- .build();
- }
-
- public Build.Target serializeRule(Rule rule) {
- Build.Rule.Builder builder = Build.Rule.newBuilder();
- builder.setName(rule.getLabel().getName());
- builder.setRuleClass(rule.getRuleClass());
- builder.setPublicByDefault(rule.getRuleClassObject().isPublicByDefault());
- for (Attribute attribute : rule.getAttributes()) {
- builder.addAttribute(
- AttributeSerializer.getAttributeProto(
- attribute,
- AttributeSerializer.getAttributeValues(rule, attribute),
- rule.isAttributeValueExplicitlySpecified(attribute),
- /*includeGlobs=*/ true));
- }
- maybeSerializeAdditionalDataForRule(rule, builder);
-
- return Build.Target.newBuilder()
- .setType(Build.Target.Discriminator.RULE)
- .setRule(builder.build())
- .build();
- }
-
- public Build.SkylarkAspect serializeSkylarkAspect(Rule rule, Aspect aspect) {
- SkylarkAspectClass aspectClass = (SkylarkAspectClass) aspect.getAspectClass();
-
- SkylarkAspect.Builder builder = SkylarkAspect.newBuilder()
- .setExtensionFileLabel(aspectClass.getExtensionLabel().toString())
- .setExportedName(aspectClass.getExportedName());
-
- AspectDefinition definition = aspect.getDefinition();
- for (Entry<String, Attribute> entry : definition.getAttributes().entrySet()) {
- Attribute attribute = entry.getValue();
- Object defaultValue = attribute.getDefaultValue(rule);
- Verify.verify(defaultValue != null, "Aspect attributes must have default values.");
- Build.Attribute attributePb =
- AttributeSerializer.getAttributeProto(
- attribute, ImmutableList.of(defaultValue), true, false);
- builder.addAttribute(attributePb);
- }
- return builder.build();
- }
-
- private static List<Build.MakeVar> serializeMakeEnvironment(MakeEnvironment makeEnv) {
- List<Build.MakeVar> result = new ArrayList<>();
-
- for (Map.Entry<String, ImmutableList<Binding>> var : makeEnv.getBindings().entrySet()) {
- Build.MakeVar.Builder varPb = Build.MakeVar.newBuilder();
- varPb.setName(var.getKey());
- for (Binding binding : var.getValue()) {
- Build.MakeVarBinding.Builder bindingPb = Build.MakeVarBinding.newBuilder();
- bindingPb.setValue(binding.getValue());
- bindingPb.setPlatformSetRegexp(binding.getPlatformSetRegexp());
- varPb.addBinding(bindingPb);
- }
-
- result.add(varPb.build());
- }
-
- return result;
- }
-
- private static Build.License serializeLicense(License license) {
- Build.License.Builder result = Build.License.newBuilder();
-
- for (License.LicenseType licenseType : license.getLicenseTypes()) {
- result.addLicenseType(licenseType.toString());
- }
-
- for (Label exception : license.getExceptions()) {
- result.addException(exception.toString());
- }
- return result.build();
- }
-
- private Build.Event serializeEvent(Event event) {
- Build.Event.Builder result = Build.Event.newBuilder();
- result.setMessage(event.getMessage());
-
- Build.Event.EventKind kind;
- switch (event.getKind()) {
- case ERROR:
- kind = Build.Event.EventKind.ERROR;
- break;
- case WARNING:
- kind = Build.Event.EventKind.WARNING;
- break;
- case INFO:
- kind = Build.Event.EventKind.INFO;
- break;
- case PROGRESS:
- kind = Build.Event.EventKind.PROGRESS;
- break;
- default: throw new IllegalArgumentException("unexpected event type: " + event.getKind());
- }
-
- result.setKind(kind);
- return result.build();
- }
-
- /** Writes targets as a series of separate TargetOrTerminator messages to out. */
- private void emitTargets(Collection<Target> targets, CodedOutputStream codedOut)
- throws IOException {
- for (Target target : targets) {
- if (target instanceof InputFile) {
- emitTarget(serializeInputFile((InputFile) target), codedOut);
- } else if (target instanceof OutputFile) {
- // Output files are not serialized; they are recreated by the RuleClass on deserialization.
- } else if (target instanceof PackageGroup) {
- emitTarget(serializePackageGroup((PackageGroup) target), codedOut);
- } else if (target instanceof Rule) {
- emitTarget(serializeRule((Rule) target), codedOut);
- }
- }
-
- // Terminate stream with isTerminator = true.
- codedOut.writeMessageNoTag(Build.TargetOrTerminator.newBuilder()
- .setIsTerminator(true)
- .build());
- }
-
- private static void emitTarget(Build.Target target, CodedOutputStream codedOut)
- throws IOException {
- codedOut.writeMessageNoTag(Build.TargetOrTerminator.newBuilder()
- .setTarget(target)
- .build());
- }
-
- private void maybeSerializeAdditionalDataForRule(Rule rule, Build.Rule.Builder builder) {
- if (rule.getRuleClassObject().isSkylark()) {
- builder.setIsSkylark(true);
- // We explicitly serialize the implicit output files for this rule so that we can recreate
- // them on deserialization via our fake placeholder rule class's
- // RuleClass#getImplicitOutputsFunction. Note that since explicit outputs are already handled
- // via the serialization of attributes with type OUTPUT or OUTPUT_LIST we don't bother with
- // those.
- Collection<OutputFile> outputsFromAttributes = rule.getOutputFileMap().values();
- Set<Label> outputLabelsFromAttributes =
- Sets.newHashSetWithExpectedSize(outputsFromAttributes.size());
- for (OutputFile outputFile : outputsFromAttributes) {
- outputLabelsFromAttributes.add(outputFile.getLabel());
- }
- for (OutputFile outputFile : rule.getOutputFiles()) {
- Label label = outputFile.getLabel();
- if (!outputLabelsFromAttributes.contains(label)) {
- // Two important notes here:
- // (i) We are co-opting the otherwise unused rule_output field.
- // (ii) We serialize with the name of the label because the logic in
- // Rule#populateImplicitOutputFiles assumes the labels aren't absolute. This is nice
- // anyways because we don't wastefully serialize the package name.
- builder.addRuleOutput(label.getName());
- }
- }
-
- // Serialize aspects.
- List<Attribute> attributes = rule.getRuleClassObject().getAttributes();
- for (Attribute attribute : attributes) {
- ImmutableList<Aspect> aspects = attribute.getAspects(rule);
- for (Aspect aspect : aspects) {
- if (aspect.getAspectClass() instanceof NativeAspectClass<?>) {
- continue;
- }
-
- SkylarkAspect skylarkAspect = serializeSkylarkAspect(rule, aspect);
- builder.addSkylarkAttributeAspects(
- AttributeAspect.newBuilder()
- .setAttributeName(attribute.getName())
- .setAspect(skylarkAspect)
- .build());
- }
- }
- }
- }
-
-}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleSerializer.java b/src/main/java/com/google/devtools/build/lib/packages/RuleSerializer.java
new file mode 100644
index 0000000000..5d49b4b420
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RuleSerializer.java
@@ -0,0 +1,37 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.query2.proto.proto2api.Build;
+
+/** Serialize a {@link Rule} as its protobuf representation. */
+public class RuleSerializer {
+
+ public static Build.Rule.Builder serializeRule(Rule rule) {
+ Build.Rule.Builder builder = Build.Rule.newBuilder();
+ builder.setName(rule.getLabel().getName());
+ builder.setRuleClass(rule.getRuleClass());
+ builder.setPublicByDefault(rule.getRuleClassObject().isPublicByDefault());
+ for (Attribute attribute : rule.getAttributes()) {
+ builder.addAttribute(
+ AttributeSerializer.getAttributeProto(
+ attribute,
+ AttributeSerializer.getAttributeValues(rule, attribute),
+ rule.isAttributeValueExplicitlySpecified(attribute),
+ /*includeGlobs=*/ true));
+ }
+ return builder;
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
index f949e0a8b1..ca746fe2b4 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
@@ -24,8 +24,8 @@ import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.Package;
-import com.google.devtools.build.lib.packages.PackageSerializer;
import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleSerializer;
import com.google.devtools.build.lib.skyframe.FileSymlinkException;
import com.google.devtools.build.lib.skyframe.FileValue;
import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException;
@@ -87,7 +87,7 @@ public abstract class RepositoryFunction implements SkyFunction {
private byte[] computeRuleKey(Rule rule, byte[] ruleSpecificData) {
return new Fingerprint()
- .addBytes(new PackageSerializer().serializeRule(rule).toByteArray())
+ .addBytes(RuleSerializer.serializeRule(rule).build().toByteArray())
.addBytes(ruleSpecificData)
.digestAndReset();
}
@@ -135,7 +135,7 @@ public abstract class RepositoryFunction implements SkyFunction {
}
}
-
+
protected Path prepareLocalRepositorySymlinkTree(Rule rule, Environment env)
throws RepositoryFunctionException {
Path repositoryDirectory = getExternalRepositoryDirectory().getRelative(rule.getName());
diff --git a/src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java b/src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java
deleted file mode 100644
index fa98a9de12..0000000000
--- a/src/test/java/com/google/devtools/build/lib/packages/PackageSerializationTest.java
+++ /dev/null
@@ -1,282 +0,0 @@
-// Copyright 2015 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.packages;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Multimap;
-import com.google.devtools.build.lib.cmdline.Label;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier;
-import com.google.devtools.build.lib.events.Event;
-import com.google.devtools.build.lib.events.Location;
-import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
-import com.google.devtools.build.lib.packages.util.PackageSerializationTestCase;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build.Attribute.Discriminator;
-import com.google.devtools.build.lib.syntax.GlobCriteria;
-import com.google.devtools.build.lib.syntax.GlobList;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.ExtensionRegistryLite;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Unit tests for package serialization and deserialization.
- */
-public class PackageSerializationTest extends PackageSerializationTestCase {
- @Override
- protected List<EnvironmentExtension> getPackageEnvironmentExtensions() {
- return ImmutableList.of();
- }
-
- public void testLocationsOmitted() throws Exception {
- Package pkg = scratchPackage("bacon",
- "sh_library(name='bacon',",
- " srcs=['bacon.sh'])");
- // By default we keep full location.
- Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
- Rule rule2 = pkg2.getRule("bacon");
-
- assertEmpty(rule2.getLocation());
- assertEmpty(rule2.getAttributeLocation("name"));
- assertEmpty(rule2.getAttributeLocation("srcs"));
- }
-
- public void testGlobInformationKept() throws Exception {
- scratch.file("ham/head.sh");
- scratch.file("ham/tbone.sh");
- Package pkg = scratchPackage("ham",
- "sh_library(name='ham',",
- " srcs=glob(['*.sh'], exclude=['tbo*.sh']))");
- Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
- Rule ham = pkg2.getRule("ham");
- GlobList<?> globs = Rule.getGlobInfo(RawAttributeMapper.of(ham)
- .get("srcs", BuildType.LABEL_LIST));
- assertThat(globs).containsExactly(Label.parseAbsolute("//ham:head.sh"));
- List<GlobCriteria> criteria = globs.getCriteria();
- assertThat(criteria).hasSize(1);
- assertThat(criteria.get(0).getIncludePatterns()).containsExactly("*.sh");
- assertThat(criteria.get(0).getExcludePatterns()).containsExactly("tbo*.sh");
- assertTrue(criteria.get(0).isGlob());
- }
-
- public void testCanConcatenateGlobs() throws Exception {
- scratch.file("serrano/sinew.sh");
- scratch.file("serrano/muscle.sh");
- scratch.file("serrano/bone.sh");
- Package pkg = scratchPackage("serrano",
- "sh_library(name='serrano',",
- " srcs=glob(['s*.sh']) + ['muscle.sh'] + glob(['b*.sh']))");
- Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
- Rule ham = pkg2.getRule("serrano");
- GlobList<?> globs = Rule.getGlobInfo(RawAttributeMapper.of(ham)
- .get("srcs", BuildType.LABEL_LIST));
- assertThat(globs).containsExactly(
- Label.parseAbsolute("//serrano:sinew.sh"),
- Label.parseAbsolute("//serrano:muscle.sh"),
- Label.parseAbsolute("//serrano:bone.sh"));
- List<GlobCriteria> criteria = globs.getCriteria();
- assertThat(criteria).hasSize(3);
- assertThat(criteria.get(0).getIncludePatterns()).containsExactly("s*.sh");
- assertTrue(criteria.get(0).isGlob());
- assertFalse(criteria.get(1).isGlob());
- assertThat(criteria.get(2).getIncludePatterns()).containsExactly("b*.sh");
- assertTrue(criteria.get(2).isGlob());
- }
-
- public void testEvents() throws Exception {
- Package pkg = scratchPackage("j", "sh_library(name='s', srcs='s.sh')");
- Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
- assertThat(pkg2.getEvents()).hasSize(1);
- Event ev = pkg2.getEvents().get(0);
- assertThat(ev.getMessage()).contains("expected value of type 'list(label)'");
- }
-
- public void testPermanentErrorBitIsKept() throws Exception {
- Package pkg = scratchPackage("g", "sh_library(name='g', srcs=g.sh)");
- Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
- assertTrue(pkg2.containsErrors());
- }
-
- public void testBasicSerialization() throws Exception {
- checkSerialization(scratchPackage("bacon",
- "sh_library(name='bacon',",
- " srcs=['bacon.sh'],",
- " data=glob(['*.txt']))"));
- }
-
- public void testIntList() throws Exception {
- Build.Attribute.Builder attrPb = Build.Attribute.newBuilder();
- attrPb.setName("has_int_list").setType(Discriminator.INTEGER_LIST);
- attrPb.addIntListValue(1).addIntListValue(2).addIntListValue(3).addIntListValue(4);
- assertThat(PackageDeserializer.deserializeAttributeValue(
- com.google.devtools.build.lib.syntax.Type.INTEGER_LIST, attrPb.build()))
- .isEqualTo(ImmutableList.of(1, 2, 3, 4));
- }
-
- public void testExternalPackageLabel() throws Exception {
- PackageIdentifier packageId = PackageIdentifier.create("@foo", new PathFragment("p"));
- Package pkg = scratchPackage(packageId, "filegroup(name = 'rumple', srcs = ['dark_one'])");
- assertEquals(packageId, pkg.getPackageIdentifier());
- Package pkg2 = deserializer.deserialize(codedInputFromPackage(pkg));
- assertEquals(pkg.getPackageIdentifier(), pkg2.getPackageIdentifier());
- }
-
- public void testConfigurableAttributes() throws Exception {
- Package pkg = scratchPackage("pkg",
- "sh_library(",
- " name = 'bread',",
- " srcs = select({",
- " ':one': ['rye.sh'],",
- " ':two': ['wheat.sh'],",
- " }))");
- Package deserialized = deserializer.deserialize(codedInputFromPackage(pkg));
- AttributeMap attrs = RawAttributeMapper.of(deserialized.getRule("bread"));
- // We expect package serialization to "flatten" configurable attributes, e.g. the deserialized
- // rule should look like "srcs = ['rye.sh', 'wheat.sh']" (without the select). Eventually
- // we'll want to preserve the original structure, but that requires syntactic changes to the
- // proto which we'll need to ensure the depserver understands.
- assertThat(attrs.get("srcs", BuildType.LABEL_LIST)).containsExactly(
- Label.parseAbsolute("//pkg:rye.sh"), Label.parseAbsolute("//pkg:wheat.sh"));
- }
-
- public void testConfigurableDictionaryAttribute() throws Exception {
- // Although we expect package serialization to "flatten" configurable attributes, as described
- // above, package deserialization should not crash if multiple configuration entries for a
- // map valued attribute have keys in common.
- checkSerialization(scratchPackage("pkg",
- "cc_library(",
- " name = 'bread',",
- " srcs = ['PROTECTED/rye.cc'],",
- " abi_deps = select({",
- " ':one': {'duplicated_key': [':value1']},",
- " ':two': {'duplicated_key': [':value2']},",
- " }))"));
- }
-
- public void testRuleAttributesWithNullValuesAreIncludedInSerializedRepresentation()
- throws Exception {
- Package pkg = scratchPackage("lettuce",
- "sh_library(name='lettuce',",
- " srcs=['romaine.sh'])");
-
- // Manually parse stream, we're interested in wire representation.
- CodedInputStream codedIn = codedInputFromPackage(pkg);
- readPackage(codedIn);
- Multimap<Build.Target.Discriminator, Build.Target> targets = readTargets(codedIn);
-
- // Check that we encoded "deprecation" in the rule proto output even though it had no value.
- Build.Rule pbRule =
- Iterables.getOnlyElement(targets.get(Build.Target.Discriminator.RULE)).getRule();
- boolean foundEmptyAttribute = false;
- for (Build.Attribute attribute : pbRule.getAttributeList()) {
- if (attribute.getName().equals("deprecation")) {
- assertFalse(attribute.hasStringValue());
- foundEmptyAttribute = true;
- }
- }
- assertTrue(foundEmptyAttribute);
- }
-
- public void testTargetsIndividuallySerializedAndDeserialized() throws Exception {
- scratchPackage("empty",
- "sh_library(name='noop')");
-
- Package pkg = scratchPackage("food",
- "sh_library(name='pork',",
- " srcs=['pig.sh'])",
- "sh_library(name='salmon',",
- " srcs=['river.sh'])",
- "package_group(name='self',",
- " packages=['//food'])");
-
- // Manually parse stream, we're interested in wire representation.
- CodedInputStream codedIn = codedInputFromPackage(pkg);
- readPackage(codedIn);
-
- Multimap<Build.Target.Discriminator, Build.Target> targets = readTargets(codedIn);
- assertThat(targets).hasSize(6);
- assertThat(targets.get(Build.Target.Discriminator.SOURCE_FILE)).hasSize(3);
- assertThat(targets.get(Build.Target.Discriminator.RULE)).hasSize(2);
- assertThat(targets.get(Build.Target.Discriminator.PACKAGE_GROUP)).hasSize(1);
-
- // Make sure we see the same thing when deserializing all the way.
- Package deserialized = deserializer.deserialize(codedInputFromPackage(pkg));
- assertThat(deserialized.getTargets(InputFile.class)).hasSize(3);
- assertThat(deserialized.getTargets(Rule.class)).hasSize(2);
- assertThat(deserialized.getTargets(PackageGroup.class)).hasSize(1);
- }
-
- public void testMassivePackageDeserializesFine() throws Exception {
- // Create a package definition which exports 2^16 files with name lengths 2^10 each, which
- // should result in 2^26 (64MB) of targets. With overhead this should push us comfortably
- // over the 64MB default protocol buffer deserialization limit.
- StringBuilder sb = new StringBuilder();
- sb.append("exports_files([");
- String srcName = Strings.repeat("x", 1 << 10);
- for (int i = 0; i < (1 << 16); i++) {
- sb.append("'").append(srcName).append(i).append("',");
- }
- sb.append("'last'])");
-
- Package pkg = scratchPackage("meat", sb.toString());
-
- // Check that we created our package correctly. There should be 2^16 + 1 dummy targets from
- // our exports_files, plus the BUILD file.
- assertThat(pkg.containsErrors()).isFalse();
- assertThat(pkg.getTargets()).hasSize((1 << 16) + 2);
-
- checkSerialization(pkg);
- }
-
- private static Multimap<Build.Target.Discriminator, Build.Target> readTargets(
- CodedInputStream codedIn) throws IOException {
- Multimap<Build.Target.Discriminator, Build.Target> targets = ArrayListMultimap.create();
- Build.TargetOrTerminator tot;
- while (!(tot = readNext(codedIn)).getIsTerminator()) {
- Build.Target pbTarget = tot.getTarget();
- targets.put(pbTarget.getType(), pbTarget);
- }
- return targets;
- }
-
- private static Build.Package readPackage(CodedInputStream codedIn) throws IOException {
- Build.Package.Builder builder = Build.Package.newBuilder();
- codedIn.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
- return builder.build();
- }
-
- private static Build.TargetOrTerminator readNext(CodedInputStream codedIn) throws IOException {
- Build.TargetOrTerminator.Builder builder = Build.TargetOrTerminator.newBuilder();
- codedIn.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
- return builder.build();
- }
-
- private void assertEmpty(Location location) {
- assertEquals(0, location.getStartOffset());
- assertEquals(0, location.getEndOffset());
- assertEquals(new PathFragment("/dev/null"), location.getPath());
- assertEquals(0, location.getStartLineAndColumn().getLine());
- assertEquals(0, location.getStartLineAndColumn().getColumn());
- assertEquals(0, location.getEndLineAndColumn().getLine());
- assertEquals(0, location.getEndLineAndColumn().getColumn());
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java b/src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java
deleted file mode 100644
index 8217e8feef..0000000000
--- a/src/test/java/com/google/devtools/build/lib/packages/util/PackageSerializationTestCase.java
+++ /dev/null
@@ -1,148 +0,0 @@
-// Copyright 2015 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.packages.util;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.devtools.build.lib.cmdline.PackageIdentifier;
-import com.google.devtools.build.lib.events.Location;
-import com.google.devtools.build.lib.events.Reporter;
-import com.google.devtools.build.lib.packages.CachingPackageLocator;
-import com.google.devtools.build.lib.packages.Package;
-import com.google.devtools.build.lib.packages.PackageDeserializer;
-import com.google.devtools.build.lib.packages.PackageDeserializer.AttributesToDeserialize;
-import com.google.devtools.build.lib.packages.PackageDeserializer.PackageDeserializationEnvironment;
-import com.google.devtools.build.lib.packages.PackageFactory;
-import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
-import com.google.devtools.build.lib.packages.PackageSerializer;
-import com.google.devtools.build.lib.packages.RuleClass;
-import com.google.devtools.build.lib.packages.RuleClassProvider;
-import com.google.devtools.build.lib.query2.proto.proto2api.Build;
-import com.google.devtools.build.lib.testutil.FoundationTestCase;
-import com.google.devtools.build.lib.testutil.TestConstants;
-import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/** Provides test infrastructure for package serialization tests. */
-public abstract class PackageSerializationTestCase extends FoundationTestCase {
-
- private CachingPackageLocator packageLocator;
- private Map<PackageIdentifier, Path> buildFileMap;
- private PackageFactory packageFactory;
- private RuleClassProvider ruleClassProvider;
-
- protected PackageSerializer serializer;
- protected PackageDeserializer deserializer;
-
- protected abstract List<EnvironmentExtension> getPackageEnvironmentExtensions();
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
-
- reporter = new Reporter();
- buildFileMap = new HashMap<>();
-
- packageLocator = new CachingPackageLocator() {
- @Override
- public Path getBuildFileForPackage(PackageIdentifier packageName) {
- return buildFileMap.get(packageName);
- }
- };
-
- ruleClassProvider = TestRuleClassProvider.getRuleClassProvider();
- packageFactory = new PackageFactory(ruleClassProvider, getPackageEnvironmentExtensions());
- serializer = new PackageSerializer();
- deserializer = new PackageDeserializer(createDeserializationEnvironment());
-
- }
-
- protected PackageDeserializationEnvironment createDeserializationEnvironment() {
- return new TestPackageDeserializationEnvironment();
- }
-
- private void registerBuildFile(PackageIdentifier packageName, Path path) {
- buildFileMap.put(packageName, path);
- }
-
- protected Package scratchPackage(String name, String... lines) throws Exception {
- return scratchPackage(PackageIdentifier.createInDefaultRepo(name), lines);
- }
-
- protected Package scratchPackage(PackageIdentifier packageId, String... lines)
- throws Exception {
- Path buildFile = scratch.file("" + packageId.getPathFragment() + "/BUILD", lines);
- registerBuildFile(packageId, buildFile);
- Package.Builder externalPkg =
- Package.newExternalPackageBuilder(buildFile.getRelative("WORKSPACE"), "TESTING");
- externalPkg.setWorkspaceName(TestConstants.WORKSPACE_NAME);
- return packageFactory.createPackageForTesting(
- packageId, externalPkg.build(), buildFile, packageLocator, reporter);
- }
-
- protected Package checkSerialization(Package pkg) throws Exception {
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- CodedOutputStream codedOut = CodedOutputStream.newInstance(bytesOut);
- serializer.serialize(pkg, codedOut);
- codedOut.flush();
-
- Package deserializedPkg = deserializer.deserialize(
- CodedInputStream.newInstance(bytesOut.toByteArray()));
-
- // Check equality of an arbitrary sample of properties.
- assertEquals(pkg.getName(), deserializedPkg.getName());
- assertEquals(pkg.getPackageIdentifier(), deserializedPkg.getPackageIdentifier());
- assertEquals(pkg.getBuildFileLabel(), deserializedPkg.getBuildFileLabel());
- assertEquals(pkg.getFilename(), deserializedPkg.getFilename());
- assertEquals(pkg.toString(), deserializedPkg.toString());
- // Not all implementations of Target implement equals, so just check sizes match up.
- assertThat(deserializedPkg.getTargets()).hasSize(pkg.getTargets().size());
- return deserializedPkg;
- }
-
- private class TestPackageDeserializationEnvironment implements PackageDeserializationEnvironment {
-
- @Override
- public Path getPath(String buildFilePath) {
- return scratch.getFileSystem().getPath(buildFilePath);
- }
-
- @Override
- public RuleClass getRuleClass(Build.Rule rulePb, Location ruleLocation) {
- return ruleClassProvider.getRuleClassMap().get(rulePb.getRuleClass());
- }
-
- @Override
- public AttributesToDeserialize attributesToDeserialize() {
- return PackageDeserializer.DESERIALIZE_ALL_ATTRS;
- }
- }
-
- protected CodedInputStream codedInputFromPackage(Package pkg) throws IOException {
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- CodedOutputStream codedOut = CodedOutputStream.newInstance(bytesOut);
- serializer.serialize(pkg, codedOut);
- codedOut.flush();
- return CodedInputStream.newInstance(bytesOut.toByteArray());
- }
-}