// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.skyframe; import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.devtools.build.lib.actions.InconsistentFilesystemException; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.concurrent.BlazeInterners; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.events.StoredEventHandler; import com.google.devtools.build.lib.packages.BuildFileNotFoundException; import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.packages.RuleClassProvider; import com.google.devtools.build.lib.packages.SkylarkExportable; import com.google.devtools.build.lib.skyframe.SkylarkImportLookupValue.SkylarkImportLookupKey; import com.google.devtools.build.lib.syntax.AssignmentStatement; import com.google.devtools.build.lib.syntax.BuildFileAST; import com.google.devtools.build.lib.syntax.Environment.Extension; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Identifier; import com.google.devtools.build.lib.syntax.LoadStatement; import com.google.devtools.build.lib.syntax.Mutability; import com.google.devtools.build.lib.syntax.SkylarkImport; import com.google.devtools.build.lib.syntax.SkylarkSemantics; import com.google.devtools.build.lib.syntax.Statement; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.RecordingSkyFunctionEnvironment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.build.skyframe.ValueOrException2; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; import javax.annotation.Nullable; /** * A Skyframe function to look up and import a single Skylark extension. * *

Given a {@link Label} referencing a Skylark file, attempts to locate the file and load it. The * Label must be absolute, and must not reference the special {@code external} package. If loading * is successful, returns a {@link SkylarkImportLookupValue} that encapsulates the loaded {@link * Extension} and {@link SkylarkFileDependency} information. If loading is unsuccessful, throws a * {@link SkylarkImportLookupFunctionException} that encapsulates the cause of the failure. */ public class SkylarkImportLookupFunction implements SkyFunction { private final RuleClassProvider ruleClassProvider; private final PackageFactory packageFactory; private Cache skylarkImportLookupValueCache; private static final Logger logger = Logger.getLogger(SkylarkImportLookupFunction.class.getName()); public SkylarkImportLookupFunction( RuleClassProvider ruleClassProvider, PackageFactory packageFactory) { this.ruleClassProvider = ruleClassProvider; this.packageFactory = packageFactory; } @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { SkylarkImportLookupKey key = (SkylarkImportLookupKey) skyKey.argument(); try { return computeInternal( key.importLabel, key.inWorkspace, env, /*alreadyVisited=*/ null, /*inlineCachedValueBuilder=*/ null); } catch (InconsistentFilesystemException e) { throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT); } catch (SkylarkImportFailedException e) { throw new SkylarkImportLookupFunctionException(e); } } @Nullable SkylarkImportLookupValue computeWithInlineCalls( SkyKey skyKey, Environment env, int expectedSizeOfVisitedSet) throws InconsistentFilesystemException, SkylarkImportFailedException, InterruptedException { // We use the visited set to track if there are any cyclic dependencies when loading the // skylark file. LinkedHashMap visited = new LinkedHashMap<>(expectedSizeOfVisitedSet); CachedSkylarkImportLookupValueAndDeps cachedSkylarkImportLookupValueAndDeps = computeWithInlineCallsInternal(skyKey, env, visited); if (cachedSkylarkImportLookupValueAndDeps == null) { return null; } for (Iterable depGroup : cachedSkylarkImportLookupValueAndDeps.deps) { // Because we automatically filter out deps we've seen before and we don't expect this to be // a super large DAG of dependencies, we iterate through without checking for already visited // deps. env.registerDependencies(depGroup); } return cachedSkylarkImportLookupValueAndDeps.getValue(); } @Nullable private CachedSkylarkImportLookupValueAndDeps computeWithInlineCallsInternal( SkyKey skyKey, Environment env, LinkedHashMap visited) throws InconsistentFilesystemException, SkylarkImportFailedException, InterruptedException { SkylarkImportLookupKey key = (SkylarkImportLookupKey) skyKey.argument(); CachedSkylarkImportLookupValueAndDeps precomputedResult = visited.get(key.importLabel); if (precomputedResult != null) { // We have already registered all the deps for this value. return precomputedResult; } // Note that we can't block other threads on the computation of this value due to a potential // deadlock on a cycle. Although we are repeating some work, it is possible we have an import // cycle where one thread starts at one side of the cycle and the other thread starts at the // other side, and they then wait forever on the results of each others computations. CachedSkylarkImportLookupValueAndDeps cachedSkylarkImportLookupValueAndDeps = skylarkImportLookupValueCache.getIfPresent(skyKey); if (cachedSkylarkImportLookupValueAndDeps != null) { return cachedSkylarkImportLookupValueAndDeps; } CachedSkylarkImportLookupValueAndDeps.Builder inlineCachedValueBuilder = CachedSkylarkImportLookupValueAndDeps.newBuilder(); Preconditions.checkState( !(env instanceof RecordingSkyFunctionEnvironment), "Found nested RecordingSkyFunctionEnvironment but it should have been stripped: %s", env); RecordingSkyFunctionEnvironment recordingEnv = new RecordingSkyFunctionEnvironment( env, inlineCachedValueBuilder::addDep, inlineCachedValueBuilder::addDeps, inlineCachedValueBuilder::noteException); SkylarkImportLookupValue value = computeInternal( key.importLabel, key.inWorkspace, recordingEnv, Preconditions.checkNotNull(visited, key.importLabel), inlineCachedValueBuilder); if (value != null) { inlineCachedValueBuilder.setValue(value); cachedSkylarkImportLookupValueAndDeps = inlineCachedValueBuilder.build(); skylarkImportLookupValueCache.put(skyKey, cachedSkylarkImportLookupValueAndDeps); visited.put(key.importLabel, cachedSkylarkImportLookupValueAndDeps); } return cachedSkylarkImportLookupValueAndDeps; } public void resetCache() { if (skylarkImportLookupValueCache != null) { logger.info( "Skylark inlining cache stats from earlier build: " + skylarkImportLookupValueCache.stats()); } skylarkImportLookupValueCache = CacheBuilder.newBuilder() .concurrencyLevel(BlazeInterners.concurrencyLevel()) .maximumSize(10000) .recordStats() .build(); } // It is vital that we don't return any value if any call to env#getValue(s)OrThrow throws an // exception. We are allowed to wrap the thrown exception and rethrow it for any calling functions // to handle though. @Nullable private SkylarkImportLookupValue computeInternal( Label fileLabel, boolean inWorkspace, Environment env, @Nullable LinkedHashMap alreadyVisited, @Nullable CachedSkylarkImportLookupValueAndDeps.Builder inlineCachedValueBuilder) throws InconsistentFilesystemException, SkylarkImportFailedException, InterruptedException { PathFragment filePath = fileLabel.toPathFragment(); SkylarkSemantics skylarkSemantics = PrecomputedValue.SKYLARK_SEMANTICS.get(env); if (skylarkSemantics == null) { return null; } // Load the AST corresponding to this file. ASTFileLookupValue astLookupValue; try { SkyKey astLookupKey = ASTFileLookupValue.key(fileLabel); astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey, ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class); } catch (ErrorReadingSkylarkExtensionException e) { throw SkylarkImportFailedException.errorReadingFile(filePath, e); } if (astLookupValue == null) { return null; } if (!astLookupValue.lookupSuccessful()) { // Skylark import files have to exist. throw SkylarkImportFailedException.noFile(astLookupValue.getErrorMsg()); } BuildFileAST ast = astLookupValue.getAST(); if (ast.containsErrors()) { throw SkylarkImportFailedException.skylarkErrors(filePath); } // Process the load statements in the file. ImmutableList imports = ast.getImports(); ImmutableMap labelsForImports; // Find the labels corresponding to the load statements. labelsForImports = findLabelsForLoadStatements(imports, fileLabel, env); if (labelsForImports == null) { return null; } // Look up and load the imports. ImmutableCollection