// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.rules.repository; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.RuleSerializer; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.skyframe.FileValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Arrays; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; /** * A {@link SkyFunction} that implements delegation to the correct repository fetcher. * *

Each repository in the WORKSPACE file is represented by a {@link SkyValue} that is computed * by this function. */ public final class RepositoryDelegatorFunction implements SkyFunction { // A special repository delegate used to handle Skylark remote repositories if present. public static final String SKYLARK_DELEGATE_NAME = "$skylark"; // Mapping of rule class name to RepositoryFunction. private final ImmutableMap handlers; // Delegate function to handle skylark remote repositories private final RepositoryFunction skylarkHandler; // This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current // command is a fetch. Remote repository lookups are only allowed during fetches. private final AtomicBoolean isFetch; private Map clientEnvironment; public RepositoryDelegatorFunction( ImmutableMap handlers, @Nullable RepositoryFunction skylarkHandler, AtomicBoolean isFetch) { this.handlers = handlers; this.skylarkHandler = skylarkHandler; this.isFetch = isFetch; } public void setClientEnvironment(Map clientEnvironment) { this.clientEnvironment = clientEnvironment; } private void setupRepositoryRoot(Path repoRoot) throws RepositoryFunctionException { try { FileSystemUtils.deleteTree(repoRoot); FileSystemUtils.createDirectoryAndParents(repoRoot.getParentDirectory()); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } } @Override public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { RepositoryName repositoryName = (RepositoryName) skyKey.argument(); Rule rule = RepositoryFunction.getRule(repositoryName, null, env); if (rule == null) { return null; } BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env); if (directories == null) { return null; } RepositoryFunction handler; if (rule.getRuleClassObject().isSkylark()) { handler = skylarkHandler; } else { handler = handlers.get(rule.getRuleClass()); } if (handler == null) { throw new RepositoryFunctionException(new EvalException( Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")), "Could not find handler for " + rule), Transience.PERSISTENT); } Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories).getRelative(rule.getName()); byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env); if (ruleSpecificData == null) { return null; } byte[] ruleKey = computeRuleKey(rule, ruleSpecificData); Path markerPath = getMarkerPath(directories, rule); handler.setClientEnvironment(clientEnvironment); if (handler.isLocal(rule)) { // Local repositories are always fetched because the operation is generally fast and they do // not depend on non-local data, so it does not make much sense to try to cache from across // server instances. setupRepositoryRoot(repoRoot); SkyValue localRepo = handler.fetch(rule, repoRoot, directories, env); if (localRepo != null) { writeMarkerFile(markerPath, ruleKey); } return localRepo; } // We check the repository root for existence here, but we can't depend on the FileValue, // because it's possible that we eventually create that directory in which case the FileValue // and the state of the file system would be inconsistent. boolean markerUpToDate = isFilesystemUpToDate(markerPath, ruleKey); if (markerUpToDate && repoRoot.exists()) { // Now that we know that it exists, we can declare a Skyframe dependency on the repository // root. RepositoryFunction.getRepositoryDirectory(repoRoot, env); if (env.valuesMissing()) { return null; } return RepositoryDirectoryValue.create(repoRoot); } if (isFetch.get()) { // Fetching enabled, go ahead. setupRepositoryRoot(repoRoot); SkyValue result = handler.fetch(rule, repoRoot, directories, env); if (env.valuesMissing()) { return null; } // No new Skyframe dependencies must be added between calling the repository implementation // and writing the marker file because if they aren't computed, it would cause a Skyframe // restart thus calling the possibly very slow (networking, decompression...) fetch() // operation again. So we write the marker file here immediately. writeMarkerFile(markerPath, ruleKey); return result; } if (!repoRoot.exists()) { // The repository isn't on the file system, there is nothing we can do. throw new RepositoryFunctionException(new IOException( "to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName + " not found and fetching repositories is disabled."), Transience.TRANSIENT); } // Declare a Skyframe dependency so that this is re-evaluated when something happens to the // directory. FileValue repoRootValue = RepositoryFunction.getRepositoryDirectory(repoRoot, env); if (env.valuesMissing()) { return null; } // Try to build with whatever is on the file system and emit a warning. env.getListener().handle(Event.warn(rule.getLocation(), String.format( "External repository '%s' is not up-to-date and fetching is disabled. To update, " + "run the build without the '--nofetch' command line option.", rule.getName()))); return RepositoryDirectoryValue.fetchingDelayed(repoRootValue.realRootedPath().asPath()); } private final byte[] computeRuleKey(Rule rule, byte[] ruleSpecificData) { return new Fingerprint() .addBytes(RuleSerializer.serializeRule(rule).build().toByteArray()) .addBytes(ruleSpecificData) // This is to make the fingerprint different after adding names to the generated // WORKSPACE files so they will get re-created, because otherwise there are // annoying warnings for all of them. // TODO(bsilver16384@gmail.com): Remove this once everybody's upgraded to the // new WORKSPACE files. .addInt(1) .digestAndReset(); } /** * Checks if the state of the repository in the file system is consistent with the rule in the * WORKSPACE file. * *

Deletes the marker file if not so that no matter what happens after, the state of the file * system stays consistent. */ private final boolean isFilesystemUpToDate(Path markerPath, byte[] ruleKey) throws RepositoryFunctionException { try { if (!markerPath.exists()) { return false; } byte[] content = FileSystemUtils.readContent(markerPath); boolean result = Arrays.equals(ruleKey, content); if (!result) { // So that we are in a consistent state if something happens while fetching the repository markerPath.delete(); } return result; } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } } private final void writeMarkerFile(Path markerPath, byte[] ruleKey) throws RepositoryFunctionException { try { FileSystemUtils.writeContent(markerPath, ruleKey); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } } private static Path getMarkerPath(BlazeDirectories directories, Rule rule) { return RepositoryFunction.getExternalRepositoryDirectory(directories) .getChild("@" + rule.getName() + ".marker"); } @Override public String extractTag(SkyKey skyKey) { return null; } }