// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.rules.repository; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.LabelValidator; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.repository.ExternalPackageException; import com.google.devtools.build.lib.repository.ExternalPackageUtil; import com.google.devtools.build.lib.repository.ExternalRuleNotFoundException; import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction; import com.google.devtools.build.lib.skyframe.FileStateValue.RegularFileStateValue; import com.google.devtools.build.lib.skyframe.FileSymlinkException; import com.google.devtools.build.lib.skyframe.FileValue; import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException; import com.google.devtools.build.lib.skyframe.PackageLookupValue; import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import java.io.IOException; import java.nio.charset.Charset; import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; /** * Implementation of fetching various external repository types. * *
These objects are called from {@link RepositoryDelegatorFunction}. * *
External repositories come in two flavors: local and non-local. * *
Local ones are those whose fetching does not require access to any external resources * (e.g. network). These are always re-fetched on Bazel server restarts. This operation is fast * (usually just a few symlinks and maybe writing a BUILD file). {@code --nofetch} does not apply * to local repositories. * *
The up-to-dateness of non-local repositories is checked using a marker file under the * output base. When such a repository is fetched, data from the rule in the WORKSPACE file is * written to the marker file which is consulted on next server startup. If the rule hasn't changed, * the repository is not re-fetched. * *
Fetching repositories can be disabled using the {@code --nofetch} command line option. If a * repository is on the file system, Bazel just tries to use it and hopes for the best. If the * repository has never been fetched, Bazel errors out for lack of a better option. This is * implemented using * {@link com.google.devtools.build.lib.bazel.BazelRepositoryModule#REPOSITORY_VALUE_CHECKER} and * a flag in {@link RepositoryDirectoryValue} that tells Bazel whether the value in Skyframe is * stale according to the value of {@code --nofetch} or not. * *
When a rule in the WORKSPACE file is changed, the corresponding
* {@link RepositoryDirectoryValue} is invalidated using the usual Skyframe route.
*/
public abstract class RepositoryFunction {
protected Map This exception should be used by child classes to limit the types of exceptions
* {@link RepositoryDelegatorFunction} has to know how to catch. When this method is called, it has already been determined that the repository is stale and
* that it needs to be re-fetched.
*
* The {@code env} argument can be used to fetch Skyframe dependencies the repository
* implementation needs on the following conditions:
*
* The {@code markerData} argument can be mutated to augment the data to write to the
* repository marker file. If any data in the {@code markerData} change between 2 execute of the
* {@link RepositoryDelegatorFunction} then this should be a reason to invalidate the repository.
* The {@link #verifyMarkerData} method is responsible for checking the value added to that map
* when checking the content of a marker file.
*/
@ThreadSafe
@Nullable
public abstract RepositoryDirectoryValue.Builder fetch(
Rule rule,
Path outputDirectory,
BlazeDirectories directories,
Environment env,
Map If this is false, Bazel may decide not to re-fetch the repository, for example when the
* {@code --nofetch} command line option is used.
*/
protected abstract boolean isLocal(Rule rule);
/**
* Returns a block of data that must be equal for two Rules for them to be considered the same.
*
* This is used for the up-to-dateness check of fetched directory trees. The only reason for
* this to exist is the {@code maven_server} rule (which should go away, but until then, we need
* to keep it working somehow)
*/
protected byte[] getRuleSpecificMarkerData(Rule rule, Environment env)
throws RepositoryFunctionException, InterruptedException {
return new byte[] {};
}
protected Path prepareLocalRepositorySymlinkTree(Rule rule, Path repositoryDirectory)
throws RepositoryFunctionException {
try {
FileSystemUtils.createDirectoryAndParents(repositoryDirectory);
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
// Add x/WORKSPACE.
createWorkspaceFile(repositoryDirectory, rule.getTargetKind(), rule.getName());
return repositoryDirectory;
}
public static void createWorkspaceFile(
Path repositoryDirectory, String ruleKind, String ruleName)
throws RepositoryFunctionException {
try {
Path workspaceFile = repositoryDirectory.getRelative("WORKSPACE");
FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"),
String.format("# DO NOT EDIT: automatically generated WORKSPACE file for %s\n"
+ "workspace(name = \"%s\")\n", ruleKind, ruleName));
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
protected static RepositoryDirectoryValue.Builder writeFile(
Path repositoryDirectory, String filename, String contents)
throws RepositoryFunctionException {
Path filePath = repositoryDirectory.getRelative(filename);
try {
// The repository could have an existing file that's either a regular file (for remote
// repositories) or a symlink (for local repositories). Either way, we want to remove it and
// write our own.
if (filePath.exists(Symlinks.NOFOLLOW)) {
filePath.delete();
}
FileSystemUtils.writeContentAsLatin1(filePath, contents);
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
return RepositoryDirectoryValue.builder().setPath(repositoryDirectory);
}
protected static RepositoryDirectoryValue.Builder writeBuildFile(
Path repositoryDirectory, String contents) throws RepositoryFunctionException {
return writeFile(repositoryDirectory, "BUILD.bazel", contents);
}
@VisibleForTesting
protected static PathFragment getTargetPath(Rule rule, Path workspace)
throws RepositoryFunctionException {
WorkspaceAttributeMapper mapper = WorkspaceAttributeMapper.of(rule);
String path;
try {
path = mapper.get("path", Type.STRING);
} catch (EvalException e) {
throw new RepositoryFunctionException(e, Transience.PERSISTENT);
}
PathFragment pathFragment = PathFragment.create(path);
return workspace.getRelative(pathFragment).asFragment();
}
/**
* Given a targetDirectory /some/path/to/y that contains files z, w, and v, create the following
* directory structure:
* Note that: - We don't add a dependency on the parent directory at the package root boundary,
* so the only transitive dependencies from files inside the package roots to external files are
* through symlinks. So the upwards transitive closure of external files is small. - The only way
* other than external repositories for external source files to get into the skyframe graph in
* the first place is through symlinks outside the package roots, which we neither want to
* encourage nor optimize for since it is not common. So the set of external files is small.
*/
public static void addExternalFilesDependencies(
RootedPath rootedPath, BlazeDirectories directories, Environment env)
throws IOException, InterruptedException {
Path externalRepoDir = getExternalRepositoryDirectory(directories);
PathFragment repositoryPath = rootedPath.asPath().relativeTo(externalRepoDir);
if (repositoryPath.segmentCount() == 0) {
// We are the top of the repository path (
*
*
*
* .external-repository/
* x/
* WORKSPACE
* BUILD -> <build_root>/x.BUILD
* z -> /some/path/to/y/z
* w -> /some/path/to/y/w
* v -> /some/path/to/y/v
*
*/
public static boolean symlinkLocalRepositoryContents(
Path repositoryDirectory, Path targetDirectory)
throws RepositoryFunctionException {
try {
FileSystemUtils.createDirectoryAndParents(repositoryDirectory);
for (Path target : targetDirectory.getDirectoryEntries()) {
Path symlinkPath = repositoryDirectory.getRelative(target.getBaseName());
createSymbolicLink(symlinkPath, target);
}
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
return true;
}
static void createSymbolicLink(Path from, Path to)
throws RepositoryFunctionException {
try {
// Remove not-symlinks that are already there.
if (from.exists()) {
from.delete();
}
FileSystemUtils.ensureSymbolicLink(from, to);
} catch (IOException e) {
throw new RepositoryFunctionException(
new IOException(String.format("Error creating symbolic link from %s to %s: %s",
from, to, e.getMessage())), Transience.TRANSIENT);
}
}
/**
* Adds the repository's directory to the graph and, if it's a symlink, resolves it to an actual
* directory.
*/
@Nullable
protected static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env)
throws RepositoryFunctionException, InterruptedException {
SkyKey outputDirectoryKey = FileValue.key(RootedPath.toRootedPath(
repositoryDirectory, PathFragment.EMPTY_FRAGMENT));
FileValue value;
try {
value = (FileValue) env.getValueOrThrow(outputDirectoryKey, IOException.class,
FileSymlinkException.class, InconsistentFilesystemException.class);
} catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
throw new RepositoryFunctionException(
new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()),
Transience.PERSISTENT);
}
return value;
}
protected static Path getExternalRepositoryDirectory(BlazeDirectories directories) {
return directories.getOutputBase().getRelative(Label.EXTERNAL_PACKAGE_NAME);
}
/**
* For files that are under $OUTPUT_BASE/external, add a dependency on the corresponding rule so
* that if the WORKSPACE file changes, the File/DirectoryStateValue will be re-evaluated.
*
*