These objects are called from {@link RepositoryDelegatorFunction}. * *
External repositories come in two flavors: local and non-local. * *
Local ones are those whose fetching does not require access to any external resources * (e.g. network). These are always re-fetched on Bazel server restarts. This operation is fast * (usually just a few symlinks and maybe writing a BUILD file). {@code --nofetch} does not apply * to local repositories. * *
The up-to-dateness of non-local repositories is checked using a marker file under the * output base. When such a repository is fetched, data from the rule in the WORKSPACE file is * written to the marker file which is consulted on next server startup. If the rule hasn't changed, * the repository is not re-fetched. * *
Fetching repositories can be disabled using the {@code --nofetch} command line option. If a * repository is on the file system, Bazel just tries to use it and hopes for the best. If the * repository has never been fetched, Bazel errors out for lack of a better option. This is * implemented using * {@link com.google.devtools.build.lib.bazel.BazelRepositoryModule#REPOSITORY_VALUE_CHECKER} and * a flag in {@link RepositoryDirectoryValue} that tells Bazel whether the value in Skyframe is * stale according to the value of {@code --nofetch} or not. * *
When a rule in the WORKSPACE file is changed, the corresponding * {@link RepositoryDirectoryValue} is invalidated using the usual Skyframe route. */ public abstract class RepositoryFunction { /** * Exception thrown when something goes wrong accessing a remote repository. * *
This exception should be used by child classes to limit the types of exceptions * {@link RepositoryDelegatorFunction} has to know how to catch.
*/ public static class RepositoryFunctionException extends SkyFunctionException { public RepositoryFunctionException(NoSuchPackageException cause, Transience transience) { super(cause, transience); } /** * Error reading or writing to the filesystem. */ public RepositoryFunctionException(IOException cause, Transience transience) { super(cause, transience); } /** * For errors in WORKSPACE file rules (e.g., malformed paths or URLs). */ public RepositoryFunctionException(EvalException cause, Transience transience) { super(cause, transience); } } /** * Exception thrown when something a repository rule cannot be found. */ public static final class RepositoryNotFoundException extends RepositoryFunctionException { public RepositoryNotFoundException(String repositoryName) { super( new BuildFileContainsErrorsException( Label.EXTERNAL_PACKAGE_IDENTIFIER, "The repository named '" + repositoryName + "' could not be resolved"), Transience.PERSISTENT); } } /** * Fetch the remote repository represented by the given rule. * *When this method is called, it has already been determined that the repository is stale and * that it needs to be re-fetched. * *
The {@code env} argument can be used to fetch Skyframe dependencies the repository * implementation needs on the following conditions: *
If this is false, Bazel may decide not to re-fetch the repository, for example when the * {@code --nofetch} command line option is used. */ protected abstract boolean isLocal(Rule rule); /** * Returns a block of data that must be equal for two Rules for them to be considered the same. * *
This is used for the up-to-dateness check of fetched directory trees. The only reason for * this to exist is the {@code maven_server} rule (which should go away, but until then, we need * to keep it working somehow) */ protected byte[] getRuleSpecificMarkerData(Rule rule, Environment env) throws RepositoryFunctionException { return new byte[] {}; } protected Path prepareLocalRepositorySymlinkTree(Rule rule, Path repositoryDirectory) throws RepositoryFunctionException { try { FileSystemUtils.createDirectoryAndParents(repositoryDirectory); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } // Add x/WORKSPACE. createWorkspaceFile(repositoryDirectory, rule); return repositoryDirectory; } protected void createWorkspaceFile(Path repositoryDirectory, Rule rule) throws RepositoryFunctionException { try { Path workspaceFile = repositoryDirectory.getRelative("WORKSPACE"); FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"), String.format("# DO NOT EDIT: automatically generated WORKSPACE file for %s\n" + "workspace(name = \"%s\")", rule, rule.getName())); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } } protected static RepositoryDirectoryValue writeBuildFile( Path repositoryDirectory, String contents) throws RepositoryFunctionException { Path buildFilePath = repositoryDirectory.getRelative("BUILD"); try { FileSystemUtils.writeContentAsLatin1(buildFilePath, contents); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } return RepositoryDirectoryValue.create(repositoryDirectory); } @VisibleForTesting protected static PathFragment getTargetPath(Rule rule, Path workspace) { AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); String path = mapper.get("path", Type.STRING); PathFragment pathFragment = new PathFragment(path); return workspace.getRelative(pathFragment).asFragment(); } /** * Given a targetDirectory /some/path/to/y that contains files z, w, and v, create the following * directory structure: *
* .external-repository/ * x/ * WORKSPACE * BUILD -> <build_root>/x.BUILD * z -> /some/path/to/y/z * w -> /some/path/to/y/w * v -> /some/path/to/y/v **/ public static boolean symlinkLocalRepositoryContents( Path repositoryDirectory, Path targetDirectory) throws RepositoryFunctionException { try { FileSystemUtils.createDirectoryAndParents(repositoryDirectory); for (Path target : targetDirectory.getDirectoryEntries()) { Path symlinkPath = repositoryDirectory.getRelative(target.getBaseName()); createSymbolicLink(symlinkPath, target); } } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } return true; } static void createSymbolicLink(Path from, Path to) throws RepositoryFunctionException { try { // Remove not-symlinks that are already there. if (from.exists()) { from.delete(); } FileSystemUtils.ensureSymbolicLink(from, to); } catch (IOException e) { throw new RepositoryFunctionException( new IOException(String.format("Error creating symbolic link from %s to %s: %s", from, to, e.getMessage())), Transience.TRANSIENT); } } /** * Uses a remote repository name to fetch the corresponding Rule describing how to get it. * * This should be the unique entry point for resolving a remote repository function. */ @Nullable public static Rule getRule(String repository, Environment env) throws RepositoryFunctionException { SkyKey packageLookupKey = PackageLookupValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER); PackageLookupValue packageLookupValue; packageLookupValue = (PackageLookupValue) env.getValue(packageLookupKey); if (packageLookupValue == null) { return null; } RootedPath workspacePath = RootedPath.toRootedPath(packageLookupValue.getRoot(), new PathFragment("WORKSPACE")); SkyKey workspaceKey = WorkspaceFileValue.key(workspacePath); do { WorkspaceFileValue value = (WorkspaceFileValue) env.getValue(workspaceKey); if (value == null) { return null; } Package externalPackage = value.getPackage(); if (externalPackage.containsErrors()) { Event.replayEventsOn(env.getListener(), externalPackage.getEvents()); throw new RepositoryFunctionException( new BuildFileContainsErrorsException( Label.EXTERNAL_PACKAGE_IDENTIFIER, "Could not load //external package"), Transience.PERSISTENT); } Rule rule = externalPackage.getRule(repository); if (rule != null) { return rule; } workspaceKey = value.next(); } while (workspaceKey != null); throw new RepositoryNotFoundException(repository); } @Nullable public static Rule getRule( String ruleName, @Nullable String ruleClassName, Environment env) throws RepositoryFunctionException { try { return getRule(RepositoryName.create("@" + ruleName), ruleClassName, env); } catch (LabelSyntaxException e) { throw new RepositoryFunctionException( new IOException("Invalid rule name " + ruleName), Transience.PERSISTENT); } } /** * Uses a remote repository name to fetch the corresponding Rule describing how to get it. * This should be called from {@link SkyFunction#compute} functions, which should return null if * this returns null. If {@code ruleClassName} is set, the rule found must have a matching rule * class name. */ @Nullable public static Rule getRule( RepositoryName repositoryName, @Nullable String ruleClassName, Environment env) throws RepositoryFunctionException { Rule rule = getRule(repositoryName.strippedName(), env); Preconditions.checkState( rule == null || ruleClassName == null || rule.getRuleClass().equals(ruleClassName), "Got %s, was expecting a %s", rule, ruleClassName); return rule; } /** * Adds the repository's directory to the graph and, if it's a symlink, resolves it to an * actual directory. */ @Nullable public static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env) throws RepositoryFunctionException { SkyKey outputDirectoryKey = FileValue.key(RootedPath.toRootedPath( repositoryDirectory, PathFragment.EMPTY_FRAGMENT)); FileValue value; try { value = (FileValue) env.getValueOrThrow(outputDirectoryKey, IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class); } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) { throw new RepositoryFunctionException( new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()), Transience.PERSISTENT); } return value; } public static Path getExternalRepositoryDirectory(BlazeDirectories directories) { return directories .getOutputBase() .getRelative(Label.EXTERNAL_PATH_PREFIX); } /** * Returns the RuleDefinition class for this type of repository. */ public abstract Class getRuleDefinition(); }