// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.rules.repository;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.RuleFormatter;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
import com.google.devtools.build.lib.skyframe.FileValue;
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
/**
* A {@link SkyFunction} that implements delegation to the correct repository fetcher.
*
*
* Each repository in the WORKSPACE file is represented by a {@link SkyValue} that is computed by
* this function.
*/
public final class RepositoryDelegatorFunction implements SkyFunction {
// The marker file version is inject in the rule key digest so the rule key is always different
// when we decide to update the format.
private static final int MARKER_FILE_VERSION = 2;
// A special repository delegate used to handle Skylark remote repositories if present.
public static final String SKYLARK_DELEGATE_NAME = "$skylark";
// Mapping of rule class name to RepositoryFunction.
private final ImmutableMap handlers;
// Delegate function to handle skylark remote repositories
private final RepositoryFunction skylarkHandler;
// This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current
// command is a fetch. Remote repository lookups are only allowed during fetches.
private final AtomicBoolean isFetch;
private Map clientEnvironment;
public RepositoryDelegatorFunction(ImmutableMap handlers,
@Nullable RepositoryFunction skylarkHandler, AtomicBoolean isFetch) {
this.handlers = handlers;
this.skylarkHandler = skylarkHandler;
this.isFetch = isFetch;
}
public void setClientEnvironment(Map clientEnvironment) {
this.clientEnvironment = clientEnvironment;
}
private void setupRepositoryRoot(Path repoRoot) throws RepositoryFunctionException {
try {
FileSystemUtils.deleteTree(repoRoot);
FileSystemUtils.createDirectoryAndParents(repoRoot.getParentDirectory());
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws SkyFunctionException, InterruptedException {
RepositoryName repositoryName = (RepositoryName) skyKey.argument();
Rule rule = RepositoryFunction.getRule(repositoryName, null, env);
if (rule == null) {
return null;
}
BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env);
if (directories == null) {
return null;
}
RepositoryFunction handler;
if (rule.getRuleClassObject().isSkylark()) {
handler = skylarkHandler;
} else {
handler = handlers.get(rule.getRuleClass());
}
if (handler == null) {
throw new RepositoryFunctionException(
new EvalException(Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")),
"Could not find handler for " + rule),
Transience.PERSISTENT);
}
handler.setClientEnvironment(clientEnvironment);
Path repoRoot =
RepositoryFunction.getExternalRepositoryDirectory(directories).getRelative(rule.getName());
byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env);
if (ruleSpecificData == null) {
return null;
}
String ruleKey = computeRuleKey(rule, ruleSpecificData);
Map markerData = new TreeMap<>();
Path markerPath = getMarkerPath(directories, rule);
if (handler.isLocal(rule)) {
// Local repositories are always fetched because the operation is generally fast and they do
// not depend on non-local data, so it does not make much sense to try to cache from across
// server instances.
setupRepositoryRoot(repoRoot);
SkyValue localRepo = handler.fetch(rule, repoRoot, directories, env, markerData);
if (localRepo != null) {
writeMarkerFile(markerPath, markerData, ruleKey);
}
return localRepo;
}
// We check the repository root for existence here, but we can't depend on the FileValue,
// because it's possible that we eventually create that directory in which case the FileValue
// and the state of the file system would be inconsistent.
Boolean markerUpToDate = isFilesystemUpToDate(markerPath, rule, ruleKey, handler, env);
if (markerUpToDate == null) {
return null;
}
if (markerUpToDate && repoRoot.exists()) {
// Now that we know that it exists, we can declare a Skyframe dependency on the repository
// root.
RepositoryFunction.getRepositoryDirectory(repoRoot, env);
if (env.valuesMissing()) {
return null;
}
return RepositoryDirectoryValue.create(repoRoot);
}
if (isFetch.get()) {
// Fetching enabled, go ahead.
setupRepositoryRoot(repoRoot);
SkyValue result = handler.fetch(rule, repoRoot, directories, env, markerData);
if (env.valuesMissing()) {
return null;
}
// No new Skyframe dependencies must be added between calling the repository implementation
// and writing the marker file because if they aren't computed, it would cause a Skyframe
// restart thus calling the possibly very slow (networking, decompression...) fetch()
// operation again. So we write the marker file here immediately.
writeMarkerFile(markerPath, markerData, ruleKey);
return result;
}
if (!repoRoot.exists()) {
// The repository isn't on the file system, there is nothing we can do.
throw new RepositoryFunctionException(
new IOException("to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName
+ " not found and fetching repositories is disabled."),
Transience.TRANSIENT);
}
// Declare a Skyframe dependency so that this is re-evaluated when something happens to the
// directory.
FileValue repoRootValue = RepositoryFunction.getRepositoryDirectory(repoRoot, env);
if (env.valuesMissing()) {
return null;
}
// Try to build with whatever is on the file system and emit a warning.
env.getListener()
.handle(Event.warn(rule.getLocation(),
String.format(
"External repository '%s' is not up-to-date and fetching is disabled. To update, "
+ "run the build without the '--nofetch' command line option.",
rule.getName())));
return RepositoryDirectoryValue.fetchingDelayed(repoRootValue.realRootedPath().asPath());
}
private final String computeRuleKey(Rule rule, byte[] ruleSpecificData) {
return new Fingerprint().addBytes(RuleFormatter.serializeRule(rule).build().toByteArray())
.addBytes(ruleSpecificData)
.addInt(MARKER_FILE_VERSION).hexDigestAndReset();
}
/**
* Checks if the state of the repository in the file system is consistent with the rule in the
* WORKSPACE file.
*
*
* Deletes the marker file if not so that no matter what happens after, the state of the file
* system stays consistent.
*
*
* Returns null if some Skyframe values need to be computed before giving a full answer.
*/
@Nullable
private final Boolean isFilesystemUpToDate(Path markerPath, Rule rule, String ruleKey,
RepositoryFunction handler, Environment env)
throws RepositoryFunctionException, InterruptedException {
try {
if (!markerPath.exists()) {
return false;
}
String[] lines = FileSystemUtils.readContent(markerPath, StandardCharsets.UTF_8).split("\n");
Map markerData = new TreeMap<>();
String markerRuleKey = "";
boolean firstLine = true;
for (String line : lines) {
if (firstLine) {
markerRuleKey = line;
firstLine = false;
} else {
int sChar = line.indexOf(' ');
String key = line;
String value = "";
if (sChar > 0) {
key = unescape(line.substring(0, sChar));
value = unescape(line.substring(sChar + 1));
}
markerData.put(key, value);
}
}
boolean result = false;
if (markerRuleKey.equals(ruleKey)) {
result = handler.verifyMarkerData(rule, markerData, env);
if (env.valuesMissing()) {
return null;
}
}
if (!result) {
// So that we are in a consistent state if something happens while fetching the repository
markerPath.delete();
}
return result;
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
// Escape a value for the marker file
@VisibleForTesting
static String escape(String str) {
return str == null ? "\\0" : str.replace("\\", "\\\\").replace("\n", "\\n").replace(" ", "\\s");
}
// Unescape a value from the marker file
@VisibleForTesting
static String unescape(String str) {
if (str.equals("\\0")) {
return null; // \0 == null string
}
StringBuffer result = new StringBuffer();
boolean escaped = false;
for (int i = 0; i < str.length(); i++) {
char c = str.charAt(i);
if (escaped) {
if (c == 'n') { // n means new line
result.append("\n");
} else if (c == 's') { // s means space
result.append(" ");
} else { // Any other escaped characters are just un-escaped
result.append(c);
}
escaped = false;
} else if (c == '\\') {
escaped = true;
} else {
result.append(c);
}
}
return result.toString();
}
private final void writeMarkerFile(
Path markerPath, Map markerData, String ruleKey)
throws RepositoryFunctionException {
try {
StringBuilder builder = new StringBuilder();
builder.append(ruleKey).append("\n");
for (Map.Entry data : markerData.entrySet()) {
String key = data.getKey();
String value = data.getValue();
builder.append(escape(key)).append(" ").append(escape(value)).append("\n");
}
FileSystemUtils.writeContent(markerPath, StandardCharsets.UTF_8, builder.toString());
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
private static Path getMarkerPath(BlazeDirectories directories, Rule rule) {
return RepositoryFunction.getExternalRepositoryDirectory(directories)
.getChild("@" + rule.getName() + ".marker");
}
@Override
public String extractTag(SkyKey skyKey) {
return null;
}
}