// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.remote; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.remote.Digests.ActionKey; import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.remoteexecution.v1test.ActionResult; import com.google.devtools.remoteexecution.v1test.Digest; import com.google.devtools.remoteexecution.v1test.Directory; import com.google.devtools.remoteexecution.v1test.DirectoryNode; import com.google.devtools.remoteexecution.v1test.FileNode; import com.google.devtools.remoteexecution.v1test.OutputDirectory; import com.google.devtools.remoteexecution.v1test.OutputFile; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.Collection; import java.util.concurrent.Semaphore; /** * A RemoteActionCache implementation that uses a concurrent map as a distributed storage for files * and action output. * *
The thread safety is guaranteed by the underlying map.
*/
@ThreadSafe
public final class SimpleBlobStoreActionCache implements RemoteActionCache {
private final SimpleBlobStore blobStore;
private static final int MAX_MEMORY_KBYTES = 512 * 1024;
private final Semaphore uploadMemoryAvailable = new Semaphore(MAX_MEMORY_KBYTES, true);
public SimpleBlobStoreActionCache(SimpleBlobStore blobStore) {
this.blobStore = blobStore;
}
@Override
public void uploadTree(TreeNodeRepository repository, Path execRoot, TreeNode root)
throws IOException, InterruptedException {
repository.computeMerkleDigests(root);
for (Directory directory : repository.treeToDirectories(root)) {
uploadBlob(directory.toByteArray());
}
// TODO(ulfjack): Only upload files that aren't in the CAS yet?
for (TreeNode leaf : repository.leaves(root)) {
uploadFileContents(leaf.getActionInput(), execRoot, repository.getInputFileCache());
}
}
@Override
public void downloadTree(Digest rootDigest, Path rootLocation)
throws IOException, CacheNotFoundException {
Directory directory = Directory.parseFrom(downloadBlob(rootDigest));
for (FileNode file : directory.getFilesList()) {
downloadFileContents(
file.getDigest(), rootLocation.getRelative(file.getName()), file.getIsExecutable());
}
for (DirectoryNode child : directory.getDirectoriesList()) {
downloadTree(child.getDigest(), rootLocation.getRelative(child.getName()));
}
}
@Override
public Digest uploadFileContents(Path file) throws IOException, InterruptedException {
// This unconditionally reads the whole file into memory first!
return uploadBlob(ByteString.readFrom(file.getInputStream()).toByteArray());
}
@Override
public Digest uploadFileContents(
ActionInput input, Path execRoot, ActionInputFileCache inputCache)
throws IOException, InterruptedException {
// This unconditionally reads the whole file into memory first!
if (input instanceof VirtualActionInput) {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
((VirtualActionInput) input).writeTo(buffer);
byte[] blob = buffer.toByteArray();
return uploadBlob(blob, Digests.computeDigest(blob));
}
return uploadBlob(
ByteString.readFrom(execRoot.getRelative(input.getExecPathString()).getInputStream())
.toByteArray(),
Digests.getDigestFromInputCache(input, inputCache));
}
@Override
public void downloadAllResults(ActionResult result, Path execRoot)
throws IOException, CacheNotFoundException {
for (OutputFile file : result.getOutputFilesList()) {
if (!file.getContent().isEmpty()) {
createFile(
file.getContent().toByteArray(),
execRoot.getRelative(file.getPath()),
file.getIsExecutable());
} else {
downloadFileContents(
file.getDigest(), execRoot.getRelative(file.getPath()), file.getIsExecutable());
}
}
for (OutputDirectory directory : result.getOutputDirectoriesList()) {
downloadTree(directory.getDigest(), execRoot.getRelative(directory.getPath()));
}
}
@Override
public void uploadAllResults(Path execRoot, Collection