// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.exec; import static java.nio.charset.StandardCharsets.US_ASCII; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.Maps; import com.google.common.io.BaseEncoding; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.DigestOfDirectoryException; import com.google.devtools.build.lib.actions.cache.Metadata; import com.google.devtools.build.lib.skyframe.FileArtifactValue; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.protobuf.ByteString; import java.io.IOException; import java.util.Map; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; /** * An in-memory cache to ensure we do I/O for source files only once during a single build. * *

Simply maintains a two-way cached mapping from digest <--> filename that may be populated * only once. */ @ThreadSafe public class SingleBuildFileCache implements ActionInputFileCache { private final Path execRoot; public SingleBuildFileCache(String cwd, FileSystem fs) { this.execRoot = fs.getPath(cwd); } // If we can't get the digest, we store the exception. This avoids extra file IO for files // that are allowed to be missing, as we first check a likely non-existent content file // first. Further we won't need to unwrap the exception in getDigest(). private final LoadingCache pathToMetadata = CacheBuilder.newBuilder() // We default to 10 disk read threads, but we don't expect them all to edit the map // simultaneously. .concurrencyLevel(8) // Even small-ish builds, as of 11/21/2011 typically have over 10k artifacts, so it's // unlikely that this default will adversely affect memory in most cases. .initialCapacity(10000) .build(new CacheLoader() { @Override public ActionInputMetadata load(ActionInput input) { Path path = null; try { path = execRoot.getRelative(input.getExecPath()); byte[] digest = path.getDigest(); BaseEncoding hex = BaseEncoding.base16().lowerCase(); ByteString hexDigest = ByteString.copyFrom(hex.encode(digest).getBytes(US_ASCII)); // Inject reverse mapping. Doing this unconditionally in getDigest() showed up // as a hotspot in CPU profiling. digestToPath.put(hexDigest, input); return new ActionInputMetadata(digest, path.getFileSize()); } catch (IOException e) { if (path != null && path.isDirectory()) { // TODO(bazel-team): This is rather presumptuous- it could have been another type of // IOException. return new ActionInputMetadata( new DigestOfDirectoryException( "Input is a directory: " + input.getExecPathString())); } else { return new ActionInputMetadata(e); } } } }); private final Map digestToPath = Maps.newConcurrentMap(); @Override public Metadata getMetadata(ActionInput input) throws IOException { return pathToMetadata.getUnchecked(input).getMetadata(); } @Nullable @Override public ActionInput getInputFromDigest(ByteString digest) { return digestToPath.get(digest); } @Override public Path getInputPath(ActionInput input) { return execRoot.getRelative(input.getExecPath()); } @Override public boolean contentsAvailableLocally(ByteString digest) { return digestToPath.containsKey(digest); } /** Container class for caching I/O around ActionInputs. */ private static class ActionInputMetadata { private final Metadata metadata; private final IOException exceptionOnAccess; /** Constructor for a successful lookup. */ ActionInputMetadata(byte[] digest, long size) { this.metadata = FileArtifactValue.createNormalFile(digest, size); this.exceptionOnAccess = null; } /** Constructor for a failed lookup, size will be 0. */ ActionInputMetadata(IOException exceptionOnAccess) { this.exceptionOnAccess = exceptionOnAccess; this.metadata = null; } Metadata getMetadata() throws IOException { maybeRaiseException(); return metadata; } private void maybeRaiseException() throws IOException { if (exceptionOnAccess != null) { throw exceptionOnAccess; } } } }