diff options
7 files changed, 40 insertions, 59 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/remote/ConcurrentMapActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/ConcurrentMapActionCache.java index e867a77411..7e27653e76 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ConcurrentMapActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ConcurrentMapActionCache.java @@ -43,24 +43,14 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { private static final int MAX_MEMORY_KBYTES = 512 * 1024; private final Semaphore uploadMemoryAvailable = new Semaphore(MAX_MEMORY_KBYTES, true); - public ConcurrentMapActionCache( - Path execRoot, RemoteOptions options, ConcurrentMap<String, byte[]> cache) { + public ConcurrentMapActionCache(Path execRoot, ConcurrentMap<String, byte[]> cache) { this.execRoot = execRoot; this.cache = cache; } @Override - public String putFileIfNotExist(Path file) throws IOException { - String contentKey = HashCode.fromBytes(file.getMD5Digest()).toString(); - if (containsFile(contentKey)) { - return contentKey; - } - putFile(contentKey, file); - return contentKey; - } - - @Override - public String putFileIfNotExist(ActionInputFileCache cache, ActionInput file) throws IOException { + public String putFileIfNotExist(ActionInputFileCache cache, ActionInput file) + throws IOException, InterruptedException { String contentKey = HashCode.fromBytes(cache.getDigest(file)).toString(); if (containsFile(contentKey)) { return contentKey; @@ -69,7 +59,7 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { return contentKey; } - private void putFile(String key, Path file) throws IOException { + private void putFile(String key, Path file) throws IOException, InterruptedException { int fileSizeKBytes = (int) (file.getFileSize() / 1024); Preconditions.checkArgument(fileSizeKBytes < MAX_MEMORY_KBYTES); try { @@ -84,8 +74,6 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { .build() .toByteArray()); } - } catch (InterruptedException e) { - throw new IOException("Failed to put file to memory cache.", e); } finally { uploadMemoryAvailable.release(fileSizeKBytes); } @@ -123,7 +111,7 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { @Override public void putActionOutput(String key, Collection<? extends ActionInput> outputs) - throws IOException { + throws IOException, InterruptedException { CacheEntry.Builder actionOutput = CacheEntry.newBuilder(); for (ActionInput output : outputs) { Path file = execRoot.getRelative(output.getExecPathString()); @@ -134,7 +122,7 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { @Override public void putActionOutput(String key, Path execRoot, Collection<Path> files) - throws IOException { + throws IOException, InterruptedException { CacheEntry.Builder actionOutput = CacheEntry.newBuilder(); for (Path file : files) { addToActionOutput(file, file.relativeTo(execRoot).getPathString(), actionOutput); @@ -144,7 +132,7 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { /** Add the file to action output cache entry. Put the file to cache if necessary. */ private void addToActionOutput(Path file, String execPathString, CacheEntry.Builder actionOutput) - throws IOException { + throws IOException, InterruptedException { if (file.isDirectory()) { // TODO(alpha): Implement this for directory. throw new UnsupportedOperationException("Storing a directory is not yet supported."); @@ -158,4 +146,13 @@ public final class ConcurrentMapActionCache implements RemoteActionCache { .setContentKey(contentKey) .setExecutable(file.isExecutable()); } + + private String putFileIfNotExist(Path file) throws IOException, InterruptedException { + String contentKey = HashCode.fromBytes(file.getMD5Digest()).toString(); + if (containsFile(contentKey)) { + return contentKey; + } + putFile(contentKey, file); + return contentKey; + } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java index 7f87803dde..20cbc6058d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java @@ -114,7 +114,7 @@ public class MemcacheWorkExecutor implements RemoteWorkExecutor { ImmutableMap<String, String> environment, Collection<? extends ActionInput> outputs, int timeout) - throws IOException, WorkTooLargeException { + throws IOException, WorkTooLargeException, InterruptedException { RemoteWorkRequest.Builder work = RemoteWorkRequest.newBuilder(); work.setOutputKey(actionOutputKey); @@ -162,7 +162,8 @@ public class MemcacheWorkExecutor implements RemoteWorkExecutor { } /** Execute a work item locally. */ - public RemoteWorkResponse executeLocally(RemoteWorkRequest work) throws IOException { + public RemoteWorkResponse executeLocally(RemoteWorkRequest work) + throws IOException, InterruptedException { ByteArrayOutputStream stdout = new ByteArrayOutputStream(); ByteArrayOutputStream stderr = new ByteArrayOutputStream(); try { diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java index 538a4c987e..e865c96e2c 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java @@ -18,7 +18,6 @@ import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.vfs.Path; - import java.io.IOException; import java.util.Collection; @@ -28,19 +27,12 @@ import java.util.Collection; @ThreadCompatible interface RemoteActionCache { /** - * Put the file in cache if it is not already in it. No-op if the file is already stored in - * cache. - * - * @return The key for fetching the file from cache. - */ - String putFileIfNotExist(Path file) throws IOException; - - /** - * Same as {@link putFileIfNotExist(Path)} but this methods takes an ActionInput. + * Put the file in cache if it is not already in it. No-op if the file is already stored in cache. * * @return The key for fetching the file from cache. */ - String putFileIfNotExist(ActionInputFileCache cache, ActionInput file) throws IOException; + String putFileIfNotExist(ActionInputFileCache cache, ActionInput file) + throws IOException, InterruptedException; /** * Write the file in cache identified by key to the file system. The key must uniquely identify @@ -58,14 +50,11 @@ interface RemoteActionCache { void writeActionOutput(String key, Path execRoot) throws IOException, CacheNotFoundException; - /** - * Update the cache with the action outputs for the specified key. - */ + /** Update the cache with the action outputs for the specified key. */ void putActionOutput(String key, Collection<? extends ActionInput> outputs) - throws IOException; + throws IOException, InterruptedException; - /** - * Update the cache with the files for the specified key. - */ - void putActionOutput(String key, Path execRoot, Collection<Path> files) throws IOException; + /** Update the cache with the files for the specified key. */ + void putActionOutput(String key, Path execRoot, Collection<Path> files) + throws IOException, InterruptedException; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index ffe27f5d94..16ce15ca65 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -69,13 +69,11 @@ public final class RemoteModule extends BlazeModule { cache = new ConcurrentMapActionCache( this.env.getDirectories().getExecRoot(), - options, HazelcastCacheFactory.create(options)); } else if (options.restCacheUrl != null) { cache = new ConcurrentMapActionCache( this.env.getDirectories().getExecRoot(), - options, RestUrlCacheFactory.create(options)); } actionCache = cache; diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java index 17b821af79..fe388f7b26 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java @@ -36,7 +36,6 @@ import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; - import java.io.IOException; import java.nio.charset.Charset; import java.util.Collection; @@ -74,12 +73,10 @@ final class RemoteSpawnStrategy implements SpawnActionContext { this.remoteWorkExecutor = workExecutor; } - /** - * Executes the given {@code spawn}. - */ + /** Executes the given {@code spawn}. */ @Override public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) - throws ExecException { + throws ExecException, InterruptedException { if (!spawn.isRemotable()) { standaloneStrategy.exec(spawn, actionExecutionContext); return; @@ -188,7 +185,7 @@ final class RemoteSpawnStrategy implements SpawnActionContext { int timeout, EventHandler eventHandler, FileOutErr outErr) - throws IOException { + throws IOException, InterruptedException { if (remoteWorkExecutor == null) { return false; } @@ -225,9 +222,8 @@ final class RemoteSpawnStrategy implements SpawnActionContext { Event.warn(mnemonic + " timed out executing work remotely (" + e + "), running locally")); return false; } catch (InterruptedException e) { - Thread.currentThread().interrupt(); eventHandler.handle(Event.warn(mnemonic + " remote work interrupted (" + e + ")")); - return false; + throw e; } catch (WorkTooLargeException e) { eventHandler.handle(Event.warn(mnemonic + " cannot be run remotely (" + e + ")")); return false; diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java index 22def42a07..198e9262a7 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java @@ -21,7 +21,6 @@ import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkResponse; import com.google.devtools.build.lib.vfs.Path; - import java.io.IOException; import java.util.Collection; @@ -31,11 +30,10 @@ import java.util.Collection; @ThreadCompatible public interface RemoteWorkExecutor { /** - * Submit the work to this work executor. - * The output of running this action should be written to {@link RemoteActionCache} indexed - * by |actionOutputKey|. + * Submit the work to this work executor. The output of running this action should be written to + * {@link RemoteActionCache} indexed by {@code actionOutputKey}. * - * Returns a future for the response of this work request. + * <p>Returns a future for the response of this work request. */ ListenableFuture<RemoteWorkResponse> executeRemotely( Path execRoot, @@ -46,5 +44,5 @@ public interface RemoteWorkExecutor { ImmutableMap<String, String> environment, Collection<? extends ActionInput> outputs, int timeout) - throws IOException, WorkTooLargeException; + throws IOException, WorkTooLargeException, InterruptedException; } diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java index 69d2716705..414737a810 100644 --- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java @@ -69,8 +69,7 @@ public class RemoteWorker extends RemoteWorkImplBase { Path tempRoot = workPath.getRelative("build-" + UUID.randomUUID().toString()); try { FileSystemUtils.createDirectoryAndParents(tempRoot); - final ConcurrentMapActionCache actionCache = - new ConcurrentMapActionCache(tempRoot, remoteOptions, cache); + final ConcurrentMapActionCache actionCache = new ConcurrentMapActionCache(tempRoot, cache); final MemcacheWorkExecutor workExecutor = MemcacheWorkExecutor.createLocalWorkExecutor(actionCache, tempRoot); if (LOG_FINER) { @@ -96,10 +95,13 @@ public class RemoteWorker extends RemoteWorkImplBase { } else { LOG.warning("Preserving work directory " + tempRoot.toString() + "."); } - } catch (IOException e) { + } catch (IOException | InterruptedException e) { RemoteWorkResponse.Builder response = RemoteWorkResponse.newBuilder(); response.setSuccess(false).setOut("").setErr("").setException(e.toString()); responseObserver.onNext(response.build()); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } } finally { responseObserver.onCompleted(); } |