From 6f32d5a7d75ff46de6efc557c1ea95777ea813ca Mon Sep 17 00:00:00 2001 From: olaola Date: Wed, 20 Sep 2017 17:12:19 +0200 Subject: Passing Bazel metadata in gRPC headers. TESTED=unit tests RELNOTES: none PiperOrigin-RevId: 169395919 --- .../build/lib/remote/ByteStreamUploaderTest.java | 116 +++++++++++++++++++- .../build/lib/remote/GrpcRemoteCacheTest.java | 5 + .../lib/remote/GrpcRemoteExecutionClientTest.java | 62 +++++++++-- .../build/lib/remote/RemoteSpawnCacheTest.java | 4 +- .../build/lib/remote/RemoteSpawnRunnerTest.java | 121 +++++++++++++++++---- 5 files changed, 276 insertions(+), 32 deletions(-) (limited to 'src/test/java/com/google/devtools/build/lib/remote') diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java index 732afac1fa..e809b74b7e 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java @@ -14,22 +14,30 @@ package com.google.devtools.build.lib.remote; import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.fail; import com.google.bytestream.ByteStreamGrpc; import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase; import com.google.bytestream.ByteStreamProto.WriteRequest; import com.google.bytestream.ByteStreamProto.WriteResponse; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.remoteexecution.v1test.Digest; +import com.google.devtools.remoteexecution.v1test.RequestMetadata; import com.google.protobuf.ByteString; +import io.grpc.BindableService; import io.grpc.Channel; +import io.grpc.Context; import io.grpc.Metadata; import io.grpc.Server; import io.grpc.ServerCall; import io.grpc.ServerCall.Listener; import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptors; import io.grpc.ServerServiceDefinition; import io.grpc.Status; import io.grpc.Status.Code; @@ -76,28 +84,36 @@ public class ByteStreamUploaderTest { private Server server; private Channel channel; + private Context withEmptyMetadata; @Mock private Retrier.Backoff mockBackoff; @Before - public void init() throws Exception { + public final void setUp() throws Exception { MockitoAnnotations.initMocks(this); String serverName = "Server for " + this.getClass(); server = InProcessServerBuilder.forName(serverName).fallbackHandlerRegistry(serviceRegistry) .build().start(); channel = InProcessChannelBuilder.forName(serverName).build(); + withEmptyMetadata = + TracingMetadataUtils.contextWithMetadata( + "none", "none", Digests.unsafeActionKeyFromDigest(Digest.getDefaultInstance())); + // Needs to be repeated in every test that uses the timeout setting, since the tests run + // on different threads than the setUp. + withEmptyMetadata.attach(); } @After - public void shutdown() { + public void tearDown() throws Exception { server.shutdownNow(); retryService.shutdownNow(); } @Test(timeout = 10000) public void singleBlobUploadShouldWork() throws Exception { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> mockBackoff, (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); @@ -166,6 +182,7 @@ public class ByteStreamUploaderTest { @Test(timeout = 20000) public void multipleBlobsUploadShouldWork() throws Exception { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> new FixedBackoff(1, 0), (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); @@ -251,10 +268,100 @@ public class ByteStreamUploaderTest { blockUntilInternalStateConsistent(uploader); } + @Test(timeout = 20000) + public void contextShouldBePreservedUponRetries() throws Exception { + withEmptyMetadata.attach(); + // We upload blobs with different context, and retry 3 times for each upload. + // We verify that the correct metadata is passed to the server with every blob. + Retrier retrier = new Retrier(() -> new FixedBackoff(3, 0), (Status s) -> true); + ByteStreamUploader uploader = + new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); + + List toUpload = ImmutableList.of("aaaaaaaaaa", "bbbbbbbbbb", "cccccccccc"); + List builders = new ArrayList<>(toUpload.size()); + Map uploadsFailed = new HashMap<>(); + for (String s : toUpload) { + Chunker chunker = new Chunker(s.getBytes(UTF_8), /* chunkSize=*/ 3); + builders.add(chunker); + uploadsFailed.put(chunker.digest().getHash(), 0); + } + + BindableService bsService = + new ByteStreamImplBase() { + @Override + public StreamObserver write(StreamObserver response) { + return new StreamObserver() { + + private String digestHash; + + @Override + public void onNext(WriteRequest writeRequest) { + String resourceName = writeRequest.getResourceName(); + if (!resourceName.isEmpty()) { + String[] components = resourceName.split("/"); + assertThat(components).hasLength(6); + digestHash = components[4]; + } + assertThat(digestHash).isNotNull(); + RequestMetadata meta = TracingMetadataUtils.fromCurrentContext(); + assertThat(meta.getCorrelatedInvocationsId()).isEqualTo("build-req-id"); + assertThat(meta.getToolInvocationId()).isEqualTo("command-id"); + assertThat(meta.getActionId()).isEqualTo(digestHash); + assertThat(meta.getToolDetails().getToolName()).isEqualTo("bazel"); + assertThat(meta.getToolDetails().getToolVersion()) + .isEqualTo(BlazeVersionInfo.instance().getVersion()); + synchronized (this) { + Integer numFailures = uploadsFailed.get(digestHash); + if (numFailures < 3) { + uploadsFailed.put(digestHash, numFailures + 1); + response.onError(Status.INTERNAL.asException()); + return; + } + } + } + + @Override + public void onError(Throwable throwable) { + fail("onError should never be called."); + } + + @Override + public void onCompleted() { + response.onNext(WriteResponse.newBuilder().setCommittedSize(10).build()); + response.onCompleted(); + } + }; + } + }; + serviceRegistry.addService( + ServerInterceptors.intercept( + bsService, new TracingMetadataUtils.ServerHeadersInterceptor())); + + List> uploads = new ArrayList<>(); + + for (Chunker chunker : builders) { + Context ctx = + TracingMetadataUtils.contextWithMetadata( + "build-req-id", "command-id", Digests.unsafeActionKeyFromDigest(chunker.digest())); + ctx.call( + () -> { + uploads.add(uploader.uploadBlobAsync(chunker)); + return null; + }); + } + + for (ListenableFuture upload : uploads) { + upload.get(); + } + + blockUntilInternalStateConsistent(uploader); + } + @Test(timeout = 10000) public void sameBlobShouldNotBeUploadedTwice() throws Exception { // Test that uploading the same file concurrently triggers only one file upload. + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> mockBackoff, (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); @@ -313,6 +420,7 @@ public class ByteStreamUploaderTest { @Test(timeout = 10000) public void errorsShouldBeReported() throws IOException, InterruptedException { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> new FixedBackoff(1, 10), (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); @@ -339,6 +447,7 @@ public class ByteStreamUploaderTest { @Test(timeout = 10000) public void shutdownShouldCancelOngoingUploads() throws Exception { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> new FixedBackoff(1, 10), (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); @@ -390,6 +499,7 @@ public class ByteStreamUploaderTest { @Test(timeout = 10000) public void failureInRetryExecutorShouldBeHandled() throws Exception { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> new FixedBackoff(1, 10), (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(INSTANCE_NAME, channel, null, 3, retrier, retryService); @@ -420,6 +530,7 @@ public class ByteStreamUploaderTest { @Test(timeout = 10000) public void resourceNameWithoutInstanceName() throws Exception { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> mockBackoff, (Status s) -> true); ByteStreamUploader uploader = new ByteStreamUploader(/* instanceName */ null, channel, null, 3, retrier, retryService); @@ -456,6 +567,7 @@ public class ByteStreamUploaderTest { @Test(timeout = 10000) public void nonRetryableStatusShouldNotBeRetried() throws Exception { + withEmptyMetadata.attach(); Retrier retrier = new Retrier(() -> new FixedBackoff(1, 0), /* No Status is retriable. */ (Status s) -> false); ByteStreamUploader uploader = diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java index fe31aa729d..5cc03be824 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java @@ -54,6 +54,7 @@ import io.grpc.Channel; import io.grpc.ClientCall; import io.grpc.ClientInterceptor; import io.grpc.ClientInterceptors; +import io.grpc.Context; import io.grpc.MethodDescriptor; import io.grpc.Server; import io.grpc.Status; @@ -103,6 +104,10 @@ public class GrpcRemoteCacheTest { FileSystemUtils.createDirectoryAndParents(stdout.getParentDirectory()); FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory()); outErr = new FileOutErr(stdout, stderr); + Context withEmptyMetadata = + TracingMetadataUtils.contextWithMetadata( + "none", "none", Digests.unsafeActionKeyFromDigest(Digest.getDefaultInstance())); + withEmptyMetadata.attach(); } @After diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java index c5d5b66f62..de6d50f908 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java @@ -34,6 +34,7 @@ import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ResourceSet; import com.google.devtools.build.lib.actions.SimpleSpawn; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; import com.google.devtools.build.lib.authandtls.GrpcUtils; import com.google.devtools.build.lib.exec.SpawnExecException; @@ -61,6 +62,7 @@ import com.google.devtools.remoteexecution.v1test.ExecutionGrpc.ExecutionImplBas import com.google.devtools.remoteexecution.v1test.FindMissingBlobsRequest; import com.google.devtools.remoteexecution.v1test.FindMissingBlobsResponse; import com.google.devtools.remoteexecution.v1test.GetActionResultRequest; +import com.google.devtools.remoteexecution.v1test.RequestMetadata; import com.google.longrunning.Operation; import com.google.protobuf.Any; import com.google.protobuf.ByteString; @@ -69,9 +71,15 @@ import com.google.watcher.v1.Change; import com.google.watcher.v1.ChangeBatch; import com.google.watcher.v1.Request; import com.google.watcher.v1.WatcherGrpc.WatcherImplBase; +import io.grpc.BindableService; import io.grpc.CallCredentials; import io.grpc.Channel; +import io.grpc.Metadata; import io.grpc.Server; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.ServerInterceptors; import io.grpc.Status; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; @@ -229,8 +237,17 @@ public class GrpcRemoteExecutionClientTest { GrpcUtils.newCallCredentials(Options.getDefaults(AuthAndTLSOptions.class)); GrpcRemoteCache remoteCache = new GrpcRemoteCache(channel, creds, options, retrier); - client = new RemoteSpawnRunner(execRoot, options, null, true, /*cmdlineReporter=*/null, - remoteCache, executor); + client = + new RemoteSpawnRunner( + execRoot, + options, + null, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + remoteCache, + executor); inputDigest = fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz"); } @@ -364,22 +381,42 @@ public class GrpcRemoteExecutionClientTest { }; } + /** Capture the request headers from a client. Useful for testing metadata propagation. */ + private static class RequestHeadersValidator implements ServerInterceptor { + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + RequestMetadata meta = headers.get(TracingMetadataUtils.METADATA_KEY); + assertThat(meta.getCorrelatedInvocationsId()).isEqualTo("build-req-id"); + assertThat(meta.getToolInvocationId()).isEqualTo("command-id"); + assertThat(meta.getActionId()).isNotEmpty(); + assertThat(meta.getToolDetails().getToolName()).isEqualTo("bazel"); + assertThat(meta.getToolDetails().getToolVersion()) + .isEqualTo(BlazeVersionInfo.instance().getVersion()); + return next.startCall(call, headers); + } + } + @Test public void remotelyExecute() throws Exception { - serviceRegistry.addService( + BindableService actionCache = new ActionCacheImplBase() { @Override public void getActionResult( GetActionResultRequest request, StreamObserver responseObserver) { responseObserver.onError(Status.NOT_FOUND.asRuntimeException()); } - }); + }; + serviceRegistry.addService( + ServerInterceptors.intercept(actionCache, new RequestHeadersValidator())); final ActionResult actionResult = ActionResult.newBuilder() .setStdoutRaw(ByteString.copyFromUtf8("stdout")) .setStderrRaw(ByteString.copyFromUtf8("stderr")) .build(); - serviceRegistry.addService( + BindableService execService = new ExecutionImplBase() { @Override public void execute(ExecuteRequest request, StreamObserver responseObserver) { @@ -395,7 +432,9 @@ public class GrpcRemoteExecutionClientTest { .build()); responseObserver.onCompleted(); } - }); + }; + serviceRegistry.addService( + ServerInterceptors.intercept(execService, new RequestHeadersValidator())); final Command command = Command.newBuilder() .addAllArguments(ImmutableList.of("/bin/echo", "Hi!")) @@ -406,7 +445,7 @@ public class GrpcRemoteExecutionClientTest { .build()) .build(); final Digest cmdDigest = Digests.computeDigest(command); - serviceRegistry.addService( + BindableService cas = new ContentAddressableStorageImplBase() { @Override public void findMissingBlobs( @@ -424,13 +463,15 @@ public class GrpcRemoteExecutionClientTest { responseObserver.onNext(b.build()); responseObserver.onCompleted(); } - }); + }; + serviceRegistry.addService(ServerInterceptors.intercept(cas, new RequestHeadersValidator())); ByteStreamImplBase mockByteStreamImpl = Mockito.mock(ByteStreamImplBase.class); when(mockByteStreamImpl.write(Mockito.>anyObject())) .thenAnswer(blobWriteAnswer(command.toByteArray())) .thenAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))); - serviceRegistry.addService(mockByteStreamImpl); + serviceRegistry.addService( + ServerInterceptors.intercept(mockByteStreamImpl, new RequestHeadersValidator())); SpawnResult result = client.exec(simpleSpawn, simplePolicy); assertThat(result.setupSuccess()).isTrue(); @@ -563,7 +604,8 @@ public class GrpcRemoteExecutionClientTest { }) .when(mockWatcherImpl) .watch(Mockito.anyObject(), Mockito.>anyObject()); - serviceRegistry.addService(mockWatcherImpl); + serviceRegistry.addService( + ServerInterceptors.intercept(mockWatcherImpl, new RequestHeadersValidator())); final Command command = Command.newBuilder() .addAllArguments(ImmutableList.of("/bin/echo", "Hi!")) diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java index 160c20fa6d..b1e34f51fe 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java @@ -166,7 +166,9 @@ public class RemoteSpawnCacheTest { Reporter reporter = new Reporter(new EventBus()); eventHandler = new StoredEventHandler(); reporter.addHandler(eventHandler); - cache = new RemoteSpawnCache(execRoot, options, remoteCache, false, reporter); + cache = + new RemoteSpawnCache( + execRoot, options, remoteCache, "build-req-id", "command-id", false, reporter); fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz"); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index 940c9d3afd..bf9dee024c 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java @@ -121,8 +121,16 @@ public class RemoteSpawnRunnerTest { options.remoteUploadLocalResults = true; RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, executor); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + executor); ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult( ActionResult.newBuilder().setExitCode(0).build()).build(); @@ -169,8 +177,16 @@ public class RemoteSpawnRunnerTest { options.remoteUploadLocalResults = true; RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, null); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + null); // Throw an IOException to trigger the local fallback. when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class); @@ -211,8 +227,17 @@ public class RemoteSpawnRunnerTest { options.remoteUploadLocalResults = true; RemoteSpawnRunner runner = - spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, null)); + spy( + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + null)); Spawn spawn = newSimpleSpawn(); SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn); @@ -249,8 +274,17 @@ public class RemoteSpawnRunnerTest { SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn); RemoteSpawnRunner runner = - spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, null)); + spy( + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + null)); try { runner.exec(spawn, policy); @@ -274,7 +308,16 @@ public class RemoteSpawnRunnerTest { reporter.addHandler(eventHandler); RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, false, reporter, cache, null); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + false, + reporter, + "build-req-id", + "command-id", + cache, + null); Spawn spawn = newSimpleSpawn(); SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn); @@ -315,8 +358,16 @@ public class RemoteSpawnRunnerTest { options.remoteLocalFallback = true; RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, null); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + null); Spawn spawn = newSimpleSpawn(); SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn); @@ -343,8 +394,16 @@ public class RemoteSpawnRunnerTest { RemoteOptions options = Options.getDefaults(RemoteOptions.class); RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, executor); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + executor); ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build(); when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(cachedResult); @@ -375,8 +434,16 @@ public class RemoteSpawnRunnerTest { options.remoteLocalFallback = false; RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, executor); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + executor); ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build(); when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(null); @@ -402,8 +469,16 @@ public class RemoteSpawnRunnerTest { options.remoteLocalFallback = false; RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, executor); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + executor); when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(null); when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(new IOException()); @@ -429,8 +504,16 @@ public class RemoteSpawnRunnerTest { options.remoteLocalFallback = false; RemoteSpawnRunner runner = - new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null, - cache, executor); + new RemoteSpawnRunner( + execRoot, + options, + localRunner, + true, + /*cmdlineReporter=*/ null, + "build-req-id", + "command-id", + cache, + executor); when(cache.getCachedActionResult(any(ActionKey.class))).thenThrow(new IOException()); -- cgit v1.2.3