diff options
author | 2017-07-04 06:01:11 -0400 | |
---|---|---|
committer | 2017-07-05 10:57:53 -0400 | |
commit | 27762e2207d2ea817499756298f08c715790d187 (patch) | |
tree | 604382da8d38f7f2f8459aa7614e050124f80e8e /src/tools | |
parent | 9409109c3016c51403ed4387ecda6c99459f1a8d (diff) |
Add an on-disk storage option for the remote worker
I'm planning to switch the remote worker over to the on-disk storage system,
and delete the in-memory option. Currently, running a Bazel build with the
in-memory storage system uses up over 50% of the total available memory on my
machine (I only have 32 GB total), and grinds it to a complete halt (unless I
close most of my apps) on a simple build of //src:bazel.
PiperOrigin-RevId: 160877546
Diffstat (limited to 'src/tools')
2 files changed, 23 insertions, 2 deletions
diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java index d9056e9932..49e63d7915 100644 --- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.remote.RemoteOptions; import com.google.devtools.build.lib.remote.SimpleBlobStoreActionCache; import com.google.devtools.build.lib.remote.SimpleBlobStoreFactory; import com.google.devtools.build.lib.remote.blobstore.ConcurrentMapBlobStore; +import com.google.devtools.build.lib.remote.blobstore.OnDiskBlobStore; import com.google.devtools.build.lib.remote.blobstore.SimpleBlobStore; import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; @@ -39,6 +40,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.JavaIoFileSystem; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.remoteexecution.v1test.ActionCacheGrpc.ActionCacheImplBase; import com.google.devtools.remoteexecution.v1test.ContentAddressableStorageGrpc.ContentAddressableStorageImplBase; @@ -165,12 +167,20 @@ public final class RemoteWorker { if (!usingRemoteCache) { logger.warning("Not using remote cache. This should be used for testing only!"); } + if ((remoteWorkerOptions.casPath != null) + && (!PathFragment.create(remoteWorkerOptions.casPath).isAbsolute() + || !getFileSystem().getPath(remoteWorkerOptions.casPath).exists())) { + logger.severe("--cas_path must refer to an existing, absolute path!"); + System.exit(1); + return; + } SimpleBlobStore blobStore = usingRemoteCache ? SimpleBlobStoreFactory.create(remoteOptions) - : new ConcurrentMapBlobStore( - new ConcurrentHashMap<String, byte[]>()); + : remoteWorkerOptions.casPath != null + ? new OnDiskBlobStore(getFileSystem().getPath(remoteWorkerOptions.casPath)) + : new ConcurrentMapBlobStore(new ConcurrentHashMap<String, byte[]>()); RemoteWorker worker = new RemoteWorker( diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java index 686761f9ed..e8bc12a781 100644 --- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java @@ -43,6 +43,17 @@ public class RemoteWorkerOptions extends OptionsBase { public String workPath; @Option( + name = "cas_path", + defaultValue = "null", + category = "build_worker", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "A directory for the build worker to store it's files in. If left unset, and if no " + + "other store is set, the worker falls back to an in-memory store." + ) + public String casPath; + + @Option( name = "debug", defaultValue = "false", category = "build_worker", |