Implement a limit for named_set_of_file events

This adds a command-line option which can be used to force Bazel to split up
very large events, e.g., events with hundreds of thousands of entries. This is
rare, but happens occasionally.

It would be better to control the maximum event size directly, but that is
significantly more difficult to do here.

PiperOrigin-RevId: 196690600

1 files changed, 60 insertions, 0 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetViewTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetViewTest.java
index 19f0135c21..8074d3ce0a 100644
--- a/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetViewTest.java
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetViewTest.java
@@ -16,6 +16,10 @@ package com.google.devtools.build.lib.collect.nestedset;
 import static com.google.common.truth.Truth.assertThat;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -133,4 +137,60 @@ public class NestedSetViewTest {
     assertThat(identifiers(view.transitives()))
         .contains((new NestedSetView<String>(multi)).identifier());
   }
+
+  @Test
+  public void testSplitFails() {
+    NestedSet<String> a = NestedSetBuilder.<String>stableOrder().add("a").add("b").build();
+    NestedSetView<String> v = new NestedSetView<>(a);
+    MoreAsserts.assertThrows(
+        IllegalArgumentException.class, () -> v.splitIfExceedsMaximumSize(-100));
+    MoreAsserts.assertThrows(
+        IllegalArgumentException.class, () -> v.splitIfExceedsMaximumSize(1));
+  }
+
+  @Test
+  public void testSplitNoSplit() {
+    NestedSet<String> a = NestedSetBuilder.<String>stableOrder().add("a").add("b").build();
+    NestedSetView<String> v = new NestedSetView<>(a);
+    assertThat(v.splitIfExceedsMaximumSize(2)).isSameAs(v);
+    assertThat(v.splitIfExceedsMaximumSize(100)).isSameAs(v);
+  }
+
+  @Test
+  public void testSplit() {
+    NestedSet<String> a =
+        NestedSetBuilder.<String>stableOrder()
+            .addAll(Arrays.asList("a", "b", "c"))
+            .build();
+    NestedSetView<String> v = new NestedSetView<>(a);
+    NestedSetView<String> s = v.splitIfExceedsMaximumSize(2);
+    assertThat(s).isNotSameAs(v);
+    assertThat(collectCheckSize(s, 2)).containsExactly("a", "b", "c");
+  }
+
+  @Test
+  public void testRecursiveSplit() {
+    NestedSet<String> a =
+        NestedSetBuilder.<String>stableOrder()
+            .addAll(Arrays.asList("a", "b", "c", "d", "e"))
+            .build();
+    NestedSetView<String> v = new NestedSetView<>(a);
+    NestedSetView<String> s = v.splitIfExceedsMaximumSize(2);
+    assertThat(s).isNotSameAs(v);
+    assertThat(collectCheckSize(s, 2)).containsExactly("a", "b", "c", "d", "e");
+  }
+
+  private <T> List<T> collectCheckSize(NestedSetView<T> view, int maxSize) {
+    return collectCheckSize(new ArrayList<>(), view, maxSize);
+  }
+
+  private <T> List<T> collectCheckSize(List<T> result, NestedSetView<T> view, int maxSize) {
+    assertThat(view.directs().size()).isAtMost(maxSize);
+    assertThat(view.transitives().size()).isAtMost(maxSize);
+    for (NestedSetView<T> t : view.transitives()) {
+      collectCheckSize(result, t, maxSize);
+    }
+    result.addAll(view.directs());
+    return result;
+  }
 }