Transform StackOverflowError thrown in ArrayCodec into SerializationException. This can happen with deep nested sets.

PiperOrigin-RevId: 191790431

2 files changed, 30 insertions, 4 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodec.java
index a9988fb362..366843587e 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodec.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodec.java
@@ -29,8 +29,13 @@ class ArrayCodec implements ObjectCodec<Object[]> {
   public void serialize(SerializationContext context, Object[] obj, CodedOutputStream codedOut)
       throws SerializationException, IOException {
     codedOut.writeInt32NoTag(obj.length);
-    for (Object item : obj) {
-      context.serialize(item, codedOut);
+    try {
+      for (Object item : obj) {
+        context.serialize(item, codedOut);
+      }
+    } catch (StackOverflowError e) {
+      // TODO(janakr): figure out if we need to handle this better and handle it better if so.
+      throw new SerializationException("StackOverflow serializing array", e);
     }
   }
 
@@ -38,8 +43,13 @@ class ArrayCodec implements ObjectCodec<Object[]> {
   public Object[] deserialize(DeserializationContext context, CodedInputStream codedIn)
       throws SerializationException, IOException {
     Object[] result = new Object[codedIn.readInt32()];
-    for (int i = 0; i < result.length; i++) {
-      result[i] = context.deserialize(codedIn);
+    try {
+      for (int i = 0; i < result.length; i++) {
+        result[i] = context.deserialize(codedIn);
+      }
+    } catch (StackOverflowError e) {
+      // TODO(janakr): figure out if we need to handle this better and handle it better if so.
+      throw new SerializationException("StackOverflow deserializing array", e);
     }
     return result;
   }
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodecTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodecTest.java
index c12e228941..58bcd4a66f 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodecTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/serialization/ArrayCodecTest.java
@@ -16,7 +16,10 @@ package com.google.devtools.build.lib.skyframe.serialization;
 
 import static com.google.common.truth.Truth.assertThat;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester;
+import com.google.devtools.build.lib.skyframe.serialization.testutils.TestUtils;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -37,6 +40,19 @@ public class ArrayCodecTest {
         .runTests();
   }
 
+  @Test
+  public void stackOverflowTransformedIntoSerializationException() {
+    int depth = 4000;
+    Object[] obj = new Object[1];
+    Object[] cur = obj;
+    for (int i = 0; i < depth; i++) {
+      cur[0] = new Object[1];
+      cur = (Object[]) cur[0];
+    }
+    MoreAsserts.assertThrows(
+        SerializationException.class, () -> TestUtils.toBytes(obj, ImmutableMap.of()));
+  }
+
   private static void verifyDeserialized(Object[] original, Object[] deserialized) {
     assertThat(deserialized).hasLength(original.length);
     for (int i = 0; i < deserialized.length; i++) {