diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/skyframe/serialization')
6 files changed, 242 insertions, 138 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java index bf11b86e69..4af9f532f0 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java @@ -14,8 +14,10 @@ package com.google.devtools.build.lib.skyframe.serialization; +import com.google.common.base.Predicates; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; import java.io.IOException; /** @@ -29,15 +31,35 @@ public class AutoRegistry { private static final Supplier<ObjectCodecRegistry> SUPPLIER = Suppliers.memoize(AutoRegistry::create); + /* Common ancestor of common.google.devtools.build and com.google.devtools.common.options, + * where Tristate lives. */ + private static final String PACKAGE_PREFIX = "com.google.devtools"; + + /** Classes outside {@link AutoRegistry#PACKAGE_PREFIX} that need to be serialized. */ + private static final ImmutableList<String> EXTERNAL_CLASS_NAMES_TO_REGISTER = + ImmutableList.of("java.io.FileNotFoundException", "java.io.IOException"); + + private static final ImmutableList<Object> CONSTANTS_TO_REGISTER = + ImmutableList.of( + Predicates.alwaysTrue(), + Predicates.alwaysFalse(), + Predicates.isNull(), + Predicates.notNull()); + public static ObjectCodecRegistry get() { return SUPPLIER.get(); } private static ObjectCodecRegistry create() { try { - return CodecScanner.initializeCodecRegistry("com.google.devtools.build") - .setAllowDefaultCodec(false) - .build(); + ObjectCodecRegistry.Builder registry = CodecScanner.initializeCodecRegistry(PACKAGE_PREFIX); + for (String className : EXTERNAL_CLASS_NAMES_TO_REGISTER) { + registry.addClassName(className); + } + for (Object constant : CONSTANTS_TO_REGISTER) { + registry.addConstant(constant); + } + return registry.build(); } catch (IOException | ReflectiveOperationException e) { throw new IllegalStateException(e); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java index 635b5ad755..319e0c1152 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java @@ -27,11 +27,8 @@ import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; -import java.util.List; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -59,62 +56,73 @@ public class CodecScanner { log.info("Building ObjectCodecRegistry"); ArrayList<Class<? extends ObjectCodec<?>>> codecs = new ArrayList<>(); ArrayList<Class<? extends CodecRegisterer<?>>> registerers = new ArrayList<>(); - List<ClassInfo> classInfos = getClassInfos(packagePrefix).collect(Collectors.toList()); - getCodecs(classInfos) - .forEach( - type -> { - if (!ObjectCodec.class.equals(type) - && ObjectCodec.class.isAssignableFrom(type) - && !Modifier.isAbstract(type.getModifiers())) { - codecs.add((Class<? extends ObjectCodec<?>>) type); - } else if (!CodecRegisterer.class.equals(type) - && CodecRegisterer.class.isAssignableFrom(type)) { - registerers.add((Class<? extends CodecRegisterer<?>>) type); - } - }); ObjectCodecRegistry.Builder builder = ObjectCodecRegistry.newBuilder(); - getMatchingClasses( - classInfos, - classInfo -> - classInfo - .getSimpleName() - .endsWith(CodecScanningConstants.REGISTERED_SINGLETON_SUFFIX)) + getClassInfos(packagePrefix) .forEach( - type -> { - if (!RegisteredSingletonDoNotUse.class.isAssignableFrom(type)) { - return; - } - Field field; - try { - field = - type.getDeclaredField( - CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME); - } catch (NoSuchFieldException e) { - throw new IllegalStateException( - type - + " inherits from " - + RegisteredSingletonDoNotUse.class - + " but does not have a field " - + CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME, - e); - } - try { - builder.addConstant( - Preconditions.checkNotNull(field.get(null), "%s %s", field, type)); - } catch (IllegalAccessException e) { - throw new IllegalStateException( - "Could not access field " + field + " for " + type, e); + classInfo -> { + if (classInfo.getName().endsWith("Codec")) { + processLikelyCodec(classInfo.load(), codecs); + } else if (classInfo.getName().endsWith("CodecRegisterer")) { + processLikelyRegisterer(classInfo.load(), registerers); + } else if (classInfo + .getName() + .endsWith(CodecScanningConstants.REGISTERED_SINGLETON_SUFFIX)) { + processLikelyConstant(classInfo.load(), builder); + } else { + // Assumes that anything with a class name matching the above won't need to be + // serialized. + builder.addClassName(classInfo.getName().intern()); } }); HashSet<Class<? extends ObjectCodec<?>>> alreadyRegistered = runRegisterers(builder, registerers); - applyDefaultRegistration(builder, alreadyRegistered, codecs); return builder; } @SuppressWarnings("unchecked") + private static void processLikelyCodec( + Class<?> type, ArrayList<Class<? extends ObjectCodec<?>>> codecs) { + if (!ObjectCodec.class.equals(type) + && ObjectCodec.class.isAssignableFrom(type) + && !Modifier.isAbstract(type.getModifiers())) { + codecs.add((Class<? extends ObjectCodec<?>>) type); + } + } + + @SuppressWarnings("unchecked") + private static void processLikelyRegisterer( + Class<?> type, ArrayList<Class<? extends CodecRegisterer<?>>> registerers) { + if (!CodecRegisterer.class.equals(type) && CodecRegisterer.class.isAssignableFrom(type)) { + registerers.add((Class<? extends CodecRegisterer<?>>) type); + } + } + + private static void processLikelyConstant(Class<?> type, ObjectCodecRegistry.Builder builder) { + if (!RegisteredSingletonDoNotUse.class.isAssignableFrom(type)) { + return; + } + Field field; + try { + field = type.getDeclaredField(CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME); + } catch (NoSuchFieldException e) { + throw new IllegalStateException( + type + + " inherits from " + + RegisteredSingletonDoNotUse.class + + " but does not have a field " + + CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME, + e); + } + try { + builder.addConstant(Preconditions.checkNotNull(field.get(null), "%s %s", field, type)); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Could not access field " + field + " for " + type, e); + } + } + + @SuppressWarnings("unchecked") private static HashSet<Class<? extends ObjectCodec<?>>> runRegisterers( ObjectCodecRegistry.Builder builder, ArrayList<Class<? extends CodecRegisterer<?>>> registerers) @@ -195,19 +203,4 @@ public class CodecScanner { .filter(c -> c.getPackageName().startsWith(packagePrefix)) .sorted(Comparator.comparing(ClassInfo::getName)); } - - /** - * Returns a stream of likely codec and registerer implementations. - * - * <p>Caller should do additional checks as this method only performs string matching. - */ - private static Stream<Class<?>> getCodecs(List<ClassInfo> classInfos) { - return getMatchingClasses( - classInfos, c -> c.getName().endsWith("Codec") || c.getName().endsWith("CodecRegisterer")); - } - - private static Stream<Class<?>> getMatchingClasses( - List<ClassInfo> classInfos, Predicate<ClassInfo> predicate) { - return classInfos.stream().filter(predicate).map(ClassInfo::load); - } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumRuntimeCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumRuntimeCodec.java deleted file mode 100644 index 0d67591077..0000000000 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumRuntimeCodec.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.skyframe.serialization; - -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import java.io.IOException; - -@SuppressWarnings("rawtypes") -class EnumRuntimeCodec implements ObjectCodec<Enum> { - - @Override - public Class<Enum> getEncodedClass() { - return Enum.class; - } - - @Override - public void serialize(SerializationContext context, Enum value, CodedOutputStream codedOut) - throws IOException, SerializationException { - context.serialize(value.getDeclaringClass(), codedOut); - codedOut.writeInt32NoTag(value.ordinal()); - } - - @Override - public Enum deserialize(DeserializationContext context, CodedInputStream codedIn) - throws SerializationException, IOException { - Class<? extends Enum> enumType = context.deserialize(codedIn); - return enumType.getEnumConstants()[codedIn.readInt32()]; - } -} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java index 95590143db..c963172e9f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java @@ -16,15 +16,18 @@ package com.google.devtools.build.lib.skyframe.serialization; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; import java.io.IOException; import java.util.Comparator; import java.util.IdentityHashMap; import java.util.Map; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -38,19 +41,27 @@ public class ObjectCodecRegistry { return new Builder(); } + private final boolean allowDefaultCodec; + private final ImmutableMap<Class<?>, CodecDescriptor> classMappedCodecs; private final ImmutableList<CodecDescriptor> tagMappedCodecs; - @Nullable - private final CodecDescriptor defaultCodecDescriptor; + private final int constantsStartTag; private final IdentityHashMap<Object, Integer> constantsMap; private final ImmutableList<Object> constants; - private final int constantsStartTag; + + /** This is sorted, but we need index-based access. */ + private final ImmutableList<String> classNames; + + private final IdentityHashMap<String, Supplier<CodecDescriptor>> dynamicCodecs; private ObjectCodecRegistry( ImmutableSet<ObjectCodec<?>> memoizingCodecs, ImmutableList<Object> constants, + ImmutableSortedSet<String> classNames, boolean allowDefaultCodec) { + this.allowDefaultCodec = allowDefaultCodec; + int nextTag = 1; // 0 is reserved for null. ImmutableMap.Builder<Class<?>, CodecDescriptor> memoizingCodecsBuilder = ImmutableMap.builderWithExpectedSize(memoizingCodecs.size()); @@ -60,14 +71,6 @@ public class ObjectCodecRegistry { processCodecs( memoizingCodecs, nextTag, tagMappedMemoizingCodecsBuilder, memoizingCodecsBuilder); - this.defaultCodecDescriptor = - allowDefaultCodec - ? new TypedCodecDescriptor<>(nextTag++, new JavaSerializableCodec()) - : null; - if (allowDefaultCodec) { - tagMappedMemoizingCodecsBuilder.add(defaultCodecDescriptor); - } - this.classMappedCodecs = memoizingCodecsBuilder.build(); this.tagMappedCodecs = tagMappedMemoizingCodecsBuilder.build(); @@ -77,16 +80,35 @@ public class ObjectCodecRegistry { constantsMap.put(constant, nextTag++); } this.constants = constants; + + this.classNames = classNames.asList(); + this.dynamicCodecs = createDynamicCodecs(classNames, nextTag); + } + + public CodecDescriptor getCodecDescriptorForObject(Object obj) + throws SerializationException.NoCodecException { + Class<?> type = obj.getClass(); + CodecDescriptor descriptor = getCodecDescriptor(type); + if (descriptor != null) { + return descriptor; + } + if (!allowDefaultCodec) { + throw new SerializationException.NoCodecException( + "No codec available for " + type + " and default fallback disabled"); + } + if (obj instanceof Enum) { + // Enums must be serialized using declaring class. + type = ((Enum) obj).getDeclaringClass(); + } + return getDynamicCodecDescriptor(type.getName()); } /** - * Returns a {@link CodecDescriptor} for the given type. + * Returns a {@link CodecDescriptor} for the given type or null if none found. * - * <p>Falls back to a codec for the nearest super type of type. Failing that, may fall back to the - * registry's default codec. + * <p>Also checks if there are codecs for a superclass of the given type. */ - public CodecDescriptor getCodecDescriptor(Class<?> type) - throws SerializationException.NoCodecException { + private @Nullable CodecDescriptor getCodecDescriptor(Class<?> type) { // TODO(blaze-team): consider caching this traversal. for (Class<?> nextType = type; nextType != null; nextType = nextType.getSuperclass()) { CodecDescriptor result = classMappedCodecs.get(nextType); @@ -94,11 +116,7 @@ public class ObjectCodecRegistry { return result; } } - if (defaultCodecDescriptor == null) { - throw new SerializationException.NoCodecException( - "No codec available for " + type + " and default fallback disabled"); - } - return defaultCodecDescriptor; + return null; } @Nullable @@ -116,17 +134,20 @@ public class ObjectCodecRegistry { /** Returns the {@link CodecDescriptor} associated with the supplied tag. */ public CodecDescriptor getCodecDescriptorByTag(int tag) throws SerializationException.NoCodecException { - int tagOffset = tag - 1; - if (tagOffset < 0 || tagOffset > tagMappedCodecs.size()) { + int tagOffset = tag - 1; // 0 reserved for null + if (tagOffset < 0) { throw new SerializationException.NoCodecException("No codec available for tag " + tag); } + if (tagOffset < tagMappedCodecs.size()) { + return tagMappedCodecs.get(tagOffset); + } - CodecDescriptor result = tagMappedCodecs.get(tagOffset); - if (result != null) { - return result; - } else { + tagOffset -= tagMappedCodecs.size(); + tagOffset -= constants.size(); + if (!allowDefaultCodec || tagOffset < 0 || tagOffset >= classNames.size()) { throw new SerializationException.NoCodecException("No codec available for tag " + tag); } + return getDynamicCodecDescriptor(classNames.get(tagOffset)); } /** @@ -137,7 +158,7 @@ public class ObjectCodecRegistry { @VisibleForTesting public Builder getBuilder() { Builder builder = newBuilder(); - builder.setAllowDefaultCodec(defaultCodecDescriptor != null); + builder.setAllowDefaultCodec(allowDefaultCodec); for (Map.Entry<Class<?>, CodecDescriptor> entry : classMappedCodecs.entrySet()) { builder.add(entry.getValue().getCodec()); } @@ -145,9 +166,17 @@ public class ObjectCodecRegistry { for (Object constant : constants) { builder.addConstant(constant); } + + for (String className : classNames) { + builder.addClassName(className); + } return builder; } + ImmutableList<String> classNames() { + return classNames; + } + /** Describes encoding logic. */ interface CodecDescriptor { void serialize(SerializationContext context, Object obj, CodedOutputStream codedOut) @@ -157,7 +186,7 @@ public class ObjectCodecRegistry { throws IOException, SerializationException; /** - * Unique identifier identifying the associated codec. + * Unique identifier for the associated codec. * * <p>Intended to be used as a compact on-the-wire representation of an encoded object's type. * @@ -214,6 +243,7 @@ public class ObjectCodecRegistry { public static class Builder { private final ImmutableSet.Builder<ObjectCodec<?>> codecBuilder = ImmutableSet.builder(); private final ImmutableList.Builder<Object> constantsBuilder = ImmutableList.builder(); + private final ImmutableSortedSet.Builder<String> classNames = ImmutableSortedSet.naturalOrder(); private boolean allowDefaultCodec = true; public Builder add(ObjectCodec<?> codec) { @@ -234,9 +264,14 @@ public class ObjectCodecRegistry { return this; } + public Builder addClassName(String className) { + classNames.add(className); + return this; + } + public ObjectCodecRegistry build() { return new ObjectCodecRegistry( - codecBuilder.build(), constantsBuilder.build(), allowDefaultCodec); + codecBuilder.build(), constantsBuilder.build(), classNames.build(), allowDefaultCodec); } } @@ -257,4 +292,51 @@ public class ObjectCodecRegistry { } return nextTag; } + + private static IdentityHashMap<String, Supplier<CodecDescriptor>> createDynamicCodecs( + ImmutableSortedSet<String> classNames, int nextTag) { + IdentityHashMap<String, Supplier<CodecDescriptor>> dynamicCodecs = + new IdentityHashMap<>(classNames.size()); + for (String className : classNames) { + int tag = nextTag++; + dynamicCodecs.put( + className, Suppliers.memoize(() -> createDynamicCodecDescriptor(tag, className))); + } + return dynamicCodecs; + } + + /** For enums, this method must only be called for the declaring class. */ + private static CodecDescriptor createDynamicCodecDescriptor(int tag, String className) { + try { + Class<?> type = Class.forName(className); + if (type.isEnum()) { + return createCodecDescriptorForEnum(tag, type); + } + return new TypedCodecDescriptor<>(tag, new DynamicCodec(Class.forName(className))); + } catch (ReflectiveOperationException e) { + new SerializationException("Could not create codec for type: " + className, e) + .printStackTrace(); + return null; + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static CodecDescriptor createCodecDescriptorForEnum(int tag, Class<?> enumType) { + return new TypedCodecDescriptor(tag, new EnumCodec(enumType)); + } + + private CodecDescriptor getDynamicCodecDescriptor(String className) + throws SerializationException.NoCodecException { + Supplier<CodecDescriptor> supplier = dynamicCodecs.get(className); + if (supplier == null) { + throw new SerializationException.NoCodecException( + "No default codec available for " + className); + } + CodecDescriptor descriptor = supplier.get(); + if (descriptor == null) { + throw new SerializationException.NoCodecException( + "There was a problem creating a codec for " + className + " check logs for details."); + } + return descriptor; + } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java index e06d6dba45..ba5b696128 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java @@ -118,7 +118,7 @@ public class SerializationContext { if (writeNullOrConstant(object, codedOut)) { return null; } - ObjectCodecRegistry.CodecDescriptor descriptor = registry.getCodecDescriptor(object.getClass()); + ObjectCodecRegistry.CodecDescriptor descriptor = registry.getCodecDescriptorForObject(object); codedOut.writeSInt32NoTag(descriptor.getTag()); return descriptor; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/NotSerializableCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/NotSerializableCodec.java new file mode 100644 index 0000000000..f8b5617670 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/NotSerializableCodec.java @@ -0,0 +1,49 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization.testutils; + +import com.google.devtools.build.lib.skyframe.serialization.DeserializationContext; +import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec; +import com.google.devtools.build.lib.skyframe.serialization.SerializationContext; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.NotSerializableException; + +/** A testing helper to force serialization errors. */ +public class NotSerializableCodec implements ObjectCodec<Object> { + private final Class<?> type; + + public NotSerializableCodec(Class<?> type) { + this.type = type; + } + + @Override + public Class<?> getEncodedClass() { + return type; + } + + @Override + public void serialize( + SerializationContext unusedContext, Object unusedObj, CodedOutputStream unusedCodedOut) + throws NotSerializableException { + throw new NotSerializableException(type + " marked not serializable"); + } + + @Override + public Object deserialize(DeserializationContext unusedContext, CodedInputStream unusedCodedIn) + throws NotSerializableException { + throw new NotSerializableException(type + " marked not serializable"); + } +} |