diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib')
11 files changed, 235 insertions, 44 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD index b9a5b4cbeb..d118b6183f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD @@ -14,6 +14,7 @@ java_library( srcs = glob(["**/*.java"]), deps = [ ":kryo", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:registered-singleton", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:guava", "//third_party:jsr305", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java index ce6db63d63..1b387134a5 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanner.java @@ -17,14 +17,19 @@ package com.google.devtools.build.lib.skyframe.serialization; import com.google.common.base.Preconditions; import com.google.common.reflect.ClassPath; import com.google.common.reflect.ClassPath.ClassInfo; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.RegisteredSingletonDoNotUse; import java.io.IOException; import java.lang.reflect.Constructor; +import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.HashSet; +import java.util.List; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -52,7 +57,8 @@ public class CodecScanner { throws IOException, ReflectiveOperationException { ArrayList<Class<? extends ObjectCodec<?>>> codecs = new ArrayList<>(); ArrayList<Class<? extends CodecRegisterer<?>>> registerers = new ArrayList<>(); - scanCodecs(packagePrefix) + List<ClassInfo> classInfos = getClassInfos(packagePrefix).collect(Collectors.toList()); + getCodecs(classInfos) .forEach( type -> { if (!ObjectCodec.class.equals(type) && ObjectCodec.class.isAssignableFrom(type)) { @@ -63,8 +69,40 @@ public class CodecScanner { registerers.add((Class<? extends CodecRegisterer<?>>) type); } }); - ObjectCodecRegistry.Builder builder = ObjectCodecRegistry.newBuilder(); + getMatchingClasses( + classInfos, + classInfo -> + classInfo + .getSimpleName() + .endsWith(CodecScanningConstants.REGISTERED_SINGLETON_SUFFIX)) + .forEach( + type -> { + if (!RegisteredSingletonDoNotUse.class.isAssignableFrom(type)) { + return; + } + Field field; + try { + field = + type.getDeclaredField( + CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME); + } catch (NoSuchFieldException e) { + throw new IllegalStateException( + type + + " inherits from " + + RegisteredSingletonDoNotUse.class + + " but does not have a field " + + CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME, + e); + } + try { + builder.addConstant(field.get(null)); + } catch (IllegalAccessException e) { + throw new IllegalStateException( + "Could not access field " + field + " for " + type, e); + } + }); + HashSet<Class<? extends ObjectCodec<?>>> alreadyRegistered = runRegisterers(builder, registerers); @@ -181,21 +219,27 @@ public class CodecScanner { return -1; } - /** - * Returns a stream of likely codec and registerer implementations. - * - * <p>Caller should do additional checks as this method only performs string matching. - * - * @param packagePrefix emits only classes in packages having this prefix - */ - private static Stream<Class<?>> scanCodecs(String packagePrefix) throws IOException { + private static Stream<ClassInfo> getClassInfos(String packagePrefix) throws IOException { return ClassPath.from(ClassLoader.getSystemClassLoader()) .getResources() .stream() .filter(r -> r instanceof ClassInfo) .map(r -> (ClassInfo) r) - .filter(c -> c.getPackageName().startsWith(packagePrefix)) - .filter(c -> c.getName().endsWith("Codec") || c.getName().endsWith("CodecRegisterer")) - .map(c -> c.load()); + .filter(c -> c.getPackageName().startsWith(packagePrefix)); + } + + /** + * Returns a stream of likely codec and registerer implementations. + * + * <p>Caller should do additional checks as this method only performs string matching. + */ + private static Stream<Class<?>> getCodecs(List<ClassInfo> classInfos) { + return getMatchingClasses( + classInfos, c -> c.getName().endsWith("Codec") || c.getName().endsWith("CodecRegisterer")); + } + + private static Stream<Class<?>> getMatchingClasses( + List<ClassInfo> classInfos, Predicate<ClassInfo> predicate) { + return classInfos.stream().filter(predicate).map(ClassInfo::load); } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanningConstants.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanningConstants.java new file mode 100644 index 0000000000..97885ee033 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecScanningConstants.java @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +/** Constants shared between {@link CodecScanner} and {@code AutoCodecProcessor}. */ +public class CodecScanningConstants { + /** + * Name of static field in RegisteredSingleton classes. Any class whose name ends in {@link + * #REGISTERED_SINGLETON_SUFFIX} and that has a field with this name will have this field + * registered as a constant by {@link CodecScanner}. Generated using uuidgen and simple + * translation of numbers to letters. + */ + public static final String REGISTERED_SINGLETON_INSTANCE_VAR_NAME = + "REGISTERED_SINGLETON_INSTANCE_VAR_NAME_GLFKMEBDQFHOJQKEHHQPGMNQBOBFEJADCMDP"; + /** Suffix for RegisteredSingleton classes. */ + public static final String REGISTERED_SINGLETON_SUFFIX = "RegisteredSingleton"; +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java index 5377d4ffc6..e867969cdf 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java @@ -43,7 +43,10 @@ public class DeserializationContext { if (tag == 0) { return null; } - return (T) registry.getCodecDescriptorByTag(tag).deserialize(this, codedIn); + T constant = (T) registry.maybeGetConstantByTag(tag); + return constant == null + ? (T) registry.getCodecDescriptorByTag(tag).deserialize(this, codedIn) + : constant; } @SuppressWarnings("unchecked") diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java index b1ff058f78..164fe92850 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java @@ -20,6 +20,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; import java.io.IOException; +import java.util.IdentityHashMap; import java.util.Map; import java.util.Map.Entry; import javax.annotation.Nullable; @@ -40,8 +41,12 @@ public class ObjectCodecRegistry { private final ImmutableList<CodecDescriptor> tagMappedCodecs; @Nullable private final CodecDescriptor defaultCodecDescriptor; + private final IdentityHashMap<Object, Integer> constantsMap; + private final ImmutableList<Object> constants; + private final int constantsStartTag; - private ObjectCodecRegistry(Map<String, CodecHolder> codecs, boolean allowDefaultCodec) { + private ObjectCodecRegistry( + Map<String, CodecHolder> codecs, ImmutableList<Object> constants, boolean allowDefaultCodec) { ImmutableMap.Builder<String, CodecDescriptor> codecMappingsBuilder = ImmutableMap.builder(); int nextTag = 1; // 0 is reserved for null. for (String classifier : ImmutableList.sortedCopyOf(codecs.keySet())) { @@ -53,8 +58,16 @@ public class ObjectCodecRegistry { this.byteStringMappedCodecs = makeByteStringMappedCodecs(stringMappedCodecs); this.defaultCodecDescriptor = - allowDefaultCodec ? new TypedCodecDescriptor<>(nextTag, new JavaSerializableCodec()) : null; + allowDefaultCodec + ? new TypedCodecDescriptor<>(nextTag++, new JavaSerializableCodec()) + : null; this.tagMappedCodecs = makeTagMappedCodecs(stringMappedCodecs, defaultCodecDescriptor); + constantsStartTag = nextTag; + constantsMap = new IdentityHashMap<>(); + for (Object constant : constants) { + constantsMap.put(constant, nextTag++); + } + this.constants = constants; } /** Returns the {@link CodecDescriptor} associated with the supplied classifier. */ @@ -107,6 +120,18 @@ public class ObjectCodecRegistry { return defaultCodecDescriptor; } + @Nullable + Object maybeGetConstantByTag(int tag) { + return tag < constantsStartTag || tag - constantsStartTag >= constants.size() + ? null + : constants.get(tag - constantsStartTag); + } + + @Nullable + Integer maybeGetTagForConstant(Object object) { + return constantsMap.get(object); + } + /** Returns the {@link CodecDescriptor} associated with the supplied tag. */ public CodecDescriptor getCodecDescriptorByTag(int tag) throws SerializationException.NoCodecException { @@ -214,6 +239,7 @@ public class ObjectCodecRegistry { /** Builder for {@link ObjectCodecRegistry}. */ public static class Builder { private final ImmutableMap.Builder<String, CodecHolder> codecsBuilder = ImmutableMap.builder(); + private final ImmutableList.Builder<Object> constantsBuilder = ImmutableList.builder(); private boolean allowDefaultCodec = true; /** @@ -240,13 +266,19 @@ public class ObjectCodecRegistry { return this; } + public Builder addConstant(Object object) { + constantsBuilder.add(object); + return this; + } + /** Wrap this builder with a {@link ClassKeyedBuilder}. */ public ClassKeyedBuilder asClassKeyedBuilder() { return new ClassKeyedBuilder(this); } public ObjectCodecRegistry build() { - return new ObjectCodecRegistry(codecsBuilder.build(), allowDefaultCodec); + return new ObjectCodecRegistry( + codecsBuilder.build(), constantsBuilder.build(), allowDefaultCodec); } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java index bb5a50492c..60df5c8140 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java @@ -43,6 +43,11 @@ public class SerializationContext { codedOut.writeSInt32NoTag(0); return; } + Integer tag = registry.maybeGetTagForConstant(object); + if (tag != null) { + codedOut.writeSInt32NoTag(tag); + return; + } ObjectCodecRegistry.CodecDescriptor descriptor = registry.getCodecDescriptor(object.getClass()); codedOut.writeSInt32NoTag(descriptor.getTag()); descriptor.serialize(this, object, codedOut); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java index af59425e7b..867dbca164 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java @@ -32,8 +32,13 @@ import java.lang.annotation.Target; * The {@code _AutoCodec} suffix is added to the {@code Target} to obtain the generated class name. * In the example, that results in a class named {@code Target_AutoCodec} but applications should * not need to directly access the generated class. + * + * <p>If applied to a field (which must be static and final), the field is stored as a "constant" + * allowing for trivial serialization of it as an integer tag (see {@code CodecScanner} and + * {@code ObjectCodecRegistery}). In order to do that, a trivial associated "RegisteredSingleton" + * class is generated. */ -@Target(ElementType.TYPE) +@Target({ElementType.TYPE, ElementType.FIELD}) // TODO(janakr): remove once serialization is complete. @Retention(RetentionPolicy.RUNTIME) public @interface AutoCodec { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecProcessor.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecProcessor.java index 41f28835c7..1df5f51451 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecProcessor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecProcessor.java @@ -21,15 +21,18 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.skyframe.serialization.CodecScanningConstants; import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec; import com.google.devtools.build.lib.skyframe.serialization.SerializationException; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationCodeGenerator.Marshaller; import com.squareup.javapoet.ClassName; +import com.squareup.javapoet.FieldSpec; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; @@ -93,33 +96,40 @@ public class AutoCodecProcessor extends AbstractProcessor { public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) { for (Element element : roundEnv.getElementsAnnotatedWith(AutoCodecUtil.ANNOTATION)) { AutoCodec annotation = element.getAnnotation(AutoCodecUtil.ANNOTATION); - TypeElement encodedType = (TypeElement) element; - TypeSpec.Builder codecClassBuilder; - switch (annotation.strategy()) { - case INSTANTIATOR: - codecClassBuilder = buildClassWithInstantiatorStrategy(encodedType); - break; - case PUBLIC_FIELDS: - codecClassBuilder = buildClassWithPublicFieldsStrategy(encodedType); - break; - case SINGLETON: - codecClassBuilder = buildClassWithSingletonStrategy(encodedType, env); - break; - default: - throw new IllegalArgumentException("Unknown strategy: " + annotation.strategy()); + TypeSpec builtClass; + if (element instanceof TypeElement) { + TypeElement encodedType = (TypeElement) element; + TypeSpec.Builder codecClassBuilder; + switch (annotation.strategy()) { + case INSTANTIATOR: + codecClassBuilder = buildClassWithInstantiatorStrategy(encodedType); + break; + case PUBLIC_FIELDS: + codecClassBuilder = buildClassWithPublicFieldsStrategy(encodedType); + break; + case SINGLETON: + codecClassBuilder = buildClassWithSingletonStrategy(encodedType, env); + break; + default: + throw new IllegalArgumentException("Unknown strategy: " + annotation.strategy()); + } + codecClassBuilder.addMethod( + AutoCodecUtil.initializeGetEncodedClassMethod(encodedType, env) + .addStatement( + "return $T.class", + TypeName.get(env.getTypeUtils().erasure(encodedType.asType()))) + .build()); + builtClass = codecClassBuilder.build(); + } else { + builtClass = buildRegisteredSingletonClass((VariableElement) element); } - codecClassBuilder.addMethod( - AutoCodecUtil.initializeGetEncodedClassMethod(encodedType, env) - .addStatement( - "return $T.class", TypeName.get(env.getTypeUtils().erasure(encodedType.asType()))) - .build()); String packageName = - env.getElementUtils().getPackageOf(encodedType).getQualifiedName().toString(); + env.getElementUtils().getPackageOf(element).getQualifiedName().toString(); try { - JavaFile file = JavaFile.builder(packageName, codecClassBuilder.build()).build(); + JavaFile file = JavaFile.builder(packageName, builtClass).build(); file.writeTo(env.getFiler()); if (env.getOptions().containsKey(PRINT_GENERATED_OPTION)) { - note("AutoCodec generated codec for " + encodedType + ":\n" + file); + note("AutoCodec generated codec for " + element + ":\n" + file); } } catch (IOException e) { env.getMessager() @@ -130,6 +140,30 @@ public class AutoCodecProcessor extends AbstractProcessor { return true; } + private static final Collection<Modifier> REQUIRED_SINGLETON_MODIFIERS = + ImmutableList.of(Modifier.STATIC, Modifier.FINAL); + + private static TypeSpec buildRegisteredSingletonClass(VariableElement symbol) { + Preconditions.checkState( + symbol.getModifiers().containsAll(REQUIRED_SINGLETON_MODIFIERS), + "Field must be static and final to be annotated with @AutoCodec: " + symbol); + return TypeSpec.classBuilder( + AutoCodecUtil.getGeneratedName( + symbol, CodecScanningConstants.REGISTERED_SINGLETON_SUFFIX)) + .addModifiers(Modifier.PUBLIC) + .addSuperinterface(RegisteredSingletonDoNotUse.class) + .addField( + FieldSpec.builder( + TypeName.get(symbol.asType()), + CodecScanningConstants.REGISTERED_SINGLETON_INSTANCE_VAR_NAME, + Modifier.PUBLIC, + Modifier.STATIC, + Modifier.FINAL) + .initializer("$T.$L", symbol.getEnclosingElement().asType(), symbol.getSimpleName()) + .build()) + .build(); + } + private TypeSpec.Builder buildClassWithInstantiatorStrategy(TypeElement encodedType) { ExecutableElement constructor = selectInstantiator(encodedType); List<? extends VariableElement> fields = constructor.getParameters(); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecUtil.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecUtil.java index 8adcddf7d7..102c1dfa4e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecUtil.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodecUtil.java @@ -104,17 +104,26 @@ class AutoCodecUtil { } /** - * Name of the generated codec class. + * Returns a class name generated from the given {@code element}. * - * <p>For {@code Foo.Bar} this is {@code Foo_Bar_AutoCodec}. + * <p>For {@code Foo.Bar} this is {@code Foo_Bar_suffix}. */ - private static String getCodecName(Element element) { + static String getGeneratedName(Element element, String suffix) { ImmutableList.Builder<String> classNamesBuilder = new ImmutableList.Builder<>(); - classNamesBuilder.add(GENERATED_CLASS_NAME_SUFFIX); + classNamesBuilder.add(suffix); do { classNamesBuilder.add(element.getSimpleName().toString()); element = element.getEnclosingElement(); } while (element instanceof TypeElement); return classNamesBuilder.build().reverse().stream().collect(Collectors.joining("_")); } + + /** + * Name of the generated codec class. + * + * <p>For {@code Foo.Bar} this is {@code Foo_Bar_AutoCodec}. + */ + private static String getCodecName(Element element) { + return getGeneratedName(element, GENERATED_CLASS_NAME_SUFFIX); + } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD index 9c7bc1945e..2fe5c58a9f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD @@ -14,6 +14,7 @@ java_library( exports = [ ":autocodec-annotation", # Generated classes have the following dependencies. + ":registered-singleton", ":unsafe-provider", "//third_party/protobuf:protobuf_java", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", @@ -22,6 +23,11 @@ java_library( ], ) +java_library( + name = "registered-singleton", + srcs = ["RegisteredSingletonDoNotUse.java"], +) + # @AutoCodec annotation only. Used by clients and the processor. java_library( name = "autocodec-annotation", @@ -53,6 +59,7 @@ java_library( ], deps = [ ":autocodec-annotation", + ":registered-singleton", ":unsafe-provider", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", "//src/main/java/com/google/devtools/build/lib/collect/nestedset:serialization", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/RegisteredSingletonDoNotUse.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/RegisteredSingletonDoNotUse.java new file mode 100644 index 0000000000..4f331e1698 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/RegisteredSingletonDoNotUse.java @@ -0,0 +1,22 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization.autocodec; + +/** + * Marker interface to indicate that an implementation provides a constant object that should be + * registered for serialization. Should never be implemented manually, only by {@code + * AutoCodecProcessor}-generated classes. + */ +public interface RegisteredSingletonDoNotUse {} |