diff options
15 files changed, 1313 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java new file mode 100644 index 0000000000..ecb8d48b80 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java @@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; + +import java.util.List; + +/** + * An implementation for the {@code py_binary} rule. + */ +public abstract class PyBinary implements RuleConfiguredTargetFactory { + /** + * Create a {@link PythonSemantics} object that governs + * the behavior of this rule. + */ + protected abstract PythonSemantics createSemantics(); + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + PyCommon common = new PyCommon(ruleContext); + common.initCommon(common.getDefaultPythonVersion()); + + RuleConfiguredTargetBuilder builder = init(ruleContext, createSemantics(), common); + if (builder == null) { + return null; + } + return builder.build(); + } + + static RuleConfiguredTargetBuilder init( + RuleContext ruleContext, PythonSemantics semantics, PyCommon common) { + List<Artifact> srcs = common.validateSrcs(); + CcLinkParamsStore ccLinkParamsStore = initializeCcLinkParamStore(ruleContext); + + common.initBinary(srcs); + semantics.validate(ruleContext, common); + if (ruleContext.hasErrors()) { + return null; + } + + semantics.createExecutable(ruleContext, common, ccLinkParamsStore); + Runfiles.Builder runfilesBuilder = collectCommonRunfiles(ruleContext, common); + semantics.collectRunfilesForBinary(ruleContext, runfilesBuilder, common); + Runfiles dataRunfiles = runfilesBuilder.build(); + semantics.collectDefaultRunfilesForBinary(ruleContext, runfilesBuilder); + Runfiles defaultRunfiles = runfilesBuilder.build(); + + RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable(ruleContext, defaultRunfiles, + common.getExecutable(), ruleContext.shouldCreateRunfilesSymlinks()); + + if (ruleContext.hasErrors()) { + return null; + } + + RunfilesProvider runfilesProvider = RunfilesProvider.withData(defaultRunfiles, dataRunfiles); + + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext); + common.addCommonTransitiveInfoProviders(builder, semantics, common.getFilesToBuild()); + + semantics.postInitBinary(ruleContext, runfilesSupport, common); + return builder + .setFilesToBuild(common.getFilesToBuild()) + .add(RunfilesProvider.class, runfilesProvider) + .setRunfilesSupport(runfilesSupport, common.getExecutable()) + .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore)); + } + + private static Runfiles.Builder collectCommonRunfiles(RuleContext ruleContext, PyCommon common) { + Runfiles.Builder builder = new Runfiles.Builder(); + builder.addArtifact(common.getExecutable()); + if (common.getConvertedFiles() != null) { + builder.addSymlinks(common.getConvertedFiles()); + } else { + builder.addTransitiveArtifacts(common.getFilesToBuild()); + } + builder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); + builder.add(ruleContext, PythonRunfilesProvider.TO_RUNFILES); + builder.setManifestExpander(PythonUtils.GET_INIT_PY_FILES); + return builder; + } + + private static CcLinkParamsStore initializeCcLinkParamStore(final RuleContext ruleContext) { + return new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + Iterable<? extends TransitiveInfoCollection> deps = + ruleContext.getPrerequisites("deps", Mode.TARGET); + builder.addTransitiveTargets(deps); + builder.addTransitiveLangTargets(deps, PyCcLinkParamsProvider.TO_LINK_PARAMS); + } + }; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCcLinkParamsProvider.java new file mode 100644 index 0000000000..1d60dfb21f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCcLinkParamsProvider.java @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides C++ libraries to be linked into Python targets. + */ +@Immutable +public final class PyCcLinkParamsProvider implements TransitiveInfoProvider { + private final CcLinkParamsStoreImpl store; + + public PyCcLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + public CcLinkParamsStore getLinkParams() { + return store; + } + + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + PyCcLinkParamsProvider provider = input.getProvider( + PyCcLinkParamsProvider.class); + return provider == null ? null : provider.getLinkParams(); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java new file mode 100644 index 0000000000..5f3a3ad9c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java @@ -0,0 +1,404 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.actions.extra.PythonInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.LanguageDependentFragment; +import com.google.devtools.build.lib.analysis.OutputGroupProvider; +import com.google.devtools.build.lib.analysis.PseudoAction; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CppFileTypes; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.protobuf.GeneratedMessage.GeneratedExtension; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * A helper class for Python rules. + */ +public final class PyCommon { + + private static final LocalMetadataCollector METADATA_COLLECTOR = new LocalMetadataCollector() { + @Override + public void collectMetadataArtifacts(Iterable<Artifact> artifacts, + AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) { + // Python doesn't do any compilation, so we simply return the empty set. + } + }; + + private final RuleContext ruleContext; + + private Artifact executable = null; + + private NestedSet<Artifact> transitivePythonSources; + + private PythonVersion sourcesVersion; + private PythonVersion version = null; + private Map<PathFragment, Artifact> convertedFiles; + + private NestedSet<Artifact> filesToBuild = null; + + public PyCommon(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + public void initCommon(PythonVersion defaultVersion) { + this.sourcesVersion = getPythonVersionAttr( + ruleContext, "srcs_version", PythonVersion.getAllValues()); + + this.version = ruleContext.getFragment(PythonConfiguration.class) + .getPythonVersion(defaultVersion); + + transitivePythonSources = collectTransitivePythonSources(); + + checkSourceIsCompatible(this.version, this.sourcesVersion, ruleContext.getLabel()); + } + + public PythonVersion getVersion() { + return version; + } + + public void initBinary(List<Artifact> srcs) { + Preconditions.checkNotNull(version); + + validatePackageName(); + executable = ruleContext.createOutputArtifact(); + if (this.version == PythonVersion.PY2AND3) { + // TODO(bazel-team): we need to create two actions + ruleContext.ruleError("PY2AND3 is not yet implemented"); + } + + filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addAll(srcs) + .add(executable) + .build(); + + if (ruleContext.hasErrors()) { + return; + } + + addPyExtraActionPseudoAction(); + } + + public void addCommonTransitiveInfoProviders(RuleConfiguredTargetBuilder builder, + PythonSemantics semantics, NestedSet<Artifact> filesToBuild) { + builder + .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl( + new InstrumentedFilesCollector(ruleContext, + semantics.getCoverageInstrumentationSpec(), METADATA_COLLECTOR, + filesToBuild))) + .add(PythonSourcesProvider.class, new PythonSourcesProvider( + transitivePythonSources, usesSharedLibraries())) + // Python targets are not really compilable. The best we can do is make sure that all + // generated source files are ready. + .addOutputGroup(OutputGroupProvider.FILES_TO_COMPILE, transitivePythonSources) + .addOutputGroup(OutputGroupProvider.COMPILATION_PREREQUISITES, transitivePythonSources); + } + + public PythonVersion getDefaultPythonVersion() { + return ruleContext.getRule() + .isAttrDefined("default_python_version", Type.STRING) + ? getPythonVersionAttr( + ruleContext, "default_python_version", PythonVersion.PY2, PythonVersion.PY3) + : null; + } + + public static PythonVersion getPythonVersionAttr(RuleContext ruleContext, + String attrName, PythonVersion... allowed) { + String stringAttr = ruleContext.attributes().get(attrName, Type.STRING); + PythonVersion version = PythonVersion.parse(stringAttr, allowed); + if (version != null) { + return version; + } + ruleContext.attributeError(attrName, + "'" + stringAttr + "' is not a valid value. Expected one of: " + Joiner.on(", ") + .join(allowed)); + return PythonVersion.defaultValue(); + } + + /** + * Returns a mutable List of the source Artifacts. + */ + public List<Artifact> validateSrcs() { + List<Artifact> sourceFiles = new ArrayList<>(); + // TODO(bazel-team): Need to get the transitive deps closure, not just the + // sources of the rule. + for (FileProvider src : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + // Make sure that none of the sources contain hyphens. + if (Util.containsHyphen(src.getLabel().getPackageFragment())) { + ruleContext.attributeError("srcs", src.getLabel() + ": package name may not contain '-'"); + } + Iterable<Artifact> pySrcs = FileType.filter(src.getFilesToBuild(), + PyRuleClasses.PYTHON_SOURCE); + Iterables.addAll(sourceFiles, pySrcs); + if (Iterables.isEmpty(pySrcs)) { + ruleContext.attributeWarning("srcs", + "rule '" + src.getLabel() + "' does not produce any Python source files"); + } + } + + LanguageDependentFragment.Checker.depsSupportsLanguage(ruleContext, PyRuleClasses.LANGUAGE); + return convertedFiles != null + ? ImmutableList.copyOf(convertedFiles.values()) + : sourceFiles; + } + + /** + * Checks that the package name of this Python rule does not contain a '-'. + */ + void validatePackageName() { + if (Util.containsHyphen(ruleContext.getLabel().getPackageFragment())) { + ruleContext.ruleError("package name may not contain '-'"); + } + } + + /** + * Adds a {@link PseudoAction} to the build graph that is only used + * for providing information to the blaze extra_action feature. + */ + void addPyExtraActionPseudoAction() { + if (ruleContext.getConfiguration().getActionListeners().isEmpty()) { + return; + } + + // Has to be unfiltered sources as filtered will give an error for + // unsupported file types where as certain tests only expect a warning. + Collection<Artifact> sources = ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(); + + // We need to do it in this convoluted way because we must not add the files declared in the + // srcs of this rule. Note that it is not enough to remove the direct members from the nested + // set of the current rule, because the same files may have been declared in a dependency, too. + NestedSetBuilder<Artifact> depBuilder = NestedSetBuilder.compileOrder(); + collectTransitivePythonSourcesFromDeps(depBuilder); + NestedSet<Artifact> dependencies = depBuilder.build(); + + PythonInfo info = PythonInfo.newBuilder() + .addAllSourceFile(Artifact.toExecPaths(sources)) + .addAllDepFile(Artifact.toExecPaths(dependencies)) + .build(); + + ruleContext.getAnalysisEnvironment() + .registerAction(new PyPseudoAction(ruleContext.getActionOwner(), + ImmutableList.copyOf(Iterables.concat(sources, dependencies)), + ImmutableList.of(PseudoAction.getDummyOutput(ruleContext)), "Python", + PythonInfo.pythonInfo, info)); + } + + private void addSourceFiles(NestedSetBuilder<Artifact> builder, Iterable<Artifact> artifacts) { + Preconditions.checkState(convertedFiles == null); + if (sourcesVersion == PythonVersion.PY2 && version == PythonVersion.PY3) { + convertedFiles = PythonUtils.generate2to3Actions(ruleContext, artifacts); + } + builder.addAll(artifacts); + } + + private void collectTransitivePythonSourcesFromDeps(NestedSetBuilder<Artifact> builder) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) { + if (dep.getProvider(PythonSourcesProvider.class) != null) { + PythonSourcesProvider provider = dep.getProvider(PythonSourcesProvider.class); + builder.addTransitive(provider.getTransitivePythonSources()); + } else { + // TODO(bazel-team): We also collect .py source files from deps (e.g. for proto_library + // rules). Rules should implement PythonSourcesProvider instead. + FileProvider provider = dep.getProvider(FileProvider.class); + builder.addAll(FileType.filter(provider.getFilesToBuild(), PyRuleClasses.PYTHON_SOURCE)); + } + } + } + + private NestedSet<Artifact> collectTransitivePythonSources() { + NestedSetBuilder<Artifact> builder = + NestedSetBuilder.compileOrder(); + collectTransitivePythonSourcesFromDeps(builder); + addSourceFiles(builder, ruleContext + .getPrerequisiteArtifacts("srcs", Mode.TARGET).filter(PyRuleClasses.PYTHON_SOURCE).list()); + return builder.build(); + } + + /** + * Checks that the source file version is compatible with the Python interpreter. + */ + private void checkSourceIsCompatible(PythonVersion targetVersion, PythonVersion sourceVersion, + Label source) { + if (targetVersion == PythonVersion.PY2 || targetVersion == PythonVersion.PY2AND3) { + if (sourceVersion == PythonVersion.PY3ONLY) { + ruleContext.ruleError("Rule '" + source + + "' can only be used with Python 3, and cannot be converted to Python 2"); + } else if (sourceVersion == PythonVersion.PY3) { + ruleContext.ruleError("Rule '" + source + + "' need to be converted to Python 2 (not yet implemented)"); + } + } + if (targetVersion == PythonVersion.PY3 || targetVersion == PythonVersion.PY2AND3) { + if (sourceVersion == PythonVersion.PY2ONLY) { + ruleContext.ruleError("Rule '" + source + + "' can only be used with Python 2, and cannot be converted to Python 3"); + } + } + } + + /** + * @return A String that is the full path to the main python entry point. + */ + public String determineMainExecutableSource() { + String mainSourceName; + Rule target = ruleContext.getRule(); + boolean explicitMain = target.isAttributeValueExplicitlySpecified("main"); + if (explicitMain) { + mainSourceName = ruleContext.attributes().get("main", Type.LABEL).getName(); + if (!mainSourceName.endsWith(".py")) { + ruleContext.attributeError("main", "main must end in '.py'"); + } + } else { + String ruleName = target.getName(); + if (ruleName.endsWith(".py")) { + ruleContext.attributeError("name", "name must not end in '.py'"); + } + mainSourceName = ruleName + ".py"; + } + PathFragment mainSourcePath = new PathFragment(mainSourceName); + + Artifact mainArtifact = null; + for (Artifact outItem : ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list()) { + if (outItem.getRootRelativePath().endsWith(mainSourcePath)) { + if (mainArtifact == null) { + mainArtifact = outItem; + } else { + ruleContext.attributeError("srcs", + buildMultipleMainMatchesErrorText(explicitMain, mainSourceName, + mainArtifact.getRootRelativePath().toString(), + outItem.getRootRelativePath().toString())); + } + } + } + + if (mainArtifact == null) { + ruleContext.attributeError("srcs", buildNoMainMatchesErrorText(explicitMain, mainSourceName)); + return null; + } + + PathFragment workspaceName = new PathFragment(ruleContext.getRule().getWorkspaceName()); + return workspaceName.getRelative(mainArtifact.getRootRelativePath()).getPathString(); + } + + public Artifact getExecutable() { + return executable; + } + + public Map<PathFragment, Artifact> getConvertedFiles() { + return convertedFiles; + } + + public NestedSet<Artifact> getFilesToBuild() { + return filesToBuild; + } + + public boolean usesSharedLibraries() { + return checkForSharedLibraries(Iterables.concat( + ruleContext.getPrerequisites("deps", Mode.TARGET), + ruleContext.getPrerequisites("data", Mode.DATA))); + } + + /** + * Returns true if this target has an .so file in its transitive dependency closure. + */ + public static boolean checkForSharedLibraries(Iterable<TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + PythonSourcesProvider provider = dep.getProvider(PythonSourcesProvider.class); + if (provider != null) { + if (provider.usesSharedLibraries()) { + return true; + } + } else if (FileType.contains( + dep.getProvider(FileProvider.class).getFilesToBuild(), CppFileTypes.SHARED_LIBRARY)) { + return true; + } + } + + return false; + } + + private static String buildMultipleMainMatchesErrorText(boolean explicit, String proposedMainName, + String match1, String match2) { + String errorText; + if (explicit) { + errorText = "file name '" + proposedMainName + + "' specified by 'main' attribute matches multiple files: e.g., '" + match1 + + "' and '" + match2 + "'"; + } else { + errorText = "default main file name '" + proposedMainName + + "' matches multiple files. Perhaps specify an explicit file with 'main' attribute? " + + "Matches were: '" + match1 + "' and '" + match2 + "'"; + } + return errorText; + } + + private static String buildNoMainMatchesErrorText(boolean explicit, String proposedMainName) { + String errorText; + if (explicit) { + errorText = "could not find '" + proposedMainName + + "' as specified by 'main' attribute"; + } else { + errorText = "corresponding default '" + proposedMainName + "' does not appear in srcs. Add it" + + " or override default file name with a 'main' attribute"; + } + return errorText; + } + + // Used purely to set the legacy ActionType of the ExtraActionInfo. + private static class PyPseudoAction extends PseudoAction<PythonInfo> { + private static final UUID ACTION_UUID = UUID.fromString("8d720129-bc1a-481f-8c4c-dbe11dcef319"); + + public PyPseudoAction(ActionOwner owner, + Collection<Artifact> inputs, Collection<Artifact> outputs, + String mnemonic, GeneratedExtension<ExtraActionInfo, PythonInfo> infoExtension, + PythonInfo info) { + super(ACTION_UUID, owner, inputs, outputs, mnemonic, infoExtension, info); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + return super.getExtraActionInfo(); + } + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java new file mode 100644 index 0000000000..89b4fc8c19 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java @@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; + +/** + * An implementation for the {@code py_library} rule. + */ +public abstract class PyLibrary implements RuleConfiguredTargetFactory { + + /** + * Create a {@link PythonSemantics} object that governs + * the behavior of this rule. + */ + protected abstract PythonSemantics createSemantics(); + + @Override + public ConfiguredTarget create(final RuleContext ruleContext) { + PythonSemantics semantics = createSemantics(); + PyCommon common = new PyCommon(ruleContext); + common.initCommon(common.getDefaultPythonVersion()); + common.validatePackageName(); + NestedSet<Artifact> filesToBuild = + NestedSetBuilder.wrap(Order.STABLE_ORDER, common.validateSrcs()); + common.addPyExtraActionPseudoAction(); + + CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + builder.addTransitiveTargets(ruleContext.getPrerequisites("deps", Mode.TARGET)); + builder.addTransitiveLangTargets( + ruleContext.getPrerequisites("deps", Mode.TARGET), + PyCcLinkParamsProvider.TO_LINK_PARAMS); + } + }; + + Runfiles.Builder runfilesBuilder = new Runfiles.Builder(); + if (common.getConvertedFiles() != null) { + runfilesBuilder.addSymlinks(common.getConvertedFiles()); + } else { + runfilesBuilder.addTransitiveArtifacts(filesToBuild); + } + runfilesBuilder.setManifestExpander(PythonUtils.GET_INIT_PY_FILES); + runfilesBuilder.add(ruleContext, PythonRunfilesProvider.TO_RUNFILES); + runfilesBuilder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); + + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext); + common.addCommonTransitiveInfoProviders(builder, semantics, filesToBuild); + return builder + .setFilesToBuild(filesToBuild) + .add(RunfilesProvider.class, RunfilesProvider.simple(runfilesBuilder.build())) + .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore)) + .build(); + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuleClasses.java new file mode 100644 index 0000000000..f927d12c85 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuleClasses.java @@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage; +import com.google.devtools.build.lib.util.FileType; + +/** + * Rule definitions for Python rules. + */ +public class PyRuleClasses { + + public static final FileType PYTHON_SOURCE = FileType.of(".py"); + public static final LibraryLanguage LANGUAGE = new LibraryLanguage("Python"); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyTest.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyTest.java new file mode 100644 index 0000000000..f433eb81e8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyTest.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * An implementation for {@code py_test} rules. + */ +public abstract class PyTest implements RuleConfiguredTargetFactory { + /** + * Create a {@link PythonSemantics} object that governs + * the behavior of this rule. + */ +protected abstract PythonSemantics createSemantics(); + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + PythonSemantics semantics = createSemantics(); + PyCommon common = new PyCommon(ruleContext); + common.initCommon(getDefaultPythonVersion(ruleContext)); + + RuleConfiguredTargetBuilder builder = PyBinary.init(ruleContext, semantics, common); + if (builder == null) { + return null; + } + return builder.build(); + } + + private PythonVersion getDefaultPythonVersion(RuleContext ruleContext) { + return ruleContext.getRule().isAttrDefined("default_python_version", Type.STRING) + ? PyCommon.getPythonVersionAttr(ruleContext, "default_python_version", PythonVersion.PY2, + PythonVersion.PY3, PythonVersion.PY2AND3) + : null; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonConfiguration.java new file mode 100644 index 0000000000..3f26c59090 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonConfiguration.java @@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; + +/** + * The configuration fragment containing information about the various pieces of infrastructure + * needed to run Python compilations. + */ +public class PythonConfiguration extends BuildConfiguration.Fragment { + private final boolean ignorePythonVersionAttribute; + private final PythonVersion defaultPythonVersion; + + PythonConfiguration(PythonVersion pythonVersion, boolean ignorePythonVersionAttribute) { + this.ignorePythonVersionAttribute = ignorePythonVersionAttribute; + + this.defaultPythonVersion = pythonVersion; + } + + @Override + public String getName() { + return "Python"; + } + + /** + * Returns the Python version (PY2 or PY3) this configuration uses. + */ + public PythonVersion getDefaultPythonVersion() { + return defaultPythonVersion; + } + + /** + * Returns the Python version to use. Command-line flag --force_python overrides + * the rule default, given as argument. + */ + public PythonVersion getPythonVersion(PythonVersion attributeVersion) { + return ignorePythonVersionAttribute || attributeVersion == null + ? defaultPythonVersion + : attributeVersion; + } + + @Override + public String cacheKey() { + return defaultPythonVersion.toString(); + } + + @Override + public String getOutputDirectoryName() { + return (defaultPythonVersion == PythonVersion.PY3) ? "py3" : null; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonConfigurationLoader.java new file mode 100644 index 0000000000..2acfe17981 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonConfigurationLoader.java @@ -0,0 +1,79 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppOptions; +import com.google.devtools.build.lib.rules.cpp.CrosstoolConfigurationLoader; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; + +import javax.annotation.Nullable; + +/** + * A factory implementation for {@link PythonConfiguration} objects. + */ +public class PythonConfigurationLoader implements ConfigurationFragmentFactory { + private final Function<String, String> cpuTransformer; + + public PythonConfigurationLoader(Function<String, String> cpuTransformer) { + this.cpuTransformer = cpuTransformer; + } + + @Nullable + private CrosstoolConfig.CToolchain getToolchain( + ConfigurationEnvironment env, BuildOptions buildOptions, Label crosstoolTop) + throws InvalidConfigurationException { + CrosstoolConfigurationLoader.CrosstoolFile file = + CrosstoolConfigurationLoader.readCrosstool(env, crosstoolTop); + if (file == null) { + return null; + } + return CrosstoolConfigurationLoader.selectToolchain( + file.getProto(), buildOptions, cpuTransformer); + } + + @Override + public PythonConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + PythonOptions pythonOptions = buildOptions.get(PythonOptions.class); + CppConfiguration cppConfiguration = env.getFragment(buildOptions, CppConfiguration.class); + if (cppConfiguration == null) { + return null; + } + + CrosstoolConfig.CToolchain toolchain = getToolchain( + env, buildOptions, buildOptions.get(CppOptions.class).crosstoolTop); + if (toolchain == null) { + return null; + } + + boolean ignorePythonVersionAttribute = pythonOptions.forcePython != null; + PythonVersion pythonVersion = pythonOptions.getPythonVersion(); + + return new PythonConfiguration(pythonVersion, ignorePythonVersionAttribute); + } + + @Override + public Class<? extends Fragment> creates() { + return PythonConfiguration.class; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonOptions.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonOptions.java new file mode 100644 index 0000000000..ba4a2388c1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonOptions.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; + +/** + * Python-related command-line options. + */ +public class PythonOptions extends FragmentOptions { + static final PythonVersion DEFAULT_PYTHON_VERSION = PythonVersion.PY2; + + /** + * Converter for the --force_python option. + */ + public static class PythonVersionConverter extends EnumConverter<PythonVersion> { + public PythonVersionConverter() { + super(PythonVersion.class, "Python version"); + } + } + @Option(name = "force_python", + defaultValue = "null", + category = "version", + converter = PythonVersionConverter.class, + help = "Overrides default_python_version attribute. Can be \"PY2\" or \"PY3\".") + public PythonVersion forcePython; + + public PythonVersion getPythonVersion() { + return (forcePython == null) ? DEFAULT_PYTHON_VERSION : forcePython; + } + + @Override + public FragmentOptions getHost(boolean fallback) { + PythonOptions hostPythonOpts = (PythonOptions) getDefault(); + hostPythonOpts.forcePython = PythonVersion.PY2; + return hostPythonOpts; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonRunfilesProvider.java new file mode 100644 index 0000000000..9fb958fa13 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonRunfilesProvider.java @@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that supplies runfiles for Python dependencies. + * + * <p>Should only be used in proto_library, and even then only until a better mechanism is found. + */ +@Immutable +public final class PythonRunfilesProvider implements TransitiveInfoProvider { + private final Runfiles pythonRunfiles; + + public PythonRunfilesProvider(Runfiles pythonRunfiles) { + this.pythonRunfiles = pythonRunfiles; + } + + public Runfiles getPythonRunfiles() { + return pythonRunfiles; + } + + /** + * Returns a function that gets the Python runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> TO_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + PythonRunfilesProvider provider = input.getProvider(PythonRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getPythonRunfiles(); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java new file mode 100644 index 0000000000..863c3ee76d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java @@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; + +/** + * Pluggable semantics for Python rules. + * + * <p>A new instance of this class is created for each configured target, therefore, it is allowed + * to keep state. + */ +public interface PythonSemantics { + /** + * Called at the beginning of the analysis of {@code py_binary} rules to validate its attributes. + */ + void validate(RuleContext ruleContext, PyCommon common); + + /** + * Extends for the default and data runfiles of {@code py_binary} rules with custom elements. + */ + void collectRunfilesForBinary(RuleContext ruleContext, Runfiles.Builder builder, PyCommon common); + + /** + * Extends the default runfiles of {@code py_binary} rules with custom elements. + */ + void collectDefaultRunfilesForBinary(RuleContext ruleContext, Runfiles.Builder builder); + + /** + * Returns the coverage instrumentation specification to be used in Python rules. + */ + InstrumentationSpec getCoverageInstrumentationSpec(); + + /** + * Create the actual executable artifact. + * + * <p>This should create a generating action for {@code common.getExecutable()}. + */ + void createExecutable(RuleContext ruleContext, PyCommon common, + CcLinkParamsStore ccLinkParamsStore); + + /** + * Called at the end of the analysis of {@code py_binary} rules. + */ + void postInitBinary(RuleContext ruleContext, RunfilesSupport runfilesSupport, + PyCommon common); +}
\ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSourcesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSourcesProvider.java new file mode 100644 index 0000000000..32bf3e38b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSourcesProvider.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A provider interface for configured targets that provide source files to + * Python targets. + */ +@Immutable +public final class PythonSourcesProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitivePythonSources; + private final boolean usesSharedLibraries; + + public PythonSourcesProvider(NestedSet<Artifact> transitivePythonSources, + boolean usesSharedLibraries) { + this.transitivePythonSources = transitivePythonSources; + this.usesSharedLibraries = usesSharedLibraries; + } + + /** + * Returns the Python sources in the transitive closure of this target. + */ + public NestedSet<Artifact> getTransitivePythonSources() { + return transitivePythonSources; + } + + /** + * Returns true if this target transitively depends on any shared libraries. + */ + public boolean usesSharedLibraries() { + return usesSharedLibraries; + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java new file mode 100644 index 0000000000..0c921c8423 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java @@ -0,0 +1,152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Various utility methods for Python support. + */ +public final class PythonUtils { + public static final PathFragment INIT_PY = new PathFragment("__init__.py"); + + private static final FileType REQUIRES_INIT_PY = FileType.of(".py", ".so", ".pyc"); + + public static final Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> + GET_INIT_PY_FILES = new Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>>() + { + @Override + public Map<PathFragment, Artifact> apply(Map<PathFragment, Artifact> input) { + return getInitDotPyFiles(input); + } + }; + + private PythonUtils() { + // This is a utility class, not to be instantiated + } + + /** + * Returns the set of empty __init__.py files to be added to a given set of files to allow + * the Python runtime to find the <code>.py</code> and <code>.so</code> files present in the + * tree. + */ + public static Set<PathFragment> getInitPyFiles(Set<PathFragment> manifestFiles) { + Set<PathFragment> result = new HashSet<>(); + + for (PathFragment source : manifestFiles) { + // If we have a python or .so file at this level... + if (!REQUIRES_INIT_PY.matches(source)) { + continue; + } + // ...then record that we need an __init__.py in this directory... + while (source.segmentCount() > 1) { + source = source.getParentDirectory(); + PathFragment initpy = source.getRelative(INIT_PY); + if (!manifestFiles.contains(initpy)) { + result.add(initpy); + } + } + } + + return ImmutableSet.copyOf(result); + } + + /** + * Creates a new map that contains all the <code>__init__.py</code> files that are necessary for + * Python to find the <code>.py</code> and <code>.so</code> files in it. + * + * @param inputManifest The input mapping of source files to absolute paths on disk. + * @return The revised mapping. Contains <code>null</code> values for the added + * <code>__init.py__</code> files. + */ + private static Map<PathFragment, Artifact> getInitDotPyFiles( + Map<PathFragment, Artifact> inputManifest) { + Map<PathFragment, Artifact> newManifest = new HashMap<>(); + + for (PathFragment initpy : getInitPyFiles(inputManifest.keySet())) { + if (newManifest.get(initpy) == null) { + newManifest.put(initpy, null); + } + } + + return newManifest; + } + + /** + * Get the artifact generated by the 2to3 action. The artifact is in a python3 + * subdirectory to avoid conflicts (eg. when the input file is generated). + */ + private static Artifact get2to3OutputArtifact(RuleContext ruleContext, Artifact input) { + Root root = ruleContext.getConfiguration().getGenfilesDirectory(); + PathFragment path = new PathFragment("python3").getRelative(input.getRootRelativePath()); + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(path, root); + } + + /** + * Create an action for each Python 2 file to convert to Python 3 + */ + public static Map<PathFragment, Artifact> generate2to3Actions(RuleContext ruleContext, + Iterable<Artifact> inputs) { + // This creates many actions, but this is fine. Creating one action per library leads + // to some problems (when the same file is generated by two different actions), with + // little benefits and negligible memory improvement. + + Map<PathFragment, Artifact> symlinks = new HashMap<>(); + for (Artifact input : inputs) { + Artifact output = generate2to3Action(ruleContext, input); + symlinks.put(input.getRootRelativePath(), output); + } + return symlinks; + } + + private static Artifact generate2to3Action(RuleContext ruleContext, Artifact input) { + FilesToRunProvider py2to3converter = + ruleContext.getExecutablePrerequisite("$python2to3", RuleConfiguredTarget.Mode.HOST); + Artifact output = get2to3OutputArtifact(ruleContext, input); + List<String> argv = new ArrayList<>(); + argv.add("--no-diffs"); + argv.add("--nobackups"); + argv.add("--write"); + argv.add("--output-dir"); + argv.add(output.getExecPath().getParentDirectory().toString()); + argv.add("--write-unchanged-files"); + argv.add(input.getExecPathString()); + + ruleContext.registerAction(new SpawnAction.Builder() + .addInput(input) + .addOutput(output) + .setExecutable(py2to3converter) + .addArguments(argv) + .setProgressMessage("Converting to Python 3: " + input.prettyPrint()) + .setMnemonic("2to3") + .build(ruleContext)); + return output; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonVersion.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonVersion.java new file mode 100644 index 0000000000..f623bc73e7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonVersion.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.python; + +import java.util.Arrays; + +/** + * Python version for Python rules. + */ +public enum PythonVersion { + PY2, + PY3, + PY2AND3, + PY2ONLY, + PY3ONLY; + + static final PythonVersion[] ALL_VALUES = + new PythonVersion[] { PY2, PY3, PY2AND3, PY2ONLY, PY3ONLY }; + + public static PythonVersion defaultValue() { + return PY2; + } + + public static PythonVersion[] getAllValues() { + return ALL_VALUES; + } + + /** + * Converts the string to PythonVersion, if it is one of the allowed values. + * Returns null if the input is not valid. + */ + public static PythonVersion parse(String str, PythonVersion... allowed) { + if (str == null) { + return null; + } + try { + PythonVersion version = PythonVersion.valueOf(str); + if (Arrays.asList(allowed).contains(version)) { + return version; + } + return null; + } catch (IllegalArgumentException e) { + return null; + } + } +} + diff --git a/src/main/protobuf/extra_actions_base.proto b/src/main/protobuf/extra_actions_base.proto index bc3b69acde..e891b56394 100644 --- a/src/main/protobuf/extra_actions_base.proto +++ b/src/main/protobuf/extra_actions_base.proto @@ -127,3 +127,14 @@ message JavaCompileInfo { repeated string processor = 6; repeated string processorpath = 7; } + +// Provides access to data that is specific to python rules. +// Usually provided by actions using the "Python" Mnemonic. +message PythonInfo { + extend ExtraActionInfo { + optional PythonInfo python_info = 1005; + } + + repeated string source_file = 1; + repeated string dep_file = 2; +} |