From ccfb2df69ecf4746f5a15e1295af995c3a45aa94 Mon Sep 17 00:00:00 2001 From: Philipp Wollermann Date: Mon, 17 Jul 2017 14:53:01 +0200 Subject: Allow py_binary to be the executable of a Skylark action or any SpawnAction on Windows. RELNOTES: None. Change-Id: I2d926447511dab5fb804051abdbef9031cb089be PiperOrigin-RevId: 162201440 --- src/main/java/com/google/devtools/build/lib/BUILD | 2 +- .../bazel/rules/python/BazelPythonSemantics.java | 23 ++- .../bazel/rules/python/python_stub_template.txt | 176 +++++++++++++++++++++ .../rules/python/python_stub_template_windows.txt | 23 +++ .../build/lib/bazel/rules/python/stub_template.txt | 176 --------------------- .../devtools/build/lib/rules/python/PyBinary.java | 8 +- .../devtools/build/lib/rules/python/PyCommon.java | 20 ++- .../build/lib/rules/python/PythonSemantics.java | 2 +- 8 files changed, 243 insertions(+), 187 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template_windows.txt delete mode 100644 src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt (limited to 'src/main/java') diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index cd5e566d61..b920246a68 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -737,7 +737,7 @@ java_library( resources = [ "bazel/rules/java/java_stub_template.txt", "bazel/rules/java/java_stub_template_windows.txt", - "bazel/rules/python/stub_template.txt", + "bazel/rules/python/python_stub_template.txt", "bazel/rules/sh/sh_stub_template_windows.txt", ], deps = [ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java index 2e3ef240cd..1202cc575f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java @@ -41,6 +41,7 @@ import com.google.devtools.build.lib.rules.python.PythonSemantics; import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; import com.google.devtools.build.lib.syntax.Type; import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.ArrayList; import java.util.Collection; @@ -51,7 +52,9 @@ import java.util.List; */ public class BazelPythonSemantics implements PythonSemantics { private static final Template STUB_TEMPLATE = - Template.forResource(BazelPythonSemantics.class, "stub_template.txt"); + Template.forResource(BazelPythonSemantics.class, "python_stub_template.txt"); + private static final Template STUB_TEMPLATE_WINDOWS = + Template.forResource(BazelPythonSemantics.class, "python_stub_template_windows.txt"); public static final InstrumentationSpec PYTHON_COLLECTION_SPEC = new InstrumentationSpec( FileTypeSet.of(BazelPyRuleClasses.PYTHON_SOURCE), "srcs", "deps", "data"); @@ -123,7 +126,7 @@ public class BazelPythonSemantics implements PythonSemantics { } @Override - public void createExecutable( + public Artifact createExecutable( RuleContext ruleContext, PyCommon common, CcLinkParamsStore ccLinkParamsStore, @@ -180,7 +183,21 @@ public class BazelPythonSemantics implements PythonSemantics { .useDefaultShellEnvironment() .setMnemonic("BuildBinary") .build(ruleContext)); + + if (OS.getCurrent() == OS.WINDOWS) { + Artifact executableWrapper = common.getExecutableWrapper(); + ruleContext.registerAction( + new TemplateExpansionAction( + ruleContext.getActionOwner(), + executableWrapper, + STUB_TEMPLATE_WINDOWS, + ImmutableList.of(Substitution.of("%python_path%", pythonBinary)), + true)); + return executableWrapper; + } } + + return executable; } @Override @@ -216,7 +233,7 @@ public class BazelPythonSemantics implements PythonSemantics { } // We put the whole runfiles tree under the ZIP_RUNFILES_DIRECTORY_NAME directory, by doing this // , we avoid the conflict between default workspace name "__main__" and __main__.py file. - // Note: This name has to be the same with the one in stub_template.txt. + // Note: This name has to be the same with the one in python_stub_template.txt. return ZIP_RUNFILES_DIRECTORY_NAME.getRelative(zipRunfilesPath).toString(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt new file mode 100644 index 0000000000..218f6aeae2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt @@ -0,0 +1,176 @@ +#!/usr/bin/env python + +import os +import re +import tempfile +import shutil +import sys +import subprocess +import zipfile + +# Return True if running on Windows +def IsWindows(): + return os.name == 'nt' + +def GetWindowsPathWithUNCPrefix(path): + """ + Adding UNC prefix after getting a normalized absolute Windows path, + it's no-op for non-Windows platforms or if running under python2. + """ + path = path.strip() + + # No need to add prefix for non-Windows platforms. + # And \\?\ doesn't work in python 2 + if not IsWindows() or sys.version_info[0] < 3: + return path + + # Lets start the unicode fun + unicode_prefix = "\\\\?\\" + if path.startswith(unicode_prefix): + return path + + # os.path.abspath returns a normalized absolute path + return unicode_prefix + os.path.abspath(path) + +PYTHON_BINARY = '%python_binary%' +if IsWindows() and not PYTHON_BINARY.endswith('.exe'): + PYTHON_BINARY = PYTHON_BINARY + '.exe' + +# Find a file in a given search path. +def SearchPath(name): + search_path = os.getenv('PATH', os.defpath).split(os.pathsep) + for directory in search_path: + if directory == '': continue + path = os.path.join(directory, name) + if os.path.isfile(path) and os.access(path, os.X_OK): + return path + return None + +def IsRunningFromZip(): + return %is_zipfile% + +# Find the real Python binary if it's not a normal absolute path +def FindPythonBinary(): + if PYTHON_BINARY.startswith('//'): + # Case 1: Path is a label. Not supported yet. + raise AssertionError( + 'Bazel does not support execution of Python interpreters via labels yet') + elif PYTHON_BINARY.startswith('/'): + # Case 2: Absolute path. + return PYTHON_BINARY + elif '/' in PYTHON_BINARY: + # Case 3: Path is relative to current working directory. + return os.path.join(os.getcwd(), PYTHON_BINARY) + else: + # Case 4: Path has to be looked up in the search path. + return SearchPath(PYTHON_BINARY) + +def CreatePythonPathEntries(python_imports, module_space): + parts = python_imports.split(':'); + return [module_space] + ["%s/%s" % (module_space, path) for path in parts] + +# Find the runfiles tree +def FindModuleSpace(): + # Follow symlinks, looking for my module space + stub_filename = os.path.abspath(sys.argv[0]) + while True: + # Found it? + module_space = stub_filename + '.runfiles' + if os.path.isdir(module_space): + break + + runfiles_pattern = "(.*\.runfiles)/.*" + if IsWindows(): + runfiles_pattern = "(.*\.runfiles)\\.*" + matchobj = re.match(runfiles_pattern, os.path.abspath(sys.argv[0])) + if matchobj: + module_space = matchobj.group(1) + break + + raise AssertionError('Cannot find .runfiles directory for %s' % + sys.argv[0]) + return module_space + +# Create the runfiles tree by extracting the zip file +def CreateModuleSpace(): + ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" + temp_dir = tempfile.mkdtemp("", "Bazel.runfiles_") + zf = zipfile.ZipFile(GetWindowsPathWithUNCPrefix(os.path.dirname(__file__))) + zf.extractall(GetWindowsPathWithUNCPrefix(temp_dir)) + return os.path.join(temp_dir, ZIP_RUNFILES_DIRECTORY_NAME) + +# Returns repository roots to add to the import path. +def GetRepositoriesImports(module_space, import_all): + if import_all: + repo_dirs = [os.path.join(module_space, d) for d in os.listdir(module_space)] + return [d for d in repo_dirs if os.path.isdir(d)] + return [os.path.join(module_space, "%workspace_name%")] + +def Main(): + args = sys.argv[1:] + + new_env = {} + + if IsRunningFromZip(): + module_space = CreateModuleSpace() + else: + module_space = FindModuleSpace() + + python_imports = '%imports%' + python_path_entries = CreatePythonPathEntries(python_imports, module_space) + python_path_entries += GetRepositoriesImports(module_space, %import_all%) + + python_path_entries = [GetWindowsPathWithUNCPrefix(d) for d in python_path_entries] + + old_python_path = os.environ.get('PYTHONPATH') + python_path = os.pathsep.join(python_path_entries) + if old_python_path: + python_path += os.pathsep + old_python_path + + if IsWindows(): + python_path = python_path.replace("/", os.sep) + + new_env['PYTHONPATH'] = python_path + + # Now look for my main python source file. + # The magic string percent-main-percent is replaced with the filename of the + # main file of the Python binary in BazelPythonSemantics.java. + rel_path = '%main%' + if IsWindows(): + rel_path = rel_path.replace("/", os.sep) + + main_filename = os.path.join(module_space, rel_path) + main_filename = GetWindowsPathWithUNCPrefix(main_filename) + assert os.path.exists(main_filename), \ + 'Cannot exec() %r: file not found.' % main_filename + assert os.access(main_filename, os.R_OK), \ + 'Cannot exec() %r: file not readable.' % main_filename + + program = python_program = FindPythonBinary() + if python_program is None: + raise AssertionError('Could not find python binary: ' + PYTHON_BINARY) + args = [python_program, main_filename] + args + + os.environ.update(new_env) + + try: + sys.stdout.flush() + if IsRunningFromZip(): + # If RUN_UNDER_RUNFILES equals 1, it means we need to + # change directory to the right runfiles directory. + # (So that the data files are accessible) + if os.environ.get("RUN_UNDER_RUNFILES") == "1": + os.chdir(os.path.join(module_space, "%workspace_name%")) + retCode = subprocess.call(args) + shutil.rmtree(os.path.dirname(module_space), True) + exit(retCode) + else: + os.execv(args[0], args) + except EnvironmentError as e: + # This exception occurs when os.execv() fails for some reason. + if not getattr(e, 'filename', None): + e.filename = program # Add info to error message + raise + +if __name__ == '__main__': + Main() diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template_windows.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template_windows.txt new file mode 100644 index 0000000000..13ce2f52a9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template_windows.txt @@ -0,0 +1,23 @@ +@rem Copyright 2017 The Bazel Authors. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem This script was generated from python_stub_template_windows.txt. Please +@rem don't edit it directly. + +@SETLOCAL ENABLEEXTENSIONS + +@rem launcher=${$0%.cmd} +@set launcher=%~dp0%~n0 + +@call %python_path% %launcher% %* diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt deleted file mode 100644 index 218f6aeae2..0000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python - -import os -import re -import tempfile -import shutil -import sys -import subprocess -import zipfile - -# Return True if running on Windows -def IsWindows(): - return os.name == 'nt' - -def GetWindowsPathWithUNCPrefix(path): - """ - Adding UNC prefix after getting a normalized absolute Windows path, - it's no-op for non-Windows platforms or if running under python2. - """ - path = path.strip() - - # No need to add prefix for non-Windows platforms. - # And \\?\ doesn't work in python 2 - if not IsWindows() or sys.version_info[0] < 3: - return path - - # Lets start the unicode fun - unicode_prefix = "\\\\?\\" - if path.startswith(unicode_prefix): - return path - - # os.path.abspath returns a normalized absolute path - return unicode_prefix + os.path.abspath(path) - -PYTHON_BINARY = '%python_binary%' -if IsWindows() and not PYTHON_BINARY.endswith('.exe'): - PYTHON_BINARY = PYTHON_BINARY + '.exe' - -# Find a file in a given search path. -def SearchPath(name): - search_path = os.getenv('PATH', os.defpath).split(os.pathsep) - for directory in search_path: - if directory == '': continue - path = os.path.join(directory, name) - if os.path.isfile(path) and os.access(path, os.X_OK): - return path - return None - -def IsRunningFromZip(): - return %is_zipfile% - -# Find the real Python binary if it's not a normal absolute path -def FindPythonBinary(): - if PYTHON_BINARY.startswith('//'): - # Case 1: Path is a label. Not supported yet. - raise AssertionError( - 'Bazel does not support execution of Python interpreters via labels yet') - elif PYTHON_BINARY.startswith('/'): - # Case 2: Absolute path. - return PYTHON_BINARY - elif '/' in PYTHON_BINARY: - # Case 3: Path is relative to current working directory. - return os.path.join(os.getcwd(), PYTHON_BINARY) - else: - # Case 4: Path has to be looked up in the search path. - return SearchPath(PYTHON_BINARY) - -def CreatePythonPathEntries(python_imports, module_space): - parts = python_imports.split(':'); - return [module_space] + ["%s/%s" % (module_space, path) for path in parts] - -# Find the runfiles tree -def FindModuleSpace(): - # Follow symlinks, looking for my module space - stub_filename = os.path.abspath(sys.argv[0]) - while True: - # Found it? - module_space = stub_filename + '.runfiles' - if os.path.isdir(module_space): - break - - runfiles_pattern = "(.*\.runfiles)/.*" - if IsWindows(): - runfiles_pattern = "(.*\.runfiles)\\.*" - matchobj = re.match(runfiles_pattern, os.path.abspath(sys.argv[0])) - if matchobj: - module_space = matchobj.group(1) - break - - raise AssertionError('Cannot find .runfiles directory for %s' % - sys.argv[0]) - return module_space - -# Create the runfiles tree by extracting the zip file -def CreateModuleSpace(): - ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" - temp_dir = tempfile.mkdtemp("", "Bazel.runfiles_") - zf = zipfile.ZipFile(GetWindowsPathWithUNCPrefix(os.path.dirname(__file__))) - zf.extractall(GetWindowsPathWithUNCPrefix(temp_dir)) - return os.path.join(temp_dir, ZIP_RUNFILES_DIRECTORY_NAME) - -# Returns repository roots to add to the import path. -def GetRepositoriesImports(module_space, import_all): - if import_all: - repo_dirs = [os.path.join(module_space, d) for d in os.listdir(module_space)] - return [d for d in repo_dirs if os.path.isdir(d)] - return [os.path.join(module_space, "%workspace_name%")] - -def Main(): - args = sys.argv[1:] - - new_env = {} - - if IsRunningFromZip(): - module_space = CreateModuleSpace() - else: - module_space = FindModuleSpace() - - python_imports = '%imports%' - python_path_entries = CreatePythonPathEntries(python_imports, module_space) - python_path_entries += GetRepositoriesImports(module_space, %import_all%) - - python_path_entries = [GetWindowsPathWithUNCPrefix(d) for d in python_path_entries] - - old_python_path = os.environ.get('PYTHONPATH') - python_path = os.pathsep.join(python_path_entries) - if old_python_path: - python_path += os.pathsep + old_python_path - - if IsWindows(): - python_path = python_path.replace("/", os.sep) - - new_env['PYTHONPATH'] = python_path - - # Now look for my main python source file. - # The magic string percent-main-percent is replaced with the filename of the - # main file of the Python binary in BazelPythonSemantics.java. - rel_path = '%main%' - if IsWindows(): - rel_path = rel_path.replace("/", os.sep) - - main_filename = os.path.join(module_space, rel_path) - main_filename = GetWindowsPathWithUNCPrefix(main_filename) - assert os.path.exists(main_filename), \ - 'Cannot exec() %r: file not found.' % main_filename - assert os.access(main_filename, os.R_OK), \ - 'Cannot exec() %r: file not readable.' % main_filename - - program = python_program = FindPythonBinary() - if python_program is None: - raise AssertionError('Could not find python binary: ' + PYTHON_BINARY) - args = [python_program, main_filename] + args - - os.environ.update(new_env) - - try: - sys.stdout.flush() - if IsRunningFromZip(): - # If RUN_UNDER_RUNFILES equals 1, it means we need to - # change directory to the right runfiles directory. - # (So that the data files are accessible) - if os.environ.get("RUN_UNDER_RUNFILES") == "1": - os.chdir(os.path.join(module_space, "%workspace_name%")) - retCode = subprocess.call(args) - shutil.rmtree(os.path.dirname(module_space), True) - exit(retCode) - else: - os.execv(args[0], args) - except EnvironmentError as e: - # This exception occurs when os.execv() fails for some reason. - if not getattr(e, 'filename', None): - e.filename = program # Add info to error message - raise - -if __name__ == '__main__': - Main() diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java index fd659781cf..181cb8511e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java @@ -77,7 +77,8 @@ public abstract class PyBinary implements RuleConfiguredTargetFactory { return null; } - semantics.createExecutable(ruleContext, common, ccLinkParamsStore, imports); + Artifact realExecutable = + semantics.createExecutable(ruleContext, common, ccLinkParamsStore, imports); Runfiles commonRunfiles = collectCommonRunfiles(ruleContext, common, semantics); Runfiles.Builder defaultRunfilesBuilder = new Runfiles.Builder( @@ -117,7 +118,7 @@ public abstract class PyBinary implements RuleConfiguredTargetFactory { return builder .setFilesToBuild(common.getFilesToBuild()) .add(RunfilesProvider.class, runfilesProvider) - .setRunfilesSupport(runfilesSupport, common.getExecutable()) + .setRunfilesSupport(runfilesSupport, realExecutable) .addNativeDeclaredProvider(new CcLinkParamsProvider(ccLinkParamsStore)) .add(PythonImportsProvider.class, new PythonImportsProvider(imports)); } @@ -127,6 +128,9 @@ public abstract class PyBinary implements RuleConfiguredTargetFactory { Runfiles.Builder builder = new Runfiles.Builder( ruleContext.getWorkspaceName(), ruleContext.getConfiguration().legacyExternalRunfiles()); builder.addArtifact(common.getExecutable()); + if (common.getExecutableWrapper() != null) { + builder.addArtifact(common.getExecutableWrapper()); + } if (common.getConvertedFiles() != null) { builder.addSymlinks(common.getConvertedFiles()); } else { diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java index 5ffa4dc610..de9d22e329 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java @@ -51,6 +51,7 @@ import com.google.devtools.build.lib.syntax.SkylarkNestedSet; import com.google.devtools.build.lib.syntax.SkylarkType; import com.google.devtools.build.lib.syntax.Type; import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.protobuf.GeneratedMessage.GeneratedExtension; @@ -80,6 +81,7 @@ public final class PyCommon { private final RuleContext ruleContext; private Artifact executable = null; + private Artifact executableWrapper = null; private NestedSet transitivePythonSources; @@ -114,15 +116,21 @@ public final class PyCommon { validatePackageName(); executable = ruleContext.createOutputArtifact(); + if (OS.getCurrent() == OS.WINDOWS) { + executableWrapper = + ruleContext.getImplicitOutputArtifact(ruleContext.getTarget().getName() + ".cmd"); + } if (this.version == PythonVersion.PY2AND3) { // TODO(bazel-team): we need to create two actions ruleContext.ruleError("PY2AND3 is not yet implemented"); } - filesToBuild = NestedSetBuilder.stableOrder() - .addAll(srcs) - .add(executable) - .build(); + NestedSetBuilder filesToBuildBuilder = + NestedSetBuilder.stableOrder().addAll(srcs).add(executable); + if (executableWrapper != null) { + filesToBuildBuilder.add(executableWrapper); + } + filesToBuild = filesToBuildBuilder.build(); if (ruleContext.hasErrors()) { return; @@ -447,6 +455,10 @@ public final class PyCommon { return executable; } + public Artifact getExecutableWrapper() { + return executableWrapper; + } + public Map getConvertedFiles() { return convertedFiles; } diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java index bbaaa8f423..a87ef4e049 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java @@ -69,7 +69,7 @@ public interface PythonSemantics { * *

This should create a generating action for {@code common.getExecutable()}. */ - void createExecutable( + Artifact createExecutable( RuleContext ruleContext, PyCommon common, CcLinkParamsStore ccLinkParamsStore, -- cgit v1.2.3