diff options
| -rwxr-xr-x | configure | 143 | ||||
| -rw-r--r-- | third_party/py/python_configure.bzl | 56 |
2 files changed, 161 insertions, 38 deletions
@@ -51,6 +51,121 @@ function write_action_env_to_bazelrc() { write_to_bazelrc "build --action_env $1=\"$2\"" } +function python_path { + "$PYTHON_BIN_PATH" - <<END +from __future__ import print_function +import site +import os + +try: + input = raw_input +except NameError: + pass + +python_paths = [] +if os.getenv('PYTHONPATH') is not None: + python_paths = os.getenv('PYTHONPATH').split(':') +try: + library_paths = site.getsitepackages() +except AttributeError: + from distutils.sysconfig import get_python_lib + library_paths = [get_python_lib()] +all_paths = set(python_paths + library_paths) + +paths = [] +for path in all_paths: + if os.path.isdir(path): + paths.append(path) + +print(",".join(paths)) +END +} + +function setup_python { + ## Set up python-related environment settings: + while true; do + fromuser="" + if [ -z "$PYTHON_BIN_PATH" ]; then + default_python_bin_path=$(which python || which python3 || true) + read -p "Please specify the location of python. [Default is $default_python_bin_path]: " PYTHON_BIN_PATH + fromuser="1" + if [ -z "$PYTHON_BIN_PATH" ]; then + PYTHON_BIN_PATH=$default_python_bin_path + fi + fi + if [ -e "$PYTHON_BIN_PATH" ]; then + break + fi + echo "Invalid python path. ${PYTHON_BIN_PATH} cannot be found" 1>&2 + if [ -z "$fromuser" ]; then + exit 1 + fi + PYTHON_BIN_PATH="" + # Retry + done + + if [ -z "$PYTHON_LIB_PATH" ]; then + # Split python_path into an array of paths, this allows path containing spaces + IFS=',' + python_lib_path=($(python_path)) + unset IFS + + if [ 1 = "$USE_DEFAULT_PYTHON_LIB_PATH" ]; then + PYTHON_LIB_PATH=${python_lib_path[0]} + echo "Using python library path: $PYTHON_LIB_PATH" + + else + echo "Found possible Python library paths:" + for x in "${python_lib_path[@]}"; do + echo " $x" + done + set -- "${python_lib_path[@]}" + echo "Please input the desired Python library path to use. Default is ["$1"]" + read b || true + if [ "$b" == "" ]; then + PYTHON_LIB_PATH=${python_lib_path[0]} + echo "Using python library path: $PYTHON_LIB_PATH" + else + PYTHON_LIB_PATH="$b" + fi + fi + fi + + if [ ! -x "$PYTHON_BIN_PATH" ] || [ -d "$PYTHON_BIN_PATH" ]; then + echo "PYTHON_BIN_PATH is not executable. Is it the python binary?" + exit 1 + fi + + local python_major_version=$("${PYTHON_BIN_PATH}" -c 'from __future__ import print_function; import sys; print(sys.version_info[0]);') + if [ "$python_major_version" == "" ]; then + echo -e "\n\nERROR: Problem getting python version. Is $PYTHON_BIN_PATH the correct python binary?" + exit 1 + fi + + # Convert python path to Windows style before writing into bazel.rc + if is_windows; then + PYTHON_BIN_PATH="$(cygpath -m "$PYTHON_BIN_PATH")" + fi + + # Set-up env variables used by python_configure.bzl + write_action_env_to_bazelrc "PYTHON_BIN_PATH" "$PYTHON_BIN_PATH" + write_action_env_to_bazelrc "PYTHON_LIB_PATH" "$PYTHON_LIB_PATH" + write_to_bazelrc "build --define PYTHON_BIN_PATH=$PYTHON_BIN_PATH" + write_to_bazelrc "build --define PYTHON_LIB_PATH=$PYTHON_LIB_PATH" + write_to_bazelrc "build --force_python=py$python_major_version" + write_to_bazelrc "build --host_force_python=py$python_major_version" + write_to_bazelrc "build --python${python_major_version}_path=$PYTHON_BIN_PATH" + write_to_bazelrc "test --force_python=py$python_major_version" + write_to_bazelrc "test --host_force_python=py$python_major_version" + write_to_bazelrc "test --define PYTHON_BIN_PATH=$PYTHON_BIN_PATH" + write_to_bazelrc "test --define PYTHON_LIB_PATH=$PYTHON_LIB_PATH" + write_to_bazelrc "run --define PYTHON_BIN_PATH=$PYTHON_BIN_PATH" + write_to_bazelrc "run --define PYTHON_LIB_PATH=$PYTHON_LIB_PATH" + + # Write tools/python_bin_path.sh + echo "export PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" > tools/python_bin_path.sh +} + # This file contains customized config settings. rm -f .tf_configure.bazelrc touch .tf_configure.bazelrc @@ -65,30 +180,7 @@ if [ -d "${MAKEFILE_DOWNLOAD_DIR}" ]; then find ${MAKEFILE_DOWNLOAD_DIR} -type f -name '*BUILD' -delete fi -## Set up python-related environment settings -while true; do - fromuser="" - if [ -z "$PYTHON_BIN_PATH" ]; then - default_python_bin_path=$(which python || which python3 || true) - read -p "Please specify the location of python. [Default is $default_python_bin_path]: " PYTHON_BIN_PATH - fromuser="1" - if [ -z "$PYTHON_BIN_PATH" ]; then - PYTHON_BIN_PATH=$default_python_bin_path - fi - fi - if [ -e "$PYTHON_BIN_PATH" ]; then - break - fi - echo "Invalid python path. ${PYTHON_BIN_PATH} cannot be found" 1>&2 - if [ -z "$fromuser" ]; then - exit 1 - fi - PYTHON_BIN_PATH="" - # Retry -done -export PYTHON_BIN_PATH -write_action_env_to_bazelrc "PYTHON_BIN_PATH" "$PYTHON_BIN_PATH" -# TODO(ngiraldo): allow the user to optionally set PYTHON_INCLUDE_PATH and NUMPY_INCLUDE_PATH +setup_python ## Set up MKL related environment settings if false; then # Disable building with MKL for now @@ -263,9 +355,6 @@ if [[ "$TF_NEED_VERBS" == "1" ]]; then write_to_bazelrc 'build --define with_verbs_support=true' fi -# Invoke python_config and set up symlinks to python includes -./util/python/python_config.sh "$PYTHON_BIN_PATH" - # Append CC optimization flags to bazel.rc echo >> tools/bazel.rc for opt in $CC_OPT_FLAGS; do diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index d49d4c1781..460a9b5d05 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -6,11 +6,13 @@ * `NUMPY_INCLUDE_PATH`: Location of Numpy libraries. * `PYTHON_BIN_PATH`: location of python binary. * `PYTHON_INCLUDE_PATH`: Location of python binaries. + * `PYTHON_LIB_PATH`: Location of python libraries. """ _NUMPY_INCLUDE_PATH = "NUMPY_INCLUDE_PATH" _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" _PYTHON_INCLUDE_PATH = "PYTHON_INCLUDE_PATH" +_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" def _tpl(repository_ctx, tpl, substitutions={}, out=None): @@ -114,6 +116,14 @@ def _genrule(src_dir, genrule_name, command, outs): ) +def _check_python_lib(repository_ctx, python_lib): + """Checks the python lib path.""" + cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) + result = repository_ctx.execute(["bash", "-c", cmd]) + if result.return_code == 1: + _python_configure_fail("Invalid python library path: %s" % python_lib) + + def _check_python_bin(repository_ctx, python_bin): """Checks the python bin path.""" cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) @@ -151,28 +161,51 @@ def _create_python_repository(repository_ctx): """Creates the repository containing files set up to build with Python.""" python_include = None numpy_include = None + empty_config = False # If local checks were requested, the python and numpy include will be auto # detected on the host config (using _PYTHON_BIN_PATH). if repository_ctx.attr.local_checks: python_bin = _get_env_var(repository_ctx, _PYTHON_BIN_PATH) _check_python_bin(repository_ctx, python_bin) - python_include = _get_python_include(repository_ctx, python_bin) - numpy_include = _get_numpy_include(repository_ctx, python_bin) + '/numpy' + python_lib = _get_env_var(repository_ctx, _PYTHON_LIB_PATH, '') + if python_lib == '': + # If we could not find the python lib we will create an empty config that + # will allow non-compilation targets to build correctly (e.g., smoke + # tests). + empty_config = True + _python_configure_warning('PYTHON_LIB_PATH was not set;' + + ' python setup cannot complete successfully.' + + ' Please run ./configure.') + else: + _check_python_lib(repository_ctx, python_lib) + python_include = _get_python_include(repository_ctx, python_bin) + numpy_include = _get_numpy_include(repository_ctx, python_bin) + '/numpy' else: # Otherwise, we assume user provides all paths (via ENV or attrs) python_include = _get_env_var(repository_ctx, _PYTHON_INCLUDE_PATH, repository_ctx.attr.python_include) numpy_include = _get_env_var(repository_ctx, _NUMPY_INCLUDE_PATH, repository_ctx.attr.numpy_include) + '/numpy' - - python_include_rule = _symlink_genrule_for_dir( - repository_ctx, python_include, 'python_include', 'python_include') - numpy_include_rule = _symlink_genrule_for_dir( - repository_ctx, numpy_include, 'numpy_include/numpy', 'numpy_include') - _tpl(repository_ctx, "BUILD", { - "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, - "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, - }) + if empty_config: + _tpl(repository_ctx, "BUILD", { + "%{PYTHON_INCLUDE_GENRULE}": ('filegroup(\n' + + ' name = "python_include",\n' + + ' srcs = [],\n' + + ')\n'), + "%{NUMPY_INCLUDE_GENRULE}": ('filegroup(\n' + + ' name = "numpy_include",\n' + + ' srcs = [],\n' + + ')\n'), + }) + else: + python_include_rule = _symlink_genrule_for_dir( + repository_ctx, python_include, 'python_include', 'python_include') + numpy_include_rule = _symlink_genrule_for_dir( + repository_ctx, numpy_include, 'numpy_include/numpy', 'numpy_include') + _tpl(repository_ctx, "BUILD", { + "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, + "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, + }) def _python_autoconf_impl(repository_ctx): @@ -190,6 +223,7 @@ python_configure = repository_rule( environ = [ _PYTHON_BIN_PATH, _PYTHON_INCLUDE_PATH, + _PYTHON_LIB_PATH, _NUMPY_INCLUDE_PATH, ], ) |