path: root/tools/cpp/wrapper/bin/pydir/msvc_tools.py.tpl

# pylint: disable=g-bad-file-header
# pylint: disable=cell-var-from-loop
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tools for working with the Microsoft Visual C++ toolchain."""

from argparse import ArgumentParser
import ntpath
import os
import re
import subprocess

MAX_PATH = 260  # The maximum number of characters in a Windows path.
MAX_OPTION_LENGTH = 10  # The maximum length of a compiler/linker option.
MAX_DRIVE_LENGTH = 3  # The maximum length of a drive.
ASSEMBLY_AS_C_SOURCE = '/Tc'
LIB_SUFFIX = '.lib'

TMP_PATH = '%{tmp}'

PATH = "%{path}"
INCLUDE = "%{include}"
LIB = "%{lib}"
LIB_TOOL = "%{lib_tool}"
supported_cuda_compute_capabilities = [ %{cuda_compute_capabilities} ]

class Error(Exception):
  """Base class for all script-specific errors."""
  pass

def Log(s):
  """Print log messages."""
  print('msvc_tools.py: {0}'.format(s))

class ArgParser(object):
  """Class that parses gcc/clang-style options for a Windows.

  The particular substitutions that are needed are passed to the object.
  """

  def __init__(self, driver, argv, substitutions):
    self.driver = driver
    self.substitutions = substitutions
    self.options = []
    self.leftover = []
    self.target_arch = None
    self.compilation_mode = None
    self.deps_file = None
    self.output_file = None
    self.params_file = None
    self.support_whole_archive = %{support_whole_archive}
    self.global_whole_archive = None
    self.is_cuda_compilation = None
    self.cuda_log = False
    self._ParseArgs(argv)

  def ReplaceLibrary(self, arg):
    """Do the actual replacement if necessary."""
    if arg == "/WHOLEARCHIVE":
      return []
    if arg.startswith("/OUT:") or os.path.splitext(arg)[1] not in ['.a', '.lo']:
      return [arg]
    if self.global_whole_archive or arg.startswith("/WHOLEARCHIVE:"):
      if arg.startswith("/WHOLEARCHIVE:"):
        arg = arg[len("/WHOLEARCHIVE:"):]
      output = subprocess.check_output([LIB_TOOL, "/list", arg]).decode("utf-8")
      object_files = []
      for line in output.split("\n"):
        line = line.strip()
        if line.endswith(".o"):
          object_files.append(line)
      return object_files
    return [arg]

  def WholeArchivePreprocess(self):
    """Replace library file with object files if /WHOLEARCHIVE is not supported."""
    if self.support_whole_archive:
      return
    options = []
    self.global_whole_archive = "/WHOLEARCHIVE" in self.options
    for arg in self.options:
      options.extend(self.ReplaceLibrary(arg))
    self.options = options

  def IsCudaCompilation(self):
    """Check if it's a cuda compilation."""
    parser = ArgumentParser()
    parser.add_argument('-x', nargs=1)
    parser.add_argument('--cuda_log', action='store_true')
    args, leftover = parser.parse_known_args(self.leftover)
    if args.x and args.x[0] == 'cuda':
      if args.cuda_log:
        Log('Using nvcc')
        self.cuda_log = True
      self.leftover = leftover
      return True
    return False

  def GetNvccOptions(self):
    """Collect the -nvcc_options values from self.leftover.

    Returns:
      The list of options that can be passed directly to nvcc.
    """

    parser = ArgumentParser()
    parser.add_argument('-nvcc_options', nargs='*', action='append')

    args, leftover = parser.parse_known_args(self.leftover)

    if args.nvcc_options:
      self.leftover = leftover
      return ['--'+a for a in sum(args.nvcc_options, [])]
    return []

  def GetOptionValue(self, option):
    """Extract the list of values for option from self.options.

    Args:
      option: The option whose value to extract, without the leading '/'.

    Returns:
      A list of values, either directly following the option,
      (eg., /opt val1 val2) or values collected from multiple occurrences of
      the option (eg., /opt val1 /opt val2).
    """

    parser = ArgumentParser(prefix_chars='/')
    parser.add_argument('/' + option, nargs='*', action='append')
    args, leftover = parser.parse_known_args(self.options)
    if args and vars(args)[option]:
      self.options = leftover
      return sum(vars(args)[option], [])
    return []

  def GetOptionsForCudaCompilation(self):
    """Get nvcc options with arguments assembled from self.options."""

    src_files = [f for f in self.options if
                 re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
    if len(src_files) == 0:
      raise Error('No source files found for cuda compilation.')

    out_file = [ f for f in self.options if f.startswith('/Fo') ]
    if len(out_file) != 1:
      raise Error('Please sepecify exactly one output file for cuda compilation.')
    out = ['-o', out_file[0][len('/Fo'):]]

    nvcc_compiler_options = self.GetNvccOptions()

    opt_option = self.GetOptionValue('O')
    opt = ['-g', '-G']
    if (len(opt_option) > 0 and opt_option[0] != 'd'):
      opt = ['-O2']

    include_options = self.GetOptionValue('I')
    includes = ["-I " + include for include in include_options]

    defines = self.GetOptionValue('D')
    defines = ['-D' + define for define in defines]

    undefines = self.GetOptionValue('U')
    undefines = ['-U' + define for define in undefines]

    # The rest of the unrecongized options should be passed to host compiler
    host_compiler_options = [option for option in self.options if option not in (src_files + out_file)]

    m_options = ["-m64"]

    nvccopts = ['-D_FORCE_INLINES']
    for capability in supported_cuda_compute_capabilities:
      capability = capability.replace('.', '')
      nvccopts += [r'-gencode=arch=compute_%s,"code=sm_%s,compute_%s"' % (
          capability, capability, capability)]
    nvccopts += nvcc_compiler_options
    nvccopts += undefines
    nvccopts += defines
    nvccopts += m_options
    nvccopts += ['--compiler-options="' + " ".join(host_compiler_options) + '"']
    nvccopts += ['-x', 'cu'] + opt + includes + out + ['-c'] + src_files

    if self.cuda_log:
      Log("Running: " + " ".join(["nvcc"] + nvccopts))

    self.options = nvccopts

  def _MatchOneArg(self, args):
    """Finds a pattern which matches the beginning elements of args.

    Args:
      args: A list of arguments to replace.

    Returns:
       A tuple of (number of arguments parsed, action, match groups).
    """
    for (regex, action) in self.substitutions:
      if isinstance(regex, str):
        regex = [regex]
      j = 0
      matches = []
      for r in regex:
        if j < len(args):
          match = re.compile('^' + r + '$').match(args[j])
        else:
          match = None
        matches.append(match)
        j += 1
      if None in matches:
        continue
      groups = []
      for m in matches:
        groups.extend(m.groups())
      return (len(regex), action, groups)
    return (0, '', [])

  def _ParseArgs(self, argv):
    """Parses argv and replaces its elements using special tokens.

    The following is a list of supported tokens. The format is $TOKEN%d, where
    %d is the 0-based group number from regex matches of the pattern.
      $CREATE_PATH%d: Touches a file at the path in the matching token.
      $LOAD_PARAMS%d: Loads an ld-style params file and appends all arguments to
                      the current argument list by recursively calling
                      _ParseArgs.
      $%d           : Numeric token that just replaces the match group with
                      the value specified in the match list.
      $PATH%d       : Replaces the match with a Windows-normalized version of
                      the match; assumes that the match is a path.
      $PATH%d_NO_EXT: Same as $PATH but strips out any file extension.
      $TARGET_ARCH  : Set self.target_arch to 'x86' or 'x64' for '-m32' and
                      '-m64', respectively.
      $DEBUG_RT     : Enforce linkage to debug runtime.
      $COMPILE_OUTPUT%d: Sets the output name of a compilation step.
      $COMPILATION_MODE: Sets self.compilation_mode from the value of a
                      '-Xcompilation-mode=' flag.
      $CREATE_PRECOMPILED_HEADER: Informs the system that we are generating a
                      precompiled header rather than an object file.
      $GENERATE_DEPS%d: Generates a gcc-style .d file containing dependencies.

    Args:
      argv: A list of arguments to replace.

    Returns:
      A list of replaced arguments to pass to the target command.

    Raises:
      Error: if wrong arguments found
    """
    i = 0
    matched = []
    unmatched = []
    files = []
    enforce_debug_rt = False
    while i < len(argv):
      num_matched, action, groups = self._MatchOneArg(argv[i:])
      arg = argv[i]
      if arg.startswith('/Fo') or arg.startswith('/Fa') or arg.startswith(
          '/Fi'):
        self.output_file = arg[3:]
        self.options.append(
            '/Fd%s.pdb' % self.NormPath(os.path.splitext(self.output_file)[0]))
      if num_matched == 0:
        # Strip out any .a's that have 0 size, they are header or intermediate
        # dependency libraries and don't contain any code. 0-length files are
        # considered corrupt by the linker (error LNK1136).
        if (os.path.isfile(arg) and os.path.splitext(arg)[1] == '.a' and
            os.path.getsize(arg) == 0):
          i += 1
          continue

        # If the argument is an absolute path, then add it directly.
        if arg[0] == '/':
          self.AddOpt(arg)
        elif os.path.isfile(arg):
          path = self.NormPath(arg)
          ext = os.path.splitext(arg)[1].lower()
          if ext in ['.s']:
            # Treat assembly files as C source files using a special option.
            path = ASSEMBLY_AS_C_SOURCE + path
          # If this is an actual file on disk then just pass it to the tool.
          files.append(path)
        elif not arg.endswith(LIB_SUFFIX):
          # Ignore .lib files.
          unmatched.append(arg)
        i += 1
        continue
      matched += argv[i:i + num_matched]
      # Handle special options.
      for entry in action:
        if entry == '$TARGET_ARCH':
          if arg == '-m32':
            self.target_arch = 'x86'
          elif arg == '-m64':
            self.target_arch = 'x64'
          else:
            raise Error('Unknown target arch flag: %r' % arg)
          continue

        if entry == '$COMPILATION_MODE':
          empty, prefix, mode = arg.partition('-Xcompilation-mode=')
          if empty or not prefix or mode not in ['dbg', 'fastbuild', 'opt']:
            raise Error('Invalid compilation mode flag: %r' % arg)
          self.compilation_mode = mode
          continue

        if entry == '$DEBUG_RT':
          enforce_debug_rt = True
          continue

        if not groups:
          self.options.append(entry)
        else:
          # Substitute special tokens.
          for g in range(0, len(groups)):
            value = groups[g]

            # Check for special tokens.
            if entry == ('$CREATE_PATH%d' % g):
              with open(value, 'a'):
                os.utime(value, None)
              continue

            if entry == ('$LOAD_PARAMS%d' % g):
              try:
                # The arguments in the params file need to be processed as
                # regular command-line arguments.
                params = [line.rstrip() for line in open(value, 'r')]
                self._ParseArgs(params)
                # Because we have no write permission to orginal params file,
                # create a new params file with addtional suffix
                self.params_file = value + '.msvc'
              except (IOError, e):
                print('Could not open', value, 'for reading:', str(e))
                exit(-1)
              continue

            if entry == ('$GENERATE_DEPS%d' % g):
              self.options.append('/showIncludes')
              self.deps_file = value
              continue

            # Regular substitution.
            patterns = {
                '$%d' % g: value,
                '$PATH%d_NO_EXT' % g: self.NormPath(os.path.splitext(value)[0]),
                '$PATH%d' % g: self.NormPath(value),
            }
            pattern = re.compile('(%s)' %
                                 '|'.join(map(re.escape, patterns.keys())))
            result = pattern.sub(lambda x: patterns[x.group(0)], entry)
            self.options.append(result)
      i += num_matched
    self.leftover = unmatched

    # Select runtime option
    # Find the last runtime option passed
    rt = None
    rt_idx = -1
    for i, opt in enumerate(reversed(self.options)):
      if opt in ['/MT', '/MTd', '/MD', '/MDd']:
        if opt[-1] == 'd':
          enforce_debug_rt = True
        rt = opt[:3]
        rt_idx = len(self.options) - i - 1
        break
    rt = rt or '/MT'  # Default to static runtime
    # Add debug if necessary
    if enforce_debug_rt:
      rt += 'd'
    # Include runtime option
    if rt_idx >= 0:
      self.options[rt_idx] = rt
    else:
      self.options.append(rt)

    # Add in any parsed files
    self.options += files

    if '/w' in self.options:
      self.options = [option for option in self.options if option not in ['/W2', '/W3', '/W4']]

    self.is_cuda_compilation = self.IsCudaCompilation()
    if self.is_cuda_compilation:
      self.GetOptionsForCudaCompilation()

    if self.leftover:
      print('Warning: Unmatched arguments: ' + ' '.join(self.leftover))

  def NormPath(self, path):
    """Uses the current WindowsRunner to normalize the passed path.

    Args:
      path: the path to normalize.

    Returns:
      A normalized string representing a path suitable for Windows.
    """
    return self.driver.NormPath(path)

  def AddOpt(self, option):
    """Adds a single option.

    Args:
      option: the option to add.
    """
    self.options.append(option)


class WindowsRunner(object):
  """Base class that encapsulates the details of running a binary."""

  def NormPath(self, path):
    """Normalizes an input unix style path to a < 260 char Windows format.

    Windows paths cannot be greater than 260 characters.

    Args:
      path: A path in unix format.

    Returns:
      An absolute path in Windows format, rooted from some
      directory.

    Raises:
      Error: if path is too long
    """
    abspath = os.path.abspath(path)
    # We must allow for the drive letter as well, which is three characters, and
    # the length of any compiler option ahead of the path,

    if len(abspath) + MAX_DRIVE_LENGTH + MAX_OPTION_LENGTH > MAX_PATH:
      print('Warning: path "' + abspath + '" is > than 260 characters (' +
            str(len(abspath)) + '); programs may crash with long arguments')
    return abspath

  def SetupEnvironment(self):
    """Setup proper path for running.

    Returns:
      An environment suitable for running on Windows.
    """

    build_env = os.environ.copy()
    build_env['PATH'] = PATH
    build_env['INCLUDE'] = INCLUDE
    build_env['LIB'] = LIB
    build_env['TEMP'] = TMP_PATH
    build_env['TMP'] = TMP_PATH
    return build_env

  def RunBinary(self, binary, args, build_arch, parser):
    """Runs binary on Windows with the passed args.

    Args:
      binary: The binary to run.
      args: The arguments to pass to binary.
      build_arch: Either 'x64' or 'x86', which binary architecture to build for.
      parser: An ArgParser that contains parsed arguments.

    Returns:
      The return code from executing binary.
    """
    # Filter out some not-so-useful cl windows messages.
    filters = [
        '.*warning LNK4006: __NULL_IMPORT_DESCRIPTOR already defined.*\n',
        '.*warning LNK4044: unrecognized option \'/MT\'; ignored.*\n',
        '.*warning LNK4044: unrecognized option \'/link\'; ignored.*\n',
        '.*warning LNK4221: This object file does not define any '
        'previously.*\n',
        # Comment the following line if you want to see warning messages
        '.*warning C.*\n',
        '\r\n',
        '\n\r',
    ]

    # Check again the arguments are within MAX_PATH.
    for arg in args:
      if os.path.splitext(arg)[1].lower() in ['.c', '.cc', '.cpp', '.s']:
        # cl.exe prints out the file name it is compiling; add that to the
        # filter.
        name = arg.rpartition(ntpath.sep)[2]
        filters.append(name)

    # Setup the Windows paths and the build environment.
    build_env = self.SetupEnvironment()

    # Construct a large regular expression for all filters.
    output_filter = re.compile('(' + ')|('.join(filters) + ')')
    includes_filter = re.compile(r'Note: including file:\s+(.*)')
    # Run the command.
    if parser.params_file:
      try:
        # Using parameter file as input when linking static libraries.
        params_file = open(parser.params_file, 'w')
        for arg in args:
          params_file.write(arg + '\n')
        params_file.close()
      except (IOError, e):
        print('Could not open', parser.params_file, 'for writing:', str(e))
        exit(-1)
      cmd = [binary] + [('@' + os.path.normpath(parser.params_file))]
    else:
      cmd = [binary] + args
    # Save stderr output to a temporary in case we need it.
    # Unconmment the following line to see what exact command is executed.
    # print("Running: " + " ".join(cmd))
    proc = subprocess.Popen(cmd,
                            env=build_env,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            shell=True)
    deps = []
    for line in proc.stdout:
      line = line.decode('utf-8')
      if not output_filter.match(line):
        includes = includes_filter.match(line)
        if includes:
          filename = includes.group(1).rstrip()
          deps += [filename]
        else:
          print(line.rstrip())
    proc.wait()

    # Generate deps file if requested.
    if parser.deps_file:
      with open(parser.deps_file, 'w') as deps_file:
        # Start with the name of the output file.
        deps_file.write(parser.output_file + ': \\\n')
        for i, dep in enumerate(deps):
          dep = dep.replace('\\', '/').replace(' ', '\\ ')
          deps_file.write('  ' + dep)
          if i < len(deps) - 1:
            deps_file.write(' \\')
          deps_file.write('\n')

    return proc.returncode