Add mips64el_toolchain_linux asset.

No-Try: true
Change-Id: Ief4e48a6bc2f126b31af3e8a381f8a6d5a73d523
Reviewed-on: https://skia-review.googlesource.com/117083
Reviewed-by: Mike Klein <mtklein@chromium.org>
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Ben Wagner <benjaminwagner@google.com>

6 files changed, 192 insertions, 0 deletions

diff --git a/infra/bots/assets/mips64el_toolchain_linux/VERSION b/infra/bots/assets/mips64el_toolchain_linux/VERSION
new file mode 100644
index 0000000000..e440e5c842
--- /dev/null
+++ b/infra/bots/assets/mips64el_toolchain_linux/VERSION
@@ -0,0 +1 @@
+3
\ No newline at end of file
diff --git a/infra/bots/assets/mips64el_toolchain_linux/common.py b/infra/bots/assets/mips64el_toolchain_linux/common.py
new file mode 100755
index 0000000000..caa0ad899c
--- /dev/null
+++ b/infra/bots/assets/mips64el_toolchain_linux/common.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Common vars used by scripts in this directory."""
+
+
+import os
+import sys
+
+FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
+
+sys.path.insert(0, INFRA_BOTS_DIR)
+from assets import assets
+
+ASSET_NAME = os.path.basename(FILE_DIR)
+
+
+def run(cmd):
+  """Run a command, eg. "upload" or "download". """
+  assets.main([cmd, ASSET_NAME] + sys.argv[1:])
diff --git a/infra/bots/assets/mips64el_toolchain_linux/create.py b/infra/bots/assets/mips64el_toolchain_linux/create.py
new file mode 100755
index 0000000000..97a8690e30
--- /dev/null
+++ b/infra/bots/assets/mips64el_toolchain_linux/create.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset."""
+
+
+import argparse
+import common
+import os
+import shutil
+import subprocess
+import utils
+
+# This is basically all the deps of g++-multilib-mips64el-linux-gnuabi64 that
+# are not already installed on the bots.
+#
+# We could try to also include packages that *are* already installed on the bots
+# as well, but that would be quite a bit, and would probably entail more hacky
+# fixes like below.
+#
+# There is probably a way to generate this list from apt, but it's not as
+# straightforward as it should be.
+PKGS = [
+    'binutils-mips64el-linux-gnuabi64',
+    'cpp-6-mips64el-linux-gnuabi64',
+    'g++-6-mips64el-linux-gnuabi64',
+    'gcc-6-cross-base',
+    'gcc-6-mips64el-linux-gnuabi64',
+    'gcc-6-mips64el-linux-gnuabi64-base',
+    'libatomic1-mips64el-cross',
+    'libc6-dev-mips64el-cross',
+    'libc6-mips64el-cross',
+    'libgcc-6-dev-mips64el-cross',
+    'libgcc1-mips64el-cross',
+    'libgomp1-mips64el-cross',
+    'libmpfr6',  # This is new in buster, so build machines don't have it yet.
+    'libstdc++-6-dev-mips64el-cross',
+    'libstdc++6-mips64el-cross',
+    'linux-libc-dev-mips64el-cross',
+]
+
+def create_asset(target_dir):
+  """Create the asset."""
+  # This is all a bit hacky. Rather than installing to a chroot, we just extract
+  # all the packages to the target dir, then fix things up so that it can be
+  # used in our recipes.
+  with utils.tmp_dir():
+    # Download required Debian packages.
+    subprocess.check_call(['apt-get', 'download'] + PKGS)
+    for f in os.listdir('.'):
+      subprocess.check_call(['dpkg-deb', '--extract', f, target_dir])
+  parent_dir = os.path.join(target_dir, 'usr')
+  # Remove unnecessary files that cause problems with zipping (due to dangling
+  # symlinks).
+  os.remove(os.path.join(parent_dir,
+                         'lib/gcc-cross/mips64el-linux-gnuabi64/6/libcc1.so'))
+  shutil.rmtree(os.path.join(parent_dir, 'share'))
+  # Remove usr/ prefix.
+  for d in os.listdir(parent_dir):
+    os.rename(os.path.join(parent_dir, d), os.path.join(target_dir, d))
+  os.rmdir(parent_dir)
+  # Remove absolute paths in GNU ld scripts.
+  lib_dir = os.path.join(target_dir, 'mips64el-linux-gnuabi64/lib')
+  ld_script_token = 'OUTPUT_FORMAT(elf64-tradlittlemips)'
+  ld_script_files = subprocess.check_output(
+    ['grep', '--recursive', '--files-with-matches',
+     '--binary-files=without-match', '--fixed-strings', ld_script_token,
+     lib_dir]).split()
+  abs_path = '/usr/mips64el-linux-gnuabi64/lib/'
+  for f in ld_script_files:
+    with open(f) as script:
+      contents = script.read()
+    contents = contents.replace(abs_path, '')
+    with open(f, 'w') as script:
+      script.write(contents)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--target_dir', '-t', required=True)
+  args = parser.parse_args()
+  create_asset(args.target_dir)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/assets/mips64el_toolchain_linux/create_and_upload.py b/infra/bots/assets/mips64el_toolchain_linux/create_and_upload.py
new file mode 100755
index 0000000000..de56a80fa8
--- /dev/null
+++ b/infra/bots/assets/mips64el_toolchain_linux/create_and_upload.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset and upload it."""
+
+
+import argparse
+import common
+import os
+import subprocess
+import sys
+import utils
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--gsutil')
+  args = parser.parse_args()
+
+  with utils.tmp_dir():
+    cwd = os.getcwd()
+    create_script = os.path.join(common.FILE_DIR, 'create.py')
+    upload_script = os.path.join(common.FILE_DIR, 'upload.py')
+
+    try:
+      subprocess.check_call(['python', create_script, '-t', cwd])
+      cmd = ['python', upload_script, '-t', cwd]
+      if args.gsutil:
+        cmd.extend(['--gsutil', args.gsutil])
+      subprocess.check_call(cmd)
+    except subprocess.CalledProcessError:
+      # Trap exceptions to avoid printing two stacktraces.
+      sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/assets/mips64el_toolchain_linux/download.py b/infra/bots/assets/mips64el_toolchain_linux/download.py
new file mode 100755
index 0000000000..ca999e0378
--- /dev/null
+++ b/infra/bots/assets/mips64el_toolchain_linux/download.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Download the current version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+  common.run('download')
diff --git a/infra/bots/assets/mips64el_toolchain_linux/upload.py b/infra/bots/assets/mips64el_toolchain_linux/upload.py
new file mode 100755
index 0000000000..bdfbda783e
--- /dev/null
+++ b/infra/bots/assets/mips64el_toolchain_linux/upload.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Upload a new version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+  common.run('upload')