docker_build: workaround the lack of LZMA in python 2.7

Some debs file actually does use the LZMA compression and reading
that format requires python 3. Backports of LZMA to python 2 are
using native deps which might be hard for the user to setup and
our support of python 3 is not really functional (it needs 2to3
to use gflags for instance). Until we fix Bazel's python 3 support,
we shell out to xzcat for supporting LZMA compressed file.

Also added test for the archive library. These tests shows some
wrong handling in the AR format padding, fixed.

--
MOS_MIGRATED_REVID=100474498

16 files changed, 276 insertions, 27 deletions

diff --git a/tools/build_defs/docker/BUILD b/tools/build_defs/docker/BUILD
index f53e1fa264..71f9df51e8 100644
--- a/tools/build_defs/docker/BUILD
+++ b/tools/build_defs/docker/BUILD
@@ -47,6 +47,16 @@ py_library(
     visibility = ["//tools/build_defs/docker:__subpackages__"],
 )
 
+py_test(
+    name = "archive_test",
+    srcs = [
+        "archive_test.py",
+        "testenv.py",
+    ],
+    data = ["//tools/build_defs/docker/testdata:archive_testdata"],
+    deps = [":archive"],
+)
+
 py_binary(
     name = "rewrite_json",
     srcs = ["rewrite_json.py"],
diff --git a/tools/build_defs/docker/archive.py b/tools/build_defs/docker/archive.py
index 744cbdd971..598562202f 100644
--- a/tools/build_defs/docker/archive.py
+++ b/tools/build_defs/docker/archive.py
@@ -53,9 +53,6 @@ class SimpleArFile(object):
     """
 
     def __init__(self, f):
-      if f.tell() % 2 != 0:
-        # AR sections are 2 bytes aligned
-        f.read(1)
       self.filename = f.read(16).strip()
       if self.filename.endswith('/'):  # SysV variant
         self.filename = self.filename[:-1]
@@ -85,7 +82,12 @@ class SimpleArFile(object):
 
   def next(self):
     """Read the next file. Returns None when reaching the end of file."""
-    if self.f.tell() == os.fstat(self.f.fileno()).st_size:
+    # AR sections are two bit aligned using new lines.
+    if self.f.tell() % 2 != 0:
+      self.f.read(1)
+    # An AR sections is at least 60 bytes. Some file might contains garbage
+    # bytes at the end of the archive, ignore them.
+    if self.f.tell() > os.fstat(self.f.fileno()).st_size - 60:
       return None
     return self.SimpleArFileEntry(self.f)
 
@@ -165,30 +167,43 @@ class TarFileWriter(object):
       compression = 'gz'
     elif compression == 'bzip2':
       compression = 'bz2'
-    elif compression not in ['gz', 'bz2']:
-      # Unfortunately xz format isn't supported in py 2.7 :(
+    elif compression == 'lzma':
+      compression = 'xz'
+    elif compression not in ['gz', 'bz2', 'xz']:
       compression = ''
-    with tarfile.open(name=tar, mode='r:' + compression) as intar:
-      for tarinfo in intar:
-        if name_filter is None or name_filter(tarinfo.name):
-          tarinfo.mtime = 0
-          if rootuid is not None and tarinfo.uid == rootuid:
-            tarinfo.uid = 0
-            tarinfo.uname = 'root'
-          if rootgid is not None and tarinfo.gid == rootgid:
-            tarinfo.gid = 0
-            tarinfo.gname = 'root'
-          if numeric:
-            tarinfo.uname = ''
-            tarinfo.gname = ''
-          name = tarinfo.name
-          if not name.startswith('/') and not name.startswith('.'):
-            tarinfo.name = './' + name
-
-          if tarinfo.isfile():
-            self.tar.addfile(tarinfo, intar.extractfile(tarinfo.name))
-          else:
-            self.tar.addfile(tarinfo)
+    if compression == 'xz':
+      # Python 2 does not support lzma, our py3 support is terrible so let's
+      # just hack around.
+      # Note that we buffer the file in memory and it can have an important
+      # memory footprint but it's probably fine as we don't use them for really
+      # large files.
+      # TODO(dmarting): once our py3 support gets better, compile this tools
+      # with py3 for proper lzma support.
+      f = StringIO(os.popen('cat %s | xzcat' % tar).read())
+      intar = tarfile.open(fileobj=f, mode='r:')
+    else:
+      intar = tarfile.open(name=tar, mode='r:' + compression)
+    for tarinfo in intar:
+      if name_filter is None or name_filter(tarinfo.name):
+        tarinfo.mtime = 0
+        if rootuid is not None and tarinfo.uid == rootuid:
+          tarinfo.uid = 0
+          tarinfo.uname = 'root'
+        if rootgid is not None and tarinfo.gid == rootgid:
+          tarinfo.gid = 0
+          tarinfo.gname = 'root'
+        if numeric:
+          tarinfo.uname = ''
+          tarinfo.gname = ''
+        name = tarinfo.name
+        if not name.startswith('/') and not name.startswith('.'):
+          tarinfo.name = './' + name
+
+        if tarinfo.isfile():
+          self.tar.addfile(tarinfo, intar.extractfile(tarinfo.name))
+        else:
+          self.tar.addfile(tarinfo)
+    intar.close()
 
   def close(self):
     """Close the output tar file.
diff --git a/tools/build_defs/docker/archive_test.py b/tools/build_defs/docker/archive_test.py
new file mode 100644
index 0000000000..4db32b5af4
--- /dev/null
+++ b/tools/build_defs/docker/archive_test.py
@@ -0,0 +1,170 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Testing for archive."""
+
+import os
+import os.path
+import tarfile
+import unittest
+
+from tools.build_defs.docker import archive
+from tools.build_defs.docker import testenv
+
+
+class SimpleArFileTest(unittest.TestCase):
+  """Testing for SimpleArFile class."""
+
+  def assertArFileContent(self, arfile, content):
+    """Assert that arfile contains exactly the entry described by `content`.
+
+    Args:
+        arfile: the path to the AR file to test.
+        content: an array describing the expected content of the AR file.
+            Each entry in that list should be a dictionary where each field
+            is a field to test in the corresponding SimpleArFileEntry. For
+            testing the presence of a file "x", then the entry could simply
+            be `{"filename": "x"}`, the missing field will be ignored.
+    """
+    with archive.SimpleArFile(arfile) as f:
+      current = f.next()
+      i = 0
+      while current:
+        error_msg = "Extraneous file at end of archive %s: %s" % (
+            arfile,
+            current.filename
+            )
+        self.assertTrue(i < len(content), error_msg)
+        for k, v in content[i].items():
+          value = getattr(current, k)
+          error_msg = " ".join([
+              "Value `%s` for key `%s` of file" % (value, k),
+              "%s in archive %s does" % (current.filename, arfile),
+              "not match expected value `%s`" % v
+              ])
+          self.assertEqual(value, v, error_msg)
+        current = f.next()
+        i += 1
+      if i < len(content):
+        self.fail("Missing file %s in archive %s" % (content[i], arfile))
+
+  def testEmptyArFile(self):
+    self.assertArFileContent(os.path.join(testenv.TESTDATA_PATH,
+                                          "archive", "empty.ar"),
+                             [])
+
+  def assertSimpleFileContent(self, names):
+    datafile = os.path.join(testenv.TESTDATA_PATH, "archive",
+                            "_".join(names) + ".ar")
+    content = [{"filename": n, "size": len(n), "data": n} for n in names]
+    self.assertArFileContent(datafile, content)
+
+  def testAFile(self):
+    self.assertSimpleFileContent(["a"])
+
+  def testBFile(self):
+    self.assertSimpleFileContent(["b"])
+
+  def testABFile(self):
+    self.assertSimpleFileContent(["ab"])
+
+  def testA_BFile(self):
+    self.assertSimpleFileContent(["a", "b"])
+
+  def testA_ABFile(self):
+    self.assertSimpleFileContent(["a", "ab"])
+
+  def testA_B_ABFile(self):
+    self.assertSimpleFileContent(["a", "b", "ab"])
+
+
+class TarFileWriterTest(unittest.TestCase):
+  """Testing for TarFileWriter class."""
+
+  def assertTarFileContent(self, tar, content):
+    """Assert that tarfile contains exactly the entry described by `content`.
+
+    Args:
+        tar: the path to the TAR file to test.
+        content: an array describing the expected content of the TAR file.
+            Each entry in that list should be a dictionary where each field
+            is a field to test in the corresponding TarInfo. For
+            testing the presence of a file "x", then the entry could simply
+            be `{"name": "x"}`, the missing field will be ignored. To match
+            the content of a file entry, use the key "data".
+    """
+    with tarfile.open(tar, "r:") as f:
+      i = 0
+      for current in f:
+        error_msg = "Extraneous file at end of archive %s: %s" % (
+            tar,
+            current.name
+            )
+        self.assertTrue(i < len(content), error_msg)
+        for k, v in content[i].items():
+          if k == "data":
+            value = f.extractfile(current).read()
+          else:
+            value = getattr(current, k)
+          error_msg = " ".join([
+              "Value `%s` for key `%s` of file" % (value, k),
+              "%s in archive %s does" % (current.name, tar),
+              "not match expected value `%s`" % v
+              ])
+          self.assertEqual(value, v, error_msg)
+        i += 1
+      if i < len(content):
+        self.fail("Missing file %s in archive %s" % (content[i], tar))
+
+  def setUp(self):
+    self.tempfile = os.path.join(os.environ["TEST_TMPDIR"], "test.tar")
+
+  def tearDown(self):
+    if os.path.exists(self.tempfile):
+      os.remove(self.tempfile)
+
+  def testEmptyTarFile(self):
+    with archive.TarFileWriter(self.tempfile):
+      pass
+    self.assertTarFileContent(self.tempfile, [])
+
+  def assertSimpleFileContent(self, names):
+    with archive.TarFileWriter(self.tempfile) as f:
+      for n in names:
+        f.add_file(n, content=n)
+    content = [{"name": n, "size": len(n), "data": n} for n in names]
+    self.assertTarFileContent(self.tempfile, content)
+
+  def testAddFile(self):
+    self.assertSimpleFileContent(["./a"])
+    self.assertSimpleFileContent(["./b"])
+    self.assertSimpleFileContent(["./ab"])
+    self.assertSimpleFileContent(["./a", "./b"])
+    self.assertSimpleFileContent(["./a", "./ab"])
+    self.assertSimpleFileContent(["./a", "./b", "./ab"])
+
+  def testMergeTar(self):
+    content = [
+        {"name": "./a", "data": "a"},
+        {"name": "./ab", "data": "ab"}
+        ]
+    for ext in ["", ".gz", ".bz2", ".xz"]:
+      with archive.TarFileWriter(self.tempfile) as f:
+        f.add_tar(os.path.join(testenv.TESTDATA_PATH, "archive",
+                               "tar_test.tar" + ext),
+                  name_filter=lambda n: n != "./b")
+      self.assertTarFileContent(self.tempfile, content)
+
+
+if __name__ == "__main__":
+  unittest.main()
diff --git a/tools/build_defs/docker/testdata/BUILD b/tools/build_defs/docker/testdata/BUILD
index e365847cff..7848a2c7a6 100644
--- a/tools/build_defs/docker/testdata/BUILD
+++ b/tools/build_defs/docker/testdata/BUILD
@@ -6,6 +6,11 @@ package(
 
 load("/tools/build_defs/docker/docker", "docker_build")
 
+filegroup(
+    name = "archive_testdata",
+    srcs = glob(["archive/**"]),
+)
+
 genrule(
     name = "gen",
     outs = ["gen.out"],
diff --git a/tools/build_defs/docker/testdata/archive/a.ar b/tools/build_defs/docker/testdata/archive/a.ar
new file mode 100644
index 0000000000..d2594dffca
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/a.ar
@@ -0,0 +1,3 @@
+!<arch>
+a/              1439231934  1000  1000  100664  1         `
+a
diff --git a/tools/build_defs/docker/testdata/archive/a_ab.ar b/tools/build_defs/docker/testdata/archive/a_ab.ar
new file mode 100644
index 0000000000..f6b72174ab
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/a_ab.ar
@@ -0,0 +1,5 @@
+!<arch>
+a/              1439231934  1000  1000  100664  1         `
+a
+ab/             1439231936  1000  1000  100664  2         `
+ab
diff --git a/tools/build_defs/docker/testdata/archive/a_b.ar b/tools/build_defs/docker/testdata/archive/a_b.ar
new file mode 100644
index 0000000000..6f2d79c439
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/a_b.ar
@@ -0,0 +1,5 @@
+!<arch>
+a/              1439231934  1000  1000  100664  1         `
+a
+b/              1439231939  1000  1000  100664  1         `
+b
diff --git a/tools/build_defs/docker/testdata/archive/a_b_ab.ar b/tools/build_defs/docker/testdata/archive/a_b_ab.ar
new file mode 100644
index 0000000000..07dc3e71f1
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/a_b_ab.ar
@@ -0,0 +1,7 @@
+!<arch>
+a/              1439231934  1000  1000  100664  1         `
+a
+b/              1439231939  1000  1000  100664  1         `
+b
+ab/             1439231936  1000  1000  100664  2         `
+ab
diff --git a/tools/build_defs/docker/testdata/archive/ab.ar b/tools/build_defs/docker/testdata/archive/ab.ar
new file mode 100644
index 0000000000..29f56beed2
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/ab.ar
@@ -0,0 +1,3 @@
+!<arch>
+ab/             1439231936  1000  1000  100664  2         `
+ab
diff --git a/tools/build_defs/docker/testdata/archive/b.ar b/tools/build_defs/docker/testdata/archive/b.ar
new file mode 100644
index 0000000000..3f41a7eb70
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/b.ar
@@ -0,0 +1,3 @@
+!<arch>
+b/              1439231939  1000  1000  100664  1         `
+b
diff --git a/tools/build_defs/docker/testdata/archive/empty.ar b/tools/build_defs/docker/testdata/archive/empty.ar
new file mode 100644
index 0000000000..8b277f0dd5
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/empty.ar
@@ -0,0 +1 @@
+!<arch>
diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar b/tools/build_defs/docker/testdata/archive/tar_test.tar
new file mode 100644
index 0000000000..70b043bd9d
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/tar_test.tar
diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar.bz2 b/tools/build_defs/docker/testdata/archive/tar_test.tar.bz2
new file mode 100644
index 0000000000..c6f7edf5a7
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/tar_test.tar.bz2
diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar.gz b/tools/build_defs/docker/testdata/archive/tar_test.tar.gz
new file mode 100644
index 0000000000..4b851716dd
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/tar_test.tar.gz
diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar.xz b/tools/build_defs/docker/testdata/archive/tar_test.tar.xz
new file mode 100644
index 0000000000..1ea3c8b836
--- /dev/null
+++ b/tools/build_defs/docker/testdata/archive/tar_test.tar.xz
diff --git a/tools/build_defs/docker/testenv.py b/tools/build_defs/docker/testenv.py
new file mode 100644
index 0000000000..4e0039e2b6
--- /dev/null
+++ b/tools/build_defs/docker/testenv.py
@@ -0,0 +1,22 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Path to the test data."""
+
+import os
+import os.path
+
+TESTDATA_PATH = os.path.join(
+    os.getcwd(),
+    "tools/build_defs/docker/testdata",
+    )