diff options
-rw-r--r-- | tools/build_defs/docker/BUILD | 10 | ||||
-rw-r--r-- | tools/build_defs/docker/archive.py | 69 | ||||
-rw-r--r-- | tools/build_defs/docker/archive_test.py | 170 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/BUILD | 5 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/a.ar | 3 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/a_ab.ar | 5 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/a_b.ar | 5 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/a_b_ab.ar | 7 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/ab.ar | 3 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/b.ar | 3 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/empty.ar | 1 | ||||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/tar_test.tar | bin | 0 -> 10240 bytes | |||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/tar_test.tar.bz2 | bin | 0 -> 134 bytes | |||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/tar_test.tar.gz | bin | 0 -> 158 bytes | |||
-rw-r--r-- | tools/build_defs/docker/testdata/archive/tar_test.tar.xz | bin | 0 -> 192 bytes | |||
-rw-r--r-- | tools/build_defs/docker/testenv.py | 22 |
16 files changed, 276 insertions, 27 deletions
diff --git a/tools/build_defs/docker/BUILD b/tools/build_defs/docker/BUILD index f53e1fa264..71f9df51e8 100644 --- a/tools/build_defs/docker/BUILD +++ b/tools/build_defs/docker/BUILD @@ -47,6 +47,16 @@ py_library( visibility = ["//tools/build_defs/docker:__subpackages__"], ) +py_test( + name = "archive_test", + srcs = [ + "archive_test.py", + "testenv.py", + ], + data = ["//tools/build_defs/docker/testdata:archive_testdata"], + deps = [":archive"], +) + py_binary( name = "rewrite_json", srcs = ["rewrite_json.py"], diff --git a/tools/build_defs/docker/archive.py b/tools/build_defs/docker/archive.py index 744cbdd971..598562202f 100644 --- a/tools/build_defs/docker/archive.py +++ b/tools/build_defs/docker/archive.py @@ -53,9 +53,6 @@ class SimpleArFile(object): """ def __init__(self, f): - if f.tell() % 2 != 0: - # AR sections are 2 bytes aligned - f.read(1) self.filename = f.read(16).strip() if self.filename.endswith('/'): # SysV variant self.filename = self.filename[:-1] @@ -85,7 +82,12 @@ class SimpleArFile(object): def next(self): """Read the next file. Returns None when reaching the end of file.""" - if self.f.tell() == os.fstat(self.f.fileno()).st_size: + # AR sections are two bit aligned using new lines. + if self.f.tell() % 2 != 0: + self.f.read(1) + # An AR sections is at least 60 bytes. Some file might contains garbage + # bytes at the end of the archive, ignore them. + if self.f.tell() > os.fstat(self.f.fileno()).st_size - 60: return None return self.SimpleArFileEntry(self.f) @@ -165,30 +167,43 @@ class TarFileWriter(object): compression = 'gz' elif compression == 'bzip2': compression = 'bz2' - elif compression not in ['gz', 'bz2']: - # Unfortunately xz format isn't supported in py 2.7 :( + elif compression == 'lzma': + compression = 'xz' + elif compression not in ['gz', 'bz2', 'xz']: compression = '' - with tarfile.open(name=tar, mode='r:' + compression) as intar: - for tarinfo in intar: - if name_filter is None or name_filter(tarinfo.name): - tarinfo.mtime = 0 - if rootuid is not None and tarinfo.uid == rootuid: - tarinfo.uid = 0 - tarinfo.uname = 'root' - if rootgid is not None and tarinfo.gid == rootgid: - tarinfo.gid = 0 - tarinfo.gname = 'root' - if numeric: - tarinfo.uname = '' - tarinfo.gname = '' - name = tarinfo.name - if not name.startswith('/') and not name.startswith('.'): - tarinfo.name = './' + name - - if tarinfo.isfile(): - self.tar.addfile(tarinfo, intar.extractfile(tarinfo.name)) - else: - self.tar.addfile(tarinfo) + if compression == 'xz': + # Python 2 does not support lzma, our py3 support is terrible so let's + # just hack around. + # Note that we buffer the file in memory and it can have an important + # memory footprint but it's probably fine as we don't use them for really + # large files. + # TODO(dmarting): once our py3 support gets better, compile this tools + # with py3 for proper lzma support. + f = StringIO(os.popen('cat %s | xzcat' % tar).read()) + intar = tarfile.open(fileobj=f, mode='r:') + else: + intar = tarfile.open(name=tar, mode='r:' + compression) + for tarinfo in intar: + if name_filter is None or name_filter(tarinfo.name): + tarinfo.mtime = 0 + if rootuid is not None and tarinfo.uid == rootuid: + tarinfo.uid = 0 + tarinfo.uname = 'root' + if rootgid is not None and tarinfo.gid == rootgid: + tarinfo.gid = 0 + tarinfo.gname = 'root' + if numeric: + tarinfo.uname = '' + tarinfo.gname = '' + name = tarinfo.name + if not name.startswith('/') and not name.startswith('.'): + tarinfo.name = './' + name + + if tarinfo.isfile(): + self.tar.addfile(tarinfo, intar.extractfile(tarinfo.name)) + else: + self.tar.addfile(tarinfo) + intar.close() def close(self): """Close the output tar file. diff --git a/tools/build_defs/docker/archive_test.py b/tools/build_defs/docker/archive_test.py new file mode 100644 index 0000000000..4db32b5af4 --- /dev/null +++ b/tools/build_defs/docker/archive_test.py @@ -0,0 +1,170 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing for archive.""" + +import os +import os.path +import tarfile +import unittest + +from tools.build_defs.docker import archive +from tools.build_defs.docker import testenv + + +class SimpleArFileTest(unittest.TestCase): + """Testing for SimpleArFile class.""" + + def assertArFileContent(self, arfile, content): + """Assert that arfile contains exactly the entry described by `content`. + + Args: + arfile: the path to the AR file to test. + content: an array describing the expected content of the AR file. + Each entry in that list should be a dictionary where each field + is a field to test in the corresponding SimpleArFileEntry. For + testing the presence of a file "x", then the entry could simply + be `{"filename": "x"}`, the missing field will be ignored. + """ + with archive.SimpleArFile(arfile) as f: + current = f.next() + i = 0 + while current: + error_msg = "Extraneous file at end of archive %s: %s" % ( + arfile, + current.filename + ) + self.assertTrue(i < len(content), error_msg) + for k, v in content[i].items(): + value = getattr(current, k) + error_msg = " ".join([ + "Value `%s` for key `%s` of file" % (value, k), + "%s in archive %s does" % (current.filename, arfile), + "not match expected value `%s`" % v + ]) + self.assertEqual(value, v, error_msg) + current = f.next() + i += 1 + if i < len(content): + self.fail("Missing file %s in archive %s" % (content[i], arfile)) + + def testEmptyArFile(self): + self.assertArFileContent(os.path.join(testenv.TESTDATA_PATH, + "archive", "empty.ar"), + []) + + def assertSimpleFileContent(self, names): + datafile = os.path.join(testenv.TESTDATA_PATH, "archive", + "_".join(names) + ".ar") + content = [{"filename": n, "size": len(n), "data": n} for n in names] + self.assertArFileContent(datafile, content) + + def testAFile(self): + self.assertSimpleFileContent(["a"]) + + def testBFile(self): + self.assertSimpleFileContent(["b"]) + + def testABFile(self): + self.assertSimpleFileContent(["ab"]) + + def testA_BFile(self): + self.assertSimpleFileContent(["a", "b"]) + + def testA_ABFile(self): + self.assertSimpleFileContent(["a", "ab"]) + + def testA_B_ABFile(self): + self.assertSimpleFileContent(["a", "b", "ab"]) + + +class TarFileWriterTest(unittest.TestCase): + """Testing for TarFileWriter class.""" + + def assertTarFileContent(self, tar, content): + """Assert that tarfile contains exactly the entry described by `content`. + + Args: + tar: the path to the TAR file to test. + content: an array describing the expected content of the TAR file. + Each entry in that list should be a dictionary where each field + is a field to test in the corresponding TarInfo. For + testing the presence of a file "x", then the entry could simply + be `{"name": "x"}`, the missing field will be ignored. To match + the content of a file entry, use the key "data". + """ + with tarfile.open(tar, "r:") as f: + i = 0 + for current in f: + error_msg = "Extraneous file at end of archive %s: %s" % ( + tar, + current.name + ) + self.assertTrue(i < len(content), error_msg) + for k, v in content[i].items(): + if k == "data": + value = f.extractfile(current).read() + else: + value = getattr(current, k) + error_msg = " ".join([ + "Value `%s` for key `%s` of file" % (value, k), + "%s in archive %s does" % (current.name, tar), + "not match expected value `%s`" % v + ]) + self.assertEqual(value, v, error_msg) + i += 1 + if i < len(content): + self.fail("Missing file %s in archive %s" % (content[i], tar)) + + def setUp(self): + self.tempfile = os.path.join(os.environ["TEST_TMPDIR"], "test.tar") + + def tearDown(self): + if os.path.exists(self.tempfile): + os.remove(self.tempfile) + + def testEmptyTarFile(self): + with archive.TarFileWriter(self.tempfile): + pass + self.assertTarFileContent(self.tempfile, []) + + def assertSimpleFileContent(self, names): + with archive.TarFileWriter(self.tempfile) as f: + for n in names: + f.add_file(n, content=n) + content = [{"name": n, "size": len(n), "data": n} for n in names] + self.assertTarFileContent(self.tempfile, content) + + def testAddFile(self): + self.assertSimpleFileContent(["./a"]) + self.assertSimpleFileContent(["./b"]) + self.assertSimpleFileContent(["./ab"]) + self.assertSimpleFileContent(["./a", "./b"]) + self.assertSimpleFileContent(["./a", "./ab"]) + self.assertSimpleFileContent(["./a", "./b", "./ab"]) + + def testMergeTar(self): + content = [ + {"name": "./a", "data": "a"}, + {"name": "./ab", "data": "ab"} + ] + for ext in ["", ".gz", ".bz2", ".xz"]: + with archive.TarFileWriter(self.tempfile) as f: + f.add_tar(os.path.join(testenv.TESTDATA_PATH, "archive", + "tar_test.tar" + ext), + name_filter=lambda n: n != "./b") + self.assertTarFileContent(self.tempfile, content) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_defs/docker/testdata/BUILD b/tools/build_defs/docker/testdata/BUILD index e365847cff..7848a2c7a6 100644 --- a/tools/build_defs/docker/testdata/BUILD +++ b/tools/build_defs/docker/testdata/BUILD @@ -6,6 +6,11 @@ package( load("/tools/build_defs/docker/docker", "docker_build") +filegroup( + name = "archive_testdata", + srcs = glob(["archive/**"]), +) + genrule( name = "gen", outs = ["gen.out"], diff --git a/tools/build_defs/docker/testdata/archive/a.ar b/tools/build_defs/docker/testdata/archive/a.ar new file mode 100644 index 0000000000..d2594dffca --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/a.ar @@ -0,0 +1,3 @@ +!<arch> +a/ 1439231934 1000 1000 100664 1 ` +a diff --git a/tools/build_defs/docker/testdata/archive/a_ab.ar b/tools/build_defs/docker/testdata/archive/a_ab.ar new file mode 100644 index 0000000000..f6b72174ab --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/a_ab.ar @@ -0,0 +1,5 @@ +!<arch> +a/ 1439231934 1000 1000 100664 1 ` +a +ab/ 1439231936 1000 1000 100664 2 ` +ab diff --git a/tools/build_defs/docker/testdata/archive/a_b.ar b/tools/build_defs/docker/testdata/archive/a_b.ar new file mode 100644 index 0000000000..6f2d79c439 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/a_b.ar @@ -0,0 +1,5 @@ +!<arch> +a/ 1439231934 1000 1000 100664 1 ` +a +b/ 1439231939 1000 1000 100664 1 ` +b diff --git a/tools/build_defs/docker/testdata/archive/a_b_ab.ar b/tools/build_defs/docker/testdata/archive/a_b_ab.ar new file mode 100644 index 0000000000..07dc3e71f1 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/a_b_ab.ar @@ -0,0 +1,7 @@ +!<arch> +a/ 1439231934 1000 1000 100664 1 ` +a +b/ 1439231939 1000 1000 100664 1 ` +b +ab/ 1439231936 1000 1000 100664 2 ` +ab diff --git a/tools/build_defs/docker/testdata/archive/ab.ar b/tools/build_defs/docker/testdata/archive/ab.ar new file mode 100644 index 0000000000..29f56beed2 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/ab.ar @@ -0,0 +1,3 @@ +!<arch> +ab/ 1439231936 1000 1000 100664 2 ` +ab diff --git a/tools/build_defs/docker/testdata/archive/b.ar b/tools/build_defs/docker/testdata/archive/b.ar new file mode 100644 index 0000000000..3f41a7eb70 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/b.ar @@ -0,0 +1,3 @@ +!<arch> +b/ 1439231939 1000 1000 100664 1 ` +b diff --git a/tools/build_defs/docker/testdata/archive/empty.ar b/tools/build_defs/docker/testdata/archive/empty.ar new file mode 100644 index 0000000000..8b277f0dd5 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/empty.ar @@ -0,0 +1 @@ +!<arch> diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar b/tools/build_defs/docker/testdata/archive/tar_test.tar Binary files differnew file mode 100644 index 0000000000..70b043bd9d --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/tar_test.tar diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar.bz2 b/tools/build_defs/docker/testdata/archive/tar_test.tar.bz2 Binary files differnew file mode 100644 index 0000000000..c6f7edf5a7 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/tar_test.tar.bz2 diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar.gz b/tools/build_defs/docker/testdata/archive/tar_test.tar.gz Binary files differnew file mode 100644 index 0000000000..4b851716dd --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/tar_test.tar.gz diff --git a/tools/build_defs/docker/testdata/archive/tar_test.tar.xz b/tools/build_defs/docker/testdata/archive/tar_test.tar.xz Binary files differnew file mode 100644 index 0000000000..1ea3c8b836 --- /dev/null +++ b/tools/build_defs/docker/testdata/archive/tar_test.tar.xz diff --git a/tools/build_defs/docker/testenv.py b/tools/build_defs/docker/testenv.py new file mode 100644 index 0000000000..4e0039e2b6 --- /dev/null +++ b/tools/build_defs/docker/testenv.py @@ -0,0 +1,22 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Path to the test data.""" + +import os +import os.path + +TESTDATA_PATH = os.path.join( + os.getcwd(), + "tools/build_defs/docker/testdata", + ) |