aboutsummaryrefslogtreecommitdiffhomepage
path: root/third_party
diff options
context:
space:
mode:
authorGravatar Damien Martin-Guillerez <dmarting@google.com>2016-01-21 14:19:07 +0000
committerGravatar Kristina Chodorow <kchodorow@google.com>2016-01-21 16:22:20 +0000
commitc88508c594d535311b092d6fdba94b2b4b9cfa12 (patch)
treec138906899daabc7620380a0479d666fe31bacc6 /third_party
parent2762919a486e48d316670313da2208d4fbd6361a (diff)
Support @ syntax for specifying the list of files to zipper
When creating an archive with //third_party/ijar:zipper, all files to add should be specified as an argument. When adding a lot of files, this leads to reaching the maximum size of the command line. This change add a @filename argument that permit to specify a file containing the list of files to zip. -- Change-Id: I02843588896969ef2d6add26c1c67ff44a27f7de Reviewed-on: https://bazel-review.googlesource.com/#/c/2695/ MOS_MIGRATED_REVID=112678600
Diffstat (limited to 'third_party')
-rwxr-xr-xthird_party/ijar/test/zip_test.sh22
-rw-r--r--third_party/ijar/zip_main.cc200
2 files changed, 161 insertions, 61 deletions
diff --git a/third_party/ijar/test/zip_test.sh b/third_party/ijar/test/zip_test.sh
index 4f3aef9fc8..379f9b7891 100755
--- a/third_party/ijar/test/zip_test.sh
+++ b/third_party/ijar/test/zip_test.sh
@@ -39,17 +39,19 @@ function assert_unzip_same_as_zipper() {
}
function assert_zipper_same_after_unzip() {
+ local dir="$1"
+ shift
local zipfile=${TEST_TMPDIR}/output.zip
- (cd $1 && $ZIPPER c ${zipfile} $(find . | sed 's|^./||' | grep -v '^.$'))
+ (cd "${dir}" && $ZIPPER c ${zipfile} "$@")
local folder=$(mktemp -d ${TEST_TMPDIR}/output.XXXXXXXX)
(cd $folder && $UNZIP -q ${zipfile} || true) # ignore CRC32 errors
- diff -r $1 $folder &> $TEST_log \
+ diff -r "${dir}" $folder &> $TEST_log \
|| fail "Unzip after zipper output differ"
# Retry with compression
- (cd $1 && $ZIPPER cC ${zipfile} $(find . | sed 's|^./||' | grep -v '^.$'))
+ (cd "${dir}" && $ZIPPER cC ${zipfile} "$@")
local folder=$(mktemp -d ${TEST_TMPDIR}/output.XXXXXXXX)
(cd $folder && $UNZIP -q ${zipfile} || true) # ignore CRC32 errors
- diff -r $1 $folder &> $TEST_log \
+ diff -r "${dir}" $folder &> $TEST_log \
|| fail "Unzip after zipper output differ"
}
@@ -63,12 +65,18 @@ function test_zipper() {
echo "titi" > ${TEST_TMPDIR}/test/path/to/some/other_file
chmod +x ${TEST_TMPDIR}/test/path/to/some/other_file
echo "tata" > ${TEST_TMPDIR}/test/file
- assert_zipper_same_after_unzip ${TEST_TMPDIR}/test
+ filelist="$(cd ${TEST_TMPDIR}/test && find . | sed 's|^./||' | grep -v '^.$')"
+
+ assert_zipper_same_after_unzip ${TEST_TMPDIR}/test ${filelist}
+ assert_unzip_same_as_zipper ${TEST_TMPDIR}/output.zip
+
+ # Test @filelist format
+ echo "${filelist}" >${TEST_TMPDIR}/test.content
+ assert_zipper_same_after_unzip ${TEST_TMPDIR}/test @${TEST_TMPDIR}/test.content
assert_unzip_same_as_zipper ${TEST_TMPDIR}/output.zip
# Test flatten option
- (cd ${TEST_TMPDIR}/test && $ZIPPER cf ${TEST_TMPDIR}/output.zip \
- $(find . | sed 's|^./||' | grep -v '^.$'))
+ (cd ${TEST_TMPDIR}/test && $ZIPPER cf ${TEST_TMPDIR}/output.zip ${filelist})
$ZIPPER v ${TEST_TMPDIR}/output.zip >$TEST_log
expect_log "file"
expect_log "other_file"
diff --git a/third_party/ijar/zip_main.cc b/third_party/ijar/zip_main.cc
index 285c0137e3..4ac13160e2 100644
--- a/third_party/ijar/zip_main.cc
+++ b/third_party/ijar/zip_main.cc
@@ -20,14 +20,14 @@
// CRC-32 of all files in the zip file will be set to 0.
//
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdint.h>
#include <stdio.h>
-#include <string.h>
#include <stdlib.h>
-#include <limits.h>
-#include <fcntl.h>
+#include <string.h>
#include <unistd.h>
-#include <sys/mman.h>
-#include <errno.h>
#include <memory>
#include "third_party/ijar/zip.h"
@@ -152,6 +152,22 @@ void basename(const char *path, char *output, size_t output_size) {
output[output_size-1] = 0;
}
+// copy size bytes from file descriptor fd into buffer.
+int copy_file_to_buffer(int fd, size_t size, void *buffer) {
+ size_t nb_read = 0;
+ while (nb_read < size) {
+ size_t to_read = size - nb_read;
+ if (to_read > 16384 /* 16K */) {
+ to_read = 16384;
+ }
+ ssize_t r = read(fd, static_cast<uint8_t *>(buffer) + nb_read, to_read);
+ if (r < 0) {
+ return -1;
+ }
+ nb_read += r;
+ }
+ return 0;
+}
// Execute the extraction (or just listing if just v is provided)
int extract(char *zipfile, bool verbose, bool extract) {
@@ -174,10 +190,122 @@ int extract(char *zipfile, bool verbose, bool extract) {
return 0;
}
+// add a file to the zip
+int add_file(std::unique_ptr<ZipBuilder> const &builder, char *file,
+ bool flatten, bool verbose, bool compress) {
+ struct stat statst;
+ if (stat(file, &statst) < 0) {
+ fprintf(stderr, "Cannot stat file %s: %s.\n", file, strerror(errno));
+ return -1;
+ }
+ bool isdir = (statst.st_mode & S_IFDIR) != 0;
+
+ if (flatten && isdir) {
+ return 0;
+ }
+
+ // Compute the path, flattening it if requested
+ char path[PATH_MAX];
+ size_t len = strlen(file);
+ if (len > PATH_MAX) {
+ fprintf(stderr, "Path too long: %s.\n", file);
+ return -1;
+ }
+ if (flatten) {
+ basename(file, path, PATH_MAX);
+ } else {
+ strncpy(path, file, PATH_MAX);
+ path[PATH_MAX - 1] = 0;
+ if (isdir && len < PATH_MAX - 1) {
+ // Add the trailing slash for folders
+ path[len] = '/';
+ path[len + 1] = 0;
+ }
+ }
+
+ if (verbose) {
+ mode_t perm = statst.st_mode & 0777;
+ printf("%c %o %s\n", isdir ? 'd' : 'f', perm, path);
+ }
+
+ u1 *buffer = builder->NewFile(path, mode_to_zipattr(statst.st_mode));
+ if (isdir || statst.st_size == 0) {
+ builder->FinishFile(0);
+ } else {
+ // read the input file
+ int fd = open(file, O_RDONLY);
+ if (fd < 0) {
+ fprintf(stderr, "Can't open file %s for reading: %s.\n", file,
+ strerror(errno));
+ return -1;
+ }
+ if (copy_file_to_buffer(fd, statst.st_size, buffer) < 0) {
+ fprintf(stderr, "Can't read file %s: %s.\n", file, strerror(errno));
+ close(fd);
+ return -1;
+ }
+ close(fd);
+ builder->FinishFile(statst.st_size, compress, true);
+ }
+ return 0;
+}
+
+// Read a list of files separated by newlines. The resulting array can be
+// freed using the free method.
+char **read_filelist(char *filename) {
+ struct stat statst;
+ int fd = open(filename, O_RDONLY);
+ if (fd < 0) {
+ fprintf(stderr, "Can't open file %s for reading: %s.\n", filename,
+ strerror(errno));
+ return NULL;
+ }
+ if (fstat(fd, &statst) < 0) {
+ fprintf(stderr, "Cannot stat file %s: %s.\n", filename, strerror(errno));
+ return NULL;
+ }
+
+ char *data = static_cast<char *>(malloc(statst.st_size));
+ if (copy_file_to_buffer(fd, statst.st_size, data) < 0) {
+ fprintf(stderr, "Can't read file %s: %s.\n", filename, strerror(errno));
+ close(fd);
+ return NULL;
+ }
+ close(fd);
+
+ int nb_entries = 1;
+ for (int i = 0; i < statst.st_size; i++) {
+ if (data[i] == '\n') {
+ nb_entries++;
+ }
+ }
+
+ size_t sizeof_array = sizeof(char *) * (nb_entries + 1);
+ void *result = malloc(sizeof_array + statst.st_size);
+ // copy the content
+ char **filelist = static_cast<char **>(result);
+ char *content = static_cast<char *>(result) + sizeof_array;
+ memcpy(content, data, statst.st_size);
+ free(data);
+ // Create the corresponding array
+ int j = 1;
+ filelist[0] = content;
+ for (int i = 0; i < statst.st_size; i++) {
+ if (content[i] == '\n') {
+ content[i] = 0;
+ if (i + 1 < statst.st_size) {
+ filelist[j] = content + i + 1;
+ j++;
+ }
+ }
+ }
+ filelist[j] = NULL;
+ return filelist;
+}
+
// Execute the create operation
int create(char *zipfile, char **files, bool flatten, bool verbose,
bool compress) {
- struct stat statst;
u8 size = ZipBuilder::EstimateSize(files);
if (size == 0) {
return -1;
@@ -189,53 +317,8 @@ int create(char *zipfile, char **files, bool flatten, bool verbose,
return -1;
}
for (int i = 0; files[i] != NULL; i++) {
- stat(files[i], &statst);
- char path[PATH_MAX];
- bool isdir = (statst.st_mode & S_IFDIR) != 0;
-
- if (flatten && isdir) {
- continue;
- }
-
- // Compute the path, flattening it if requested
- if (flatten) {
- basename(files[i], path, PATH_MAX);
- } else {
- strncpy(path, files[i], PATH_MAX);
- path[PATH_MAX-1] = 0;
- size_t len = strlen(path);
- if (isdir && len < PATH_MAX - 1) {
- // Add the trailing slash for folders
- path[len] = '/';
- path[len+1] = 0;
- }
- }
-
- if (verbose) {
- mode_t perm = statst.st_mode & 0777;
- printf("%c %o %s\n", isdir ? 'd' : 'f', perm, path);
- }
-
- u1 *buffer = builder->NewFile(path, mode_to_zipattr(statst.st_mode));
- if (isdir || statst.st_size == 0) {
- builder->FinishFile(0);
- } else {
- // mmap the input file and memcpy
- int fd = open(files[i], O_RDONLY);
- if (fd < 0) {
- fprintf(stderr, "Can't open file %s for reading: %s.\n",
- files[i], strerror(errno));
- return -1;
- }
- void *data = mmap(NULL, statst.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
- if (data == MAP_FAILED) {
- fprintf(stderr, "Can't mmap file %s for reading: %s.\n",
- files[i], strerror(errno));
- return -1;
- }
- memcpy(buffer, data, statst.st_size);
- munmap(data, statst.st_size);
- builder->FinishFile(statst.st_size, compress, true);
+ if (add_file(builder, files[i], flatten, verbose, compress) < 0) {
+ return -1;
}
}
if (builder->Finish() < 0) {
@@ -299,7 +382,16 @@ int main(int argc, char **argv) {
usage(argv[0]);
}
// Create a zip
- return devtools_ijar::create(argv[2], argv + 3, flatten, verbose, compress);
+ char **filelist = argv + 3;
+ if (argc == 4 && argv[3][0] == '@') {
+ // We never free that list because it needs to be allocated during the
+ // whole execution, the system will reclaim memory.
+ filelist = devtools_ijar::read_filelist(argv[3] + 1);
+ if (filelist == NULL) {
+ return -1;
+ }
+ }
+ return devtools_ijar::create(argv[2], filelist, flatten, verbose, compress);
} else {
if (flatten) {
usage(argv[0]);