diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/tools/singlejar/BUILD | 4 | ||||
-rw-r--r-- | src/tools/singlejar/output_jar.cc | 110 | ||||
-rw-r--r-- | src/tools/singlejar/output_jar.h | 2 | ||||
-rw-r--r-- | src/tools/singlejar/output_jar_simple_test.cc | 152 |
4 files changed, 217 insertions, 51 deletions
diff --git a/src/tools/singlejar/BUILD b/src/tools/singlejar/BUILD index 3ebb690fd1..069b462ede 100644 --- a/src/tools/singlejar/BUILD +++ b/src/tools/singlejar/BUILD @@ -122,12 +122,14 @@ cc_test( srcs = [ "output_jar_simple_test.cc", ], - copts = ["-Ithird_party/bazel"], + copts = ["-DJAR_TOOL_PATH=\\\"external/local_jdk/bin/jar\\\""], data = [ ":data1", ":data2", ":test1", ":test2", + "//external:jar", + "//external:jdk-default", ], deps = [ ":input_jar", diff --git a/src/tools/singlejar/output_jar.cc b/src/tools/singlejar/output_jar.cc index 5620fa9406..7314e5f6c7 100644 --- a/src/tools/singlejar/output_jar.cc +++ b/src/tools/singlejar/output_jar.cc @@ -25,7 +25,7 @@ #include <sys/sendfile.h> #endif #include <sys/stat.h> -#include <sys/times.h> +#include <time.h> #include <unistd.h> #include "src/main/cpp/util/file.h" @@ -78,7 +78,6 @@ int OutputJar::Doit(Options *options) { // TODO(asmundak): handle these options. TODO(!options_->force_compression, "Handle --compression"); - TODO(!options_->normalize_timestamps, "Handle --normalize"); TODO(!options_->preserve_compression, "Handle --dont_change_compression"); build_properties_.AddProperty("build.target", options_->output_jar.c_str()); @@ -343,6 +342,35 @@ bool OutputJar::AddJar(int jar_path_index) { num_bytes += lh->compressed_file_size(); } off_t output_position = Position(); + + // When normalize_timestamps is set, entry's timestamp is to be set to + // 01/01/1980 00:00:00 (or to 01/01/1980 00:00:02, if an entry is a .class + // file). This is somewhat expensive because we have to copy the local + // header to memory as input jar is memory mapped as read-only. Try to copy + // as little as possible. + uint16_t normalized_time = 0; + bool fix_timestamp = false; + if (options_->normalize_timestamps) { + if (ends_with(file_name, file_name_length, ".class")) { + normalized_time = 1; + } + fix_timestamp = jar_entry->last_mod_file_date() != 0 || + jar_entry->last_mod_file_time() != normalized_time; + } + if (fix_timestamp) { + LH lh_new; + memcpy(&lh_new, lh, sizeof(lh_new)); + lh_new.last_mod_file_date(33); + lh_new.last_mod_file_time(normalized_time); + // Now write these few bytes and adjust read/write positions accordingly. + if (!WriteBytes(reinterpret_cast<uint8_t *>(&lh_new), sizeof(lh_new))) { + diag_err(1, "%s:%d: Cannot copy modified local header for %.*s", + __FILE__, __LINE__, file_name_length, file_name); + } + copy_from += sizeof(lh_new); + num_bytes -= sizeof(lh_new); + } + // Do the actual copy. Use sendfile, avoiding copying the data to user // space and back. ssize_t n_copied = AppendFile(input_jar.fd(), ©_from, num_bytes); @@ -358,7 +386,12 @@ bool OutputJar::AddJar(int jar_path_index) { // Append central directory header for this file to the output central // directory we are building. TODO(output_position < 0xFFFFFFFF, "Handle Zip64"); - AppendToDirectoryBuffer(jar_entry)->local_header_offset32(output_position); + CDH *out_cdh = AppendToDirectoryBuffer(jar_entry); + out_cdh->local_header_offset32(output_position); + if (fix_timestamp) { + out_cdh->last_mod_file_time(normalized_time); + out_cdh->last_mod_file_date(33); + } ++entries_; } return input_jar.Close(); @@ -389,16 +422,33 @@ void OutputJar::WriteEntry(void *buffer) { : "compressed", entry->compressed_file_size()); } - uint8_t *data_end = entry->data() + entry->in_zip_size(); + + // Set this entry's timestamp. + // MSDOS file timestamp format that Zip uses is described here: + // https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx + // ("32-Bit Windows Time/Date Formats") + if (options_->normalize_timestamps) { + // Regular "normalized" timestamp is 01/01/1980 00:00:00. No need to handle + // .class files here, they are always copied from the input jars. + entry->last_mod_file_time(0); + entry->last_mod_file_date(33); + } else { + struct tm tm; + // Time has 2-second resolution, so round up: + time_t t_adjusted = (time(nullptr) + 1) & ~1; + localtime_r(&t_adjusted, &tm); + uint16_t dos_date = + ((tm.tm_year - 80) << 9) | ((tm.tm_mon + 1) << 5) | tm.tm_mday; + uint16_t dos_time = + (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec >> 1); + entry->last_mod_file_time(dos_time); + entry->last_mod_file_date(dos_date); + } + uint8_t *data = reinterpret_cast<uint8_t *>(entry); off_t output_position = Position(); - while (data < data_end) { - ssize_t written = write(fd_, data, data_end - data); - if (written >= 0) { - data += written; - } else if (errno != EINTR) { - diag_err(1, "%s:%d: write", __FILE__, __LINE__); - } + if (!WriteBytes(data, entry->data() + entry->in_zip_size() - data)) { + diag_err(1, "%s:%d: write", __FILE__, __LINE__); } // Data written, allocate CDH space and populate CDH. CDH *cdh = reinterpret_cast<CDH *>( @@ -434,8 +484,6 @@ void OutputJar::AddDirectory(const char *path) { lh->version(20); lh->bit_flag(0); // TODO(asmundak): should I set UTF8 flag? lh->compression_method(Z_NO_COMPRESSION); - lh->last_mod_file_time(0); - lh->last_mod_file_date(33); lh->crc32(0); lh->compressed_file_size32(0); lh->uncompressed_file_size32(0); @@ -504,15 +552,9 @@ bool OutputJar::Close() { TODO(output_position < 0xFFFFFFFF, "Handle Zip64"); ecd->cen_offset32(output_position); - // Write Central Directory. - uint8_t *cen_end = cen_ + cen_size_; - uint8_t *cen = cen_; - while (cen < cen_end) { - ssize_t n = write(fd_, cen, cen_end - cen); - if (n < 0) { - diag_err(1, "%s:%d: Cannot write central directory", __FILE__, __LINE__); - } - cen += n; + // Save Central Directory and wrap up. + if (!WriteBytes(cen_, cen_size_)) { + diag_err(1, "%s:%d: Cannot write central directory", __FILE__, __LINE__); } free(cen_); @@ -581,16 +623,8 @@ ssize_t OutputJar::AppendFile(int in_fd, off_t *in_offset, size_t count) { ssize_t n_read = read(in_fd, buffer, std::min(sizeof(buffer), count - total_written)); if (n_read > 0) { - uint8_t *write_buffer = buffer; - uint8_t *write_buffer_end = write_buffer + n_read; - while (write_buffer < write_buffer_end) { - ssize_t n_written = - write(fd_, write_buffer, write_buffer_end - write_buffer); - if (n_written > 0) { - write_buffer += n_written; - } else if (EAGAIN != errno) { - return -1; - } + if (!WriteBytes(buffer, n_read)) { + return -1; } total_written += n_read; } else if (n_read == 0) { @@ -633,3 +667,15 @@ void OutputJar::ExtraCombiner(const std::string &entry_name, extra_combiners_.emplace_back(combiner); known_members_.emplace(entry_name, EntryInfo{combiner}); } + +bool OutputJar::WriteBytes(uint8_t *buffer, size_t count) { + for (uint8_t *buffer_end = buffer + count; buffer < buffer_end;) { + ssize_t n_written = write(fd_, buffer, buffer_end - buffer); + if (n_written > 0) { + buffer += n_written; + } else if (EAGAIN == errno) { + return false; + } + } + return true; +} diff --git a/src/tools/singlejar/output_jar.h b/src/tools/singlejar/output_jar.h index 8bad75813c..f9ba4511e7 100644 --- a/src/tools/singlejar/output_jar.h +++ b/src/tools/singlejar/output_jar.h @@ -65,6 +65,8 @@ class OutputJar { const std::string& resource_path); // Copy the bytes from the given file. ssize_t AppendFile(int in_fd, off_t *in_offset, size_t count); + // Write bytes to the output file, return true on success. + bool WriteBytes(uint8_t *buffer, size_t count); // The purpose of these two tiny utility methods is to avoid creating a // std::string instance (which always involves allocating an object on the diff --git a/src/tools/singlejar/output_jar_simple_test.cc b/src/tools/singlejar/output_jar_simple_test.cc index ccd22a1a3f..cb7f82baf1 100644 --- a/src/tools/singlejar/output_jar_simple_test.cc +++ b/src/tools/singlejar/output_jar_simple_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <stdlib.h> + #include "src/main/cpp/blaze_util.h" #include "src/main/cpp/util/file.h" #include "src/main/cpp/util/port.h" @@ -22,10 +24,17 @@ #include "src/tools/singlejar/test_util.h" #include "gtest/gtest.h" +#if !defined(JAR_TOOL_PATH) +#error "The path to jar tool has to be defined via -DJAR_TOOL_PATH=" +#endif + namespace { +using singlejar_test_util::CreateTextFile; +using singlejar_test_util::GetEntryContents; using singlejar_test_util::GetEntryContents; using singlejar_test_util::OutputFilePath; +using singlejar_test_util::RunCommand; using singlejar_test_util::VerifyZip; using std::string; @@ -90,6 +99,41 @@ TEST_F(OutputJarSimpleTest, Empty) { EXPECT_EQ(lh->uncompressed_file_size(), cdh->uncompressed_file_size()) << "Entry: " << cdh->file_name_string(); } + // Verify that each entry has a reasonable timestamp. + EXPECT_EQ(lh->last_mod_file_date(), cdh->last_mod_file_date()) + << "Entry: " << lh->file_name_string(); + EXPECT_EQ(lh->last_mod_file_time(), cdh->last_mod_file_time()) + << "Entry: " << lh->file_name_string(); + uint16_t dos_time = lh->last_mod_file_time(); + uint16_t dos_date = lh->last_mod_file_date(); + + // Current time, rounded to even number of seconds because MSDOS timestamp + // does this, too. + time_t now = (time(nullptr) + 1) & ~1; + struct tm tm_now; + localtime_r(&now, &tm_now); + char now_time_str[50]; + strftime(now_time_str, sizeof(now_time_str), "%c", &tm_now); + + // Unpack MSDOS file timestamp. See the comment about its format in + // output_jar.cc. + struct tm tm; + tm.tm_sec = (dos_time & 31) << 1; + tm.tm_min = (dos_time >> 5) & 63; + tm.tm_hour = (dos_time >> 11) & 31; + tm.tm_mday = (dos_date & 31); + tm.tm_mon = ((dos_date >> 5) & 15) - 1; + tm.tm_year = ((dos_date >> 9) & 127) + 80; + tm.tm_isdst = tm_now.tm_isdst; + time_t entry_time = mktime(&tm); + char entry_time_str[50]; + strftime(entry_time_str, sizeof(entry_time_str), "%c", &tm); + + // Without --normalize option all the entries should have reasonably + // current timestamp (which we arbitrarily choose to be <5 minutes). + EXPECT_GE(now, entry_time) << now_time_str << " vs. " << entry_time_str; + EXPECT_LE(now, entry_time + 300) << now_time_str << " vs. " + << entry_time_str; } input_jar.Close(); string manifest = GetEntryContents(out_path, "META-INF/MANIFEST.MF"); @@ -112,6 +156,7 @@ TEST_F(OutputJarSimpleTest, Source) { ASSERT_TRUE(input_jar.Open(out_path)); const LH *lh; const CDH *cdh; + int file_count = 0; while ((cdh = input_jar.NextEntry(&lh))) { ASSERT_TRUE(cdh->is()) << "No expected tag in the Central Directory Entry."; ASSERT_NE(nullptr, lh) << "No local header."; @@ -123,7 +168,11 @@ TEST_F(OutputJarSimpleTest, Source) { EXPECT_EQ(lh->uncompressed_file_size(), cdh->uncompressed_file_size()) << "Entry: " << cdh->file_name_string(); } + if (lh->file_name()[lh->file_name_length() - 1] != '/') { + ++file_count; + } } + ASSERT_LE(4, file_count); input_jar.Close(); } @@ -187,12 +236,11 @@ TEST_F(OutputJarSimpleTest, ExtraBuildInfo) { // --build_info_file and --extra_build_info options. TEST_F(OutputJarSimpleTest, BuildInfoFile) { - string build_info_path1 = OutputFilePath("buildinfo1"); - ASSERT_TRUE(blaze::WriteFile("property11=value11\nproperty12=value12\n", - build_info_path1)); - string build_info_path2 = OutputFilePath("buildinfo2"); - ASSERT_TRUE(blaze::WriteFile("property21=value21\nproperty22=value22\n", - build_info_path2)); + string build_info_path1 = + CreateTextFile("buildinfo1", "property11=value11\nproperty12=value12\n"); + string build_info_path2 = + CreateTextFile("buildinfo2", "property21=value21\nproperty22=value22\n"); + string out_path = OutputFilePath("out.jar"); CreateOutput(out_path, "--build_info_file", build_info_path1.c_str(), "--extra_build_info", "property=value", "--build_info_file", @@ -207,16 +255,13 @@ TEST_F(OutputJarSimpleTest, BuildInfoFile) { // --resources option. TEST_F(OutputJarSimpleTest, Resources) { - string res11_path = OutputFilePath("res11"); + string res11_path = CreateTextFile("res11", "res11.line1\nres11.line2\n"); string res11_spec = string("res1:") + res11_path; - ASSERT_TRUE(blaze::WriteFile("res11.line1\nres11.line2\n", res11_path)); - string res12_path = OutputFilePath("res12"); + string res12_path = CreateTextFile("res12", "res12.line1\nres12.line2\n"); string res12_spec = string("res1:") + res12_path; - ASSERT_TRUE(blaze::WriteFile("res12.line1\nres12.line2\n", res12_path)); - string res2_path = OutputFilePath("res2"); - ASSERT_TRUE(blaze::WriteFile("res2.line1\nres2.line2\n", res2_path)); + string res2_path = CreateTextFile("res2", "res2.line1\nres2.line2\n"); string out_path = OutputFilePath("out.jar"); CreateOutput(out_path, "--resources", res11_spec.c_str(), res12_spec.c_str(), @@ -244,16 +289,13 @@ TEST_F(OutputJarSimpleTest, ClasspathResources) { // Duplicate entries for --resources or --classpath_resources TEST_F(OutputJarSimpleTest, DuplicateResources) { - string cp_res_path = OutputFilePath("cp_res"); - ASSERT_TRUE(blaze::WriteFile("line1\nline2\n", cp_res_path)); + string cp_res_path = CreateTextFile("cp_res", "line1\nline2\n"); - string res1_path = OutputFilePath("res1"); + string res1_path = CreateTextFile("res1", "resline1\nresline2\n"); string res1_spec = "foo:" + res1_path; - ASSERT_TRUE(blaze::WriteFile("resline1\nresline2\n", res1_path)); - string res2_path = OutputFilePath("res2"); + string res2_path = CreateTextFile("res2", "line3\nline4\n"); string res2_spec = "foo:" + res2_path; - ASSERT_TRUE(blaze::WriteFile("line3\nline4\n", res2_path)); string out_path = OutputFilePath("out.jar"); CreateOutput(out_path, "--warn_duplicate_resources", "--resources", @@ -308,4 +350,78 @@ TEST_F(OutputJarSimpleTest, IncludeHeaders) { EXPECT_EQ(expected_entries, jar_entries); } +// --normalize +TEST_F(OutputJarSimpleTest, Normalize) { + // Creates output jar containing entries from all possible sources: + // * archives created by java_library rule, by jar tool, by zip + // * resource files + // * classpath resource files + // * + string out_path = OutputFilePath("out.jar"); + string testjar_path = OutputFilePath("testinput.jar"); + { + char *jar_tool_path = realpath(JAR_TOOL_PATH, nullptr); + string textfile_path = CreateTextFile("jar_testinput.txt", "jar_inputtext"); + string classfile_path = CreateTextFile("JarTestInput.class", "Dummy"); + unlink(testjar_path.c_str()); + ASSERT_EQ( + 0, RunCommand(jar_tool_path, "-cf", testjar_path.c_str(), + textfile_path.c_str(), classfile_path.c_str(), nullptr)); + free(jar_tool_path); + } + + string testzip_path = OutputFilePath("testinput.zip"); + { + string textfile_path = CreateTextFile("zip_testinput.txt", "zip_inputtext"); + string classfile_path = CreateTextFile("ZipTestInput.class", "Dummy"); + unlink(testzip_path.c_str()); + ASSERT_EQ( + 0, RunCommand("zip", "-m", testzip_path.c_str(), textfile_path.c_str(), + classfile_path.c_str(), nullptr)); + } + + string resource_path = CreateTextFile("resource", "resource_text"); + string cp_resource_path = CreateTextFile("cp_resource", "cp_resource_text"); + + // TODO(asmundak): check the following generated entries, too: + // * services + // * spring.schemas + // * spring.handlers + // * protobuf.meta + // * extra combiner + + CreateOutput(out_path, "--normalize", "--sources", + DATA_DIR_TOP "src/tools/singlejar/libtest1.jar", + testjar_path.c_str(), testzip_path.c_str(), "--resources", + resource_path.c_str(), "--classpath_resources", + cp_resource_path.c_str(), nullptr); + + // Scan all entries, verify that *.class entries have timestamp + // 01/01/1980 00:00:02 and the rest have the timestamp of 01/01/1980 00:00:00. + InputJar input_jar; + ASSERT_TRUE(input_jar.Open(out_path)); + const LH *lh; + const CDH *cdh; + while ((cdh = input_jar.NextEntry(&lh))) { + string entry_name = cdh->file_name_string(); + EXPECT_EQ(lh->last_mod_file_date(), cdh->last_mod_file_date()) + << entry_name << " modification date"; + EXPECT_EQ(lh->last_mod_file_time(), cdh->last_mod_file_time()) + << entry_name << " modification time"; + EXPECT_EQ(33, cdh->last_mod_file_date()) + << entry_name << " modification date should be 01/01/1980"; + auto n = entry_name.size() - strlen(".class"); + if (0 == strcmp(entry_name.c_str() + n, ".class")) { + EXPECT_EQ(1, cdh->last_mod_file_time()) + << entry_name + << " modification time for .class entry should be 00:00:02"; + } else { + EXPECT_EQ(0, cdh->last_mod_file_time()) + << entry_name + << " modification time for non .class entry should be 00:00:00"; + } + } + input_jar.Close(); +} + } // namespace |