diff options
author | Sasha Smundak <asmundak@google.com> | 2016-08-09 17:17:57 +0000 |
---|---|---|
committer | Yue Gan <yueg@google.com> | 2016-08-10 08:37:31 +0000 |
commit | 3f814d72af4c86edfe997a5bb2217e6f753f16b1 (patch) | |
tree | 3920b3dbe20474609d8229985ed0dc2dc94501a8 /src | |
parent | 2e6c76e27125a4326e25dd7375f9a1058d30da9d (diff) |
Combiners overhaul: introduce Combiner interface common to all combiners, use it instead of switch statement in output_jar. Move implementations to combiners.cc
--
MOS_MIGRATED_REVID=129763019
Diffstat (limited to 'src')
-rw-r--r-- | src/tools/singlejar/BUILD | 29 | ||||
-rw-r--r-- | src/tools/singlejar/combiners.cc | 134 | ||||
-rw-r--r-- | src/tools/singlejar/combiners.h | 138 | ||||
-rw-r--r-- | src/tools/singlejar/combiners_test.cc | 11 | ||||
-rw-r--r-- | src/tools/singlejar/output_jar.cc | 96 | ||||
-rw-r--r-- | src/tools/singlejar/output_jar.h | 9 |
6 files changed, 247 insertions, 170 deletions
diff --git a/src/tools/singlejar/BUILD b/src/tools/singlejar/BUILD index b16ec59ed6..7f0e446bab 100644 --- a/src/tools/singlejar/BUILD +++ b/src/tools/singlejar/BUILD @@ -26,11 +26,11 @@ cc_test( size = "large", srcs = [ "combiners_test.cc", - ":combiners", ":zip_headers", ":zlib_interface", ], deps = [ + ":combiners", ":input_jar", "//third_party:gtest", "//third_party/zlib", @@ -192,6 +192,17 @@ cc_test( ) cc_library( + name = "combiners", + srcs = [ + "combiners.cc", + ":transient_bytes", + ":zip_headers", + ], + hdrs = ["combiners.h"], + deps = ["//third_party/zlib"], +) + +cc_library( name = "input_jar", srcs = [ "diag.h", @@ -218,15 +229,15 @@ cc_library( cc_library( name = "output_jar", srcs = [ + "diag.h", + "mapped_file.h", "output_jar.cc", "output_jar.h", - ":combiners", - ":mapped_file.h", - ":options.h", ":zip_headers", ], hdrs = ["output_jar.h"], deps = [ + ":combiners", ":input_jar", ":options", "//src/main/cpp/util", @@ -246,16 +257,6 @@ cc_library( ) filegroup( - name = "combiners", - srcs = [ - "combiners.h", - ":transient_bytes", - ":zip_headers", - ":zlib_interface", - ], -) - -filegroup( name = "token_stream", srcs = [ "diag.h", diff --git a/src/tools/singlejar/combiners.cc b/src/tools/singlejar/combiners.cc new file mode 100644 index 0000000000..736e504df9 --- /dev/null +++ b/src/tools/singlejar/combiners.cc @@ -0,0 +1,134 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/tools/singlejar/combiners.h" +#include "src/tools/singlejar/diag.h" + +Combiner::~Combiner() {} + +Concatenator::~Concatenator() {} + +bool Concatenator::Merge(const CDH *cdh, const LH *lh) { + CreateBuffer(); + if (Z_NO_COMPRESSION == lh->compression_method()) { + buffer_->ReadEntryContents(lh); + } else if (Z_DEFLATED == lh->compression_method()) { + if (!inflater_.get()) { + inflater_.reset(new Inflater()); + } + buffer_->DecompressEntryContents(cdh, lh, inflater_.get()); + } else { + errx(2, "%s is neither stored nor deflated", filename_.c_str()); + } + return true; +} + +void *Concatenator::OutputEntry() { + if (!buffer_.get()) { + return nullptr; + } + + // Allocate a contiguous buffer for the local file header and + // deflated data. We assume that deflate decreases the size, so if + // the deflater reports overflow, we just save original data. + size_t deflated_buffer_size = + sizeof(LH) + filename_.size() + buffer_->data_size(); + + // Huge entry (>4GB) needs Zip64 extension field with 64-bit original + // and compressed size values. + uint8_t + zip64_extension_buffer[sizeof(Zip64ExtraField) + 2 * sizeof(uint64_t)]; + bool huge_buffer = (buffer_->data_size() >= 0xFFFFFFFF); + if (huge_buffer) { + deflated_buffer_size += sizeof(zip64_extension_buffer); + } + LH *lh = reinterpret_cast<LH *>(malloc(deflated_buffer_size)); + if (lh == nullptr) { + return nullptr; + } + lh->signature(); + lh->version(20); + lh->bit_flag(0x0); + lh->last_mod_file_time(1); // 00:00:01 + lh->last_mod_file_date(33); // 1980-01-01 + lh->crc32(0x12345678); + lh->compressed_file_size32(0); + lh->file_name(filename_.c_str(), filename_.size()); + + if (huge_buffer) { + // Add Z64 extension if this is a huge entry. + lh->uncompressed_file_size32(0xFFFFFFFF); + Zip64ExtraField *z64 = + reinterpret_cast<Zip64ExtraField *>(zip64_extension_buffer); + z64->signature(); + z64->payload_size(2 * sizeof(uint64_t)); + z64->attr64(0, buffer_->data_size()); + lh->extra_fields(reinterpret_cast<uint8_t *>(z64), z64->size()); + } else { + lh->uncompressed_file_size32(buffer_->data_size()); + lh->extra_fields(nullptr, 0); + } + + uint32_t checksum; + uint64_t compressed_size; + uint16_t method = buffer_->Write(lh->data(), &checksum, &compressed_size); + lh->crc32(checksum); + lh->compression_method(method); + if (huge_buffer) { + lh->compressed_file_size32(compressed_size < 0xFFFFFFFF ? compressed_size + : 0xFFFFFFFF); + // Not sure if this has to be written in the small case, but it shouldn't + // hurt. + const_cast<Zip64ExtraField *>(lh->zip64_extra_field()) + ->attr64(1, compressed_size); + } else { + // If original data is <4GB, the compressed one is, too. + lh->compressed_file_size32(compressed_size); + } + return reinterpret_cast<void *>(lh); +} + +NullCombiner::~NullCombiner() {} + +bool NullCombiner::Merge(const CDH *cdh, const LH *lh) { return true; } + +void *NullCombiner::OutputEntry() { return nullptr; } + +XmlCombiner::~XmlCombiner() {} + +bool XmlCombiner::Merge(const CDH *cdh, const LH *lh) { + if (!concatenator_.get()) { + concatenator_.reset(new Concatenator(filename_)); + concatenator_->Append("<"); + concatenator_->Append(xml_tag_); + concatenator_->Append(">\n"); + } + return concatenator_->Merge(cdh, lh); +} + +void *XmlCombiner::OutputEntry() { + if (!concatenator_.get()) { + return nullptr; + } + concatenator_->Append("</"); + concatenator_->Append(xml_tag_); + concatenator_->Append(">\n"); + return concatenator_->OutputEntry(); +} + +PropertyCombiner::~PropertyCombiner() {} + +bool PropertyCombiner::Merge(const CDH *cdh, const LH *lh) { + return false; // This should not be called. +} diff --git a/src/tools/singlejar/combiners.h b/src/tools/singlejar/combiners.h index 1bc2b16aac..15baa78c7f 100644 --- a/src/tools/singlejar/combiners.h +++ b/src/tools/singlejar/combiners.h @@ -20,96 +20,29 @@ #include "src/tools/singlejar/transient_bytes.h" #include "src/tools/singlejar/zip_headers.h" -#include "src/tools/singlejar/zlib_interface.h" + +// An interface for combining the files. +class Combiner { + public: + virtual ~Combiner(); + // Merges the contents of the given Zip entry to this instance. + virtual bool Merge(const CDH *cdh, const LH *lh) = 0; + // Returns a point to the buffer containing Local Header followed by the + // payload. The caller is responsible of freeing the buffer. + virtual void *OutputEntry() = 0; +}; // An output jar entry consisting of a concatenation of the input jar // entries. Byte sequences can be appended to it, too. -class Concatenator { +class Concatenator : public Combiner { public: Concatenator(const std::string &filename) : filename_(filename) {} - // Appends the contents of the given input entry. - bool Merge(const CDH *cdh, const LH *lh) { - CreateBuffer(); - if (Z_NO_COMPRESSION == lh->compression_method()) { - buffer_->ReadEntryContents(lh); - } else if (Z_DEFLATED == lh->compression_method()) { - if (!inflater_.get()) { - inflater_.reset(new Inflater()); - } - buffer_->DecompressEntryContents(cdh, lh, inflater_.get()); - } else { - errx(2, "%s is neither stored nor deflated", filename_.c_str()); - } - return true; - } + ~Concatenator() override; - // Returns a point to the buffer containing Local Header followed by the - // payload. The caller is responsible of freeing the buffer. - void *OutputEntry() { - if (!buffer_.get()) { - return nullptr; - } + bool Merge(const CDH *cdh, const LH *lh) override; - // Allocate a contiguous buffer for the local file header and - // deflated data. We assume that deflate decreases the size, so if - // the deflater reports overflow, we just save original data. - size_t deflated_buffer_size = - sizeof(LH) + filename_.size() + buffer_->data_size(); - - // Huge entry (>4GB) needs Zip64 extension field with 64-bit original - // and compressed size values. - uint8_t - zip64_extension_buffer[sizeof(Zip64ExtraField) + 2 * sizeof(uint64_t)]; - bool huge_buffer = (buffer_->data_size() >= 0xFFFFFFFF); - if (huge_buffer) { - deflated_buffer_size += sizeof(zip64_extension_buffer); - } - LH *lh = reinterpret_cast<LH *>(malloc(deflated_buffer_size)); - if (lh == nullptr) { - return nullptr; - } - lh->signature(); - lh->version(20); - lh->bit_flag(0x0); - lh->last_mod_file_time(1); // 00:00:01 - lh->last_mod_file_date(33); // 1980-01-01 - lh->crc32(0x12345678); - lh->compressed_file_size32(0); - lh->file_name(filename_.c_str(), filename_.size()); - - if (huge_buffer) { - // Add Z64 extension if this is a huge entry. - lh->uncompressed_file_size32(0xFFFFFFFF); - Zip64ExtraField *z64 = - reinterpret_cast<Zip64ExtraField *>(zip64_extension_buffer); - z64->signature(); - z64->payload_size(2 * sizeof(uint64_t)); - z64->attr64(0, buffer_->data_size()); - lh->extra_fields(reinterpret_cast<uint8_t *>(z64), z64->size()); - } else { - lh->uncompressed_file_size32(buffer_->data_size()); - lh->extra_fields(nullptr, 0); - } - - uint32_t checksum; - uint64_t compressed_size; - uint16_t method = buffer_->Write(lh->data(), &checksum, &compressed_size); - lh->crc32(checksum); - lh->compression_method(method); - if (huge_buffer) { - lh->compressed_file_size32(compressed_size < 0xFFFFFFFF ? compressed_size - : 0xFFFFFFFF); - // Not sure if this has to be written in the small case, but it shouldn't - // hurt. - const_cast<Zip64ExtraField *>(lh->zip64_extra_field()) - ->attr64(1, compressed_size); - } else { - // If original data is <4GB, the compressed one is, too. - lh->compressed_file_size32(compressed_size); - } - return reinterpret_cast<void *>(lh); - } + void *OutputEntry() override; void Append(const char *s, size_t n) { CreateBuffer(); @@ -133,35 +66,27 @@ class Concatenator { std::unique_ptr<Inflater> inflater_; }; +// The combiner that does nothing. Useful to represent for instance directory +// entries: once a directory entry has been created and added to the output +// jar, the subsequent entries are ignored on input, and nothing is output. +class NullCombiner : public Combiner { + public: + ~NullCombiner() override; + bool Merge(const CDH *cdh, const LH *lh) override; + void *OutputEntry() override; +}; + // Combines the contents of the multiple input entries which are XML // files into a single XML output entry with given top level XML tag. -class XmlCombiner { +class XmlCombiner : public Combiner { public: XmlCombiner(const std::string &filename, const char *xml_tag) : filename_(filename), xml_tag_(xml_tag) {} + ~XmlCombiner() override; - bool Merge(const CDH *cdh, const LH *lh) { - if (!concatenator_.get()) { - concatenator_.reset(new Concatenator(filename_)); - concatenator_->Append("<"); - concatenator_->Append(xml_tag_); - concatenator_->Append(">\n"); - } - return concatenator_->Merge(cdh, lh); - } + bool Merge(const CDH *cdh, const LH *lh) override; - // Returns a pointer to the buffer containing LocalHeader for the entry, - // immediately followed by entry payload. The caller is responsible for - // freeing the buffer. - void *OutputEntry() { - if (!concatenator_.get()) { - return nullptr; - } - concatenator_->Append("</"); - concatenator_->Append(xml_tag_); - concatenator_->Append(">\n"); - return concatenator_->OutputEntry(); - } + void *OutputEntry() override; const std::string filename() const { return filename_; } @@ -175,9 +100,14 @@ class XmlCombiner { // A wrapper around Concatenator allowing to append // NAME=VALUE // lines to the contents. +// NOTE that it does not allow merging existing entries. class PropertyCombiner : public Concatenator { public: PropertyCombiner(const std::string &filename) : Concatenator(filename) {} + ~PropertyCombiner(); + + bool Merge(const CDH *cdh, const LH *lh) override; + void AddProperty(const char *key, const char *value) { // TODO(asmundak): deduplicate properties. Append(key); diff --git a/src/tools/singlejar/combiners_test.cc b/src/tools/singlejar/combiners_test.cc index 9b6701c9ea..f6da247721 100644 --- a/src/tools/singlejar/combiners_test.cc +++ b/src/tools/singlejar/combiners_test.cc @@ -118,6 +118,13 @@ TEST_F(CombinersTest, ConcatenatorHuge) { free(reinterpret_cast<void *>(entry)); } +// Test NullCombiner. +TEST_F(CombinersTest, NullCombiner) { + NullCombiner null_combiner; + ASSERT_TRUE(null_combiner.Merge(nullptr, nullptr)); + ASSERT_EQ(nullptr, null_combiner.OutputEntry()); +} + // Test XmlCombiner. TEST_F(CombinersTest, XmlCombiner) { InputJar input_jar; @@ -155,6 +162,7 @@ TEST_F(CombinersTest, XmlCombiner) { free(reinterpret_cast<void *>(entry)); } + // Test PropertyCombiner. TEST_F(CombinersTest, PropertyCombiner) { static char kProperties[] = @@ -165,6 +173,9 @@ TEST_F(CombinersTest, PropertyCombiner) { property_combiner.AddProperty(std::string("name_str"), std::string("value_str")); + // Merge should not be called. + ASSERT_FALSE(property_combiner.Merge(nullptr, nullptr)); + // Create output, verify Local Header contents. LH *entry = reinterpret_cast<LH *>(property_combiner.OutputEntry()); EXPECT_TRUE(entry->is()); diff --git a/src/tools/singlejar/output_jar.cc b/src/tools/singlejar/output_jar.cc index a81e35f06b..b0748ab153 100644 --- a/src/tools/singlejar/output_jar.cc +++ b/src/tools/singlejar/output_jar.cc @@ -57,16 +57,14 @@ OutputJar::OutputJar() manifest_("META-INF/MANIFEST.MF"), build_properties_("build-data.properties") { known_members_.emplace(spring_handlers_.filename(), - EntryInfo{EntryInfo::CONCATENATE, &spring_handlers_}); + EntryInfo{&spring_handlers_}); known_members_.emplace(spring_schemas_.filename(), - EntryInfo{EntryInfo::CONCATENATE, &spring_schemas_}); - known_members_.emplace(manifest_.filename(), - EntryInfo{EntryInfo::SKIP, &manifest_}); - known_members_.emplace( - protobuf_meta_handler_.filename(), - EntryInfo{EntryInfo::CONCATENATE, &protobuf_meta_handler_}); + EntryInfo{&spring_schemas_}); + known_members_.emplace(manifest_.filename(), EntryInfo{&manifest_}); + known_members_.emplace(protobuf_meta_handler_.filename(), + EntryInfo{&protobuf_meta_handler_}); known_members_.emplace(build_properties_.filename(), - EntryInfo{EntryInfo::SKIP, &build_properties_}); + EntryInfo{&build_properties_}); manifest_.Append( "Manifest-Version: 1.0\r\n" "Created-By: singlejar\r\n"); @@ -203,7 +201,7 @@ int OutputJar::Doit(Options *options) { } // Then copy source files' contents. - for (size_t ix = 0; ix < options_->input_jars.size(); ++ix) { + for (int ix = 0; ix < options_->input_jars.size(); ++ix) { if (!AddJar(ix)) { exit(1); } @@ -239,7 +237,7 @@ bool OutputJar::Open() { return true; } -bool OutputJar::AddJar(size_t jar_path_index) { +bool OutputJar::AddJar(int jar_path_index) { const std::string& input_jar_path = options_->input_jars[jar_path_index]; InputJar input_jar; if (!input_jar.Open(input_jar_path)) { @@ -259,56 +257,57 @@ bool OutputJar::AddJar(size_t jar_path_index) { // Special files that cannot be handled by looking up known_members_ map: // * ignore *.SF, *.RSA, *.DSA // (TODO(asmundak): should this be done only in META-INF? - // * concatenate the contents of each file META-INF/services/ directory // if (ends_with(file_name, file_name_length, ".SF") || ends_with(file_name, file_name_length, ".RSA") || ends_with(file_name, file_name_length, ".DSA")) { continue; - } else if (file_name[file_name_length - 1] != '/' && - begins_with(file_name, file_name_length, "META-INF/services/")) { + } + + bool is_dir = (file_name[file_name_length - 1] != '/'); + if (is_dir && + begins_with(file_name, file_name_length, "META-INF/services/")) { + // The contents of the META-INF/services/<SERVICE> on the output is the + // concatenation of the META-INF/services/<SERVICE> files from all inputs. std::string service_path(file_name, file_name_length); if (!known_members_.count(service_path)) { + // Create a concatenator and add it to the known_members_ map. + // The call to Merge() below will then take care of the rest. Concatenator *service_handler = new Concatenator(service_path); service_handlers_.emplace_back(service_handler); - known_members_.emplace( - service_path, EntryInfo{EntryInfo::CONCATENATE, service_handler}); + known_members_.emplace(service_path, EntryInfo{service_handler}); } } - auto got = known_members_.emplace( - std::string(file_name, file_name_length), - EntryInfo{EntryInfo::PLAIN, reinterpret_cast<void *>(jar_path_index)}); + + // Install a new entry unless it is already present. All the plain (non-dir) + // entries that require a combiner have been already installed, so the call + // will add either a directory entry whose handler will ignore subsequent + // duplicates, or an ordinary plain entry, for which we save the index of + // the first input jar (in order to provide diagnostics on duplicate). + auto got = + known_members_.emplace(std::string(file_name, file_name_length), + EntryInfo{is_dir ? &null_combiner_ : nullptr, + is_dir ? -1 : jar_path_index}); if (!got.second) { - // We allow duplicate entries in special cases: - // - various combiners - // - directory entries - // - manifest files - if (got.first->second.type_ == EntryInfo::XML_COMBINE) { - reinterpret_cast<XmlCombiner *>(got.first->second.data_) - ->Merge(jar_entry, lh); - continue; - } else if (got.first->second.type_ == EntryInfo::CONCATENATE) { - reinterpret_cast<Concatenator *>(got.first->second.data_) - ->Merge(jar_entry, lh); - continue; - } else if (got.first->second.type_ == EntryInfo::SKIP) { - continue; - } else if (file_name[file_name_length - 1] == '/') { + auto &entry_info = got.first->second; + // Handle special entries (the ones that have a combiner. + if (entry_info.combiner_ != nullptr) { + entry_info.combiner_->Merge(jar_entry, lh); continue; + } + + // Plain file entry. If duplicates are not allowed, bail out. Otherwise + // just ignore this entry. + if (options_->no_duplicates || + (options_->no_duplicate_classes && + ends_with(file_name, file_name_length, ".class"))) { + diag_errx(1, "%s:%d: %.*s is present both in %s and %s", __FILE__, + __LINE__, file_name_length, file_name, + options_->input_jars[entry_info.input_jar_index_].c_str(), + input_jar_path.c_str()); } else { - if (options_->no_duplicates || - (options_->no_duplicate_classes && - ends_with(file_name, file_name_length, ".class"))) { - auto previous_input_jar_index = - reinterpret_cast<size_t>(got.first->second.data_); - diag_errx(1, "%s:%d: %.*s is present both in %s and %s", __FILE__, - __LINE__, file_name_length, file_name, - options_->input_jars[previous_input_jar_index].c_str(), - input_jar_path.c_str()); - } else { - duplicate_entries_++; - continue; - } + duplicate_entries_++; + continue; } } @@ -429,7 +428,7 @@ void OutputJar::AddDirectory(const char *path) { lh->uncompressed_file_size32(0); lh->file_name(path, n_path); lh->extra_fields(nullptr, 0); - known_members_.emplace(path, EntryInfo{EntryInfo::SKIP, nullptr}); + known_members_.emplace(path, EntryInfo{&null_combiner_}); WriteEntry(lh); } @@ -540,8 +539,7 @@ void OutputJar::ClasspathResource(const std::string &resource_name, classpath_resource->Append( reinterpret_cast<const char *>(mapped_file.start()), mapped_file.size()); classpath_resources_.emplace_back(classpath_resource); - known_members_.emplace(resource_name, - EntryInfo{EntryInfo::PLAIN, classpath_resource}); + known_members_.emplace(resource_name, EntryInfo{classpath_resource}); } #if defined(__APPLE__) diff --git a/src/tools/singlejar/output_jar.h b/src/tools/singlejar/output_jar.h index f723e41077..b46a37d078 100644 --- a/src/tools/singlejar/output_jar.h +++ b/src/tools/singlejar/output_jar.h @@ -40,7 +40,7 @@ class OutputJar { // Open output jar. bool Open(); // Add the contents of the given input jar. - bool AddJar(size_t jar_path_index); + bool AddJar(int jar_path_index); // Returns the current output position. off_t Position(); // Write Jar entry. @@ -78,8 +78,10 @@ class OutputJar { Options *options_; struct EntryInfo { - enum EntryType { PLAIN, XML_COMBINE, CONCATENATE, SKIP } type_; - void *data_; // TODO(asmundak): use virtual dispatch instead. + EntryInfo(Combiner *combiner, int index = -1) + : combiner_(combiner), input_jar_index_(index) {} + Combiner *combiner_; + int input_jar_index_; // Input jar index for the plain entry or -1. }; std::unordered_map<std::string, struct EntryInfo> known_members_; @@ -94,6 +96,7 @@ class OutputJar { Concatenator protobuf_meta_handler_; Concatenator manifest_; PropertyCombiner build_properties_; + NullCombiner null_combiner_; std::vector<std::unique_ptr<Concatenator> > service_handlers_; std::vector<std::unique_ptr<Concatenator> > classpath_resources_; }; |