// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/tools/singlejar/combiners.h" #include "src/tools/singlejar/diag.h" #include "src/main/protobuf/desugar_deps.pb.h" Combiner::~Combiner() {} Concatenator::~Concatenator() {} bool Concatenator::Merge(const CDH *cdh, const LH *lh) { if (insert_newlines_ && buffer_.get() && buffer_->data_size() && '\n' != buffer_->last_byte()) { Append("\n", 1); } CreateBuffer(); if (Z_NO_COMPRESSION == lh->compression_method()) { buffer_->ReadEntryContents(lh); } else if (Z_DEFLATED == lh->compression_method()) { if (!inflater_.get()) { inflater_.reset(new Inflater()); } buffer_->DecompressEntryContents(cdh, lh, inflater_.get()); } else { errx(2, "%s is neither stored nor deflated", filename_.c_str()); } return true; } void *Concatenator::OutputEntry(bool compress) { if (!buffer_.get()) { return nullptr; } // Allocate a contiguous buffer for the local file header and // deflated data. We assume that deflate decreases the size, so if // the deflater reports overflow, we just save original data. size_t deflated_buffer_size = sizeof(LH) + filename_.size() + buffer_->data_size(); // Huge entry (>4GB) needs Zip64 extension field with 64-bit original // and compressed size values. uint8_t zip64_extension_buffer[sizeof(Zip64ExtraField) + 2 * sizeof(uint64_t)]; bool huge_buffer = ziph::zfield_needs_ext64(buffer_->data_size()); if (huge_buffer) { deflated_buffer_size += sizeof(zip64_extension_buffer); } LH *lh = reinterpret_cast(malloc(deflated_buffer_size)); if (lh == nullptr) { return nullptr; } lh->signature(); lh->version(20); lh->bit_flag(0x0); lh->last_mod_file_time(1); // 00:00:01 lh->last_mod_file_date(33); // 1980-01-01 lh->crc32(0x12345678); lh->compressed_file_size32(0); lh->file_name(filename_.c_str(), filename_.size()); if (huge_buffer) { // Add Z64 extension if this is a huge entry. lh->uncompressed_file_size32(0xFFFFFFFF); Zip64ExtraField *z64 = reinterpret_cast(zip64_extension_buffer); z64->signature(); z64->payload_size(2 * sizeof(uint64_t)); z64->attr64(0, buffer_->data_size()); lh->extra_fields(reinterpret_cast(z64), z64->size()); } else { lh->uncompressed_file_size32(buffer_->data_size()); lh->extra_fields(nullptr, 0); } uint32_t checksum; uint64_t compressed_size; uint16_t method; if (compress) { method = buffer_->CompressOut(lh->data(), &checksum, &compressed_size); } else { buffer_->CopyOut(lh->data(), &checksum); method = Z_NO_COMPRESSION; compressed_size = buffer_->data_size(); } lh->crc32(checksum); lh->compression_method(method); if (huge_buffer) { lh->compressed_file_size32(ziph::zfield_needs_ext64(compressed_size) ? 0xFFFFFFFF : compressed_size); // Not sure if this has to be written in the small case, but it shouldn't // hurt. const_cast(lh->zip64_extra_field()) ->attr64(1, compressed_size); } else { // If original data is <4GB, the compressed one is, too. lh->compressed_file_size32(compressed_size); } return reinterpret_cast(lh); } NullCombiner::~NullCombiner() {} bool NullCombiner::Merge(const CDH *cdh, const LH *lh) { return true; } void *NullCombiner::OutputEntry(bool compress) { return nullptr; } XmlCombiner::~XmlCombiner() {} bool XmlCombiner::Merge(const CDH *cdh, const LH *lh) { if (!concatenator_.get()) { concatenator_.reset(new Concatenator(filename_, false)); concatenator_->Append(start_tag_); concatenator_->Append("\n"); } // To ensure xml concatentation is idempotent, read in the entry being added // and remove the start and end tags if they are present. TransientBytes bytes_; if (Z_NO_COMPRESSION == lh->compression_method()) { bytes_.ReadEntryContents(lh); } else if (Z_DEFLATED == lh->compression_method()) { if (!inflater_.get()) { inflater_.reset(new Inflater()); } bytes_.DecompressEntryContents(cdh, lh, inflater_.get()); } else { errx(2, "%s is neither stored nor deflated", filename_.c_str()); } uint32_t checksum; char *buf = reinterpret_cast(malloc(bytes_.data_size())); // TODO(b/37631490): optimize this to avoid copying the bytes twice bytes_.CopyOut(reinterpret_cast(buf), &checksum); int start_offset = 0; if (strncmp(buf, start_tag_.c_str(), start_tag_.length()) == 0) { start_offset = start_tag_.length(); } uint64_t end = bytes_.data_size(); while (end >= end_tag_.length() && std::isspace(buf[end - 1])) end--; if (strncmp(buf + end - end_tag_.length(), end_tag_.c_str(), end_tag_.length()) == 0) { end -= end_tag_.length(); } else { // Leave trailing whitespace alone if we didn't find a match. end = bytes_.data_size(); } concatenator_->Append(buf + start_offset, end - start_offset); free(buf); return true; } void *XmlCombiner::OutputEntry(bool compress) { if (!concatenator_.get()) { return nullptr; } concatenator_->Append(end_tag_); concatenator_->Append("\n"); return concatenator_->OutputEntry(compress); } PropertyCombiner::~PropertyCombiner() {} bool PropertyCombiner::Merge(const CDH *cdh, const LH *lh) { return false; // This should not be called. } bool Java8DesugarDepsChecker::Merge(const CDH *cdh, const LH *lh) { // Throw away anything previously read, no need to concatenate buffer_.reset(new TransientBytes()); if (Z_NO_COMPRESSION == lh->compression_method()) { buffer_->ReadEntryContents(lh); } else if (Z_DEFLATED == lh->compression_method()) { if (!inflater_.get()) { inflater_.reset(new Inflater()); } buffer_->DecompressEntryContents(cdh, lh, inflater_.get()); } else { errx(2, "META-INF/desugar_deps is neither stored nor deflated"); } // TODO(kmb): Wrap buffer_ as ZeroCopyInputStream to avoid copying out. // Note we only copy one file at a time, so overhead should be modest. uint32_t checksum; const size_t data_size = buffer_->data_size(); uint8_t *buf = reinterpret_cast(malloc(data_size)); buffer_->CopyOut(reinterpret_cast(buf), &checksum); buffer_.reset(); // release buffer eagerly bazel::tools::desugar::DesugarDepsInfo deps_info; google::protobuf::io::CodedInputStream content(buf, data_size); if (!deps_info.ParseFromCodedStream(&content)) { errx(2, "META-INF/desugar_deps: unable to parse"); } if (!content.ConsumedEntireMessage()) { errx(2, "META-INF/desugar_deps: unexpected trailing content"); } free(buf); for (const auto &assume_present : deps_info.assume_present()) { // This means we need file named .class in the output. Remember // the first origin of this requirement for error messages, drop others. needed_deps_.emplace(assume_present.target().binary_name() + ".class", assume_present.origin().binary_name()); } for (const auto &missing : deps_info.missing_interface()) { // Remember the first origin of this requirement for error messages, drop // subsequent ones. missing_interfaces_.emplace(missing.target().binary_name(), missing.origin().binary_name()); } for (const auto &extends : deps_info.interface_with_supertypes()) { // Remember interface hierarchy the first time we see this interface, drop // subsequent ones for consistency with how singlejar will keep the first // occurrence of the file defining the interface. We'll lazily derive // whether missing_interfaces_ inherit default methods with this data later. if (extends.extended_interface_size() > 0) { std::vector extended; extended.reserve(extends.extended_interface_size()); for (const auto &itf : extends.extended_interface()) { extended.push_back(itf.binary_name()); } extended_interfaces_.emplace(extends.origin().binary_name(), std::move(extended)); } } for (const auto &companion : deps_info.interface_with_companion()) { // Only remember interfaces that definitely have default methods for now. // For all other interfaces we'll transitively check extended interfaces // in HasDefaultMethods. if (companion.num_default_methods() > 0) { has_default_methods_[companion.origin().binary_name()] = true; } } return true; } void *Java8DesugarDepsChecker::OutputEntry(bool compress) { if (verbose_) { fprintf(stderr, "Needed deps: %lu\n", needed_deps_.size()); fprintf(stderr, "Interfaces to check: %lu\n", missing_interfaces_.size()); fprintf(stderr, "Sub-interfaces: %lu\n", extended_interfaces_.size()); fprintf(stderr, "Interfaces w/ default methods: %lu\n", has_default_methods_.size()); } for (auto needed : needed_deps_) { if (verbose_) { fprintf(stderr, "Looking for %s\n", needed.first.c_str()); } if (!known_member_(needed.first)) { if (fail_on_error_) { errx(2, "%s referenced by %s but not found. Is the former defined in " "a neverlink library?", needed.first.c_str(), needed.second.c_str()); } else { error_ = true; } } } for (auto missing : missing_interfaces_) { if (verbose_) { fprintf(stderr, "Checking %s\n", missing.first.c_str()); } if (HasDefaultMethods(missing.first)) { if (fail_on_error_) { errx(2, "%s needed on the classpath for desugaring %s. Please add the " "missing dependency to the target containing the latter.", missing.first.c_str(), missing.second.c_str()); } else { error_ = true; } } } // We don't want these files in the output, just check them for consistency return nullptr; } bool Java8DesugarDepsChecker::HasDefaultMethods( const std::string &interface_name) { auto cached = has_default_methods_.find(interface_name); if (cached != has_default_methods_.end()) { return cached->second; } // Prime with false in case there's a cycle. We'll update with the true value // (ignoring the cycle) below. has_default_methods_.emplace(interface_name, false); for (const std::string &extended : extended_interfaces_[interface_name]) { if (HasDefaultMethods(extended)) { has_default_methods_[interface_name] = true; return true; } } has_default_methods_[interface_name] = false; return false; }