path: root/src/tools/android/java/com/google/devtools/build/android/ZipFilterAction.java

// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android;

import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.IValueValidator;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Multimap;
import com.google.devtools.build.singlejar.ZipCombiner;
import com.google.devtools.build.singlejar.ZipCombiner.OutputMode;
import com.google.devtools.build.zip.ZipFileEntry;
import com.google.devtools.build.zip.ZipReader;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import java.util.regex.Pattern;

/**
 * Action to filter entries out of a Zip file.
 *
 * <p>The entries to remove are determined from the filterZips and filterTypes. All entries from the
 * filter Zip files that have an extension listed in filterTypes will be removed. If no filterZips
 * are specified, no entries will be removed. Specifying no filterTypes is treated as if an
 * extension of '.*' was specified.
 *
 * <p>Assuming each Zip as a set of entries, the result is:
 *
 * <pre> outputZip = inputZip - union[x intersect filterTypes for x in filterZips]</pre>
 *
 * <p>
 *
 * <pre>
 * Example Usage:
 *   java/com/google/build/android/ZipFilterAction\
 *      --inputZip path/to/inputZip
 *      --outputZip path/to/outputZip
 *      --filterZips [path/to/filterZip[,path/to/filterZip]...]
 *      --filterTypes [fileExtension[,fileExtension]...]
 *      --explicitFilters [fileRegex[,fileRegex]...]
 *      --outputMode [DONT_CARE|FORCE_DEFLATE|FORCE_STORED]
 *      --checkHashMismatch [IGNORE|WARN|ERROR]
 * </pre>
 */
public class ZipFilterAction {

  private static final Logger logger = Logger.getLogger(ZipFilterAction.class.getName());

  /** Modes of performing content hash checking during zip filtering. */
  public enum HashMismatchCheckMode {
    /** Filter file from input zip iff a file is found with the same filename in filter zips. */
    IGNORE,

    /**
     * Filter file from input zip iff a file is found with the same filename and content hash in
     * filter zips. Print warning if the filename is identical but content hash is not.
     */
    WARN,

    /**
     * Same behavior as WARN, but throw an error if a file is found with the same filename with
     * different content hash.
     */
    ERROR
  }

  @Parameters(optionPrefixes = "--")
  static class Options {
    @Parameter(
      names = "--inputZip",
      description = "Path of input zip.",
      converter = PathFlagConverter.class,
      validateValueWith = PathExistsValidator.class
    )
    Path inputZip;

    @Parameter(
      names = "--outputZip",
      description = "Path to write output zip.",
      converter = PathFlagConverter.class
    )
    Path outputZip;

    @Parameter(
      names = "--filterZips",
      description = "Filter zips.",
      converter = PathFlagConverter.class,
      validateValueWith = AllPathsExistValidator.class
    )
    List<Path> filterZips = ImmutableList.of();

    @Parameter(names = "--filterTypes", description = "Filter file types.")
    List<String> filterTypes = ImmutableList.of();

    @Parameter(names = "--explicitFilters", description = "Explicitly specified filters.")
    List<String> explicitFilters = ImmutableList.of();

    @Parameter(names = "--outputMode", description = "Output zip compression mode.")
    OutputMode outputMode = OutputMode.DONT_CARE;

    @Parameter(
      names = "--checkHashMismatch",
      description =
          "Ignore, warn or throw an error if the content hashes of two files with the "
              + "same name are different."
    )
    HashMismatchCheckMode hashMismatchCheckMode = HashMismatchCheckMode.WARN;

    /**
     * @deprecated please use --checkHashMismatch ERROR instead. Other options are IGNORE and WARN.
     */
    @Deprecated
    @Parameter(
      names = "--errorOnHashMismatch",
      description = "Error on entry filter with hash mismatch."
    )
    boolean errorOnHashMismatch = false;

    /**
     * @deprecated please use --checkHashMismatch WARN instead. Other options are IGNORE and WARN.
     *     <p>This is a hack to support existing users of --noerrorOnHashMismatch. JCommander does
     *     not support setting boolean flags with "--no", so instead we set the default to false and
     *     just ignore anyone who passes --noerrorOnHashMismatch.
     */
    @Deprecated
    @Parameter(names = "--noerrorOnHashMismatch")
    boolean ignored = false;
  }

  /** Converts string flags to paths. Public because JCommander invokes this by reflection. */
  public static class PathFlagConverter implements IStringConverter<Path> {

    @Override
    public Path convert(String text) {
      return FileSystems.getDefault().getPath(text);
    }
  }

  /** Validates that a path exists. Public because JCommander invokes this by reflection. */
  public static class PathExistsValidator implements IValueValidator<Path> {

    @Override
    public void validate(String s, Path path) {
      if (!Files.exists(path)) {
        throw new ParameterException(String.format("%s is not a valid path.", path.toString()));
      }
    }
  }

  /** Validates that a set of paths exist. Public because JCommander invokes this by reflection. */
  public static class AllPathsExistValidator implements IValueValidator<List<Path>> {

    @Override
    public void validate(String s, List<Path> paths) {
      for (Path path : paths) {
        if (!Files.exists(path)) {
          throw new ParameterException(String.format("%s is not a valid path.", path.toString()));
        }
      }
    }
  }

  @VisibleForTesting
  static Multimap<String, Long> getEntriesToOmit(
      Collection<Path> filterZips, Collection<String> filterTypes) throws IOException {
    // Escape filter types to prevent regex abuse
    Set<String> escapedFilterTypes = new HashSet<>();
    for (String filterType : filterTypes) {
      escapedFilterTypes.add(Pattern.quote(filterType));
    }
    // Match any string that ends with any of the filter file types
    String filterRegex = String.format(".*(%s)$", Joiner.on("|").join(escapedFilterTypes));

    ImmutableSetMultimap.Builder<String, Long> entriesToOmit = ImmutableSetMultimap.builder();
    for (Path filterZip : filterZips) {
      try (ZipReader zip = new ZipReader(filterZip.toFile())) {
        for (ZipFileEntry entry : zip.entries()) {
          if (filterTypes.isEmpty() || entry.getName().matches(filterRegex)) {
            entriesToOmit.put(entry.getName(), entry.getCrc());
          }
        }
      }
    }
    return entriesToOmit.build();
  }

  public static void main(String[] args) throws IOException {
    System.exit(run(args));
  }

  static int run(String[] args) throws IOException {
    Options options = new Options();
    new JCommander(options).parse(args);
    logger.fine(
        String.format(
            "Creating filter from entries of type %s, in zip files %s.",
            options.filterTypes, options.filterZips));

    final Stopwatch timer = Stopwatch.createStarted();
    Multimap<String, Long> entriesToOmit =
        getEntriesToOmit(options.filterZips, options.filterTypes);
    final String explicitFilter =
        options.explicitFilters.isEmpty()
            ? ""
            : String.format(".*(%s).*", Joiner.on("|").join(options.explicitFilters));
    logger.fine(String.format("Filter created in %dms", timer.elapsed(TimeUnit.MILLISECONDS)));

    ImmutableMap.Builder<String, Long> inputEntries = ImmutableMap.builder();
    try (ZipReader input = new ZipReader(options.inputZip.toFile())) {
      for (ZipFileEntry entry : input.entries()) {
        inputEntries.put(entry.getName(), entry.getCrc());
      }
    }

    // TODO(jingwen): Remove --errorOnHashMismatch when Blaze release with --checkHashMismatch
    // is checked in.
    if (options.errorOnHashMismatch) {
      options.hashMismatchCheckMode = HashMismatchCheckMode.ERROR;
    }
    ZipFilterEntryFilter entryFilter =
        new ZipFilterEntryFilter(
            explicitFilter, entriesToOmit, inputEntries.build(), options.hashMismatchCheckMode);

    try (OutputStream out = Files.newOutputStream(options.outputZip);
        ZipCombiner combiner = new ZipCombiner(options.outputMode, entryFilter, out)) {
      combiner.addZip(options.inputZip.toFile());
    }
    logger.fine(String.format("Filtering completed in %dms", timer.elapsed(TimeUnit.MILLISECONDS)));
    return entryFilter.sawErrors() ? 1 : 0;
  }
}