path: root/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileAggregator.java

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android.dexer;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;

import com.android.dex.Dex;
import com.android.dx.command.dexer.DxContext;
import com.android.dx.merge.CollisionPolicy;
import com.android.dx.merge.DexMerger;
import com.google.common.base.Throwables;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import java.io.Closeable;
import java.io.IOException;
import java.nio.BufferOverflowException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.zip.ZipEntry;

/**
 * Merger for {@code .dex} files into larger chunks subject to {@code .dex} file limits on methods
 * and fields.
 */
class DexFileAggregator implements Closeable {

  /**
   * File extension of a {@code .dex} file.
   */
  private static final String DEX_EXTENSION = ".dex";

  private final ArrayList<Dex> currentShard = new ArrayList<>();
  private final boolean forceJumbo;
  private final int wasteThresholdPerDex;
  private final MultidexStrategy multidex;
  private final DxContext context;
  private final ListeningExecutorService executor;
  private final DexFileArchive dest;
  private final String dexPrefix;
  private final DexLimitTracker tracker;

  private int nextDexFileIndex = 0;
  private ListenableFuture<Void> lastWriter = Futures.<Void>immediateFuture(null);

  public DexFileAggregator(
      DxContext context,
      DexFileArchive dest,
      ListeningExecutorService executor,
      MultidexStrategy multidex,
      boolean forceJumbo,
      int maxNumberOfIdxPerDex,
      int wasteThresholdPerDex,
      String dexPrefix) {
    this.context = context;
    this.dest = dest;
    this.executor = executor;
    this.multidex = multidex;
    this.forceJumbo = forceJumbo;
    this.wasteThresholdPerDex = wasteThresholdPerDex;
    this.dexPrefix = dexPrefix;
    tracker = new DexLimitTracker(maxNumberOfIdxPerDex);
  }

  public DexFileAggregator add(Dex dexFile) {
    if (multidex.isMultidexAllowed()) {
      // To determine whether currentShard is "full" we track unique field and method signatures,
      // which predicts precisely the number of field and method indices.
      if (tracker.track(dexFile) && !currentShard.isEmpty()) {
        // For simplicity just start a new shard to fit the given file.
        // Don't bother with waiting for a later file that might fit the old shard as in the extreme
        // we'd have to wait until the end to write all shards.
        rotateDexFile();
        tracker.track(dexFile);
      }
    }
    currentShard.add(dexFile);
    return this;
  }

  @Override
  public void close() throws IOException {
    try {
      if (!currentShard.isEmpty()) {
        rotateDexFile();
      }
      // Wait for last shard to be written before closing underlying archive
      lastWriter.get();
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    } catch (ExecutionException e) {
      Throwables.throwIfInstanceOf(e.getCause(), IOException.class);
      Throwables.throwIfUnchecked(e.getCause());
      throw new AssertionError("Unexpected execution exception", e);
    } finally {
      dest.close();
    }
  }

  public void flush() {
    checkState(multidex.isMultidexAllowed());
    if (!currentShard.isEmpty()) {
      rotateDexFile();
    }
  }

  public int getDexFilesWritten() {
    return nextDexFileIndex;
  }

  private void rotateDexFile() {
    writeMergedFile(currentShard.toArray(/* apparently faster than pre-sized array */ new Dex[0]));
    currentShard.clear();
    tracker.clear();
  }

  private void writeMergedFile(Dex... dexes) {
    checkArgument(0 < dexes.length);
    checkState(multidex.isMultidexAllowed() || nextDexFileIndex == 0);
    String filename = getDexFileName(nextDexFileIndex++);
    ListenableFuture<Dex> merged =
        dexes.length == 1 && !forceJumbo
            ? Futures.immediateFuture(dexes[0])
            : executor.submit(new RunDexMerger(dexes));
    lastWriter =
        Futures.whenAllSucceed(lastWriter, merged)
            .call(new WriteFile(filename, merged, dest), executor);
  }

  private Dex merge(Dex... dexes) throws IOException {
    switch (dexes.length) {
      case 0:
        return new Dex(0);
      case 1:
        // Need to actually process the single given file for forceJumbo :(
        return forceJumbo ? merge(dexes[0], new Dex(0)) : dexes[0];
      default: // fall out
    }
    DexMerger dexMerger = new DexMerger(dexes, CollisionPolicy.FAIL, context);
    dexMerger.setCompactWasteThreshold(wasteThresholdPerDex);
    if (forceJumbo) {
      try {
        DexMerger.class.getMethod("setForceJumbo", Boolean.TYPE).invoke(dexMerger, true);
      } catch (ReflectiveOperationException e) {
        throw new IllegalStateException("--forceJumbo flag not supported", e);
      }
    }

    try {
      return dexMerger.merge();
    } catch (BufferOverflowException e) {
      if (dexes.length <= 2) {
        throw e;
      }
      // Bug in dx can cause this for ~1500 or more classes
      Dex[] left = Arrays.copyOf(dexes, dexes.length / 2);
      Dex[] right = Arrays.copyOfRange(dexes, left.length, dexes.length);
      System.err.printf("Couldn't merge %d classes, trying %d%n", dexes.length, left.length);
      try {
        return merge(merge(left), merge(right));
      } catch (RuntimeException e2) {
        e2.addSuppressed(e);
        throw e2;
      }
    }
  }

  // More or less copied from from com.android.dx.command.dexer.Main
  private String getDexFileName(int i) {
    return dexPrefix + (i == 0 ? "" : i + 1) + DEX_EXTENSION;
  }


  private class RunDexMerger implements Callable<Dex> {

    private final Dex[] dexes;

    public RunDexMerger(Dex... dexes) {
      this.dexes = dexes;
    }

    @Override
    public Dex call() throws IOException {
      try {
        return merge(dexes);
      } catch (Throwable t) {
        // Print out exceptions so they don't get swallowed completely
        t.printStackTrace();
        Throwables.throwIfInstanceOf(t, IOException.class);
        Throwables.throwIfUnchecked(t);
        throw new AssertionError(t);  // shouldn't get here
      }
    }
  }

  private static class WriteFile implements Callable<Void> {

    private final ListenableFuture<Dex> dex;
    private final String filename;
    @SuppressWarnings ("hiding") private final DexFileArchive dest;

    public WriteFile(String filename, ListenableFuture<Dex> dex, DexFileArchive dest) {
      this.filename = filename;
      this.dex = dex;
      this.dest = dest;
    }

    @Override
    public Void call() throws Exception {
      try {
        checkState(dex.isDone());
        ZipEntry entry = new ZipEntry(filename);
        entry.setTime(0L); // Use simple stable timestamps for deterministic output
        dest.addFile(entry, dex.get());
        return null;
      } catch (Exception e) {
        // Print out exceptions so they don't get swallowed completely
        e.printStackTrace();
        throw e;
      } catch (Throwable t) {
        t.printStackTrace();
        Throwables.throwIfUnchecked(t);
        throw new AssertionError(t);  // shouldn't get here
      }
    }
  }
}