Rollback of unknown previous commit.

*** Reason for rollback ***

Broke bazel build.

*** Original change description ***

Update iossim for Xcode 7 support.
As of Xcode 7, supportedDeviceTypesByName was replaced by supportedDeviceTypesByAlias.

This is from latest chromium build.
Downstream hash is 9dd179a339c0457f8754069e0774b38f69c258a8.
The latest merge was to upstream ef05b7da00844c0d500c4a7f20d4095dab56e7fe

***


Also includes the following changes:

Size the Lexer tokenization to minimize internal resizing. This value is chosen empirically.

--
Fixes toolchain selection in the generated Android NDK crosstools by making
each target_cpu and compiler field unique.

Note that there are some problems with the clang compilers (e.g. can't find ld), which I'll fix in a subsequent change.

--
Update iossim for Xcode 7 support.
As of Xcode 7, supportedDeviceTypesByName was replaced by supportedDeviceTypesByAlias.

This is from latest chromium build.
Downstream hash is 9dd179a339c0457f8754069e0774b38f69c258a8.
The latest merge was to upstream ef05b7da00844c0d500c4a7f20d4095dab56e7fe

--
MOS_MIGRATED_REVID=105337154

7 files changed, 71 insertions, 75 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/ArmCrosstools.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/ArmCrosstools.java
index 9fdb49d67b..ed4dafc5b9 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/ArmCrosstools.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/ArmCrosstools.java
@@ -143,7 +143,7 @@ class ArmCrosstools {
         .setToolchainIdentifier("aarch64-linux-android-clang" + clangVersion)
         .setTargetSystemName("aarch64-linux-android")
         .setTargetCpu("arm64-v8a")
-        .setCompiler("gcc-4.9")
+        .setCompiler("gcc-4.9-clang" + clangVersion)
 
         .addAllToolPath(ndkPaths.createClangToolpaths(toolchainName, targetPlatform, clangVersion))
 
@@ -201,7 +201,7 @@ class ArmCrosstools {
         .add(createBaseArmeabiToolchain(thumb, gccVersion, stackProtectorFlag, excludedTools)
               .setToolchainIdentifier(
                   createArmeabiName("arm-linux-androideabi-%s", gccVersion, thumb))
-              .setTargetCpu("armeabi")
+              .setTargetCpu(createArmeabiCpuName("armeabi", thumb))
 
               .addCompilerFlag("-march=armv5te")
               .addCompilerFlag("-mtune=xscale")
@@ -210,7 +210,7 @@ class ArmCrosstools {
         .add(createBaseArmeabiToolchain(thumb, gccVersion, stackProtectorFlag, excludedTools)
             .setToolchainIdentifier(
                 createArmeabiName("arm-linux-androideabi-%s-v7a", gccVersion, thumb))
-            .setTargetCpu("armeabi-v7a")
+            .setTargetCpu(createArmeabiCpuName("armeabi-v7a", thumb))
 
             .addCompilerFlag("-march=armv7-a") 
             .addCompilerFlag("-mfpu=vfpv3-d16")
@@ -222,7 +222,7 @@ class ArmCrosstools {
         .add(createBaseArmeabiToolchain(thumb, gccVersion, stackProtectorFlag, excludedTools)
             .setToolchainIdentifier(
                 createArmeabiName("arm-linux-androideabi-%s-v7a-hard", gccVersion, thumb))
-            .setTargetCpu("armeabi-v7a-hard")
+            .setTargetCpu(createArmeabiCpuName("armeabi-v7a-hard", thumb))
     
             .addCompilerFlag("-march=armv7-a") 
             .addCompilerFlag("-mfpu=vfpv3-d16")
@@ -236,15 +236,10 @@ class ArmCrosstools {
 
         .build();
 
-    stlImpl.addStlImpl(toolchains, gccVersion, thumb);
+    stlImpl.addStlImpl(toolchains, gccVersion);
     return toolchains;
   }
 
-  private String createArmeabiName(String base, String gccVersion, boolean thumb) {
-    String thumbString = thumb ? "-thumb" : "";
-    return String.format(base, gccVersion) + thumbString; 
-  }
-
   /**
    * Flags common to arm-linux-androideabi*
    */
@@ -335,7 +330,7 @@ class ArmCrosstools {
         .add(createBaseArmeabiClangToolchain(clangVersion, thumb)
             .setToolchainIdentifier(
                 createArmeabiName("arm-linux-androideabi-clang%s", clangVersion, thumb))
-            .setTargetCpu("armeabi")
+            .setTargetCpu(createArmeabiCpuName("armeabi", thumb))
     
             .addCompilerFlag("-target")
             .addCompilerFlag("armv5te-none-linux-androideabi") // LLVM_TRIPLE
@@ -350,7 +345,7 @@ class ArmCrosstools {
       .add(createBaseArmeabiClangToolchain(clangVersion, thumb)
           .setToolchainIdentifier(
               createArmeabiName("arm-linux-androideabi-clang%s-v7a", clangVersion, thumb))
-          .setTargetCpu("armeabi-v7a")
+          .setTargetCpu(createArmeabiCpuName("armeabi-v7a", thumb))
   
           .addCompilerFlag("-target")
           .addCompilerFlag("armv7-none-linux-androideabi") // LLVM_TRIPLE
@@ -365,7 +360,7 @@ class ArmCrosstools {
       .add(createBaseArmeabiClangToolchain(clangVersion, thumb)
           .setToolchainIdentifier(
               createArmeabiName("arm-linux-androideabi-clang%s-v7a-hard", clangVersion, thumb))
-          .setTargetCpu("armeabi-v7a-hard")
+          .setTargetCpu(createArmeabiCpuName("armeabi-v7a-hard", thumb))
   
           .addCompilerFlag("-target")
           .addCompilerFlag("armv7-none-linux-androideabi") // LLVM_TRIPLE
@@ -381,7 +376,7 @@ class ArmCrosstools {
           .addLinkerFlag("-lm_hard"))
       .build();
 
-    stlImpl.addStlImpl(toolchains, "4.9", thumb);
+    stlImpl.addStlImpl(toolchains, "4.9");
     return toolchains;
   }
 
@@ -393,7 +388,7 @@ class ArmCrosstools {
 
     CToolchain.Builder builder = CToolchain.newBuilder()
         .setTargetSystemName("arm-linux-androideabi")
-        .setCompiler("gcc-4.8")
+        .setCompiler("gcc-4.8-clang" + clangVersion)
 
         .addAllToolPath(ndkPaths.createClangToolpaths(
             toolchainName,
@@ -462,4 +457,12 @@ class ArmCrosstools {
     
     return builder;
   }
+
+  private static String createArmeabiName(String base, String gccVersion, boolean thumb) {
+    return String.format(base, gccVersion) + (thumb ? "-thumb" : "");
+  }
+
+  private static String createArmeabiCpuName(String base, boolean thumb) {
+    return base + (thumb ? "-thumb" : "");
+  }
 }
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/MipsCrosstools.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/MipsCrosstools.java
index 7389409427..a18acffbec 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/MipsCrosstools.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/MipsCrosstools.java
@@ -70,11 +70,10 @@ class MipsCrosstools {
     // given in their tool paths.
     for (String clangVersion : new String[] { "3.5", "3.6" }) {
 
-      toolchainsListBuilder.add(createBaseMipsClangToolchain("mips64", "4.9")
+      toolchainsListBuilder.add(createBaseMipsClangToolchain("mips64", "4.9", clangVersion)
           .setToolchainIdentifier("mips64el-linux-android-clang" + clangVersion)
           .setTargetSystemName("mips64el-linux-android")
           .setTargetCpu("mips64")
-          .setCompiler("gcc-4.9")
 
           .addAllToolPath(ndkPaths.createClangToolpaths(
               "mips64el-linux-android-4.9", "mips64el-linux-android", clangVersion,
@@ -87,7 +86,7 @@ class MipsCrosstools {
     }
 
     List<CToolchain.Builder> toolchains = toolchainsListBuilder.build();
-    stlImpl.addStlImpl(toolchains, "4.9", false);
+    stlImpl.addStlImpl(toolchains, "4.9");
     return toolchains;
   }
 
@@ -106,12 +105,11 @@ class MipsCrosstools {
     // given in their tool paths.
     for (String clangVersion : new String[] { "3.5", "3.6" }) {
 
-      CToolchain.Builder mipsClang = createBaseMipsClangToolchain("mips", "4.8")
+      CToolchain.Builder mipsClang = createBaseMipsClangToolchain("mips", "4.8", clangVersion)
           // Purposefully no hyphen between "clang" and clang version.
           .setToolchainIdentifier("mipsel-linux-android-clang" + clangVersion)
           .setTargetSystemName("mipsel-linux-android")
           .setTargetCpu("mips")
-          .setCompiler("gcc-4.8")
     
           .addAllToolPath(ndkPaths.createClangToolpaths(
               "mipsel-linux-android-4.8", "mipsel-linux-android", clangVersion,
@@ -187,7 +185,8 @@ class MipsCrosstools {
             .addCompilerFlag("-fno-omit-frame-pointer"));
   }
 
-  private CToolchain.Builder createBaseMipsClangToolchain(String mipsArch, String gccVersion) {
+  private CToolchain.Builder createBaseMipsClangToolchain(
+      String mipsArch, String gccVersion, String clangVersion) {
 
     String gccToolchain = ndkPaths.createGccToolchainPath(
         String.format("%s-linux-android-%s", mipsArch, gccVersion));
@@ -195,7 +194,7 @@ class MipsCrosstools {
     String llvmTriple = mipsArch + "-none-linux-android";
     
     return CToolchain.newBuilder()
-        .setCompiler("gcc-" + gccVersion)
+        .setCompiler("gcc-" + gccVersion + "-clang" + clangVersion)
 
         // Compiler flags
         .addCompilerFlag("-gcc-toolchain")
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/NdkPaths.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/NdkPaths.java
index fabdfd251c..6166ccdc8f 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/NdkPaths.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/NdkPaths.java
@@ -160,9 +160,7 @@ public class NdkPaths {
 
   ImmutableList<String> createGnuLibstdcIncludePaths(String gccVersion, String targetCpu) {
 
-    if (targetCpu.equals("arm64")) {
-      targetCpu = "arm64-v8a";
-    }
+    String cpuNoThumb = targetCpu.replaceAll("-thumb$", "");
 
     String prefix = "external/%repositoryName%/ndk/sources/cxx-stl/gnu-libstdc++/%gccVersion%/";
     List<String> includePathTemplates = Arrays.asList(
@@ -176,7 +174,7 @@ public class NdkPaths {
           template
             .replace("%repositoryName%", repositoryName)
             .replace("%gccVersion%", gccVersion)
-            .replace("%targetCpu%", targetCpu));
+            .replace("%targetCpu%", cpuNoThumb));
     }
     return includePaths.build();
   }
@@ -205,20 +203,17 @@ public class NdkPaths {
    * @param stl The STL name as it appears in the NDK path
    * @param gccVersion The GCC version "4.8" or "4.9", applicable only to gnu-libstdc++, or null
    * @param targetCpu Target CPU
-   * @param armThumb Thumb mode for ARM
    * @param fileExtension "a" or "so"
    * @return A glob pattern for the STL runtime libs in the NDK.
    */
   static String createStlRuntimeLibsGlob(
-      String stl, String gccVersion, String targetCpu, boolean armThumb, String fileExtension) {
+      String stl, String gccVersion, String targetCpu, String fileExtension) {
     
     if (gccVersion != null) {
       stl += "/" + gccVersion;
     }
 
-    if (armThumb) {
-      targetCpu += "/thumb";
-    }
+    targetCpu = targetCpu.replaceAll("-thumb$", "/thumb");
 
     String template =
         "ndk/sources/cxx-stl/%stl%/libs/%targetCpu%/*.%fileExtension%";
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpl.java
index 6d4606afa7..d8e793df78 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpl.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpl.java
@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools;
 
+import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain;
@@ -54,42 +55,36 @@ public abstract class StlImpl {
   }
 
   public void addStlImpl(
-      List<CToolchain.Builder> baseToolchains, String gccVersion, boolean armThumb) {
+      List<CToolchain.Builder> baseToolchains, String gccVersion) {
 
     for (CToolchain.Builder baseToolchain : baseToolchains) {
-      addStlImpl(baseToolchain, gccVersion, armThumb);
+      addStlImpl(baseToolchain, gccVersion);
     }    
   }
 
-  public void addStlImpl(CToolchain.Builder baseToolchain, String gccVersion) {
-    addStlImpl(baseToolchain, gccVersion, false);
-  }
-
   /**
    * Adds an Android NDK C++ STL implementation to the given CToolchain builder.
    *
    * @param toolchain the toolchain to add the STL implementation to
    * @param gccVersion the gcc version for the STL impl. Applicable only to gnu-libstdc++
-   * @param armThumb whether in thumb mode, applicable only when creating a CToolchain for ARM
    */
-  public abstract void addStlImpl(
-      CToolchain.Builder toolchain, String gccVersion, boolean armThumb);
+  public abstract void addStlImpl(CToolchain.Builder toolchain, String gccVersion);
 
-  protected void addBaseStlImpl(CToolchain.Builder toolchain, String gccVersion, boolean armThumb) {
+  protected void addBaseStlImpl(CToolchain.Builder toolchain, String gccVersion) {
  
     toolchain
       .setToolchainIdentifier(toolchain.getToolchainIdentifier() + "-" + name)
       .setSupportsEmbeddedRuntimes(true)
       .setDynamicRuntimesFilegroup(
           createRuntimeLibrariesFilegroup(
-              name, gccVersion, toolchain.getTargetCpu(), armThumb, RuntimeType.DYNAMIC))
+              name, gccVersion, toolchain.getTargetCpu(), RuntimeType.DYNAMIC))
       .setStaticRuntimesFilegroup(
           createRuntimeLibrariesFilegroup(
-              name, gccVersion, toolchain.getTargetCpu(), armThumb, RuntimeType.STATIC));
+              name, gccVersion, toolchain.getTargetCpu(), RuntimeType.STATIC));
   }
 
   private String createRuntimeLibrariesFilegroup(
-      String stl, String gccVersion, String targetCpu, boolean armThumb, RuntimeType type) {
+      String stl, String gccVersion, String targetCpu, RuntimeType type) {
 
     // gnu-libstlc++ has separate libraries for 4.8 and 4.9
     String fullStlName = stl;
@@ -97,16 +92,10 @@ public abstract class StlImpl {
       fullStlName += "-" + gccVersion;
     }
 
-    // arm has separate libraries for thumb mode
-    String fullTargetCpu = targetCpu;
-    if (armThumb) {
-      fullTargetCpu += "-thumb";
-    }
-
     String filegroupNameTemplate = "%stl%-%targetCpu%-%type%-runtime-libraries";
     String filegroupName = filegroupNameTemplate
         .replace("%stl%", fullStlName)
-        .replace("%targetCpu%", fullTargetCpu)
+        .replace("%targetCpu%", targetCpu)
         .replace("%type%", type.name);
 
     // At the same time that the filegroup name is created, record a corresponding file glob
@@ -121,9 +110,16 @@ public abstract class StlImpl {
     if (stl.equals("libcpp")) {
       stl = "llvm-libc++";
     }
-    fileGroupNameToFileGlobs.put(filegroupName, NdkPaths.createStlRuntimeLibsGlob(
-        stl, gccVersion, targetCpu, armThumb, type.fileExtension));
 
+    String glob = NdkPaths.createStlRuntimeLibsGlob(stl, gccVersion, targetCpu, type.fileExtension);
+    String previousValue = fileGroupNameToFileGlobs.put(filegroupName, glob);
+
+    // Some STL filegroups will end up being duplicates, but a filegroup should never be registered
+    // with a different glob, otherwise one toolchain would get the wrong glob.
+    Verify.verify(previousValue == null || previousValue.equals(glob),
+        "STL filegroup glob being replaced with a different glob:\nname: %s\n%s\n%s",
+        filegroupName, glob, previousValue);
+    
     return filegroupName;
   }
 
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpls.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpls.java
index b81795c292..90cb21f793 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpls.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/StlImpls.java
@@ -36,8 +36,8 @@ public final class StlImpls {
     }
 
     @Override
-    public void addStlImpl(Builder toolchain, String gccVersion, boolean armThumb) {
-      addBaseStlImpl(toolchain, gccVersion, armThumb);
+    public void addStlImpl(Builder toolchain, String gccVersion) {
+      addBaseStlImpl(toolchain, gccVersion);
       toolchain.addAllUnfilteredCxxFlag(createIncludeFlags(
           ndkPaths.createGnuLibstdcIncludePaths(gccVersion, toolchain.getTargetCpu())));
     }
@@ -50,8 +50,8 @@ public final class StlImpls {
     }
 
     @Override
-    public void addStlImpl(Builder toolchain, String gccVersion, boolean armThumb) {
-      addBaseStlImpl(toolchain, null, armThumb);
+    public void addStlImpl(Builder toolchain, String gccVersion) {
+      addBaseStlImpl(toolchain, null);
       toolchain.addAllUnfilteredCxxFlag(createIncludeFlags(ndkPaths.createLibcxxIncludePaths()));
     }
   }
@@ -63,8 +63,8 @@ public final class StlImpls {
     }
 
     @Override
-    public void addStlImpl(Builder toolchain, String gccVersion, boolean armThumb) {
-      addBaseStlImpl(toolchain, null, armThumb);
+    public void addStlImpl(Builder toolchain, String gccVersion) {
+      addBaseStlImpl(toolchain, null);
       toolchain.addAllUnfilteredCxxFlag(createIncludeFlags(ndkPaths.createStlportIncludePaths()));
     }
   }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/X86Crosstools.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/X86Crosstools.java
index 217d929b79..f3caab34bb 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/X86Crosstools.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/X86Crosstools.java
@@ -51,7 +51,7 @@ class X86Crosstools {
     // Add Clang toolchains. x86 uses gcc-4.8.
     for (String clangVersion : new String[] { "3.5", "3.6" }) {
 
-      CToolchain.Builder x86Clang = createBaseX86ClangToolchain("x86", "i686", "4.8")
+      CToolchain.Builder x86Clang = createBaseX86ClangToolchain("x86", "i686", "4.8", clangVersion)
           .setToolchainIdentifier("x86-clang" + clangVersion)
           .setTargetCpu("x86")
 
@@ -94,17 +94,18 @@ class X86Crosstools {
     // Add Clang toolchains. x86_64 uses gcc-4.9.
     for (String clangVersion : new String[] { "3.5", "3.6" }) {
 
-      CToolchain.Builder x8664Clang = createBaseX86ClangToolchain("x86_64", "x86_64", "4.9")
-            .setToolchainIdentifier("x86_64-clang" + clangVersion)
-            .setTargetCpu("x86_64")
-  
-            .addAllToolPath(ndkPaths.createClangToolpaths(
-                "x86_64-4.9", "x86_64-linux-android", clangVersion))
-  
-            .addAllCxxBuiltinIncludeDirectory(ndkPaths.createToolchainIncludePaths(
-                "x86_64-4.9", "x86_64-linux-android", "4.9"))
-  
-            .setBuiltinSysroot(ndkPaths.createBuiltinSysroot("x86_64"));
+      CToolchain.Builder x8664Clang =
+          createBaseX86ClangToolchain("x86_64", "x86_64", "4.9", clangVersion)
+              .setToolchainIdentifier("x86_64-clang" + clangVersion)
+              .setTargetCpu("x86_64")
+    
+              .addAllToolPath(ndkPaths.createClangToolpaths(
+                  "x86_64-4.9", "x86_64-linux-android", clangVersion))
+    
+              .addAllCxxBuiltinIncludeDirectory(ndkPaths.createToolchainIncludePaths(
+                  "x86_64-4.9", "x86_64-linux-android", "4.9"))
+    
+              .setBuiltinSysroot(ndkPaths.createBuiltinSysroot("x86_64"));
 
       stlImpl.addStlImpl(x8664Clang, "4.9");
       toolchains.add(x8664Clang);
@@ -173,7 +174,7 @@ class X86Crosstools {
   }
 
   private CToolchain.Builder createBaseX86ClangToolchain(
-      String x86Arch, String llvmArch, String gccVersion) {
+      String x86Arch, String llvmArch, String gccVersion, String clangVersion) {
 
     String gccToolchain = ndkPaths.createGccToolchainPath(
         String.format("%s-linux-android-%s", x86Arch, gccVersion));
@@ -181,7 +182,7 @@ class X86Crosstools {
     String llvmTriple = llvmArch + "-none-linux-android";
 
     return CToolchain.newBuilder()
-        .setCompiler("gcc-" + gccVersion)
+        .setCompiler("gcc-" + gccVersion + "-clang" + clangVersion)
 
         // Compiler flags
         .addCompilerFlag("-gcc-toolchain")
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
index bc59e2e17f..2f739493b3 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -15,6 +15,7 @@
 package com.google.devtools.build.lib.syntax;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
@@ -24,7 +25,6 @@ import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.util.Pair;
 import com.google.devtools.build.lib.vfs.PathFragment;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -77,7 +77,7 @@ public final class Lexer {
   // bottom.
   private final Stack<Integer> indentStack = new Stack<>();
 
-  private final List<Token> tokens = new ArrayList<>();
+  private final List<Token> tokens;
 
   // The number of unclosed open-parens ("(", '{', '[') at the current point in
   // the stream. Whitespace is handled differently when this is nonzero.
@@ -94,6 +94,8 @@ public final class Lexer {
   public Lexer(ParserInputSource input, EventHandler eventHandler, boolean parsePython,
       LineNumberTable lineNumberTable) {
     this.buffer = input.getContent();
+    // Empirical measurements show roughly 1 token per 8 characters in buffer.
+    this.tokens = Lists.newArrayListWithExpectedSize(buffer.length / 8);
     this.pos = 0;
     this.parsePython = parsePython;
     this.eventHandler = eventHandler;