Add SMT detection for mac. Fixes #963.

--
Change-Id: Ib83af0d0a04dc6b173bef1df28d17abc7a3c824d
Reviewed-on: https://bazel-review.googlesource.com/#/c/3120/
MOS_MIGRATED_REVID=119027507

6 files changed, 292 insertions, 110 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD
index e51fbea1af..8f717e78da 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD
@@ -18,6 +18,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:packages-internal",
         "//src/main/java/com/google/devtools/build/lib:shell",
         "//src/main/java/com/google/devtools/build/lib:skylarkinterface",
+        "//src/main/java/com/google/devtools/build/lib:unix",
         "//src/main/java/com/google/devtools/build/lib:util",
         "//src/main/java/com/google/devtools/build/lib:vfs",
         "//src/main/java/com/google/devtools/build/skyframe",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
index 43509b5cd9..74ef6830c9 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
@@ -14,17 +14,8 @@
 
 package com.google.devtools.build.lib.actions;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Splitter;
-import com.google.common.io.Files;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
-import com.google.devtools.build.lib.util.ProcMeminfoParser;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.Set;
+import com.google.devtools.build.lib.util.OS;
 
 /**
  * This class estimates the local host's resource capacity.
@@ -32,115 +23,34 @@ import java.util.Set;
 @ThreadCompatible
 public final class LocalHostCapacity {
 
-  /* If /proc/* information is not available, guess based on what the JVM thinks.  Anecdotally,
-   * the JVM picks 0.22 the total available memory as maxMemory (tested on a standard Mac), so
-   * multiply by 3, and divide by 2^20 because we want megabytes.
-   */
-  private static final ResourceSet DEFAULT_RESOURCES = ResourceSet.create(
-      3.0 * (Runtime.getRuntime().maxMemory() >> 20),
-      Runtime.getRuntime().availableProcessors(), 1.0,
-      Integer.MAX_VALUE);
+  private static final OS currentOS = OS.getCurrent();
+  private static ResourceSet localHostCapacity;
 
   private LocalHostCapacity() {}
 
-  /**
-   * Estimates of the local host's resource capacity,
-   * obtained by reading /proc/cpuinfo and /proc/meminfo.
-   */
-  private static ResourceSet localHostCapacity;
-
-  /**
-   * Estimates of the local host's resource capacity,
-   * obtained by reading /proc/cpuinfo and /proc/meminfo.
-   */
   public static ResourceSet getLocalHostCapacity() {
     if (localHostCapacity == null) {
-      localHostCapacity = getLocalHostCapacity("/proc/cpuinfo", "/proc/meminfo");
+      localHostCapacity = getNewLocalHostCapacity();
     }
     return localHostCapacity;
   }
 
-  private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
-
-  @VisibleForTesting
-  static int getLogicalCpuCount(String cpuinfoContent) {
-    Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent);
-    int count = 0;
-    for (String line : lines) {
-      if(line.startsWith("processor")) {
-        count++;
-      }
-    }
-    if (count == 0) {
-      throw new IllegalArgumentException("Can't locate processor in the /proc/cpuinfo");
-    }
-    return count;
-  }
-
-  @VisibleForTesting
-  static int getPhysicalCpuCount(String cpuinfoContent, int logicalCpuCount) {
-    Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent);
-    Set<String> uniq = new HashSet<>();
-    for (String line : lines) {
-      if(line.startsWith("physical id")) {
-        uniq.add(line);
-      }
-    }
-    int physicalCpuCount = uniq.size();
-    if (physicalCpuCount == 0) {
-      physicalCpuCount = logicalCpuCount;
-    }
-    return physicalCpuCount;
-  }
-
-  @VisibleForTesting
-  static int getCoresPerCpu(String cpuinfoFileContent) {
-    Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoFileContent);
-    Set<String> uniq = new HashSet<>();
-    for (String line : lines) {
-      if(line.startsWith("core id")) {
-        uniq.add(line);
-      }
-    }
-    int coresPerCpu = uniq.size();
-    if (coresPerCpu == 0) {
-      coresPerCpu = 1;
-    }
-    return coresPerCpu;
-  }
-
-  @VisibleForTesting
-  static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) {
-    try {
-      String cpuinfoContent = readContent(cpuinfoFile);
-      ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile);
-      int logicalCpuCount = getLogicalCpuCount(cpuinfoContent);
-      int physicalCpuCount = getPhysicalCpuCount(cpuinfoContent, logicalCpuCount);
-      int coresPerCpu = getCoresPerCpu(cpuinfoContent);
-      int totalCores = coresPerCpu * physicalCpuCount;
-      boolean hyperthreading = (logicalCpuCount != totalCores);
-      double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
-      final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
-      return ResourceSet.create(
-          ramMb,
-          logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
-          1.0,
-          Integer.MAX_VALUE);
-    } catch (IOException | IllegalArgumentException e) {
-      return DEFAULT_RESOURCES;
-    }
-  }
-
-  /**
-   * For testing purposes only. Do not use it.
-   */
-  @VisibleForTesting
-  static void setLocalHostCapacity(ResourceSet resources) {
-    localHostCapacity = resources;
-  }
-
-  private static String readContent(String filename) throws IOException {
-    return Files.toString(new File(filename), Charset.defaultCharset());
+  private static ResourceSet getNewLocalHostCapacity() {
+    ResourceSet localResources = null;
+    switch (currentOS) {
+      case DARWIN:
+        localResources = LocalHostResourceManagerDarwin.getLocalHostResources();
+        break;
+      case LINUX:
+        localResources = LocalHostResourceManagerLinux.getLocalHostResources();
+        break;
+      default:
+        break;
+    }
+    if (localResources == null) {
+      localResources = LocalHostResourceFallback.getLocalHostResources();
+    }
+    return localResources;
   }
 
 }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceFallback.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceFallback.java
new file mode 100644
index 0000000000..2829cb4dd2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceFallback.java
@@ -0,0 +1,36 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * This class provide a fallback of the local host's resource capacity.
+ */
+public class LocalHostResourceFallback {
+
+  /* If /proc/* information is not available, guess based on what the JVM thinks.  Anecdotally,
+   * the JVM picks 0.22 the total available memory as maxMemory (tested on a standard Mac), so
+   * multiply by 3, and divide by 2^20 because we want megabytes.
+   */
+  private static final ResourceSet DEFAULT_RESOURCES =
+      ResourceSet.create(
+          3.0 * (Runtime.getRuntime().maxMemory() >> 20),
+          Runtime.getRuntime().availableProcessors(),
+          1.0,
+          Integer.MAX_VALUE);
+
+  public static ResourceSet getLocalHostResources() {
+    return DEFAULT_RESOURCES;
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerDarwin.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerDarwin.java
new file mode 100644
index 0000000000..1ada1bfd0c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerDarwin.java
@@ -0,0 +1,60 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.unix.NativePosixSystem;
+
+import java.io.IOException;
+
+/**
+ * This class estimates the local host's resource capacity for Darwin.
+ */
+public class LocalHostResourceManagerDarwin {
+  private static final Boolean JNI_UNAVAILABLE =
+      "0".equals(System.getProperty("io.bazel.UnixFileSystem"));
+  private static final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
+
+  private static int getLogicalCpuCount() throws IOException {
+    return (int) NativePosixSystem.sysctlbynameGetLong("hw.logicalcpu");
+  }
+
+  private static int getPhysicalCpuCount() throws IOException {
+    return (int) NativePosixSystem.sysctlbynameGetLong("hw.physicalcpu");
+  }
+
+  private static double getMemoryInMb() throws IOException {
+    return NativePosixSystem.sysctlbynameGetLong("hw.memsize") / 1E6;
+  }
+
+  public static ResourceSet getLocalHostResources() {
+    if (JNI_UNAVAILABLE) {
+      return null;
+    }
+    try {
+      int logicalCpuCount = getLogicalCpuCount();
+      int physicalCpuCount = getPhysicalCpuCount();
+      double ramMb = getMemoryInMb();
+      boolean hyperthreading = (logicalCpuCount != physicalCpuCount);
+
+      return ResourceSet.create(
+          ramMb,
+          logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
+          1.0,
+          Integer.MAX_VALUE);
+    } catch (IOException e) {
+      return null;
+    }
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java
new file mode 100644
index 0000000000..a9b50f98bb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java
@@ -0,0 +1,132 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Splitter;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.util.ProcMeminfoParser;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * This class estimates the local host's resource capacity for Linux.
+ */
+public class LocalHostResourceManagerLinux {
+  private static String cpuInfoContent = null;
+
+  private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
+  private static final String CPU_INFO_FILE = "/proc/cpuinfo";
+  private static final String MEM_INFO_FILE = "/proc/meminfo";
+
+  private static int getLogicalCpuCount() throws IOException {
+    String content = getCpuInfoContent();
+    return getLogicalCpuCountHelper(content);
+  }
+
+  private static int getPhysicalCpuCount(int logicalCpuCount) throws IOException {
+    String content = getCpuInfoContent();
+    return getPhysicalCpuCountHelper(logicalCpuCount, content);
+  }
+
+  private static double getMemoryInMb() throws IOException {
+    return getMemoryInMbHelper(MEM_INFO_FILE);
+  }
+
+  public static ResourceSet getLocalHostResources() {
+    try {
+      int logicalCpuCount = getLogicalCpuCount();
+      int physicalCpuCount = getPhysicalCpuCount(logicalCpuCount);
+      double ramMb = getMemoryInMb();
+
+      boolean hyperthreading = (logicalCpuCount != physicalCpuCount);
+      final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
+      return ResourceSet.create(
+          ramMb,
+          logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
+          1.0,
+          Integer.MAX_VALUE);
+    } catch (IOException | IllegalArgumentException e) {
+      return null;
+    }
+  }
+
+  private static String getCpuInfoContent() throws IOException {
+    if (cpuInfoContent == null) {
+      cpuInfoContent = readContent(CPU_INFO_FILE);
+    }
+    return cpuInfoContent;
+  }
+
+  private static String readContent(String filename) throws IOException {
+    return Files.toString(new File(filename), Charset.defaultCharset());
+  }
+
+  /**
+   * For testing purposes only. Do not use it.
+   */
+  public static int getLogicalCpuCountHelper(String content) throws IOException {
+    int count = 0;
+    Iterable<String> lines = NEWLINE_SPLITTER.split(content);
+    for (String line : lines) {
+      if (line.startsWith("processor")) {
+        count++;
+      }
+    }
+    if (count == 0) {
+      throw new IllegalArgumentException("Can't get logical CPU count");
+    }
+    return count;
+  }
+
+  public static int getPhysicalCpuCountHelper(int logicalCpuCount, String content)
+      throws IOException {
+    // CPU count
+    Iterable<String> lines = NEWLINE_SPLITTER.split(content);
+    Set<String> uniq = new HashSet<>();
+    for (String line : lines) {
+      if (line.startsWith("physical id")) {
+        uniq.add(line);
+      }
+    }
+    int cpuCount = uniq.size();
+    if (cpuCount == 0) {
+      cpuCount = logicalCpuCount;
+    }
+
+    // core per CPU
+    uniq = new HashSet<>();
+    for (String line : lines) {
+      if (line.startsWith("core id")) {
+        uniq.add(line);
+      }
+    }
+    int coresPerCpu = uniq.size();
+    if (coresPerCpu == 0) {
+      coresPerCpu = 1;
+    }
+
+    return cpuCount * coresPerCpu;
+  }
+
+  public static double getMemoryInMbHelper(String memInfoFileName) throws IOException {
+    ProcMeminfoParser memInfo = new ProcMeminfoParser(memInfoFileName);
+    double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
+    return ramMb;
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/NativePosixSystem.java b/src/main/java/com/google/devtools/build/lib/unix/NativePosixSystem.java
new file mode 100644
index 0000000000..9834f24ada
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/NativePosixSystem.java
@@ -0,0 +1,43 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.UnixJniLoader;
+
+import java.io.IOException;
+
+/**
+ * Utility methods for access to UNIX system calls not exposed by the Java
+ * SDK. Exception messages are selected to be consistent with those generated
+ * by the java.io package where appropriate--see package javadoc for details.
+ */
+public class NativePosixSystem {
+
+  private NativePosixSystem() {}
+
+  static {
+    if (!"0".equals(System.getProperty("io.bazel.UnixFileSystem"))) {
+      UnixJniLoader.loadJni();
+    }
+  }
+
+  /**
+   * Native wrapper around POSIX sysctlbyname(3) syscall.
+   *
+   * @param name the name for value to get from sysctl
+   * @throws IOException iff the sysctlbyname() syscall failed.
+   */
+  public static native long sysctlbynameGetLong(String name) throws IOException;
+}