Bazel client: implement PathExists on Windows

Checking if a path exists is surprisingly hard on
Windows. The most convenient API functions are
PathFileExists and GetFileAttributes but neither
of them follows junctions. To check if a junction
is dangling, we have to resolve it all the way.

This change adds a JunctionResolver class to
file_windows, which can resolve junctions (if they
aren't dangling) and non-junctions (in this case
just checks their existence).

See https://github.com/bazelbuild/bazel/issues/2107
See https://github.com/bazelbuild/bazel/issues/2181

--
PiperOrigin-RevId: 143645274
MOS_MIGRATED_REVID=143645274

5 files changed, 287 insertions, 33 deletions

diff --git a/src/main/cpp/blaze_util_windows.cc b/src/main/cpp/blaze_util_windows.cc
index 47882e6434..052c7d6c5b 100644
--- a/src/main/cpp/blaze_util_windows.cc
+++ b/src/main/cpp/blaze_util_windows.cc
@@ -845,6 +845,9 @@ typedef struct {
   WCHAR PathBuffer[ANYSIZE_ARRAY];
 } REPARSE_MOUNTPOINT_DATA_BUFFER, *PREPARSE_MOUNTPOINT_DATA_BUFFER;
 
+// TODO(laszlocsomor): get rid of this method in favor of OpenDirectory in
+// file_windows, as part of fixing
+// https://github.com/bazelbuild/bazel/issues/2181.
 HANDLE OpenDirectory(const string& path, bool readWrite) {
   HANDLE result = ::CreateFileA(
       /* lpFileName */ path.c_str(),
@@ -942,6 +945,7 @@ bool SymlinkDirectories(const string &posix_target, const string &posix_name) {
   return result;
 }
 
+// TODO(laszlocsomor): use JunctionResolver in file_windows.cc
 bool ReadDirectorySymlink(const string &posix_name, string* result) {
   string name = ConvertPath(posix_name);
   HANDLE directory = OpenDirectory(name, false);
@@ -995,6 +999,7 @@ bool ReadDirectorySymlink(const string &posix_name, string* result) {
   }
 }
 
+// TODO(laszlocsomor): use IsAbsolute from file_windows.cc
 static bool IsAbsoluteWindowsPath(const string& p) {
   if (p.size() < 3) {
     return false;
diff --git a/src/main/cpp/util/file_platform.h b/src/main/cpp/util/file_platform.h
index 5354052d81..b0458e8fa6 100644
--- a/src/main/cpp/util/file_platform.h
+++ b/src/main/cpp/util/file_platform.h
@@ -44,7 +44,7 @@ bool WriteFile(const std::string &content, const std::string &filename);
 // Returns true on success. In case of failure sets errno.
 bool UnlinkPath(const std::string &file_path);
 
-// Returns true if this path exists.
+// Returns true if this path exists, following symlinks.
 bool PathExists(const std::string& path);
 
 // Returns the real, absolute path corresponding to `path`.
diff --git a/src/main/cpp/util/file_posix.cc b/src/main/cpp/util/file_posix.cc
index 127f811d32..475de9ef18 100644
--- a/src/main/cpp/util/file_posix.cc
+++ b/src/main/cpp/util/file_posix.cc
@@ -213,9 +213,15 @@ bool UnlinkPath(const string &file_path) {
   return unlink(file_path.c_str()) == 0;
 }
 
+// TODO(bazel-team): implement all functions in file_windows.cc, use them from
+// MSYS, remove file_posix.cc from the `srcs` of
+// //src/main/cpp/util:file when building for MSYS, and remove all
+// #ifndef __CYGWIN__ directives.
+#ifndef __CYGWIN__
 bool PathExists(const string& path) {
   return access(path.c_str(), F_OK) == 0;
 }
+#endif  // not __CYGWIN__
 
 string MakeCanonical(const char *path) {
   char *resolved_path = realpath(path, NULL);
diff --git a/src/main/cpp/util/file_windows.cc b/src/main/cpp/util/file_windows.cc
index 1eef9fc95f..f947df1508 100644
--- a/src/main/cpp/util/file_windows.cc
+++ b/src/main/cpp/util/file_windows.cc
@@ -17,6 +17,7 @@
 #include <windows.h>
 
 #include <memory>  // unique_ptr
+#include <vector>
 
 #include "src/main/cpp/util/errors.h"
 #include "src/main/cpp/util/exit_code.h"
@@ -28,6 +29,7 @@ namespace blaze_util {
 using std::pair;
 using std::string;
 using std::unique_ptr;
+using std::vector;
 using std::wstring;
 
 class WindowsPipe : public IPipe {
@@ -139,23 +141,37 @@ pair<string, string> SplitPath(const string& path) {
 
 class MsysRoot {
  public:
-  static bool IsValid() { return instance_.data_.first; }
-  static const string& GetPath() { return instance_.data_.second; }
-  static void ReInitForTesting() { instance_.data_ = Get(); }
+  static bool IsValid();
+  static const string& GetPath();
+  static void ResetForTesting() { instance_.initialized_ = false; }
 
  private:
-  std::pair<bool, string> data_;
+  bool initialized_;
+  bool valid_;
+  string path_;
   static MsysRoot instance_;
 
-  static std::pair<bool, string> Get();
-  MsysRoot() : data_(Get()) {}
+  static bool Get(string* path);
+
+  MsysRoot() : initialized_(false) {}
+  void InitIfNecessary();
 };
 
 MsysRoot MsysRoot::instance_;
 
-void ReinitMsysRootForTesting() { MsysRoot::ReInitForTesting(); }
+void ResetMsysRootForTesting() { MsysRoot::ResetForTesting(); }
+
+bool MsysRoot::IsValid() {
+  instance_.InitIfNecessary();
+  return instance_.valid_;
+}
+
+const string& MsysRoot::GetPath() {
+  instance_.InitIfNecessary();
+  return instance_.path_;
+}
 
-std::pair<bool, string> MsysRoot::Get() {
+bool MsysRoot::Get(string* path) {
   string result;
   char value[MAX_PATH];
   DWORD len = GetEnvironmentVariableA("BAZEL_SH", value, MAX_PATH);
@@ -167,7 +183,7 @@ std::pair<bool, string> MsysRoot::Get() {
       PrintError(
           "BAZEL_SH environment variable is not defined, cannot convert MSYS "
           "paths to Windows paths");
-      return std::make_pair(false, "");
+      return false;
     }
     result = value2;
   }
@@ -181,9 +197,17 @@ std::pair<bool, string> MsysRoot::Get() {
     result = Dirname(result);
   }
   if (IsRootDirectory(result)) {
-    return std::make_pair(false, "");
+    return false;
+  }
+  *path = result;
+  return true;
+}
+
+void MsysRoot::InitIfNecessary() {
+  if (!initialized_) {
+    valid_ = Get(&path_);
+    initialized_ = true;
   }
-  return std::make_pair(true, std::move(result));
 }
 
 bool AsWindowsPath(const string& path, wstring* result) {
@@ -285,23 +309,165 @@ bool UnlinkPath(const string& file_path) {
 #else  // not COMPILER_MSVC
 #endif  // COMPILER_MSVC
 
-#ifdef COMPILER_MSVC
-bool PathExists(const string& path) {
-  // TODO(bazel-team): implement this.
-  pdie(255, "blaze_util::PathExists is not implemented on Windows");
-  return false;
+HANDLE OpenDirectory(const WCHAR* path, bool read_write) {
+  return ::CreateFileW(
+      /* lpFileName */ path,
+      /* dwDesiredAccess */ read_write ? (GENERIC_READ | GENERIC_WRITE)
+                                       : GENERIC_READ,
+      /* dwShareMode */ 0,
+      /* lpSecurityAttributes */ NULL,
+      /* dwCreationDisposition */ OPEN_EXISTING,
+      /* dwFlagsAndAttributes */ FILE_FLAG_OPEN_REPARSE_POINT |
+          FILE_FLAG_BACKUP_SEMANTICS,
+      /* hTemplateFile */ NULL);
 }
-#else  // not COMPILER_MSVC
-#endif  // COMPILER_MSVC
 
-#ifdef COMPILER_MSVC
-string MakeCanonical(const char *path) {
-  // TODO(bazel-team): implement this.
-  pdie(255, "blaze_util::MakeCanonical is not implemented on Windows");
-  return "";
+class JunctionResolver {
+ public:
+  JunctionResolver();
+
+  // Resolves junctions, or simply checks file existence (if not a junction).
+  //
+  // Returns true if `path` is not a junction and it exists.
+  // Returns true if `path` is a junction and can be successfully resolved and
+  // its target exists.
+  // Returns false otherwise.
+  //
+  // If `result` is not nullptr and the method returned false, then this will be
+  // reset to point to a new WCHAR buffer containing the final resolved path.
+  // If `path` was a junction, this will be the fully resolved path, otherwise
+  // it will be a copy of `path`.
+  bool Resolve(const WCHAR* path, std::unique_ptr<WCHAR[]>* result);
+
+ private:
+  static const int kMaximumJunctionDepth;
+
+  // This struct is a simplified version of REPARSE_DATA_BUFFER, defined by
+  // the <Ntifs.h> header file, which is not available on some systems.
+  // This struct removes the original one's union keeping only
+  // MountPointReparseBuffer, while also renames some fields to reflect how
+  // ::DeviceIoControl actually uses them when reading junction data.
+  typedef struct _ReparseMountPointData {
+    static const int kSize = MAXIMUM_REPARSE_DATA_BUFFER_SIZE;
+
+    ULONG ReparseTag;
+    USHORT Dummy1;
+    USHORT Dummy2;
+    USHORT Dummy3;
+    USHORT Dummy4;
+    // Length of string in PathBuffer, in WCHARs, including the "\??\" prefix
+    // and the null-terminator.
+    //
+    // Reparse points use the "\??\" prefix instead of "\\?\", presumably
+    // because the junction is resolved by the kernel and it points to a Device
+    // Object path (which is what the kernel understands), and "\??" is a device
+    // path. ("\??" is shorthand for "\DosDevices" under which disk drives
+    // reside, e.g. "C:" is a symlink to "\DosDevices\C:" aka "\??\C:").
+    // See (on 2017-01-04):
+    // https://msdn.microsoft.com/en-us/library/windows/hardware/ff565384(v=vs.85).aspx
+    // https://msdn.microsoft.com/en-us/library/windows/hardware/ff557762(v=vs.85).aspx
+    USHORT Size;
+    USHORT Dummy5;
+    // First character of the string returned by ::DeviceIoControl. The rest of
+    // the string follows this in memory, that's why the caller must allocate
+    // kSize bytes and cast that data to ReparseMountPointData.
+    WCHAR PathBuffer[1];
+  } ReparseMountPointData;
+
+  uint8_t reparse_buffer_bytes_[ReparseMountPointData::kSize];
+  ReparseMountPointData* reparse_buffer_;
+
+  bool Resolve(const WCHAR* path, std::unique_ptr<WCHAR[]>* result,
+               int max_junction_depth);
+};
+
+// Maximum reparse point depth on Windows 8 and above is 63.
+// Source (on 2016-12-20):
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365503(v=vs.85).aspx
+const int JunctionResolver::kMaximumJunctionDepth = 63;
+
+JunctionResolver::JunctionResolver()
+    : reparse_buffer_(
+          reinterpret_cast<ReparseMountPointData*>(reparse_buffer_bytes_)) {
+  reparse_buffer_->ReparseTag = IO_REPARSE_TAG_MOUNT_POINT;
+}
+
+bool JunctionResolver::Resolve(const WCHAR* path, unique_ptr<WCHAR[]>* result,
+                               int max_junction_depth) {
+  DWORD attributes = ::GetFileAttributesW(path);
+  if (attributes == INVALID_FILE_ATTRIBUTES) {
+    // `path` does not exist.
+    return false;
+  } else {
+    if ((attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 &&
+        (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) {
+      // `path` is a junction. GetFileAttributesW succeeds for these even if
+      // their target does not exist. We need to resolve the target and check if
+      // that exists. (There seems to be no API function for this.)
+      if (max_junction_depth <= 0) {
+        // Too many levels of junctions. Simply say this file doesn't exist.
+        return false;
+      }
+      // Get a handle to the directory.
+      HANDLE handle = OpenDirectory(path, /* read_write */ false);
+      if (handle == INVALID_HANDLE_VALUE) {
+        // Opening the junction failed for whatever reason. For all intents and
+        // purposes we can treat this file as if it didn't exist.
+        return false;
+      }
+      // Read out the junction data.
+      DWORD bytes_returned;
+      BOOL ok = ::DeviceIoControl(
+          handle, FSCTL_GET_REPARSE_POINT, NULL, 0, reparse_buffer_,
+          MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &bytes_returned, NULL);
+      CloseHandle(handle);
+      if (!ok) {
+        // Reading the junction data failed. For all intents and purposes we can
+        // treat this file as if it didn't exist.
+        return false;
+      }
+      reparse_buffer_->PathBuffer[reparse_buffer_->Size - 1] = UNICODE_NULL;
+      // Check if the junction target exists.
+      return Resolve(reparse_buffer_->PathBuffer, result,
+                     max_junction_depth - 1);
+    }
+  }
+  // `path` is a normal file or directory.
+  if (result) {
+    size_t len = wcslen(path) + 1;
+    result->reset(new WCHAR[len]);
+    memcpy(result->get(), path, len * sizeof(WCHAR));
+  }
+  return true;
+}
+
+bool JunctionResolver::Resolve(const WCHAR* path, unique_ptr<WCHAR[]>* result) {
+  return Resolve(path, result, kMaximumJunctionDepth);
+}
+
+bool PathExists(const string& path) {
+  if (path.empty()) {
+    return false;
+  }
+  wstring wpath;
+  if (!AsWindowsPath(NormalizePath(path), &wpath)) {
+    PrintError("could not convert path to widechar, path=(%s), err=%d\n",
+               path.c_str(), GetLastError());
+    return false;
+  }
+  if (!IsAbsolute(path)) {
+    DWORD len = ::GetCurrentDirectoryW(0, nullptr);
+    unique_ptr<WCHAR[]> cwd(new WCHAR[len]);
+    if (!GetCurrentDirectoryW(len, cwd.get())) {
+      PrintError("could not make the path absolute, path=(%s), err=%d\n",
+                 path.c_str(), GetLastError());
+      return false;
+    }
+    wpath = wstring(cwd.get()) + L"\\" + wpath;
+  }
+  wpath = wstring(L"\\\\?\\") + wpath;
+  return JunctionResolver().Resolve(wpath.c_str(), nullptr);
 }
-#else  // not COMPILER_MSVC
-#endif  // COMPILER_MSVC
 
 #ifdef COMPILER_MSVC
 bool CanAccess(const string& path, bool read, bool write, bool exec) {
diff --git a/src/test/cpp/util/file_windows_test.cc b/src/test/cpp/util/file_windows_test.cc
index fe23a92b67..66c1af481f 100644
--- a/src/test/cpp/util/file_windows_test.cc
+++ b/src/test/cpp/util/file_windows_test.cc
@@ -11,8 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#include <stdio.h>
 #include <string.h>
-#include <windows.h>  // SetEnvironmentVariableA
+#include <windows.h>
 
 #include "src/main/cpp/util/file.h"
 #include "src/main/cpp/util/file_platform.h"
@@ -24,7 +25,9 @@
 
 namespace blaze_util {
 
-void ReinitMsysRootForTesting();  // defined in file_windows.cc
+using std::string;
+
+void ResetMsysRootForTesting();  // defined in file_windows.cc
 
 TEST(FileTest, TestDirname) {
   ASSERT_EQ("", Dirname(""));
@@ -98,7 +101,7 @@ TEST(FileTest, IsRootDirectory) {
 
 TEST(FileTest, TestAsWindowsPath) {
   SetEnvironmentVariableA("BAZEL_SH", "c:\\msys\\some\\long\\path\\bash.exe");
-  ReinitMsysRootForTesting();
+  ResetMsysRootForTesting();
   std::wstring actual;
 
   ASSERT_TRUE(AsWindowsPath("", &actual));
@@ -141,21 +144,95 @@ TEST(FileTest, TestMsysRootRetrieval) {
   std::wstring actual;
 
   SetEnvironmentVariableA("BAZEL_SH", "c:/foo/msys/bar/qux.exe");
-  ReinitMsysRootForTesting();
+  ResetMsysRootForTesting();
   ASSERT_TRUE(AsWindowsPath("/blah", &actual));
   ASSERT_EQ(std::wstring(L"c:\\foo\\msys\\blah"), actual);
 
   SetEnvironmentVariableA("BAZEL_SH", "c:/foo/MSYS64/bar/qux.exe");
-  ReinitMsysRootForTesting();
+  ResetMsysRootForTesting();
   ASSERT_TRUE(AsWindowsPath("/blah", &actual));
   ASSERT_EQ(std::wstring(L"c:\\foo\\msys64\\blah"), actual);
 
   SetEnvironmentVariableA("BAZEL_SH", "c:/qux.exe");
-  ReinitMsysRootForTesting();
+  ResetMsysRootForTesting();
   ASSERT_FALSE(AsWindowsPath("/blah", &actual));
 
   SetEnvironmentVariableA("BAZEL_SH", nullptr);
-  ReinitMsysRootForTesting();
+  ResetMsysRootForTesting();
+}
+
+static void RunCommand(const string& cmdline) {
+  STARTUPINFOA startupInfo = {sizeof(STARTUPINFO)};
+  PROCESS_INFORMATION processInfo;
+  // command line maximum size is 32K
+  // Source (on 2017-01-04):
+  // https://msdn.microsoft.com/en-us/library/windows/desktop/ms682425(v=vs.85).aspx
+  char mutable_cmdline[0x8000];
+  strncpy(mutable_cmdline, cmdline.c_str(), 0x8000);
+  BOOL ok = CreateProcessA(
+      /* lpApplicationName */ NULL,
+      /* lpCommandLine */ mutable_cmdline,
+      /* lpProcessAttributes */ NULL,
+      /* lpThreadAttributes */ NULL,
+      /* bInheritHandles */ TRUE,
+      /* dwCreationFlags */ 0,
+      /* lpEnvironment */ NULL,
+      /* lpCurrentDirectory */ NULL,
+      /* lpStartupInfo */ &startupInfo,
+      /* lpProcessInformation */ &processInfo);
+  ASSERT_TRUE(ok);
+
+  // Wait 1 second for the process to finish.
+  ASSERT_EQ(WAIT_OBJECT_0, WaitForSingleObject(processInfo.hProcess, 1000));
+
+  DWORD exit_code = 1;
+  ASSERT_TRUE(GetExitCodeProcess(processInfo.hProcess, &exit_code));
+  ASSERT_EQ(0, exit_code);
+}
+
+TEST(FileTest, TestPathExistsWindows) {
+  ASSERT_FALSE(PathExists(""));
+  ASSERT_TRUE(PathExists("."));
+  ASSERT_FALSE(PathExists("non.existent"));
+
+  char buf[MAX_PATH] = {0};
+  DWORD len = GetEnvironmentVariableA("TEST_TMPDIR", buf, MAX_PATH);
+  ASSERT_GT(len, 0);
+  string tmpdir(buf);
+  ASSERT_TRUE(PathExists(tmpdir));
+
+  // Create a fake msys root. We'll also use it as a junction target.
+  string fake_msys_root(tmpdir + "/fake_msys");
+  ASSERT_EQ(0, mkdir(fake_msys_root.c_str()));
+  ASSERT_TRUE(PathExists(fake_msys_root));
+
+  // Set the BAZEL_SH root so we can resolve MSYS paths.
+  SetEnvironmentVariableA("BAZEL_SH",
+                          (fake_msys_root + "/fake_bash.exe").c_str());
+  ResetMsysRootForTesting();
+
+  // Assert existence check for MSYS paths.
+  ASSERT_FALSE(PathExists("/this/should/not/exist/mkay"));
+  ASSERT_TRUE(PathExists("/"));
+
+  // Create a junction pointing to an existing directory.
+  RunCommand(string("cmd.exe /C mklink /J \"") + tmpdir + "/junc1\" \"" +
+             fake_msys_root + "\" >NUL 2>NUL");
+  ASSERT_TRUE(PathExists(fake_msys_root));
+  ASSERT_TRUE(PathExists(JoinPath(tmpdir, "junc1")));
+
+  // Create a junction pointing to a non-existent directory.
+  RunCommand(string("cmd.exe /C mklink /J \"") + tmpdir + "/junc2\" \"" +
+             fake_msys_root + "/i.dont.exist\" >NUL 2>NUL");
+  ASSERT_FALSE(PathExists(JoinPath(fake_msys_root, "i.dont.exist")));
+  ASSERT_FALSE(PathExists(JoinPath(tmpdir, "junc2")));
+
+  // Clean up.
+  ASSERT_EQ(0, rmdir(JoinPath(tmpdir, "junc1").c_str()));
+  ASSERT_EQ(0, rmdir(JoinPath(tmpdir, "junc2").c_str()));
+  ASSERT_EQ(0, rmdir(fake_msys_root.c_str()));
+  ASSERT_FALSE(PathExists(JoinPath(tmpdir, "junc1")));
+  ASSERT_FALSE(PathExists(JoinPath(tmpdir, "junc2")));
 }
 
 }  // namespace blaze_util