Remove word2vec's dependency on RE2. This is primarily to allow inputs

over 2GiB in size.
As part of this, add ConsumeNonWhitespace() to str_util.
Change: 112436569

1 files changed, 20 insertions, 0 deletions

diff --git a/tensorflow/core/lib/strings/str_util_test.cc b/tensorflow/core/lib/strings/str_util_test.cc
index 23ae27cfd9..f9ce25d6aa 100644
--- a/tensorflow/core/lib/strings/str_util_test.cc
+++ b/tensorflow/core/lib/strings/str_util_test.cc
@@ -168,6 +168,26 @@ TEST(ConsumeLeadingDigits, Basic) {
                            "xyz");
 }
 
+void TestConsumeNonWhitespace(StringPiece s, StringPiece expected,
+                              StringPiece remaining) {
+  StringPiece v;
+  StringPiece input(s);
+  if (str_util::ConsumeNonWhitespace(&input, &v)) {
+    EXPECT_EQ(v, expected);
+    EXPECT_EQ(input, remaining);
+  } else {
+    EXPECT_EQ(expected, "");
+    EXPECT_EQ(input, remaining);
+  }
+}
+
+TEST(ConsumeNonWhitespace, Basic) {
+  TestConsumeNonWhitespace("", "", "");
+  TestConsumeNonWhitespace(" ", "", " ");
+  TestConsumeNonWhitespace("abc", "abc", "");
+  TestConsumeNonWhitespace("abc ", "abc", " ");
+}
+
 TEST(ConsumePrefix, Basic) {
   string s("abcdef");
   StringPiece input(s);