Remove word2vec's dependency on RE2. This is primarily to allow inputs

over 2GiB in size.
As part of this, add ConsumeNonWhitespace() to str_util.
Change: 112436569

1 files changed, 19 insertions, 0 deletions

diff --git a/tensorflow/core/lib/strings/str_util.cc b/tensorflow/core/lib/strings/str_util.cc
index 562a2cfb1d..5a6a888d3f 100644
--- a/tensorflow/core/lib/strings/str_util.cc
+++ b/tensorflow/core/lib/strings/str_util.cc
@@ -311,6 +311,25 @@ bool ConsumeLeadingDigits(StringPiece* s, uint64* val) {
   }
 }
 
+bool ConsumeNonWhitespace(StringPiece* s, StringPiece* val) {
+  const char* p = s->data();
+  const char* limit = p + s->size();
+  while (p < limit) {
+    const char c = *p;
+    if (isspace(c)) break;
+    p++;
+  }
+  const size_t n = p - s->data();
+  if (n > 0) {
+    val->set(s->data(), n);
+    s->remove_prefix(n);
+    return true;
+  } else {
+    val->clear();
+    return false;
+  }
+}
+
 bool SplitAndParseAsInts(StringPiece text, char delim,
                          std::vector<int32>* result) {
   result->clear();