diff options
| author |  liujisi@google.com <liujisi@google.com@630680e5-0e50-0410-840e-4b1c322b438d> | 2010-11-02 13:14:58 +0000 |
|---|---|---|
| committer | liujisi@google.com <liujisi@google.com@630680e5-0e50-0410-840e-4b1c322b438d> | 2010-11-02 13:14:58 +0000 |
| commit | 33165fe0d5c265c92f2a67fc2b437b567c24e294 (patch) | |
| tree | 52def0850ddd2e976da238d1a437fbda79c96e44 /src/google/protobuf/io/tokenizer.cc | |
| parent | 80aa23df6c63750e8cdfdcf3996fbc37d63cac61 (diff) | |
Submit recent changes from internal branch. See CHANGES.txt for more details.
Diffstat (limited to 'src/google/protobuf/io/tokenizer.cc')
| -rw-r--r-- | src/google/protobuf/io/tokenizer.cc | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index 38fa351c..513831d5 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -193,6 +193,7 @@ Tokenizer::Tokenizer(ZeroCopyInputStream* input, current_.line = 0; current_.column = 0; + current_.end_column = 0; current_.type = TYPE_START; Refresh(); @@ -277,6 +278,7 @@ inline void Tokenizer::EndToken() { current_.text.append(buffer_ + token_start_, buffer_pos_ - token_start_); } token_start_ = -1; + current_.end_column = column_; } // ------------------------------------------------------------------- @@ -462,7 +464,7 @@ void Tokenizer::ConsumeBlockComment() { // ------------------------------------------------------------------- bool Tokenizer::Next() { - TokenType last_token_type = current_.type; + previous_ = current_; // Did we skip any characters after the last token? bool skipped_stuff = false; @@ -517,7 +519,7 @@ bool Tokenizer::Next() { if (TryConsumeOne<Digit>()) { // It's a floating-point number. - if (last_token_type == TYPE_IDENTIFIER && !skipped_stuff) { + if (previous_.type == TYPE_IDENTIFIER && !skipped_stuff) { // We don't accept syntax like "blah.123". error_collector_->AddError(line_, column_ - 2, "Need space between identifier and decimal point."); @@ -551,6 +553,7 @@ bool Tokenizer::Next() { current_.text.clear(); current_.line = line_; current_.column = column_; + current_.end_column = column_; return false; } |