1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
#include "tensorflow/core/public/env.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/platform/protobuf.h"
namespace tensorflow {
Env::~Env() {}
RandomAccessFile::~RandomAccessFile() {}
WritableFile::~WritableFile() {}
Thread::~Thread() {}
EnvWrapper::~EnvWrapper() {}
Status ReadFileToString(Env* env, const string& fname, string* data) {
data->clear();
RandomAccessFile* file;
Status s = env->NewRandomAccessFile(fname, &file);
if (!s.ok()) {
return s;
}
int64 offset = 0;
static const int kBufferSize = 8192;
char* space = new char[kBufferSize];
while (true) {
StringPiece fragment;
s = file->Read(offset, kBufferSize, &fragment, space);
if (!s.ok()) {
if (errors::IsOutOfRange(s)) { // No more bytes, but not an error
s = Status::OK();
data->append(fragment.data(), fragment.size());
}
break;
}
offset += fragment.size();
data->append(fragment.data(), fragment.size());
if (fragment.empty()) {
break;
}
}
delete[] space;
delete file;
return s;
}
Status WriteStringToFile(Env* env, const string& fname,
const StringPiece& data) {
WritableFile* file;
Status s = env->NewWritableFile(fname, &file);
if (!s.ok()) {
return s;
}
s = file->Append(data);
if (s.ok()) {
s = file->Close();
}
delete file;
return s;
}
// A ZeroCopyInputStream on a RandomAccessFile.
namespace {
class FileStream : public ::tensorflow::protobuf::io::ZeroCopyInputStream {
public:
explicit FileStream(RandomAccessFile* file) : file_(file), pos_(0) {}
void BackUp(int count) override { pos_ -= count; }
bool Skip(int count) override {
pos_ += count;
return true;
}
int64 ByteCount() const override { return pos_; }
Status status() const { return status_; }
bool Next(const void** data, int* size) override {
StringPiece result;
Status s = file_->Read(pos_, kBufSize, &result, scratch_);
if (result.empty()) {
status_ = s;
return false;
}
pos_ += result.size();
*data = result.data();
*size = result.size();
return true;
}
private:
static const int kBufSize = 512 << 10;
RandomAccessFile* file_;
int64 pos_;
Status status_;
char scratch_[kBufSize];
};
} // namespace
Status ReadBinaryProto(Env* env, const string& fname,
::tensorflow::protobuf::MessageLite* proto) {
RandomAccessFile* file;
auto s = env->NewRandomAccessFile(fname, &file);
if (!s.ok()) {
return s;
}
std::unique_ptr<RandomAccessFile> file_holder(file);
std::unique_ptr<FileStream> stream(new FileStream(file));
// TODO(jiayq): the following coded stream is for debugging purposes to allow
// one to parse arbitrarily large messages for MessageLite. One most likely
// doesn't want to put protobufs larger than 64MB on Android, so we should
// eventually remove this and quit loud when a large protobuf is passed in.
::tensorflow::protobuf::io::CodedInputStream coded_stream(stream.get());
// Total bytes hard limit / warning limit are set to 1GB and 512MB
// respectively.
coded_stream.SetTotalBytesLimit(1024LL << 20, 512LL << 20);
if (!proto->ParseFromCodedStream(&coded_stream)) {
s = stream->status();
if (s.ok()) {
s = Status(error::DATA_LOSS, "Parse error");
}
}
return s;
}
} // namespace tensorflow
|
