1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android;
import com.google.common.base.Stopwatch;
import com.google.devtools.build.android.proto.SerializeFormat;
import com.google.devtools.build.android.proto.SerializeFormat.Header;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
/** Serializes {@link DataKey},{@link DataValue} entries to a binary file. */
public class AndroidDataSerializer {
private static final Logger logger = Logger.getLogger(AndroidDataSerializer.class.getName());
private final NavigableMap<DataKey, DataValue> entries = new TreeMap<>();
public static AndroidDataSerializer create() {
return new AndroidDataSerializer();
}
private AndroidDataSerializer() {}
/**
* Writes all of the collected DataKey -> DataValue.
*
* <p>The binary format will be:
*
* <pre>
* {@link Header}
* {@link com.google.devtools.build.android.proto.SerializeFormat.DataKey} keys...
* {@link com.google.devtools.build.android.proto.SerializeFormat.DataValue} entries...
* </pre>
*
* The key and values will be written in comparable order, allowing for the optimization of not
* converting the DataValue from binary, only writing it into a merged serialized binary.
*/
public void flushTo(Path out) throws IOException {
Stopwatch timer = Stopwatch.createStarted();
// Ensure the parent directory exists, if any.
if (out.getParent() != null) {
Files.createDirectories(out.getParent());
}
try (OutputStream outStream =
new BufferedOutputStream(
Files.newOutputStream(out, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))) {
// Set the header for the deserialization process.
SerializeFormat.Header.Builder headerBuilder =
Header.newBuilder().setEntryCount(entries.size());
// Create table of source paths to allow references in the serialization format via an index.
ByteArrayOutputStream sourceTableOutputStream = new ByteArrayOutputStream(2048);
DataSourceTable sourceTable =
DataSourceTable.createAndWrite(entries, sourceTableOutputStream, headerBuilder);
headerBuilder.build().writeDelimitedTo(outStream);
writeKeyValuesTo(entries, outStream, sourceTable, sourceTableOutputStream.toByteArray());
}
logger.fine(String.format("Serialized merged in %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
}
private void writeKeyValuesTo(
NavigableMap<DataKey, DataValue> map,
OutputStream outStream,
DataSourceTable sourceTable,
byte[] sourceTableBytes)
throws IOException {
Set<Map.Entry<DataKey, DataValue>> entries = map.entrySet();
int[] orderedValueSizes = new int[entries.size()];
int valueSizeIndex = 0;
// Serialize all the values in sorted order to a intermediate buffer, so that the keys
// can be associated with a value size.
// TODO(corysmith): Tune the size of the byte array.
ByteArrayOutputStream valuesOutputStream = new ByteArrayOutputStream(2048);
for (Map.Entry<DataKey, DataValue> entry : entries) {
orderedValueSizes[valueSizeIndex++] =
entry.getValue().serializeTo(sourceTable, valuesOutputStream);
}
// Serialize all the keys in sorted order
valueSizeIndex = 0;
for (Map.Entry<DataKey, DataValue> entry : entries) {
entry.getKey().serializeTo(outStream, orderedValueSizes[valueSizeIndex++]);
}
// write the source table
outStream.write(sourceTableBytes);
// write the values to the output stream.
outStream.write(valuesOutputStream.toByteArray());
}
/** Queues the key and value for serialization as a entries entry. */
public void queueForSerialization(DataKey key, DataValue value) {
entries.put(key, value);
}
}
|
