1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.android.AndroidResourceMerger.MergingException;
import com.google.devtools.build.android.ParsedAndroidData.KeyValueConsumer;
import com.google.devtools.build.android.proto.SerializeFormat;
import com.google.devtools.build.android.proto.SerializeFormat.Header;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
/** Deserializes {@link DataKey}, {@link DataValue} entries from a binary file. */
public class AndroidParsedDataDeserializer implements AndroidDataDeserializer {
/** Task to deserialize resources from a path. */
static final class Deserialize implements Callable<Boolean> {
private final Path symbolPath;
private final ParsedAndroidData.Builder finalDataBuilder;
private final AndroidParsedDataDeserializer deserializer;
private Deserialize(
AndroidParsedDataDeserializer deserializer,
Path symbolPath,
ParsedAndroidData.Builder finalDataBuilder) {
this.deserializer = deserializer;
this.symbolPath = symbolPath;
this.finalDataBuilder = finalDataBuilder;
}
@Override
public Boolean call() throws Exception {
final ParsedAndroidData.Builder parsedDataBuilder = ParsedAndroidData.Builder.newBuilder();
deserializer.read(symbolPath, parsedDataBuilder.consumers());
// The builder isn't threadsafe, so synchronize the copyTo call.
synchronized (finalDataBuilder) {
// All the resources are sorted before writing, so they can be aggregated in
// whatever order here.
parsedDataBuilder.copyTo(finalDataBuilder);
}
return Boolean.TRUE;
}
}
private static final Logger logger =
Logger.getLogger(AndroidParsedDataDeserializer.class.getName());
private final ImmutableSet<String> filteredResources;
/**
* @param filteredResources resources that were filtered out of this target and should be ignored
* if they are referenced in symbols files.
*/
public static AndroidParsedDataDeserializer withFilteredResources(
Collection<String> filteredResources) {
return new AndroidParsedDataDeserializer(ImmutableSet.copyOf(filteredResources));
}
public static AndroidParsedDataDeserializer create() {
return new AndroidParsedDataDeserializer(ImmutableSet.<String>of());
}
private AndroidParsedDataDeserializer(ImmutableSet<String> filteredResources) {
this.filteredResources = filteredResources;
}
/**
* Reads the serialized {@link DataKey} and {@link DataValue} to the {@link KeyValueConsumers}.
*
* @param inPath The path to the serialized protocol buffer.
* @param consumers The {@link KeyValueConsumers} for the entries {@link DataKey} -> {@link
* DataValue}.
* @throws DeserializationException Raised for an IOException or when the inPath is not a valid
* proto buffer.
*/
@Override
public void read(Path inPath, KeyValueConsumers consumers) {
Stopwatch timer = Stopwatch.createStarted();
try (InputStream in = Files.newInputStream(inPath, StandardOpenOption.READ)) {
FileSystem currentFileSystem = inPath.getFileSystem();
Header header = Header.parseDelimitedFrom(in);
if (header == null) {
throw new DeserializationException("No Header found in " + inPath);
}
readEntriesSegment(consumers, in, currentFileSystem, header);
} catch (IOException e) {
throw new DeserializationException("Error deserializing " + inPath, e);
} finally {
logger.fine(
String.format("Deserialized in merged in %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
}
}
private void readEntriesSegment(
KeyValueConsumers consumers, InputStream in, FileSystem currentFileSystem, Header header)
throws IOException {
int numberOfEntries = header.getEntryCount();
Map<DataKey, KeyValueConsumer<DataKey, ? extends DataValue>> keys =
Maps.newLinkedHashMapWithExpectedSize(numberOfEntries);
for (int i = 0; i < numberOfEntries; i++) {
SerializeFormat.DataKey protoKey = SerializeFormat.DataKey.parseDelimitedFrom(in);
if (protoKey.hasResourceType()) {
FullyQualifiedName resourceName = FullyQualifiedName.fromProto(protoKey);
keys.put(
resourceName,
resourceName.isOverwritable()
? consumers.overwritingConsumer
: consumers.combiningConsumer);
} else {
keys.put(RelativeAssetPath.fromProto(protoKey, currentFileSystem), consumers.assetConsumer);
}
}
// Read back the sources table.
DataSourceTable sourceTable = DataSourceTable.read(in, currentFileSystem, header);
// TODO(corysmith): Make this a lazy read of the values.
for (Map.Entry<DataKey, KeyValueConsumer<DataKey, ?>> entry : keys.entrySet()) {
SerializeFormat.DataValue protoValue = SerializeFormat.DataValue.parseDelimitedFrom(in);
DataSource source = sourceTable.sourceFromId(protoValue.getSourceId());
// Compose the `shortPath` manually to ensure it uses a forward slash.
// Using Path.subpath would return a backslash-using path on Windows.
String shortPath =
source.getPath().getParent().getFileName() + "/" + source.getPath().getFileName();
if (filteredResources.contains(shortPath) && !Files.exists(source.getPath())) {
// Skip files that were filtered out during analysis.
// TODO(asteinb): Properly filter out these files from android_library symbol files during
// analysis instead, and remove this list.
continue;
}
if (protoValue.hasXmlValue()) {
// TODO(corysmith): Figure out why the generics are wrong.
// If I use Map<DataKey, KeyValueConsumer<DataKey, ? extends DataValue>>, I can put
// consumers into the map, but I can't call accept.
// If I use Map<DataKey, KeyValueConsumer<DataKey, ? super DataValue>>, I can consume
// but I can't put.
// Same for below.
@SuppressWarnings("unchecked")
KeyValueConsumer<DataKey, DataValue> value =
(KeyValueConsumer<DataKey, DataValue>) entry.getValue();
value.accept(entry.getKey(), DataResourceXml.from(protoValue, source));
} else {
@SuppressWarnings("unchecked")
KeyValueConsumer<DataKey, DataValue> value =
(KeyValueConsumer<DataKey, DataValue>) entry.getValue();
value.accept(entry.getKey(), DataValueFile.of(source));
}
}
}
/**
* Deserializes a list of serialized resource paths to a {@link
* com.google.devtools.build.android.ParsedAndroidData}.
*/
public static ParsedAndroidData deserializeSymbolsToData(List<Path> symbolPaths)
throws IOException {
AndroidParsedDataDeserializer deserializer = create();
final ListeningExecutorService executorService =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(15));
final ParsedAndroidData.Builder deserializedDataBuilder =
ParsedAndroidData.Builder.newBuilder();
try (Closeable closeable = ExecutorServiceCloser.createWith(executorService)) {
List<ListenableFuture<Boolean>> deserializing = new ArrayList<>();
for (final Path symbolPath : symbolPaths) {
deserializing.add(
executorService.submit(
new AndroidParsedDataDeserializer.Deserialize(
deserializer, symbolPath, deserializedDataBuilder)));
}
FailedFutureAggregator<MergingException> aggregator =
FailedFutureAggregator.createForMergingExceptionWithMessage(
"Failure(s) during dependency parsing");
aggregator.aggregateAndMaybeThrow(deserializing);
}
return deserializedDataBuilder.build();
}
public static ParsedAndroidData deserializeSingleAndroidData(SerializedAndroidData data) {
final ParsedAndroidData.Builder builder = ParsedAndroidData.Builder.newBuilder();
final AndroidParsedDataDeserializer deserializer = create();
data.deserialize(deserializer, builder.consumers());
return builder.build();
}
}
|
