aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
blob: cecc089d5b675345ab36b84aa23b5def1eed7505 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;

import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata.MiddlemanType;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.MissingInputFileException;
import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey;
import com.google.devtools.build.lib.skyframe.ArtifactSkyKey.OwnedArtifact;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
import java.util.Map;

/** A builder of values for {@link ArtifactSkyKey} keys. */
class ArtifactFunction implements SkyFunction {

  private final Predicate<PathFragment> allowedMissingInputs;

  ArtifactFunction(Predicate<PathFragment> allowedMissingInputs) {
    this.allowedMissingInputs = allowedMissingInputs;
  }

  @Override
  public SkyValue compute(SkyKey skyKey, Environment env)
      throws ArtifactFunctionException, InterruptedException {
    OwnedArtifact ownedArtifact = (OwnedArtifact) skyKey.argument();
    Artifact artifact = ownedArtifact.getArtifact();
    if (artifact.isSourceArtifact()) {
      try {
        return createSourceValue(artifact, ownedArtifact.isMandatory(), env);
      } catch (MissingInputFileException e) {
        // The error is not necessarily truly transient, but we mark it as such because we have
        // the above side effect of posting an event to the EventBus. Importantly, that event
        // is potentially used to report root causes.
        throw new ArtifactFunctionException(e, Transience.TRANSIENT);
      }
    }

    ActionAnalysisMetadata actionMetadata = extractActionFromArtifact(artifact, env);
    if (actionMetadata == null) {
      return null;
    }

    // If the action is an ActionTemplate, we need to expand the ActionTemplate into concrete
    // actions, execute those actions in parallel and then aggregate the action execution results.
    if (artifact.isTreeArtifact() && actionMetadata instanceof SpawnActionTemplate) {
      // Create the directory structures for the output TreeArtifact first.
      try {
        FileSystemUtils.createDirectoryAndParents(artifact.getPath());
      } catch (IOException e) {
        env.getListener().handle(
            Event.error(
                String.format(
                    "Failed to create output directory for TreeArtifact %s: %s",
                    artifact,
                    e.getMessage())));
        throw new ArtifactFunctionException(e, Transience.TRANSIENT);
      }

      return createTreeArtifactValueFromActionTemplate(
          (SpawnActionTemplate) actionMetadata, artifact, env);
    } else {
      Preconditions.checkState(
          actionMetadata instanceof Action,
          "%s is not a proper Action object and therefore cannot be executed",
          actionMetadata);
      Action action = (Action) actionMetadata;
      ActionExecutionValue actionValue =
        (ActionExecutionValue) env.getValue(ActionExecutionValue.key(action));
      if (actionValue == null) {
        return null;
      }

      if (artifact.isTreeArtifact()) {
        // We get a request for the whole tree artifact. We can just return the associated
        // TreeArtifactValue.
        return Preconditions.checkNotNull(actionValue.getTreeArtifactValue(artifact), artifact);
      } else if (isAggregatingValue(action)) {
        return createAggregatingValue(artifact, action,
            actionValue.getArtifactValue(artifact), env);
      } else {
        return createSimpleFileArtifactValue(artifact, action, actionValue, env);
      }
    }
  }

  private static TreeArtifactValue createTreeArtifactValueFromActionTemplate(
      SpawnActionTemplate actionTemplate, Artifact treeArtifact, Environment env)
      throws ArtifactFunctionException, InterruptedException {
    // Request the list of expanded actions from the ActionTemplate.
    ActionTemplateExpansionValue expansionValue = (ActionTemplateExpansionValue) env.getValue(
        ActionTemplateExpansionValue.key(actionTemplate));

    // The expanded actions are not yet available.
    if (env.valuesMissing()) {
      return null;
    }

    // Execute the expanded actions in parallel.
    Iterable<SkyKey> expandedActionExecutionKeys = ActionExecutionValue.keys(
        expansionValue.getExpandedActions());
    Map<SkyKey, SkyValue> expandedActionValueMap = env.getValues(expandedActionExecutionKeys);

    // The execution values of the expanded actions are not yet all available.
    if (env.valuesMissing()) {
      return null;
    }

    // Aggregate the ArtifactValues for individual TreeFileArtifacts into a TreeArtifactValue for
    // the parent TreeArtifact.
    ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> map = ImmutableMap.builder();
    for (Map.Entry<SkyKey, SkyValue> entry : expandedActionValueMap.entrySet()) {
      SkyKey expandedActionExecutionKey = entry.getKey();
      ActionExecutionValue actionExecutionValue = (ActionExecutionValue) entry.getValue();
      Action expandedAction = (Action) expandedActionExecutionKey.argument();

      Iterable<TreeFileArtifact> treeFileArtifacts = findActionOutputsWithMatchingParent(
          expandedAction, treeArtifact);

      Preconditions.checkState(
          !Iterables.isEmpty(treeFileArtifacts),
          "Action %s does not output TreeFileArtifact under %s",
          expandedAction,
          treeArtifact);

      for (TreeFileArtifact treeFileArtifact : treeFileArtifacts) {
        FileArtifactValue value  = createSimpleFileArtifactValue(
            treeFileArtifact, expandedAction, actionExecutionValue, env);
        map.put(treeFileArtifact, value);
      }
    }

    // Return the aggregated TreeArtifactValue.
    return TreeArtifactValue.create(map.build());
  }

  private FileArtifactValue createSourceValue(Artifact artifact, boolean mandatory, Environment env)
      throws MissingInputFileException, InterruptedException {
    SkyKey fileSkyKey = FileValue.key(RootedPath.toRootedPath(artifact.getRoot().getPath(),
        artifact.getPath()));
    FileValue fileValue;
    try {
      fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
          InconsistentFilesystemException.class, FileSymlinkException.class);
    } catch (IOException | InconsistentFilesystemException | FileSymlinkException e) {
      throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
    }
    if (fileValue == null) {
      return null;
    }
    if (!fileValue.exists()) {
      if (isAllowedMissingInput(fileSkyKey)) {
        return FileArtifactValue.MISSING_FILE_MARKER;
      } else {
        return missingInputFile(artifact, mandatory, null, env.getListener());
      }
    }
    try {
      return FileArtifactValue.create(artifact, fileValue);
    } catch (IOException e) {
      if (isAllowedMissingInput(fileSkyKey)) {
        return FileArtifactValue.MISSING_FILE_MARKER;
      }
      throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
    }
  }

  private boolean isAllowedMissingInput(SkyKey fileSkyKey) {
    return allowedMissingInputs.apply(((RootedPath) fileSkyKey.argument()).getRelativePath());
  }

  private static FileArtifactValue missingInputFile(
      Artifact artifact, boolean mandatory, Exception failure, EventHandler reporter)
      throws MissingInputFileException {
    if (!mandatory) {
      return FileArtifactValue.MISSING_FILE_MARKER;
    }
    throw makeMissingInputFileExn(artifact, mandatory, failure, reporter);
  }

  private static MissingInputFileException makeMissingInputFileExn(Artifact artifact,
      boolean mandatory, Exception failure, EventHandler reporter) {
    String extraMsg = (failure == null) ? "" : (":" + failure.getMessage());
    MissingInputFileException ex = new MissingInputFileException(
        constructErrorMessage(artifact) + extraMsg, null);
    if (mandatory) {
      reporter.handle(Event.error(ex.getLocation(), ex.getMessage()));
    }
    return ex;
  }

  // Non-aggregating artifact -- should contain at most one piece of artifact data.
  // data may be null if and only if artifact is a middleman artifact.
  private static FileArtifactValue createSimpleFileArtifactValue(Artifact artifact,
      Action generatingAction, ActionExecutionValue actionValue, Environment env)
      throws ArtifactFunctionException {
    FileArtifactValue value = actionValue.getArtifactValue(artifact);
    if (value != null) {
      return value;
    }
    // Middleman artifacts have no corresponding files, so their ArtifactValues should have already
    // been constructed during execution of the action.
    Preconditions.checkState(!artifact.isMiddlemanArtifact(), artifact);
    FileValue data = Preconditions.checkNotNull(actionValue.getData(artifact),
        "%s %s", artifact, actionValue);
    Preconditions.checkNotNull(data.getDigest(),
          "Digest should already have been calculated for %s (%s)", artifact, data);

    try {
      return FileArtifactValue.create(artifact, data);
    } catch (IOException e) {
      ActionExecutionException ex = new ActionExecutionException(e, generatingAction,
          /*catastrophe=*/false);
      env.getListener().handle(Event.error(ex.getLocation(), ex.getMessage()));
      // This is a transient error since we did the work that led to the IOException.
      throw new ArtifactFunctionException(ex, Transience.TRANSIENT);
    }
  }

  private static AggregatingArtifactValue createAggregatingValue(
      Artifact artifact,
      ActionAnalysisMetadata action,
      FileArtifactValue value,
      SkyFunction.Environment env)
      throws InterruptedException {
    // This artifact aggregates other artifacts. Keep track of them so callers can find them.
    ImmutableList.Builder<Pair<Artifact, FileArtifactValue>> inputs = ImmutableList.builder();
    for (Map.Entry<SkyKey, SkyValue> entry :
        env.getValues(ArtifactSkyKey.mandatoryKeys(action.getInputs())).entrySet()) {
      Artifact input = ArtifactSkyKey.artifact(entry.getKey());
      SkyValue inputValue = entry.getValue();
      Preconditions.checkNotNull(inputValue, "%s has null dep %s", artifact, input);
      if (!(inputValue instanceof FileArtifactValue)) {
        // We do not recurse in aggregating middleman artifacts.
        Preconditions.checkState(!(inputValue instanceof AggregatingArtifactValue),
            "%s %s %s", artifact, action, inputValue);
        continue;
      }
      inputs.add(Pair.of(input, (FileArtifactValue) inputValue));
    }
    return new AggregatingArtifactValue(inputs.build(), value);
  }

  /**
   * Returns whether this value needs to contain the data of all its inputs. Currently only tests to
   * see if the action is an aggregating middleman action. However, may include runfiles middleman
   * actions and Fileset artifacts in the future.
   */
  private static boolean isAggregatingValue(ActionAnalysisMetadata action) {
    return action.getActionType() == MiddlemanType.AGGREGATING_MIDDLEMAN;
  }

  @Override
  public String extractTag(SkyKey skyKey) {
    return Label.print(((OwnedArtifact) skyKey.argument()).getArtifact().getOwner());
  }

  private static ActionAnalysisMetadata extractActionFromArtifact(
      Artifact artifact, SkyFunction.Environment env) throws InterruptedException {
    ArtifactOwner artifactOwner = artifact.getArtifactOwner();

    Preconditions.checkState(artifactOwner instanceof ActionLookupKey, "", artifact, artifactOwner);
    SkyKey actionLookupKey = ActionLookupValue.key((ActionLookupKey) artifactOwner);
    ActionLookupValue value = (ActionLookupValue) env.getValue(actionLookupKey);
    if (value == null) {
      Preconditions.checkState(artifactOwner == CoverageReportValue.ARTIFACT_OWNER,
          "Not-yet-present artifact owner: %s", artifactOwner);
      return null;
    }
    // The value should already exist (except for the coverage report action output artifacts):
    // ConfiguredTargetValues were created during the analysis phase, and BuildInfo*Values
    // were created during the first analysis of a configured target.
    Preconditions.checkNotNull(value,
        "Owner %s of %s not in graph %s", artifactOwner, artifact, actionLookupKey);

    ActionAnalysisMetadata action = value.getGeneratingAction(artifact);
    if (artifact.hasParent()) {
      // We are trying to resolve the generating action for a TreeFileArtifact. It may not have
      // a generating action in the action graph at analysis time. In that case, we get the
      // generating action for its parent TreeArtifact, which contains this TreeFileArtifact.
      if (action == null) {
        action = value.getGeneratingAction(artifact.getParent());
      }
    }

    return Preconditions.checkNotNull(action,
        "Value %s does not contain generating action of %s", value, artifact);
  }

  private static Iterable<TreeFileArtifact> findActionOutputsWithMatchingParent(
      Action action, Artifact treeArtifact) {
    ImmutableList.Builder<TreeFileArtifact> matchingOutputs = ImmutableList.builder();
    for (Artifact output : action.getOutputs()) {
      Preconditions.checkState(output.hasParent(), "%s must be a TreeFileArtifact", output);
      if (output.getParent().equals(treeArtifact)) {
        matchingOutputs.add((TreeFileArtifact) output);
      }
    }
    return matchingOutputs.build();
  }

  private static final class ArtifactFunctionException extends SkyFunctionException {
    ArtifactFunctionException(MissingInputFileException e, Transience transience) {
      super(e, transience);
    }

    ArtifactFunctionException(ActionExecutionException e, Transience transience) {
      super(e, transience);
    }

    ArtifactFunctionException(IOException e, Transience transience) {
      super(e, transience);
    }
  }

  private static String constructErrorMessage(Artifact artifact) {
    if (artifact.getOwner() == null) {
      return String.format("missing input file '%s'", artifact.getPath().getPathString());
    } else {
      return String.format("missing input file '%s'", artifact.getOwner());
    }
  }
}