1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
/** Common utilities for dealing with files outside the package roots. */
public class ExternalFilesHelper {
private final AtomicReference<PathPackageLocator> pkgLocator;
private final Set<Path> immutableDirs;
private final boolean errorOnExternalFiles;
public ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator) {
this(pkgLocator, ImmutableSet.<Path>of(), /*errorOnExternalFiles=*/false);
}
@VisibleForTesting
ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator,
boolean errorOnExternalFiles) {
this(pkgLocator, ImmutableSet.<Path>of(), errorOnExternalFiles);
}
/**
* @param pkgLocator an {@link AtomicReference} to a {@link PathPackageLocator} used to
* determine what files are internal
* @param immutableDirs directories whose contents may be considered unchangeable
* @param errorOnExternalFiles whether or not to allow references to files outside of
* the directories provided by pkgLocator or immutableDirs. See
* {@link #maybeHandleExternalFile(RootedPath, SkyFunction.Environment)} for more details.
*/
ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator, Set<Path> immutableDirs,
boolean errorOnExternalFiles) {
this.pkgLocator = pkgLocator;
this.immutableDirs = immutableDirs;
this.errorOnExternalFiles = errorOnExternalFiles;
}
private enum FileType {
// A file inside the package roots.
INTERNAL_FILE,
// A file outside the package roots that users of ExternalFilesHelper may pretend is immutable.
EXTERNAL_IMMUTABLE_FILE,
// A file outside the package roots about which we may make no other assumptions.
EXTERNAL_MUTABLE_FILE,
}
private FileType getFileType(RootedPath rootedPath) {
// TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a
// lot of immutable directories. Consider either explicitly preventing this case or using a more
// efficient approach here (e.g. use a trie for determing if a file is under an immutable
// directory).
if (!pkgLocator.get().getPathEntries().contains(rootedPath.getRoot())) {
Path path = rootedPath.asPath();
for (Path immutableDir : immutableDirs) {
if (path.startsWith(immutableDir)) {
return FileType.EXTERNAL_IMMUTABLE_FILE;
}
}
return FileType.EXTERNAL_MUTABLE_FILE;
}
return FileType.INTERNAL_FILE;
}
public boolean shouldAssumeImmutable(RootedPath rootedPath) {
return getFileType(rootedPath) == FileType.EXTERNAL_IMMUTABLE_FILE;
}
/**
* Potentially adds a dependency on build_id to env if this instance is configured
* with errorOnExternalFiles=false and rootedPath is an external mutable file.
* If errorOnExternalFiles=true and rootedPath is an external mutable file then
* a FileOutsidePackageRootsException is thrown. If the file is an external file that is
* referenced by the WORKSPACE, it gets a dependency on the //external package (and, thus,
* WORKSPACE file changes). This method is a no-op for any rootedPaths that fall within the known
* package roots.
*
* @param rootedPath
* @param env
* @throws FileOutsidePackageRootsException
*/
public void maybeHandleExternalFile(RootedPath rootedPath, SkyFunction.Environment env)
throws FileOutsidePackageRootsException {
if (getFileType(rootedPath) == FileType.EXTERNAL_MUTABLE_FILE) {
if (!errorOnExternalFiles) {
// For files outside the package roots that are not assumed to be immutable, add a
// dependency on the build_id. This is sufficient for correctness; all other files
// will be handled by diff awareness of their respective package path, but these
// files need to be addressed separately.
//
// Using the build_id here seems to introduce a performance concern because the upward
// transitive closure of these external files will get eagerly invalidated on each
// incremental build (e.g. if every file had a transitive dependency on the filesystem root,
// then we'd have a big performance problem). But this a non-issue by design:
// - We don't add a dependency on the parent directory at the package root boundary, so the
// only transitive dependencies from files inside the package roots to external files are
// through symlinks. So the upwards transitive closure of external files is small.
// - The only way external source files get into the skyframe graph in the first place is
// through symlinks outside the package roots, which we neither want to encourage nor
// optimize for since it is not common. So the set of external files is small.
//
// The above reasoning doesn't hold for bazel, because external repositories
// (e.g. new_local_repository) cause lots of external symlinks to be present in the build.
// So bazel pretends that these external repositories are immutable to avoid the performance
// penalty described above.
PrecomputedValue.dependOnBuildId(env);
} else {
throw new FileOutsidePackageRootsException(rootedPath);
}
} else if (getFileType(rootedPath) == FileType.EXTERNAL_IMMUTABLE_FILE) {
PackageValue pkgValue =
(PackageValue)
Preconditions.checkNotNull(
env.getValue(PackageValue.key(Package.EXTERNAL_PACKAGE_IDENTIFIER)));
Preconditions.checkState(!pkgValue.getPackage().containsErrors());
}
}
}
|
