1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.actions;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertTrue;
import com.google.common.base.Strings;
import com.google.devtools.build.lib.actions.cache.DigestUtils;
import com.google.devtools.build.lib.testutil.TestThread;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.util.BlazeClock;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* Tests for DigestUtils.
*/
@RunWith(JUnit4.class)
public class DigestUtilsTest {
private static void assertMd5CalculationConcurrency(boolean expectConcurrent,
final boolean fastDigest, final int fileSize1, final int fileSize2) throws Exception {
final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads.
final CountDownLatch readyLatch = new CountDownLatch(1); // Used to block main thread.
FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) {
@Override
protected byte[] getMD5Digest(Path path) throws IOException {
try {
barrierLatch.countDown();
readyLatch.countDown();
// Either both threads will be inside getMD5Digest at the same time or they
// both will be blocked.
barrierLatch.await();
} catch (Exception e) {
throw new IOException(e);
}
return super.getMD5Digest(path);
}
@Override
protected String getFastDigestFunctionType(Path path) {
return "MD5";
}
@Override
protected byte[] getFastDigest(Path path) throws IOException {
return fastDigest ? super.getMD5Digest(path) : null;
}
};
final Path myFile1 = myfs.getPath("/f1.dat");
final Path myFile2 = myfs.getPath("/f2.dat");
FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));
TestThread thread1 = new TestThread () {
@Override public void runTest() throws Exception {
DigestUtils.getDigestOrFail(myFile1, fileSize1);
}
};
TestThread thread2 = new TestThread () {
@Override public void runTest() throws Exception {
DigestUtils.getDigestOrFail(myFile2, fileSize2);
}
};
thread1.start();
thread2.start();
if (!expectConcurrent) { // Synchronized case.
// Wait until at least one thread reached getMD5Digest().
assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
// Only 1 thread should be inside getMD5Digest().
assertEquals(1, barrierLatch.getCount());
barrierLatch.countDown(); // Release barrier latch, allowing both threads to proceed.
}
// Test successful execution within 5 seconds.
thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
}
/**
* Ensures that MD5 calculation is synchronized for files
* greater than 4096 bytes if MD5 is not available cheaply,
* so machines with rotating drives don't become unusable.
*/
@Test
public void testMd5CalculationConcurrency() throws Exception {
assertMd5CalculationConcurrency(true, true, 4096, 4096);
assertMd5CalculationConcurrency(true, true, 4097, 4097);
assertMd5CalculationConcurrency(true, false, 4096, 4096);
assertMd5CalculationConcurrency(false, false, 4097, 4097);
assertMd5CalculationConcurrency(true, false, 1024, 4097);
assertMd5CalculationConcurrency(true, false, 1024, 1024);
}
@Test
public void testRecoverFromMalformedDigest() throws Exception {
final byte[] malformed = {0, 0, 0};
FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) {
@Override
protected String getFastDigestFunctionType(Path path) {
return "MD5";
}
@Override
protected byte[] getFastDigest(Path path) throws IOException {
// MD5 digests are supposed to be 16 bytes.
return malformed;
}
};
Path path = myFS.getPath("/file");
FileSystemUtils.writeContentAsLatin1(path, "a");
byte[] result = DigestUtils.getDigestOrFail(path, 1);
assertArrayEquals(path.getMD5Digest(), result);
assertNotSame(malformed, result);
assertEquals(16, result.length);
}
}
|
