path: root/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java

// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.actions;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.fail;

import com.google.common.base.Strings;
import com.google.common.cache.CacheStats;
import com.google.devtools.build.lib.actions.cache.DigestUtils;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.testutil.TestThread;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.CheckReturnValue;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/**
 * Tests for DigestUtils.
 */
@RunWith(JUnit4.class)
public class DigestUtilsTest {

  @After
  public void tearDown() {
    DigestUtils.configureCache(0);
  }

  private static void assertDigestCalculationConcurrency(boolean expectConcurrent,
      final boolean fastDigest, final int fileSize1, final int fileSize2,
      HashFunction hf) throws Exception {
    final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads.
    final CountDownLatch readyLatch = new CountDownLatch(1);   // Used to block main thread.

    FileSystem myfs =
        new InMemoryFileSystem(BlazeClock.instance(), hf) {
          @Override
          protected byte[] getDigest(Path path, HashFunction hashFunction) throws IOException {
            try {
              barrierLatch.countDown();
              readyLatch.countDown();
              // Either both threads will be inside getDigest at the same time or they
              // both will be blocked.
              barrierLatch.await();
            } catch (Exception e) {
              throw new IOException(e);
            }
            return super.getDigest(path, hashFunction);
          }

          @Override
          protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            return fastDigest ? super.getDigest(path, hashFunction) : null;
          }
        };

    final Path myFile1 = myfs.getPath("/f1.dat");
    final Path myFile2 = myfs.getPath("/f2.dat");
    FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
    FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));

     TestThread thread1 = new TestThread () {
       @Override public void runTest() throws Exception {
         DigestUtils.getDigestOrFail(myFile1, fileSize1);
       }
     };

     TestThread thread2 = new TestThread () {
       @Override public void runTest() throws Exception {
         DigestUtils.getDigestOrFail(myFile2, fileSize2);
       }
     };

     thread1.start();
     thread2.start();
     if (!expectConcurrent) { // Synchronized case.
      // Wait until at least one thread reached getDigest().
      assertThat(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)).isTrue();
      // Only 1 thread should be inside getDigest().
      assertThat(barrierLatch.getCount()).isEqualTo(1);
       barrierLatch.countDown(); // Release barrier latch, allowing both threads to proceed.
     }
     // Test successful execution within 5 seconds.
     thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
     thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
  }

  /**
   * Ensures that digest calculation is synchronized for files
   * greater than 4096 bytes if the digest is not available cheaply,
   * so machines with rotating drives don't become unusable.
   */
  @Test
  public void testCalculationConcurrency() throws Exception {
    for (HashFunction hf : Arrays.asList(HashFunction.MD5, HashFunction.SHA1)) {
      assertDigestCalculationConcurrency(true, true, 4096, 4096, hf);
      assertDigestCalculationConcurrency(true, true, 4097, 4097, hf);
      assertDigestCalculationConcurrency(true, false, 4096, 4096, hf);
      assertDigestCalculationConcurrency(false, false, 4097, 4097, hf);
      assertDigestCalculationConcurrency(true, false, 1024, 4097, hf);
      assertDigestCalculationConcurrency(true, false, 1024, 1024, hf);
    }
  }

  public void assertRecoverFromMalformedDigest(HashFunction... hashFunctions) throws Exception {
    for (HashFunction hf : hashFunctions) {
      final byte[] malformed = {0, 0, 0};
      FileSystem myFS =
          new InMemoryFileSystem(BlazeClock.instance(), hf) {
            @Override
            protected byte[] getFastDigest(Path path, HashFunction hashFunction)
                throws IOException {
              // Digest functions have more than 3 bytes, usually at least 16.
              return malformed;
            }
          };
      Path path = myFS.getPath("/file");
      FileSystemUtils.writeContentAsLatin1(path, "a");
      byte[] result = DigestUtils.getDigestOrFail(path, 1);
      assertThat(result).isEqualTo(path.getDigest());
      assertThat(result).isNotSameAs(malformed);
      assertThat(path.isValidDigest(result)).isTrue();
    }
  }

  @Test
  public void testRecoverFromMalformedDigestWithoutCache() throws Exception {
    try {
      DigestUtils.getCacheStats();
      fail("Digests cache should remain disabled until configureCache is called");
    } catch (NullPointerException expected) {
    }
    assertRecoverFromMalformedDigest(HashFunction.MD5, HashFunction.SHA1);
    try {
      DigestUtils.getCacheStats();
      fail("Digests cache was unexpectedly enabled through the test");
    } catch (NullPointerException expected) {
    }
  }

  @Test
  public void testRecoverFromMalformedDigestWithCache() throws Exception {
    DigestUtils.configureCache(10);
    assertThat(DigestUtils.getCacheStats()).isNotNull(); // Ensure the cache is enabled.

    // When using the cache, we cannot run our test using different hash functions because the
    // hash function is not part of the cache key. This is intentional: the hash function is
    // essentially final and can only be changed for tests. Therefore, just test the same hash
    // function twice to further exercise the cache code.
    assertRecoverFromMalformedDigest(HashFunction.MD5, HashFunction.MD5);

    assertThat(DigestUtils.getCacheStats()).isNotNull(); // Ensure the cache remains enabled.
  }

  /** Helper class to assert the cache statistics. */
  private static class CacheStatsChecker {
    /** Cache statistics, grabbed at construction time. */
    private final CacheStats stats;

    private int expectedEvictionCount;
    private int expectedHitCount;
    private int expectedMissCount;

    CacheStatsChecker() {
      this.stats = DigestUtils.getCacheStats();
    }

    @CheckReturnValue
    CacheStatsChecker evictionCount(int count) {
      expectedEvictionCount = count;
      return this;
    }

    @CheckReturnValue
    CacheStatsChecker hitCount(int count) {
      expectedHitCount = count;
      return this;
    }

    @CheckReturnValue
    CacheStatsChecker missCount(int count) {
      expectedMissCount = count;
      return this;
    }

    void check() throws Exception {
      assertThat(stats.evictionCount()).isEqualTo(expectedEvictionCount);
      assertThat(stats.hitCount()).isEqualTo(expectedHitCount);
      assertThat(stats.missCount()).isEqualTo(expectedMissCount);
    }
  }

  @Test
  public void testCache() throws Exception {
    final AtomicInteger getFastDigestCounter = new AtomicInteger(0);
    final AtomicInteger getDigestCounter = new AtomicInteger(0);

    FileSystem tracingFileSystem =
        new InMemoryFileSystem(BlazeClock.instance()) {
          @Override
          protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            getFastDigestCounter.incrementAndGet();
            return null;
          }

          @Override
          protected byte[] getDigest(Path path, HashFunction hashFunction) throws IOException {
            getDigestCounter.incrementAndGet();
            return super.getDigest(path, hashFunction);
          }
        };

    DigestUtils.configureCache(2);

    final Path file1 = tracingFileSystem.getPath("/1.txt");
    final Path file2 = tracingFileSystem.getPath("/2.txt");
    final Path file3 = tracingFileSystem.getPath("/3.txt");
    FileSystemUtils.writeContentAsLatin1(file1, "some contents");
    FileSystemUtils.writeContentAsLatin1(file2, "some other contents");
    FileSystemUtils.writeContentAsLatin1(file3, "and something else");

    byte[] digest1 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    assertThat(getFastDigestCounter.get()).isEqualTo(1);
    assertThat(getDigestCounter.get()).isEqualTo(1);
    new CacheStatsChecker().evictionCount(0).hitCount(0).missCount(1).check();

    byte[] digest2 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    assertThat(getFastDigestCounter.get()).isEqualTo(2);
    assertThat(getDigestCounter.get()).isEqualTo(1);
    new CacheStatsChecker().evictionCount(0).hitCount(1).missCount(1).check();

    assertThat(digest2).isEqualTo(digest1);

    // Evict the digest for the previous file.
    DigestUtils.getDigestOrFail(file2, file2.getFileSize());
    DigestUtils.getDigestOrFail(file3, file3.getFileSize());
    new CacheStatsChecker().evictionCount(1).hitCount(1).missCount(3).check();

    // And now try to recompute it.
    byte[] digest3 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    new CacheStatsChecker().evictionCount(2).hitCount(1).missCount(4).check();

    assertThat(digest3).isEqualTo(digest1);
  }
}