@Test
  public void testHardDeleteRecovery() throws Exception {
    List<MockId> keys = setupIndexForTest();

    // indexes: **[1 2] [3 4] [3d 5] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();
    index.persistAndAdvanceStartTokenSafeToPersist();

    // indexes: [1 2**] [3 4] [3d 5] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();

    // indexes: [1 2*] [3 4*] [3d 5] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();
    index.persistAndAdvanceStartTokenSafeToPersist();

    // indexes: [1 2*] [3 4] [3d 5*] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();

    // Here we are going to perform a hard delete. We set the exception for diskIOScheduler so that we can persist
    // HardDeleteRecoveryRange, but not changing the log data. Since diskIOScheduler will be invoked before every log
    // write, we can use it to throw an exception and stop the writes from happening. Then we have to perform the recovery
    // and make sure recovery works.
    Set<StoreKey> hardDeletedKeys = new HashSet<>();

    // indexes: [1 2] [3 4] [3d 5*] [6 7*] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    diskIOScheduler.setException(new IOException("Stopping log writes"));
    try {
      index.hardDelete();
      fail("Should throw an exception here");
    } catch (RuntimeException e) {
      // make sure we nothing is changed in the log
      checkRecordHardDeleted(keys, hardDeletedKeys);
    }

    // Now recovery from the "crash"
    diskIOScheduler.setException(null);
    hardDeletedKeys.add(keys.get(1)); // blobId02
    hardDeletedKeys.add(keys.get(5)); // blobId06
    index.performHardDeleteRecovery();
    checkRecordHardDeleted(keys, hardDeletedKeys);

    StoreFindToken startToken = (StoreFindToken) index.hardDeleter.getStartToken();
    StoreFindToken endToken = (StoreFindToken) index.hardDeleter.getEndToken();
    Assert.assertEquals("StartToken should match EndToken", startToken, endToken);
    assertEquals("Token type mismatch", startToken.getType(), FindTokenType.IndexBased);
    assertEquals("Token key mismatch", startToken.getStoreKey(), keys.get(5)); // blobId06
    assertEquals("Token offset mismatch", startToken.getOffset().getOffset(), 4 * 2 * helper.sizeOfEntry);

    // Now make sure we can move on with hard deleter.
    // After perform recovery, the start token is now equal to end token.
    // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d**] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();

    // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d*] [8 9*] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();

    // inject this supplier to mimic the persistor thread to persist recovery range.
    // The reason to set it here and not above is the hardDelete method below would actually trigger a
    // change of recovery range.
    index.hardDeleter.setInjectedBeforeHardDeleteSupplier(() -> {
      index.persistAndAdvanceStartTokenSafeToPersist();
      return null;
    });
    // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d*] [8 9] [1d 10*] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u 10d 8d]
    index.hardDelete();
    index.hardDeleter.setInjectedBeforeHardDeleteSupplier(null);

    // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d*] [8 9] [1d 10] [1u 3u] [8d 10d]
    // journal:                                                    [3u 1u* 10d 8d]
    index.hardDelete();
    hardDeletedKeys.add(keys.get(9)); // blobId10
    hardDeletedKeys.add(keys.get(7)); // blobId08
    checkRecordHardDeleted(keys, hardDeletedKeys);
  }
Test for recovery in HardDeleter.
@throws Exception
ambry-store/src/test/java/com/github/ambry/store/HardDeleterTest.java