@Test
  public void testHardDelete() {
    // Create a mock index with regular log.
    // perform puts to the index.
    // perform deletes to the index.
    // call hardDelete() explicitly (set the thread frequency time to a large number if required) -
    // that will do findDeleted, prune and performHardDelete()
    // perform will need a messageStoreHardDelete implementation that just returns something
    // that will be written back. - need to implement that.
    // disable index persistor if that can be done.
    // call persist cleanup token and close the index. Reopen it to execute recovery path.
    // perform hard deletes upto some point.
    // perform more deletes.
    // do recovery.
    try {
      List<MockId> keys = setupIndexForTest();
      boolean recordContentChecked = false;
      for (int i = 0; i < 3; i++) {
        assertEquals("There should have been no progress reported for hard deletes", 0, index.getHardDeleteProgress());
        // The first * shows where startTokenSafeToPersist is
        // The second * shows where startToken/endToken are before the operations.
        // Note since we are only depicting values before and after hardDelete() is done, startToken and endToken
        // will be the same.
        StoreFindToken token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        assertEquals("Token type mismatch", token.getType(), FindTokenType.Uninitialized);
        Set<StoreKey> hardDeletedKeys = new HashSet<>();

        // indexes: **[1 2] [3 4] [3d 5] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        boolean tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize());
        // startToken = endToken = 2.

        // call into the log flush hooks so that startTokenSafeToPersist = startToken = 2.
        index.persistAndAdvanceStartTokenSafeToPersist();
        token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased);
        assertEquals("Token key mismatch", token.getStoreKey(), keys.get(1)); // blobId02
        // Token still points to the first index segment, and the start offset of the first segment is 0
        assertEquals("Token offset mismatch", token.getOffset().getOffset(), 0);

        // indexes: [1 2**] [3 4] [3d 5] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize());

        // Without moving the token forward, make sure it's still the old value.
        token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased);
        assertEquals("Token key mismatch", token.getStoreKey(), keys.get(1)); // blobId02
        assertEquals("Token offset mismatch", token.getOffset().getOffset(), 0);
        // startToken = endToken = 4.

        // indexes: [1 2*] [3 4*] [3d 5] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        // blob03 is deleted in this index segment, but it's undeleted later.
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize());
        // startToken = 5, endToken = 5, startTokenSafeToPersist = 2

        // indexes: [1 2*] [3 4] [3d 5*] [6 7] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        index.persistAndAdvanceStartTokenSafeToPersist();
        token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased);
        assertEquals("Token key mismatch", token.getStoreKey(), keys.get(4)); // blobId05
        // Token moves to third segment, the start offset of the third segment is 4 * sizeOfEntry
        assertEquals("Token offset mismatch", token.getOffset().getOffset(), 2 * 2 * helper.sizeOfEntry);
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize());
        // startToken = 7, endToken = 7, starttokenSafeToPersist = 5
        // 3d just got pruned.

        // indexes: [1 2] [3 4] [3d 5*] [6 7*] [2d 6d] [8 9] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        hardDeletedKeys.add(keys.get(1)); // blobId02
        hardDeletedKeys.add(keys.get(5)); // blobId06
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        // blobid02 and blobid06
        assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize());

        // indexes: [1 2] [3 4] [3d 5*] [6 7] [2d 6d*] [8 9] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        // Didn't perform hard delete, so blobid02 and blobid06
        assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize());

        // indexes: [1 2] [3 4] [3d 5*] [6 7] [2d 6d] [8 9*] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        index.persistAndAdvanceStartTokenSafeToPersist();
        token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased);
        assertEquals("Token key mismatch", token.getStoreKey(), keys.get(8)); // blobId09
        assertEquals("Token offset mismatch", token.getOffset().getOffset(), 5 * 2 * helper.sizeOfEntry);
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        // blob01 is deleted in this index segment, but it's undeleted later
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize());

        // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d] [8 9*] [1d 10*] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d]
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }

        // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d] [8 9*] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u* 10d 8d]
        tokenMovedForward = index.hardDelete();
        Assert.assertTrue(tokenMovedForward);
        hardDeletedKeys.add(keys.get(9)); // blobId10
        hardDeletedKeys.add(keys.get(7)); // blobId08
        if (!recordContentChecked) {
          checkRecordHardDeleted(keys, hardDeletedKeys);
        }
        // blobid08 and blobid10
        assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize());
        // Persist safeToken here since now we have two items in the recovery range and their startToken is journal-based.
        token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        index.hardDeleter.getHardDeleteRecoveryRange().pruneTill(token);
        // After prune we still have two
        assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize());

        // indexes: [1 2] [3 4] [3d 5] [6 7] [2d 6d] [8 9*] [1d 10] [1u 3u] [8d 10d]
        // journal:                                                    [3u 1u 10d 8d*]
        // All caught up, hardDelete should not move token forward.
        tokenMovedForward = index.hardDelete();
        Assert.assertFalse(tokenMovedForward);

        index.persistAndAdvanceStartTokenSafeToPersist();
        token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo();
        assertEquals("Token type mismatch", token.getType(), FindTokenType.JournalBased);
        // JournalBased Token is not inclusive, so offset pointing to 1u is the offset of 1u.
        assertEquals("Token offset mismatch", token.getOffset().getOffset(), 17 * helper.sizeOfEntry);
        assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize());

        // directly prune the recovery range completely (which should happen since we flushed till the endToken).
        index.pruneHardDeleteRecoveryRange();

        // Test recovery - this tests reading from the persisted token, filling up the hard delete recovery range and
        // then actually redoing the hard deletes on the range.
        index.performHardDeleteRecovery();

        // reset the internal tokens
        index.resetHardDeleterTokens();
        recordContentChecked = true;
      }
      index.close(false);
    } catch (Exception e) {
      e.printStackTrace();
      fail("Unexpected exception");
    }
  }
Test hard delete and recovery failure due to IOExceptions.
@throws Exception
ambry-store/src/test/java/com/github/ambry/store/HardDeleterTest.java