private boolean isDuplicate(IndexEntry copyCandidate, FileSpan duplicateSearchSpan, Offset indexSegmentStartOffset,
      boolean checkAlreadyCopied) {
    // Duplicates can exist because of two reasons
    // 1. Duplicates in src: These can occur because compaction creates temporary duplicates
    // Consider the following situation
    // Segments 0_0, 1_0 and 2_0 are being compacted. All the data in 0_0 and half the data in 1_0 fit in 0_1 and the
    // rest in 1_1. When 0_1 is full and put into the "main" log, there will be some duplicated data in 0_1 and 1_0.
    // If a shutdown occurs after this point, then the code has to make sure not to copy any of the data already copied
    // 2. Duplicates in tgt: the checkpointing logic during copying works at the IndexSegment level
    // This means that a crash/shutdown could occur in the middle of copying the data represented by an IndexSegment and
    // the incremental progress inside an IndexSegment is not logged. After restart, compaction is resumed from the
    // checkpoint and will have to detect data that it has already copied
    try {
      boolean isDuplicate = false;
      IndexValue copyCandidateValue = copyCandidate.getValue();
      if (duplicateSearchSpan != null && validEntryFilter.alreadyExists(srcIndex, duplicateSearchSpan,
          copyCandidate.getKey(), copyCandidateValue)) {
        // is a duplicate because it already is in the store
        logger.trace("{} in segment with start offset {} in {} is a duplicate because it already exists in store",
            copyCandidate, indexSegmentStartOffset, storeId);
        isDuplicate = true;
      } else if (checkAlreadyCopied && validEntryFilter.alreadyExists(tgtIndex, null, copyCandidate.getKey(),
          copyCandidateValue)) {
        // is a duplicate because it has already been copied
        logger.trace("{} in segment with start offset {} in {} is a duplicate because it has already been copied",
            copyCandidate, indexSegmentStartOffset, storeId);
        isDuplicate = true;
      } else {
        // not a duplicate
        logger.trace("{} in index segment with start offset {} in {} is not a duplicate", copyCandidate,
            indexSegmentStartOffset, storeId);
      }
      return isDuplicate;
    } catch (StoreException e) {
      throw new IllegalStateException(e);
    }
  }
Determines if {@code copyCandidate} is a duplicate.
@param copyCandidate the {@link IndexEntry} to check
@param duplicateSearchSpan the search span for duplicates in the source index
@param indexSegmentStartOffset the start offset of the {@link IndexSegment} being processed
@param checkAlreadyCopied if {@code true}, checks if {@code copyCandidate} has already been copied.
@return {@code true} if {@code copyCandidate} is a duplicate of something that exists or has been copied
ambry-store/src/main/java/com/github/ambry/store/BlobStoreCompactor.java