@Test
  public void testCompactionFailure()
      throws ReflectiveOperationException, StoreException, CloudStorageException {
    v2TestOnly();
    this.compactionDryRun = false;
    setupCloudStore(false, false, 0, true);
    clearContainer(partitionId, (AzureCloudDestinationSync) dest, new VerifiableProperties(properties));
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();

    /*
      Add some permanent blobs.
      set isVcr = true so that the lifeVersion is 0, else it is -1
     */
    int numBlobs = 10;
    IntStream.range(0,numBlobs).forEach(i -> CloudTestUtil.addBlobToMessageSet(messageWriteSet, 1024,
        Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId,
        System.currentTimeMillis(), isVcr));

    /*
      Spy on these objects to intercept certain calls. We don't want to mock all of them because we want
      to execute the real methods by contacting Azurite - storage emulator. However, we want to inject failures at
      certain points for testing. Besides, mocking would involve a lot of interface change which is just ugly.
      Might be easier if dependency injection is used.

      Object map:
      CloudBlobStore -> AzureCloudDestinationSync -> BlobContainerClient -> BlobClient
     */
    MetricRegistry metricRegistry = new MetricRegistry();
    AzureMetrics azureMetrics = new AzureMetrics(metricRegistry);
    VcrMetrics vcrMetrics = new VcrMetrics(metricRegistry);
    AzureCloudDestinationSync spyDest = spy(azuriteUtils.getAzuriteClient(properties, metricRegistry,
        clusterMap, accountService));
    CloudBlobStore spyStore = spy(new CloudBlobStore(new VerifiableProperties(properties), partitionId, spyDest,
        clusterMap, vcrMetrics));
    spyStore.start();
    BlobContainerClient spyContainerClient = spy(getTestBlobContainerClient(spyDest, partitionId));
    when(spyDest.getBlobStore(any())).thenReturn(spyContainerClient);

    // Put blobs
    spyStore.put(messageWriteSet);

    // Delete blobs
    spyStore.delete(messageWriteSet.getMessageSetInfo());

    // Compact blobs, return error
    BlobClient mockBlobClient = mock(BlobClient.class);
    Response<Void> mockResponse = mock(Response.class);
    when(spyContainerClient.getBlobClient(any())).thenReturn(mockBlobClient);
    when(mockBlobClient.deleteWithResponse(any(), any(), any(), any())).thenReturn(mockResponse);
    // Return error when compaction tries to delete blobs and check nothing is deleted
    when(mockResponse.getStatusCode()).thenReturn(HttpStatus.SC_NOT_FOUND);
    assertEquals(0, spyDest.compactPartition(String.valueOf(partitionId.getId())));
    assertEquals(numBlobs, azureMetrics.blobCompactionErrorCount.getCount());
    when(mockResponse.getStatusCode()).thenReturn(HttpStatus.SC_INTERNAL_SERVER_ERROR);
    assertEquals(0, spyDest.compactPartition(String.valueOf(partitionId.getId())));
    assertEquals(numBlobs * 2, azureMetrics.blobCompactionErrorCount.getCount());
    // Check blob exists
    assertEquals(0, azureMetrics.blobCompactionSuccessRate.getCount());
    when(spyContainerClient.getBlobClient(any())).thenCallRealMethod();
    messageWriteSet.getMessageSetInfo().forEach(messageInfo -> {
      try {
        assertTrue(spyDest.doesBlobExist((BlobId) messageInfo.getStoreKey()));
      } catch (CloudStorageException e) {
        throw new RuntimeException(e);
      }
    });

    // Really compact and check blobs are gone
    when(mockBlobClient.deleteWithResponse(any(), any(), any(), any())).thenCallRealMethod();
    assertEquals(numBlobs, spyDest.compactPartition(String.valueOf(partitionId.getId())));
    assertEquals(numBlobs, azureMetrics.blobCompactionSuccessRate.getCount());
    messageWriteSet.getMessageSetInfo().forEach(messageInfo -> {
      try {
        assertFalse(spyDest.doesBlobExist((BlobId) messageInfo.getStoreKey()));
      } catch (CloudStorageException e) {
        throw new RuntimeException(e);
      }
    });
  }
Tests that compaction handles failures incl. 404 and other errors
@throws ReflectiveOperationException
@throws StoreException
@throws CloudStorageException
ambry-vcr/src/test/java/com/github/ambry/vcr/CloudBlobStoreTest.java