@Test
  public void testUndeleteBasic() throws Exception {
    try {
      assumeTrue(!testEncryption && !includeCloudDc);
      // Setting put and delete parallelism to 2, same as the put and delete success target.
      // If put or delete requests' parallelism is 3 (default value), when we ensure all the servers has the correspoding
      // requests, it might have override by the dangling request.
      // For example, when deleting a blob, there are three delete requests being sent to the mock server. But only two of
      // them required to be acknowledged. There is a chance that when we undelete this blob, this third unacknowledged
      // delete request would override the undelete state.
      setRouter(getNonBlockingRouterProperties(localDcName, 3, 2), mockServerLayout, new LoggingNotificationSystem());
      assertExpectedThreadCounts(2, 1);

      // 1. Test undelete a composite blob
      List<String> blobIds = new ArrayList<>();
      for (int i = 0; i < 2; i++) {
        setOperationParams();
        String blobId =
            router.putBlob(putBlobProperties, putUserMetadata, putChannel, putOptionsForChunkedUpload).get();
        ensurePutInAllServers(blobId, mockServerLayout);
        logger.debug("Put blob {}", blobId);
        blobIds.add(blobId);
      }
      setOperationParams();
      List<ChunkInfo> chunksToStitch = blobIds.stream()
          .map(blobId -> new ChunkInfo(blobId, PUT_CONTENT_SIZE, Utils.Infinite_Time, null))
          .collect(Collectors.toList());
      String stitchedBlobId = router.stitchBlob(putBlobProperties, putUserMetadata, chunksToStitch).get();
      ensureStitchInAllServers(stitchedBlobId, mockServerLayout, chunksToStitch, PUT_CONTENT_SIZE);
      blobIds.add(stitchedBlobId);

      for (String blobId : blobIds) {
        router.getBlob(blobId, new GetBlobOptionsBuilder().build()).get();
        router.deleteBlob(blobId, null).get();
        ensureDeleteInAllServers(blobId, mockServerLayout);
        try {
          router.getBlob(blobId, new GetBlobOptionsBuilder().build()).get();
        } catch (ExecutionException e) {
          RouterException r = (RouterException) e.getCause();
          Assert.assertEquals("BlobDeleted error is expected", RouterErrorCode.BlobDeleted, r.getErrorCode());
        }
        router.getBlob(blobId, new GetBlobOptionsBuilder().getOption(GetOption.Include_Deleted_Blobs).build()).get();
        router.getBlob(blobId, new GetBlobOptionsBuilder().getOption(GetOption.Include_All).build()).get();
      }
      // StitchedBlob is a composite blob
      router.undeleteBlob(stitchedBlobId, "undelete_server_id").get();
      for (String blobId : blobIds) {
        ensureUndeleteInAllServers(blobId, mockServerLayout);
      }
      // Now we should be able to fetch all the blobs
      for (String blobId : blobIds) {
        router.getBlob(blobId, new GetBlobOptionsBuilder().build()).get();
      }

      // 2. Test undelete a simple blob
      setOperationParams();
      String simpleBlobId =
          router.putBlob(putBlobProperties, putUserMetadata, putChannel, PutBlobOptions.DEFAULT).get();
      ensurePutInAllServers(simpleBlobId, mockServerLayout);
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().build()).get();
      router.deleteBlob(simpleBlobId, null).get();
      ensureDeleteInAllServers(simpleBlobId, mockServerLayout);

      try {
        router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().build()).get();
      } catch (ExecutionException e) {
        RouterException r = (RouterException) e.getCause();
        Assert.assertEquals("BlobDeleted error is expected", RouterErrorCode.BlobDeleted, r.getErrorCode());
      }
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().getOption(GetOption.Include_Deleted_Blobs).build())
          .get();
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().getOption(GetOption.Include_All).build()).get();
      router.undeleteBlob(simpleBlobId, "undelete_server_id").get();
      ensureUndeleteInAllServers(simpleBlobId, mockServerLayout);
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().build()).get();

      // 3. Test delete after undelete
      router.deleteBlob(simpleBlobId, null).get();
      ensureDeleteInAllServers(simpleBlobId, mockServerLayout);

      try {
        router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().build()).get();
      } catch (ExecutionException e) {
        RouterException r = (RouterException) e.getCause();
        Assert.assertEquals("BlobDeleted error is expected", RouterErrorCode.BlobDeleted, r.getErrorCode());
      }
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().getOption(GetOption.Include_Deleted_Blobs).build())
          .get();
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().getOption(GetOption.Include_All).build()).get();

      // 4. Test undelete more than once
      router.undeleteBlob(simpleBlobId, "undelete_server_id").get();
      ensureUndeleteInAllServers(simpleBlobId, mockServerLayout);
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().build()).get();

      // 5. Undelete the same blob again
      router.undeleteBlob(simpleBlobId, "undelete_server_id").get();

      // 6. Test ttl update after undelete
      router.updateBlobTtl(simpleBlobId, null, Utils.Infinite_Time);
      router.getBlob(simpleBlobId, new GetBlobOptionsBuilder().build()).get();
    } finally {
      if (router != null) {
        router.close();
        assertExpectedThreadCounts(0, 0);

        //submission after closing should return a future that is already done.
        assertClosed();
      }
    }
  }
Test undelete with Router with a single scaling unit.
@throws Exception
ambry-router/src/test/java/com/github/ambry/router/NonBlockingRouterTest.java