@Test
  public void testCompositeBlobDelete503AndThenRetryFailure() throws Exception {
    // CompositeBlob with four chunks.
    maxPutChunkSize = PUT_CONTENT_SIZE / 4;
    String serviceID = "delete-service";
    MockServerLayout layout = new MockServerLayout(mockClusterMap);
    String localDcName = "DC3";
    setRouter(getNonBlockingRouterPropertiesForRepairRequests(localDcName, true, false, false), layout,
        new LoggingNotificationSystem());
    setOperationParams();
    String compositeBlobId =
        router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build())
            .get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);

    // set two replicas Replica_Unavailable. Leave one local replica available as the source of the ReplicationBlob
    // But all remote replicas fail the ReplicateBlob.
    DataNodeId sourceDataNode = null;
    List<ServerErrorCode> localServerErrors = new ArrayList<>();
    List<ServerErrorCode> remoteServerErrors = new ArrayList<>();
    localServerErrors.add(ServerErrorCode.ReplicaUnavailable); // return Replica_Unavailable for the first Delete
    localServerErrors.add(
        ServerErrorCode.BlobAuthorizationFailure); // return Blob_Authorization_Failure for the ReplicateBlob Request
    remoteServerErrors.add(ServerErrorCode.BlobNotFound); // return NOT_FOUND for the first Delete Request
    remoteServerErrors.add(
        ServerErrorCode.BlobAuthorizationFailure); // return Blob_Authorization_Failure for the ReplicateBlob Request
    for (MockServer server : layout.getMockServers()) {
      if (server.getDataCenter().equals(localDcName)) {
        // local DC, return NO_ERROR for one replica,
        if (sourceDataNode == null) {
          sourceDataNode = mockClusterMap.getDataNodeId(server.getHostName(), server.getHostPort());
        } else {
          server.setServerErrors(localServerErrors);
        }
      } else {
        server.setServerErrors(remoteServerErrors);
      }
    }

    // Delete should fail with Unavailable. The error status should come from the deletion, not from the ReplicateBlob.
    try {
      router.deleteBlob(compositeBlobId, serviceID).get();
      fail("Deletion should fail");
    } catch (Exception e) {
      RouterException r = (RouterException) e.getCause();
      Assert.assertEquals(RouterErrorCode.AmbryUnavailable, r.getErrorCode());
    } finally {
      layout.getMockServers().forEach(mockServer -> mockServer.setServerErrorForAllRequests(null));
      router.getNotFoundCache().invalidateAll();
      if (router != null) {
        router.close();
      }
    }
  }
CompositeBlob deletion first hits 503. Then run ReplicateBlob and Retry.
ReplicateBlob fails and then deletion fails.
@throws Exception
ambry-router/src/test/java/com/github/ambry/router/NonBlockingRouterTest.java