@Test
  public void testBlobTtlUpdate503ReplicateBlobSuccessRetryFailure() throws Exception {
    try {
      String updateServiceId = "update-service";
      MockServerLayout layout = new MockServerLayout(mockClusterMap);
      setRouter(getNonBlockingRouterPropertiesForRepairRequests(localDcName, true, false, false), layout,
          new LoggingNotificationSystem());
      setOperationParams();
      String blobId =
          router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build())
              .get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);

      // set error status for all the replicas except one local replica
      boolean oneSuccess = false;
      List<ServerErrorCode> serverErrors = new ArrayList<>();
      serverErrors.add(ServerErrorCode.BlobNotFound); // for the first TtlUpdate request
      serverErrors.add(ServerErrorCode.NoError); // for the ReplicateBlob request
      serverErrors.add(ServerErrorCode.ReplicaUnavailable); // for the TtlUpdate retry
      for (MockServer server : layout.getMockServers()) {
        if (server.getDataCenter().equals(localDcName)) {
          // local DC, return NO_ERROR for one replica,
          if (!oneSuccess) {
            oneSuccess = true;
          } else {
            server.setServerErrors(serverErrors);
          }
        }
      }

      TestCallback<Void> testCallback = new TestCallback<>();
      Future<Void> future =
          router.updateBlobTtl(null, blobId, updateServiceId, Utils.Infinite_Time, testCallback, null);
      assertFailureAndCheckErrorCode(future, testCallback, RouterErrorCode.AmbryUnavailable);

      layout.getMockServers().forEach(mockServer -> mockServer.setServerErrorForAllRequests(null));
    } finally {
      // Since we are using same blob ID for all tests, clear cache which stores blobs that are not found in router.
      router.getNotFoundCache().invalidateAll();
      if (router != null) {
        router.close();
      }
    }
  }
Test TTLUpdate first hits 503. The following ReplicateBlob is successful but retry fails.
@throws Exception
ambry-router/src/test/java/com/github/ambry/router/NonBlockingRouterTest.java