@Test
  public void testReplicateBlobToRemoteSuccess() throws Exception {
    try {
      MockServerLayout layout = new MockServerLayout(mockClusterMap);
      String serviceId = "replicate-blob-service";
      setRouter(getNonBlockingRouterProperties(localDcName), layout, new LoggingNotificationSystem());
      setOperationParams();
      // PutBlob to the three local replicas
      String blobIdStr =
          router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build())
              .get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
      GetBlobResult localGetBlobResult =
          router.getBlob(blobIdStr, new GetBlobOptionsBuilder().build(), null, null).get();

      // set error status for all the local replica. read from the remote replica, it should fail
      for (MockServer server : layout.getMockServers()) {
        if (server.getDataCenter().equals(localDcName)) {
          server.setServerErrorForAllRequests(ServerErrorCode.ReplicaUnavailable);
        }
      }
      try {
        router.getBlob(blobIdStr, new GetBlobOptionsBuilder().build(), null, null).get();
        Assert.fail("Should return Ambry unavailable.");
      } catch (ExecutionException e) {
        Throwable t = e.getCause();
        assertTrue("Cause should be RouterException", t instanceof RouterException);
        assertEquals("ErrorCode mismatch", RouterErrorCode.AmbryUnavailable, ((RouterException) t).getErrorCode());
      }
      layout.getMockServers().forEach(mockServer -> mockServer.setServerErrorForAllRequests(null));

      // Replicate Blob from the local replica to some remote replicas.
      // choose one local replica as the source DataNode. Set the other two replicas as Replica_Unavailable
      DataNodeId sourceDataNode = null;
      for (MockServer server : layout.getMockServers()) {
        if (server.getDataCenter().equals(localDcName)) {
          if (sourceDataNode == null) {
            sourceDataNode = mockClusterMap.getDataNodeId(server.getHostName(), server.getHostPort());
          } else {
            server.setServerErrorForAllRequests(ServerErrorCode.ReplicaUnavailable);
          }
        }
      }
      // since there is only one local available, router will replicate the blob to at least 1 remote replicas.
      router.replicateBlob(blobIdStr, serviceId, sourceDataNode).get();

      // set unavailable to all local replicas. So router.getBlob will get the Blob from remote replicas.
      for (MockServer server : layout.getMockServers()) {
        if (server.getDataCenter().equals(localDcName)) {
          server.setServerErrorForAllRequests(ServerErrorCode.ReplicaUnavailable);
        }
      }
      GetBlobResult remoteGetBlobResult =
          router.getBlob(blobIdStr, new GetBlobOptionsBuilder().build(), null, null).get();

      assertEqual(localGetBlobResult, remoteGetBlobResult);
      layout.getMockServers().forEach(mockServer -> mockServer.setServerErrorForAllRequests(null));
    } finally {
      // Since we are using same blob ID for all tests, clear cache which stores blobs that are not found in router.
      router.getNotFoundCache().invalidateAll();

      if (router != null) {
        router.close();
      }
    }
  }
Test for ReplicateBlob success case
Replicating one Blob to the remote replicas when two local replicas were unavailable.
@throws Exception
ambry-router/src/test/java/com/github/ambry/router/NonBlockingRouterTest.java