@Test
  public void testGetBlobRetryUntilSuccess() throws Exception {
    // Make sure we are work on a simple blob
    blobSize = maxChunkSize - 1;
    doPut();
    int retryLimitInSeconds = 10;
    RouterConfig originalConfig = routerConfig;
    Properties props = getDefaultNonBlockingRouterProperties(true);
    props.setProperty(RouterConfig.ROUTER_GET_BLOB_RETRY_LIMIT_COUNT, "10");
    props.setProperty(RouterConfig.ROUTER_GET_BLOB_RETRY_LIMIT_IN_SEC, String.valueOf(retryLimitInSeconds));
    routerConfig = new RouterConfig(new VerifiableProperties(props));
    try {
      List<ServerErrorCode> errorCodes = new ArrayList<>();
      errorCodes.add(ServerErrorCode.ReplicaUnavailable); // First get return unavailable
      errorCodes.add(ServerErrorCode.NoError); // Second get return no_error
      for (MockServer server : mockServerLayout.getMockServers()) {
        server.setServerErrors(errorCodes);
      }
      AtomicReference<GetBlobResult> resultRef = new AtomicReference<>();
      AtomicReference<Exception> exceptionRef = new AtomicReference<>();
      Callback<GetBlobResult> callback = (result, exception) -> {
        resultRef.set(result);
        exceptionRef.set(exception);
      };

      long retryCountBefore = routerMetrics.getBlobRetryCount.getCount();
      GetBlobOperation op = createOperation(routerConfig, callback);
      while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
        for (ResponseInfo responseInfo : responses) {
          GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(
              new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
          op.handleResponse(responseInfo, getResponse);
          responseInfo.release();
        }
        if (resultRef.get() != null || exceptionRef.get() != null) {
          break;
        }
      }
      Assert.assertNull("Operation should succeed now", op.getOperationException());
      Assert.assertNull("Operation should succeed now", exceptionRef.get());
      Assert.assertNotNull("Operation should succeed now", resultRef.get());
      resultRef.get().getBlobDataChannel().close();
      assertEquals(1, routerMetrics.getBlobRetryCount.getCount() - retryCountBefore);
    } finally {
      routerConfig = originalConfig;
    }
  }
Test the case where all requests time out within the SocketNetworkClient.
@throws Exception
ambry-router/src/test/java/com/github/ambry/router/GetBlobOperationTest.java