@Test
  public void testHADRFailOver() throws Exception {
    addIgnoredException(DistributedSystemDisconnectedException.class);

    // set notifyBySubscription=true to test register interest
    int serverPort1 = server1.invoke(() -> new ServerBuilder()
        .regionShortcut(REPLICATE)
        .create());
    int serverPort2 = server2.invoke(() -> new ServerBuilder()
        .regionShortcut(REPLICATE)
        .create());

    client1.invoke(() -> new ClientBuilder()
        .serverPorts(serverPort1, serverPort2)
        .subscriptionEnabled(true)
        .subscriptionAckInterval()
        .subscriptionRedundancy()
        .create());
    client2.invoke(() -> new ClientBuilder()
        .serverPorts(serverPort1, serverPort2)
        .subscriptionAckInterval()
        .subscriptionEnabled(true)
        .subscriptionRedundancy()
        .create());

    // client1 registerInterest
    client1.invoke(() -> {
      Region<String, TickerData> region = getClientCache().getRegion(regionName);
      region.registerInterest("ALL_KEYS");
    });

    // client2 registerInterest
    client2.invoke(() -> {
      Region<String, TickerData> region = getClientCache().getRegion(regionName);
      region.registerInterest("ALL_KEYS");
    });

    // async putAll1 from server2
    AsyncInvocation<Void> putAllInServer2 = server2
        .invokeAsync(() -> doPutAll(getCache().getRegion(regionName), "server2-key-", ONE_HUNDRED));

    // async putAll1 from client1
    AsyncInvocation<Void> putAllInClient1 = client1.invokeAsync(
        () -> doPutAll(getClientCache().getRegion(regionName), "client1-key-", ONE_HUNDRED));

    // stop cache server 1
    server1.invoke(() -> {
      Region<String, TickerData> region = getCache().getRegion(regionName);
      await().untilAsserted(() -> {
        assertThat(region.get("server2-key-1")).isNotNull();
        assertThat(region.get("client1-key-1")).isNotNull();
      });
      stopCacheServer(serverPort1);
    });

    putAllInServer2.await();
    putAllInClient1.await();

    // verify Bridge client2 for async keys
    client2.invokeAsync(() -> {
      Region<String, TickerData> region = getClientCache().getRegion(regionName);
      await().untilAsserted(() -> assertThat(region.size()).isEqualTo(ONE_HUNDRED * 2));
    });
  }
Tests while putAll to 2 distributed servers, one server failed over Add a listener to slow down
the processing of putAll
geode-cq/src/distributedTest/java/org/apache/geode/internal/cache/PutAllClientServerDistributedTest.java