@Test
  public void testRecoveryOfSingleMemberFailure() throws Exception {
    localMaxMemory = 20;
    redundancy = 2;
    recoveryDelay = 0;

    // Create PR in all VMs
    createPartitionRegionsInAllVMs(uniqueName, 1, localMaxMemory, recoveryDelay, redundancy);

    // Create some buckets, pick one and get one of the members hosting it
    DistributedMember bucketHost = vms[0].invoke(() -> {
      PartitionedRegion region = (PartitionedRegion) getCache().getRegion(uniqueName + "0");

      // Create some buckets
      for (int k = 0; region.getRegionAdvisor().getBucketSet().size() < 2; k++) {
        assertThat(k).isLessThanOrEqualTo(region.getTotalNumberOfBuckets());
        region.put(k, k);
      }

      // Grab a bucket id
      int bucketId = region.getRegionAdvisor().getBucketSet().iterator().next();
      assertThat(bucketId).isNotNull();

      // Find a host for the bucket
      Set<InternalDistributedMember> bucketOwners =
          region.getRegionAdvisor().getBucketOwners(bucketId);
      assertThat(bucketOwners).hasSize(redundancy + 1);

      InternalDistributedMember bucketOwner = bucketOwners.iterator().next();
      assertThat(bucketOwner).isNotNull();
      return bucketOwner;
    });

    assertThat(bucketHost).isNotNull();

    // Disconnect the selected host
    Map<VM, Boolean> stillHasDS = invokeInEveryVM(() -> {
      if (getSystem().getDistributedMember().equals(bucketHost)) {
        disconnectFromDS();
        return FALSE;
      }
      return TRUE;
    });

    // Wait for each member to finish recovery of redundancy for the selected bucket
    int count = 0;
    for (int i = 0; i < 4; i++) {
      if (awaitRedundancyRecovery(stillHasDS, vms[i])) {
        count++;
      }
    }
    assertThat(count).isEqualTo(3);

    // Validate all buckets have proper redundancy
    for (int i = 0; i < 4; i++) {
      validateBucketRedundancy(stillHasDS, vms[i]);
    }
  }
Test for peer recovery of buckets when a member is removed from the distributed system
geode-core/src/distributedTest/java/org/apache/geode/internal/cache/PartitionedRegionHAFailureAndRecoveryDUnitTest.java