@Test
  public void multipleVcrTest() throws Exception {
    StrictMatchExternalViewVerifier helixBalanceVerifier =
        new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName,
            Collections.singleton(VcrTestUtil.helixResource), null);
    int initialNumOfVcrs = 5;
    // create a shared in memory destination.
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination =
        new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory =
        new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    // 1st phase: Start VCRs to do backup.
    List<VcrServer> vcrServers = new ArrayList<>();
    List<MockNotificationSystem> vcrNotificationSystems = new ArrayList<>();
    for (int port = 12310; port < 12310 + initialNumOfVcrs; port++) {
      Properties props =
          VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, port,
              port + 100, port + 200, null, vcrHelixStateModelFactoryClass, true);
      MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
      VcrServer vcrServer =
          VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(),
              vcrNotificationSystem, cloudDestinationFactory);
      vcrServer.startup();
      vcrServers.add(vcrServer);
      vcrNotificationSystems.add(vcrNotificationSystem);
    }
    makeSureHelixBalance(vcrServers.get(vcrServers.size() - 1), helixBalanceVerifier);
    int numOfBlobs = 100;
    sendBlobToDataNode(dataNode, numOfBlobs);
    // Make sure blobs are backed up.
    TestUtils.checkAndSleep(numOfBlobs,
        () -> vcrNotificationSystems.stream().mapToInt(i -> i.getBlobIds().size()).sum(), 5000);

    // verify each VCR is only replicating partitions assigned to it.
    for (int i = 0; i < initialNumOfVcrs; i++) {
      Set<PartitionId> partitionIdSet = vcrNotificationSystems.get(i).getBlobIds().stream().map(blobIdStr -> {
        try {
          return new BlobId(blobIdStr, mockCluster.getClusterMap()).getPartition();
        } catch (IOException e) {
          e.printStackTrace();
          return null;
        }
      }).collect(Collectors.toSet());
      assertTrue("Each VCR should have some assignment.",
          vcrServers.get(i).getVcrClusterParticipant().getAssignedPartitionIds().size() > 0);
      assertTrue("Each VCR should only backup its assigned partitions.",
          new HashSet<>(vcrServers.get(i).getVcrClusterParticipant().getAssignedPartitionIds()).containsAll(
              partitionIdSet));
    }
    logger.info("Phase 1 done.");

    // 2nd phase: Add a new VCR to cluster.
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString,
        12310 + initialNumOfVcrs, 12310 + initialNumOfVcrs + 100, 12310 + initialNumOfVcrs + 200, null,
        vcrHelixStateModelFactoryClass, true);
    MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    VcrServer vcrServer =
        VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(),
            vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    vcrServers.add(vcrServer);
    vcrNotificationSystems.add(vcrNotificationSystem);

    makeSureHelixBalance(vcrServers.get(vcrServers.size() - 1), helixBalanceVerifier);
    int secondNumOfBlobs = 100;
    sendBlobToDataNode(dataNode, secondNumOfBlobs);
    Assert.assertTrue("All blobs should be back up.", TestUtils.checkAndSleep(numOfBlobs + secondNumOfBlobs,
        () -> vcrNotificationSystems.stream().mapToInt(i -> i.getBlobIds().size()).sum(), 5000));
    logger.info("Phase 2 done.");

    // 3rd phase: Remove last VCR from cluster.
    vcrServers.get(vcrServers.size() - 1).shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServers.get(vcrServers.size() - 1).awaitShutdown(5000));
    // Error metrics should be zero.
    assertEquals("Error count should be zero", 0, vcrServers.get(vcrServers.size() - 1)
        .getVcrReplicationManager()
        .getVcrMetrics().addPartitionErrorCount.getCount());
    assertEquals("Error count should be zero", 0, vcrServers.get(vcrServers.size() - 1)
        .getVcrReplicationManager()
        .getVcrMetrics().removePartitionErrorCount.getCount());
    int temp = vcrNotificationSystems.get(vcrNotificationSystems.size() - 1).getBlobIds().size();

    assertTrue("Helix balance timeout.", helixBalanceVerifier.verify(5000));
    int thirdNumOfBlobs = 100;
    sendBlobToDataNode(dataNode, thirdNumOfBlobs);
    Assert.assertTrue("All blobs should be back up.",
        TestUtils.checkAndSleep(numOfBlobs + secondNumOfBlobs + thirdNumOfBlobs,
            () -> vcrNotificationSystems.stream().mapToInt(i -> i.getBlobIds().size()).sum(), 5000));
    Assert.assertEquals("The removed vcr shouldn't have any change", temp,
        vcrNotificationSystems.get(vcrNotificationSystems.size() - 1).getBlobIds().size());
    logger.info("Phase 3 done.");

    // Shutdown all others.
    for (int i = 0; i < initialNumOfVcrs; i++) {
      // Error metrics should be zero.
      assertEquals("Error count should be zero", 0,
          vcrServers.get(i).getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
      assertEquals("Error count should be zero", 0,
          vcrServers.get(i).getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
      vcrServers.get(i).shutdown();
      assertTrue("VCR server shutdown timeout.", vcrServers.get(i).awaitShutdown(5000));
    }

    logger.info("Test done.");
  }
A multiple VCR test to test helix assignment and backup.
ambry-vcr/src/test/java/com/github/ambry/vcr/VcrBackupTest.java