@Test
  public void singleNodeUpDownTestWithPersist() throws Exception {
    StrictMatchExternalViewVerifier helixBalanceVerifier =
        new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName,
            Collections.singleton(VcrTestUtil.helixResource), null);
    int numberOfBlobs = 20;
    sendBlobToDataNode(dataNode, numberOfBlobs);
    // Create in memory cloud destination.
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination =
        new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory =
        new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    // Start the VCR with token persistor on.
    Properties props =
        VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, clusterMapPort,
            12410, 12510, null, vcrHelixStateModelFactoryClass, true);
    props.setProperty("replication.persist.token.on.shutdown.or.replica.remove", "true");
    MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    VcrServer vcrServer =
        VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(),
            vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy = vcrNotificationSystem;
    assertTrue("Blob count is not correct.",
        TestUtils.checkAndSleep(numberOfBlobs, () -> vcrNotificationSystemCopy.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    assertEquals("Error count should be zero", 0,
        vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    assertEquals("Error count should be zero", 0,
        vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    assertTrue("Token is expected.", latchBasedInMemoryCloudDestination.getTokenMap().size() > 0);

    // Start VCR again with same cloud destination
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(),
        vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();

    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    // Because token is reloaded, back up number is 0.
    Assert.assertEquals("Number of blobs doesn't match", 0, vcrNotificationSystem.getBlobIds().size());
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    assertEquals("Error count should be zero", 0,
        vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    assertEquals("Error count should be zero", 0,
        vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());

    // Start VCR again with token.
    LatchBasedInMemoryCloudDestination newLatchBasedInMemoryCloudDestination =
        new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    for (Map.Entry<String, byte[]> entry : latchBasedInMemoryCloudDestination.getTokenMap().entrySet()) {
      newLatchBasedInMemoryCloudDestination.getTokenMap().put(entry.getKey(), entry.getValue());
    }
    cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);

    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(),
        vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();

    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy2 = vcrNotificationSystem;
    assertTrue("Blob count is not correct.",
        TestUtils.checkAndSleep(0, () -> vcrNotificationSystemCopy2.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    assertEquals("Error count should be zero", 0,
        vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    assertEquals("Error count should be zero", 0,
        vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
  }
Test single VCR up and down with persisted token.
ambry-vcr/src/test/java/com/github/ambry/vcr/VcrBackupTest.java