@Test
  public void testPrimaryFailover() throws Exception {

    createClientCache(getName(), NetworkUtils.getServerHostName(server1.getHost()));
    // primary
    server1.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsAlive);

    // secondaries
    server2.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsNotAlive);
    server3.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsNotAlive);

    setClientServerObserver();

    server1.invoke(HAStartupAndFailoverDUnitTest::stopServer);

    waitForPrimaryIdentification();
    // primary
    server2.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsAlive);
    unSetClientServerObserver();
    // secondary
    server3.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsNotAlive);

    setClientServerObserver();
    server2.invoke(HAStartupAndFailoverDUnitTest::stopServer);
    // primary
    waitForPrimaryIdentification();
    server3.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsAlive);
    unSetClientServerObserver();
    server3.invoke(HAStartupAndFailoverDUnitTest::stopServer);
    // All servers are dead at this point , no primary in the system.
    verifyDeadAndLiveServers(3, 0);

    // now start one of the servers
    server3.invoke(HAStartupAndFailoverDUnitTest::startServer);
    // make sure that the server3 which was started recenty was marked live.
    verifyDeadAndLiveServers(2, 1);

    // verify that is it primary , and dispatche is running
    server3.invoke(HAStartupAndFailoverDUnitTest::verifyDispatcherIsAlive);
  }
Stops primary server one by one to ensure new primary is selected
geode-core/src/distributedTest/java/org/apache/geode/internal/cache/tier/sockets/HAStartupAndFailoverDUnitTest.java