@Test
  public void testLeadShutdownAndCoordFailure() throws Exception {
    VM memberThatWillBeShutdownVM = vm1;
    VM memberVM = vm2;
    VM locatorThatWillBeShutdownVM = vm3;

    String locators = hostName + "[" + port1 + "]," + hostName + "[" + port2 + "]";

    Properties properties = getClusterProperties(locators, "true");
    addDSProps(properties);

    locatorThatWillBeShutdownVM.invoke(() -> {
      Locator localLocator = Locator.startLocatorAndDS(port2, null, properties);
      system = (InternalDistributedSystem) localLocator.getDistributedSystem();
      assertThat(localLocator.getDistributedSystem().isConnected()).isTrue();
    });

    // Test runner will be locator 2
    Locator locator = Locator.startLocatorAndDS(port1, null, properties);
    system = (InternalDistributedSystem) locator.getDistributedSystem();
    assertThat(locator.getDistributedSystem().isConnected()).isTrue();
    DistributedSystem testRunnerLocatorDS = locator.getDistributedSystem();
    addIgnoredException(ForcedDisconnectException.class);
    assertThat(MembershipManagerHelper.getLeadMember(testRunnerLocatorDS))
        .describedAs("There was a lead member when there should not be.").isNull();

    DistributedMember distributedMemberThatWillBeShutdown =
        memberThatWillBeShutdownVM.invoke(() -> getDistributedMember(properties));
    memberThatWillBeShutdownVM
        .invoke(() -> MembershipManagerHelper.inhibitForcedDisconnectLogging(true));

    DistributedMember distributedMember = memberVM.invoke(() -> getDistributedMember(properties));

    DistributedMember locatorMemberToBeShutdown =
        locatorThatWillBeShutdownVM.invoke(LocatorDUnitTest::getLocatorDistributedMember);

    waitForMemberToBecomeLeadMemberOfDistributedSystem(distributedMemberThatWillBeShutdown,
        testRunnerLocatorDS);
    DistributedMember oldLeader = MembershipManagerHelper.getLeadMember(testRunnerLocatorDS);

    assertThat(locatorMemberToBeShutdown)
        .isEqualTo(MembershipManagerHelper.getCoordinator(testRunnerLocatorDS));
    DistributedMember oldCoordinator =
        MembershipManagerHelper.getCoordinator(testRunnerLocatorDS);

    // crash the lead locator. Should be okay
    locatorThatWillBeShutdownVM.invoke("crash locator", () -> {
      Locator loc = Locator.getLocator();
      DistributedSystem distributedSystem = loc.getDistributedSystem();
      addIgnoredException("service failure");
      addIgnoredException(ForcedDisconnectException.class);
      MembershipManagerHelper.crashDistributedSystem(distributedSystem);
      loc.stop();
    });

    await().until(testRunnerLocatorDS::isConnected);

    waitUntilTheSystemIsConnected(memberThatWillBeShutdownVM, memberVM);

    // disconnect the first vm and demonstrate that the non-lead vm and the
    // locator notice the failure and continue to run
    memberThatWillBeShutdownVM.invoke(LocatorDUnitTest::disconnectDistributedSystem);
    await().until(
        () -> memberThatWillBeShutdownVM.invoke(() -> !isSystemConnected()));
    await().until(() -> memberVM.invoke(LocatorDUnitTest::isSystemConnected));

    assertThat(memberVM.invoke(LocatorDUnitTest::isSystemConnected))
        .describedAs("Distributed system should not have disconnected").isTrue();

    await("waiting for the old coordinator to drop out").until(
        () -> MembershipManagerHelper.getCoordinator(testRunnerLocatorDS) != oldCoordinator);

    await().untilAsserted(() -> {
      DistributedMember survivingDistributedMember = testRunnerLocatorDS.getDistributedMember();
      DistributedMember coordinator = MembershipManagerHelper.getCoordinator(testRunnerLocatorDS);
      assertThat(survivingDistributedMember).isEqualTo(coordinator);
    });

    await("Waiting for the old leader to drop out")
        .pollInterval(1, SECONDS).until(() -> {
          DistributedMember leader = MembershipManagerHelper.getLeadMember(testRunnerLocatorDS);
          return leader != oldLeader;
        });

    await().untilAsserted(() -> assertThat(distributedMember)
        .isEqualTo(MembershipManagerHelper.getLeadMember(testRunnerLocatorDS)));
  }
test lead member shutdown and coordinator crashing with network partition detection enabled.
<p>
Start two locators with admin distributed systems, then start two regular distributed members.
<p>
We kill the coordinator and shut down the lead member and observe the second locator pick up
the job and the remaining member continue to operate normally.
geode-core/src/distributedTest/java/org/apache/geode/distributed/LocatorDUnitTest.java