@Test
  public void testFailAndRollback() throws Exception {
    HRegionServer rsWithMeta = UTIL.getRSForFirstRegionInTable(TableName.META_TABLE_NAME);
    UTIL.getMiniHBaseCluster().killRegionServer(rsWithMeta.getServerName());
    UTIL.waitFor(15000, () -> getSCPForServer(rsWithMeta.getServerName()) != null);
    ServerCrashProcedure scp = getSCPForServer(rsWithMeta.getServerName());
    ProcedureExecutor<MasterProcedureEnv> procExec =
      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    // wait for the procedure to stop, as we inject a code bug and also set kill before store update
    UTIL.waitFor(30000, () -> !procExec.isRunning());
    // make sure that finally we could successfully rollback the procedure
    while (scp.getState() != ProcedureState.FAILED || !procExec.isRunning()) {
      MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
      ProcedureTestingUtility.waitProcedure(procExec, scp);
    }
    assertEquals(scp.getState(), ProcedureState.FAILED);
    assertThat(scp.getException().getMessage(), containsString("inject code bug"));
    // make sure all sub procedures are cleaned up
    assertThat(procExec.getProcedures(), everyItem(not(subProcOf(scp))));
  }
SCP does not support rollback actually, here we just want to simulate that when there is a code
bug, SCP and its sub procedures will not hang there forever, and it will not mess up the
procedure store.
hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRollbackSCP.java