protected void testRecoveryAndDoubleExecution(boolean carryingMeta, boolean doubleExecution)
    throws Exception {
    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution-carryingMeta-"
      + carryingMeta + "-doubleExecution-" + doubleExecution);
    try (Table t = createTable(tableName)) {
      // Load the table with a bit of data so some logs to split and some edits in each region.
      this.util.loadTable(t, HBaseTestingUtil.COLUMNS[0]);
      final int count = HBaseTestingUtil.countRows(t);
      assertTrue("expected some rows", count > 0);
      final String checksum = util.checksumRows(t);
      // Run the procedure executor outside the master so we can mess with it. Need to disable
      // Master's running of the server crash processing.
      final HMaster master = this.util.getHBaseCluster().getMaster();
      final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
      // find the first server that match the request and executes the test
      ServerName rsToKill = null;
      for (RegionInfo hri : util.getAdmin().getRegions(tableName)) {
        final ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(util, hri);
        if (AssignmentTestingUtil.isServerHoldingMeta(util, serverName) == carryingMeta) {
          rsToKill = serverName;
          break;
        }
      }
      // Enable test flags and then queue the crash procedure.
      ProcedureTestingUtility.waitNoProcedureRunning(procExec);
      if (doubleExecution) {
        // For SCP, if you enable this then we will enter an infinite loop, as we will crash between
        // queue and open for TRSP, and then going back to queue, as we will use the crash rs as the
        // target server since it is recored in hbase:meta.
        ProcedureTestingUtility.setKillIfHasParent(procExec, false);
        ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
        // kill the RS
        AssignmentTestingUtil.killRs(util, rsToKill);
        long procId = getSCPProcId(procExec);
        // Now run through the procedure twice crashing the executor on each step...
        MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
      } else {
        // kill the RS
        AssignmentTestingUtil.killRs(util, rsToKill);
        long procId = getSCPProcId(procExec);
        ProcedureTestingUtility.waitProcedure(procExec, procId);
      }
      RegionReplicaTestHelper.assertReplicaDistributed(util, t);
      assertEquals(count, HBaseTestingUtil.countRows(t));
      assertEquals(checksum, util.checksumRows(t));
    }
  }
Run server crash procedure steps twice to test idempotency and that we are persisting all
needed state.
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPBase.java