@Test
  public void testRetryBackoff() throws IOException, InterruptedException {
    HRegionServer srcRs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
    RegionInfo region = srcRs.getRegions(TABLE_NAME).get(0).getRegionInfo();
    HRegionServer dstRs = UTIL.getOtherRegionServer(srcRs);
    ProcedureExecutor<MasterProcedureEnv> procExec =
      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    procExec.submitProcedure(new DummyServerProcedure(srcRs.getServerName()));
    ARRIVE.await();
    UTIL.getMiniHBaseCluster().killRegionServer(srcRs.getServerName());
    UTIL.waitFor(30000,
      () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof ServerCrashProcedure));
    Thread t = new Thread(() -> {
      try {
        UTIL.getAdmin().move(region.getEncodedNameAsBytes(), dstRs.getServerName());
      } catch (IOException e) {
        LOG.info("Failed move of {}", region.getRegionNameAsString(), e);
      }
    });
    t.start();
    // wait until we enter the WAITING_TIMEOUT state
    ProcedureTestUtil.waitUntilProcedureWaitingTimeout(UTIL, TransitRegionStateProcedure.class,
      30000);
    // wait until the timeout value increase three times
    ProcedureTestUtil.waitUntilProcedureTimeoutIncrease(UTIL, TransitRegionStateProcedure.class, 3);
    // close connection to make sure that we can not finish the TRSP
    final HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    master.getConnection().close();
    RESUME.countDown();
    UTIL.waitFor(30000, () -> !master.isAlive());
    // here we start a new master
    HMaster master2 = UTIL.getMiniHBaseCluster().startMaster().getMaster();
    LOG.info("Master2 {}, joining move thread", master2.getServerName());
    t.join();
    // Make sure that the region is online, it may not on the original target server, as we will set
    // forceNewPlan to true if there is a server crash
    try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
      table.put(new Put(Bytes.toBytes(1)).addColumn(CF, Bytes.toBytes("cq"), Bytes.toBytes(1)));
    }
    // Make sure that the region is online, it may not be on the original target server, as we will
    // set forceNewPlan to true if there is a server crash.
    try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
      table.put(new Put(Bytes.toBytes(1)).addColumn(CF, Bytes.toBytes("cq"), Bytes.toBytes(1)));
    }
  }
Confirm that we will do backoff when retrying on closing a region, to avoid consuming all the
CPUs.
hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestCloseRegionWhileRSCrash.java