@Test
  public void testRetryBackoff() throws IOException, InterruptedException {
    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
    ProcedureExecutor<MasterProcedureEnv> procExec =
      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    RegionInfo regionInfo = UTIL.getAdmin().getRegions(TABLE_NAME).get(0);
    RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo);
    // just a dummy one
    TransitRegionStateProcedure trsp =
      TransitRegionStateProcedure.unassign(procExec.getEnvironment(), regionInfo);
    long openSeqNum;
    regionNode.lock();
    try {
      openSeqNum = regionNode.getOpenSeqNum();
      // make a fake state to let the procedure wait.
      regionNode.setState(State.OPENING);
      regionNode.setOpenSeqNum(-1L);
      regionNode.setProcedure(trsp);
    } finally {
      regionNode.unlock();
    }
    ReopenTableRegionsProcedure proc = new ReopenTableRegionsProcedure(TABLE_NAME);
    procExec.submitProcedure(proc);
    UTIL.waitFor(10000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT);
    long oldTimeout = 0;
    int timeoutIncrements = 0;
    for (;;) {
      long timeout = proc.getTimeout();
      if (timeout > oldTimeout) {
        LOG.info("Timeout incremented, was {}, now is {}, increments={}", timeout, oldTimeout,
          timeoutIncrements);
        oldTimeout = timeout;
        timeoutIncrements++;
        if (timeoutIncrements > 3) {
          // If we incremented at least twice, break; the backoff is working.
          break;
        }
      }
      Thread.sleep(1000);
    }
    regionNode.lock();
    try {
      // reset to the correct state
      regionNode.setState(State.OPEN);
      regionNode.setOpenSeqNum(openSeqNum);
      regionNode.unsetProcedure(trsp);
    } finally {
      regionNode.unlock();
    }
    ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60000);
    assertTrue(regionNode.getOpenSeqNum() > openSeqNum);
  }
Confirm that we will do backoff when retrying on reopening table regions, to avoid consuming all
the CPUs.
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBackoff.java