@Test
  public void test() throws Exception {
    RegionServerThread rsWithMetaThread = UTIL.getMiniHBaseCluster().getRegionServerThreads()
      .stream().filter(t -> !t.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty())
      .findAny().get();
    HRegionServer rsNoMeta = UTIL.getOtherRegionServer(rsWithMetaThread.getRegionServer());
    FAIL = true;
    UTIL.getMiniHBaseCluster().killRegionServer(rsNoMeta.getServerName());
    ProcedureExecutor<?> executor =
      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    // wait until we have way more TRSPs than the core pool size, and then make sure we can recover
    // normally
    UTIL.waitFor(60000, new ExplainingPredicate<Exception>() {

      @Override
      public boolean evaluate() throws Exception {
        return executor.getProcedures().stream().filter(p -> !p.isFinished())
          .filter(p -> p.getState() != ProcedureState.INITIALIZING)
          .filter(p -> p instanceof TransitRegionStateProcedure).count() > 5 * CORE_POOL_SIZE;
      }

      @Override
      public String explainFailure() throws Exception {
        return "Not enough TRSPs scheduled";
      }
    });
    // sleep more time to make sure the TRSPs have been executed
    Thread.sleep(10000);
    UTIL.getMiniHBaseCluster().killRegionServer(rsWithMetaThread.getRegionServer().getServerName());
    rsWithMetaThread.join();
    FAIL = false;
    // verify that the cluster is back
    UTIL.waitUntilNoRegionsInTransition(480000);
    for (int i = 0; i < TABLE_COUNT; i++) {
      try (Table table = UTIL.getConnection().getTable(TableName.valueOf(TABLE_NAME_PREFIX + i))) {
        table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
      }
    }
    UTIL.waitFor(60000, new ExplainingPredicate<Exception>() {

      @Override
      public boolean evaluate() throws Exception {
        return executor.getWorkerThreadCount() == CORE_POOL_SIZE;
      }

      @Override
      public String explainFailure() throws Exception {
        return "The new workers do not timeout";
      }
    });
  }
Test to ensure that the priority for procedures and stuck checker can partially solve the problem
describe in HBASE-19976, that is, RecoverMetaProcedure can finally be executed within a certain
period of time.
<p>
As of HBASE-28199, we no longer block a worker when updating meta now, so this test can not test
adding procedure worker now, but it could still be used to make sure that we could make progress
when meta is gone and we have a lot of pending TRSPs.
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java