@Test
  public void testDynamicMasterConfigurationRefresh() throws Exception {
    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
    String currentMasterAddrs = Preconditions.checkNotNull(conf.get(HConstants.MASTER_ADDRS_KEY));
    HMaster activeMaster = TEST_UTIL.getHBaseCluster().getMaster();
    String clusterId = activeMaster.getClusterId();
    // Add a non-working master
    ServerName badServer = ServerName.valueOf("localhost", 1234, -1);
    conf.set(HConstants.MASTER_ADDRS_KEY, badServer.toShortString() + "," + currentMasterAddrs);
    // Set the hedging fan out so that all masters are queried.
    conf.setInt(MasterRegistry.MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY, 4);
    // Do not limit the number of refreshes during the test run.
    conf.setLong(MasterRegistry.MASTER_REGISTRY_MIN_SECS_BETWEEN_REFRESHES, 0);
    try (MasterRegistry registry = new MasterRegistry(conf, User.getCurrent())) {
      final Set<ServerName> masters = registry.getParsedServers();
      assertTrue(masters.contains(badServer));
      // Make a registry RPC, this should trigger a refresh since one of the hedged RPC fails.
      assertEquals(registry.getClusterId().get(), clusterId);
      // Wait for new set of masters to be populated.
      TEST_UTIL.waitFor(5000,
        (Waiter.Predicate<Exception>) () -> !registry.getParsedServers().equals(masters));
      // new set of masters should not include the bad server
      final Set<ServerName> newMasters = registry.getParsedServers();
      // Bad one should be out.
      assertEquals(3, newMasters.size());
      assertFalse(newMasters.contains(badServer));
      // Kill the active master
      activeMaster.stopMaster();
      TEST_UTIL.waitFor(10000,
        () -> TEST_UTIL.getMiniHBaseCluster().getLiveMasterThreads().size() == 2);
      TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(10000);
      // Wait until the killed master de-registered. This should also trigger another refresh.
      TEST_UTIL.waitFor(10000, () -> registry.getMasters().get().size() == 2);
      TEST_UTIL.waitFor(20000, () -> registry.getParsedServers().size() == 2);
      final Set<ServerName> newMasters2 = registry.getParsedServers();
      assertEquals(2, newMasters2.size());
      assertFalse(newMasters2.contains(activeMaster.getServerName()));
    } finally {
      // Reset the state, add a killed master.
      TEST_UTIL.getMiniHBaseCluster().startMaster();
    }
  }
Tests that the list of masters configured in the MasterRegistry is dynamically refreshed in the
event of errors.
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMasterRegistry.java