@Test
  public void testConsistency() throws IOException {
    CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().getCatalogJanitor();
    RegionStateStore regionStateStore =
      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStateStore();
    janitor.scan();
    CatalogJanitorReport report = janitor.getLastReport();
    // Assert no problems.
    assertTrue(report.isEmpty());
    // Now remove first region in table t2 to see if catalogjanitor scan notices.
    List<RegionInfo> t2Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T2);
    regionStateStore.deleteRegion(t2Ris.get(0));
    janitor.scan();
    report = janitor.getLastReport();
    assertFalse(report.isEmpty());
    assertEquals(1, report.getHoles().size());
    assertTrue(report.getHoles().get(0).getFirst().getTable()
      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
    assertTrue(report.getHoles().get(0).getSecond().getTable().equals(T2));
    assertEquals(0, report.getOverlaps().size());
    // Next, add overlaps to first row in t3
    List<RegionInfo> t3Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T3);
    RegionInfo ri = t3Ris.get(0);
    RegionInfo newRi1 = RegionInfoBuilder.newBuilder(ri.getTable())
      .setStartKey(incrementRow(ri.getStartKey())).setEndKey(incrementRow(ri.getEndKey())).build();
    Put p1 = MetaTableAccessor.makePutFromRegionInfo(newRi1, EnvironmentEdgeManager.currentTime());
    RegionInfo newRi2 = RegionInfoBuilder.newBuilder(newRi1.getTable())
      .setStartKey(incrementRow(newRi1.getStartKey())).setEndKey(incrementRow(newRi1.getEndKey()))
      .build();
    Put p2 = MetaTableAccessor.makePutFromRegionInfo(newRi2, EnvironmentEdgeManager.currentTime());
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(p1, p2));
    janitor.scan();
    report = janitor.getLastReport();
    assertFalse(report.isEmpty());
    // We added two overlaps so total three.
    assertEquals(3, report.getOverlaps().size());
    // Assert hole is still there.
    assertEquals(1, report.getHoles().size());
    // Assert other attributes are empty still.
    assertTrue(report.getEmptyRegionInfo().isEmpty());
    assertTrue(report.getUnknownServers().isEmpty());
    // Now make bad server in t1.
    List<RegionInfo> t1Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T1);
    RegionInfo t1Ri1 = t1Ris.get(1);
    Put pServer = new Put(t1Ri1.getRegionName());
    pServer.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(0),
      Bytes.toBytes("bad.server.example.org:1234"));
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pServer));
    janitor.scan();
    report = janitor.getLastReport();
    assertFalse(report.isEmpty());
    assertEquals(1, report.getUnknownServers().size());
    // Test what happens if we blow away an info:server row, if it is null. Should not kill CJ
    // and we should log the row that had the problem. HBASE-23192. Just make sure we don't
    // break if this happens.
    LOG.info("Make null info:server");
    Put emptyInfoServerPut = new Put(t1Ri1.getRegionName());
    emptyInfoServerPut.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(0),
      Bytes.toBytes(""));
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(emptyInfoServerPut));
    janitor.scan();
    report = janitor.getLastReport();
    assertEquals(0, report.getUnknownServers().size());
    // Mke an empty regioninfo in t1.
    RegionInfo t1Ri2 = t1Ris.get(2);
    Put pEmptyRI = new Put(t1Ri2.getRegionName());
    pEmptyRI.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
      HConstants.EMPTY_BYTE_ARRAY);
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pEmptyRI));
    janitor.scan();
    report = janitor.getLastReport();
    assertEquals(1, report.getEmptyRegionInfo().size());

    int holesReported = report.getHoles().size();
    int overlapsReported = report.getOverlaps().size();

    // Test the case for T4
    // r1: [aa, bb), r2: [cc, dd), r3: [a, cc)
    // Make sure only overlaps and no holes are reported.
    List<RegionInfo> t4Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T4);
    // delete the region [bb, cc)
    regionStateStore.deleteRegion(t4Ris.get(2));

    // add a new region [a, cc)
    RegionInfo newRiT4 = RegionInfoBuilder.newBuilder(T4).setStartKey("a".getBytes())
      .setEndKey("cc".getBytes()).build();
    Put putForT4 =
      MetaTableAccessor.makePutFromRegionInfo(newRiT4, EnvironmentEdgeManager.currentTime());
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT4));

    janitor.scan();
    report = janitor.getLastReport();
    // there is no new hole reported, 2 more overLaps added.
    assertEquals(holesReported, report.getHoles().size());
    assertEquals(overlapsReported + 2, report.getOverlaps().size());

    holesReported = report.getHoles().size();
    overlapsReported = report.getOverlaps().size();

    // Test the case for T5
    // r0: [, bb), r1: [a, g), r2: [bb, cc), r3: [dd, )
    // Make sure only overlaps and no holes are reported.
    List<RegionInfo> t5Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T5);
    // delete the region [cc, dd)
    regionStateStore.deleteRegion(t5Ris.get(2));

    // add a new region [a, g)
    RegionInfo newRiT5 = RegionInfoBuilder.newBuilder(T5).setStartKey("a".getBytes())
      .setEndKey("g".getBytes()).build();
    Put putForT5 =
      MetaTableAccessor.makePutFromRegionInfo(newRiT5, EnvironmentEdgeManager.currentTime());
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT5));

    janitor.scan();
    report = janitor.getLastReport();
    // there is no new hole reported, 3 more overLaps added.
    // ([a, g), [, bb)), ([a, g), [bb, cc)), ([a, g), [dd, ))
    assertEquals(holesReported, report.getHoles().size());
    assertEquals(overlapsReported + 3, report.getOverlaps().size());
  }
Fat method where we start with a fat hbase:meta and then gradually intro problems running
catalogjanitor for each to ensure it triggers complaint. Do one big method because takes a
while to build up the context we need. We create three tables and then make holes, overlaps,
add unknown servers and empty out regioninfo columns. Each should up counts in the
CatalogJanitor.Report produced.
hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitorCluster.java