@Test
  public void testClientTracksServerPushback() throws Exception {
    HRegionServer rs = UTIL.getHBaseCluster().getRegionServer(0);
    Region region = rs.getRegions(tableName).get(0);

    LOG.debug("Writing some data to " + tableName);
    // write some data
    Put p = new Put(Bytes.toBytes("row"));
    p.addColumn(family, qualifier, Bytes.toBytes("value1"));
    mutate(p);

    // get the current load on RS. Hopefully memstore isn't flushed since we wrote the the data
    int load = (int) ((region.getMemStoreHeapSize() * 100) / flushSizeBytes);
    LOG.debug("Done writing some data to " + tableName);

    // get the stats for the region hosting our table
    ClientBackoffPolicy backoffPolicy = getBackoffPolicy();
    assertTrue("Backoff policy is not correctly configured",
      backoffPolicy instanceof ExponentialClientBackoffPolicy);

    ServerStatisticTracker stats = getStatisticsTracker();
    assertNotNull("No stats configured for the client!", stats);
    // get the names so we can query the stats
    ServerName server = rs.getServerName();
    byte[] regionName = region.getRegionInfo().getRegionName();

    // check to see we found some load on the memstore
    ServerStatistics serverStats = stats.getStats(server);
    ServerStatistics.RegionStatistics regionStats = serverStats.getStatsForRegion(regionName);
    assertEquals("We did not find some load on the memstore", load,
      regionStats.getMemStoreLoadPercent());
    // check that the load reported produces a nonzero delay
    long backoffTime = backoffPolicy.getBackoffTime(server, regionName, serverStats);
    assertNotEquals("Reported load does not produce a backoff", 0, backoffTime);
    LOG.debug("Backoff calculated for " + region.getRegionInfo().getRegionNameAsString() + " @ "
      + server + " is " + backoffTime);

    CountDownLatch latch = new CountDownLatch(1);
    AtomicLong endTime = new AtomicLong();
    long startTime = EnvironmentEdgeManager.currentTime();
    mutate(p, endTime, latch);
    // Currently the ExponentialClientBackoffPolicy under these test conditions
    // produces a backoffTime of 151 milliseconds. This is long enough so the
    // wait and related checks below are reasonable. Revisit if the backoff
    // time reported by above debug logging has significantly deviated.
    MetricsConnection metrics = getConnectionMetrics();
    String name = server.getServerName() + "," + Bytes.toStringBinary(regionName);
    MetricsConnection.RegionStats rsStats = metrics.getServerStats().get(server).get(regionName);
    assertEquals(name, rsStats.name);
    assertEquals(rsStats.heapOccupancyHist.getSnapshot().getMean(),
      (double) regionStats.getHeapOccupancyPercent(), 0.1);
    assertEquals(rsStats.memstoreLoadHist.getSnapshot().getMean(),
      (double) regionStats.getMemStoreLoadPercent(), 0.1);

    MetricsConnection.RunnerStats runnerStats = metrics.getRunnerStats();

    assertEquals(1, runnerStats.delayRunners.getCount());
    assertEquals(1, runnerStats.normalRunners.getCount());
    assertEquals("", runnerStats.delayIntevalHist.getSnapshot().getMean(), (double) backoffTime,
      0.1);

    latch.await(backoffTime * 2, TimeUnit.MILLISECONDS);
    assertNotEquals("AsyncProcess did not submit the work time", 0, endTime.get());
    assertTrue("AsyncProcess did not delay long enough", endTime.get() - startTime >= backoffTime);
  }
Test that we can actually send and use region metrics to slowdown client writes
hbase-server/src/test/java/org/apache/hadoop/hbase/client/ClientPushbackTestBase.java