@Test
  public void testLockupAroundBadAssignSync() throws IOException {
    // Make up mocked server and services.
    RegionServerServices services = mock(RegionServerServices.class);
    when(services.getConfiguration()).thenReturn(CONF);
    when(services.isStopped()).thenReturn(false);
    when(services.isAborted()).thenReturn(false);
    // OK. Now I have my mocked up Server and RegionServerServices and my dodgy WAL, go ahead with
    // the test.
    FileSystem fs = FileSystem.get(CONF);
    Path rootDir = new Path(dir + getName());
    DodgyFSLog dodgyWAL = new DodgyFSLog(fs, (Server) services, rootDir, getName(), CONF);
    dodgyWAL.init();
    LogRoller logRoller = new LogRoller(services);
    logRoller.addWAL(dodgyWAL);
    logRoller.start();

    boolean threwOnSync = false;
    boolean threwOnAppend = false;
    boolean threwOnBoth = false;

    HRegion region = initHRegion(tableName, null, null, CONF, dodgyWAL);
    try {
      // Get some random bytes.
      byte[] value = Bytes.toBytes(getName());
      try {
        // First get something into memstore
        Put put = new Put(value);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), value);
        region.put(put);
      } catch (IOException ioe) {
        fail();
      }
      long rollsCount = dodgyWAL.rolls.get();
      try {
        dodgyWAL.throwAppendException = true;
        dodgyWAL.throwSyncException = false;
        Put put = new Put(value);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("3"), value);
        region.put(put);
      } catch (IOException ioe) {
        threwOnAppend = true;
      }
      while (rollsCount == dodgyWAL.rolls.get()) {
        Threads.sleep(100);
      }
      rollsCount = dodgyWAL.rolls.get();

      // When we get to here.. we should be ok. A new WAL has been put in place. There were no
      // appends to sync. We should be able to continue.

      try {
        dodgyWAL.throwAppendException = true;
        dodgyWAL.throwSyncException = true;
        Put put = new Put(value);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("4"), value);
        region.put(put);
      } catch (IOException ioe) {
        threwOnBoth = true;
      }
      while (rollsCount == dodgyWAL.rolls.get()) {
        Threads.sleep(100);
      }

      // Again, all should be good. New WAL and no outstanding unsync'd edits so we should be able
      // to just continue.

      // So, should be no abort at this stage. Verify.
      verify(services, atLeast(0)).abort(anyString(), any(Throwable.class));
      try {
        dodgyWAL.throwAppendException = false;
        dodgyWAL.throwSyncException = true;
        Put put = new Put(value);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("2"), value);
        region.rsServices = services;
        region.put(put);
      } catch (IOException ioe) {
        threwOnSync = true;
      }

      region.rsServices = null;
      // An append in the WAL but the sync failed is a server abort condition. That is our
      // current semantic. Verify.
      verify(services, atLeast(1)).abort(anyString(), any());
      try {
        dodgyWAL.throwAppendException = false;
        dodgyWAL.throwSyncException = false;
        dodgyWAL.throwArchiveException = true;
        Pair<Path, Long> pair = new Pair<Path, Long>();
        pair.setFirst(new Path("/a/b/"));
        pair.setSecond(100L);
        dodgyWAL.archive(pair);
      } catch (Throwable ioe) {
      }
      while (true) {
        try {
          // one more abort needs to be called
          verify(services, atLeast(2)).abort(anyString(), any());
          break;
        } catch (WantedButNotInvoked t) {
          Threads.sleep(1);
        }
      }
    } finally {
      // To stop logRoller, its server has to say it is stopped.
      when(services.isStopped()).thenReturn(true);
      if (logRoller != null) logRoller.close();
      if (region != null) {
        try {
          region.close(true);
        } catch (DroppedSnapshotException e) {
          LOG.info("On way out; expected!", e);
        }
      }
      if (dodgyWAL != null) dodgyWAL.close();
      assertTrue("The regionserver should have thrown an exception", threwOnBoth);
      assertTrue("The regionserver should have thrown an exception", threwOnAppend);
      assertTrue("The regionserver should have thrown an exception", threwOnSync);
    }
  }
Reproduce locking up that happens when we get an exceptions appending and syncing. See
HBASE-14317. First I need to set up some mocks for Server and RegionServerServices. I also need
to set up a dodgy WAL that will throw an exception when we go to append to it.
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java