@Test
  public void testUnreachableControllerDuringCommit() throws Exception {
    coordinator = buildNewCoordinator();

    // setup the task and spy on it
    List<String> expected = Arrays.asList("cohort");
    final Procedure spy =
      spy(new Procedure(coordinator, WAKE_FREQUENCY, TIMEOUT, procName, procData, expected));

    when(coordinator.createProcedure(any(), eq(procName), eq(procData), anyList())).thenReturn(spy);

    // use the passed controller responses
    IOException cause = new IOException("Failed to reach controller during prepare");
    doAnswer(new AcquireBarrierAnswer(procName, new String[] { "cohort" })).when(controller)
      .sendGlobalBarrierAcquire(eq(spy), eq(procData), anyList());
    doThrow(cause).when(controller).sendGlobalBarrierReached(eq(spy), anyList());

    // run the operation
    Procedure task =
      coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
    // and wait for it to finish
    while (!task.completedLatch.await(WAKE_FREQUENCY, TimeUnit.MILLISECONDS))
      ;
    verify(spy, atLeastOnce()).receive(any());
    verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
    verify(controller, times(1)).sendGlobalBarrierAcquire(eq(spy), eq(procData), anyList());
    verify(controller, times(1)).sendGlobalBarrierReached(any(), anyList());
  }
Check handling a connection failure correctly if we get it during the barrier phase
hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java