@Test
  public void testUnreachableControllerDuringPrepare() throws Exception {
    coordinator = buildNewCoordinator();
    // setup the proc
    List<String> expected = Arrays.asList("cohort");
    Procedure proc =
      new Procedure(coordinator, WAKE_FREQUENCY, TIMEOUT, procName, procData, expected);
    final Procedure procSpy = spy(proc);

    when(coordinator.createProcedure(any(), eq(procName), eq(procData), anyList()))
      .thenReturn(procSpy);

    // use the passed controller responses
    IOException cause = new IOException("Failed to reach comms during acquire");
    doThrow(cause).when(controller).sendGlobalBarrierAcquire(eq(procSpy), eq(procData), anyList());

    // run the operation
    proc = coordinator.startProcedure(proc.getErrorMonitor(), procName, procData, expected);
    // and wait for it to finish
    while (!proc.completedLatch.await(WAKE_FREQUENCY, TimeUnit.MILLISECONDS))
      ;
    verify(procSpy, atLeastOnce()).receive(any());
    verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
    verify(controller, times(1)).sendGlobalBarrierAcquire(procSpy, procData, expected);
    verify(controller, never()).sendGlobalBarrierReached(any(), anyList());
  }
Check handling a connection failure correctly if we get it during the acquiring phase
hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java