@Test
  public void testRememberLastSucceededServerPrincipal() throws Exception {
    // after this call we will remember the last succeeded server principal
    assertEquals("a", echo("a"));
    // shutdown the connection, but does not remove it from pool
    RpcConnection conn =
      Iterables.getOnlyElement(((AbstractRpcClient<?>) rpcClient).getConnections().values());
    conn.shutdown();
    // recreate rpc server with server principal2
    int port = rpcServer.getListenerAddress().getPort();
    rpcServer.stop();
    serverUGI.logoutUserFromKeytab();
    loginAndStartRpcServer(SERVER_PRINCIPAL2, port);
    // this time we will still use the remembered server principal, so we will get a sasl exception
    UndeclaredThrowableException error =
      assertThrows(UndeclaredThrowableException.class, () -> echo("a"));
    assertThat(error.getCause(), instanceOf(ServiceException.class));
    // created by IPCUtil.wrap, to prepend the server address
    assertThat(error.getCause().getCause(), instanceOf(IOException.class));
    // wraped IPCUtil.toIOE
    assertThat(error.getCause().getCause().getCause(), instanceOf(IOException.class));
    Throwable cause = error.getCause().getCause().getCause().getCause();
    // for netty rpc client, it is DecoderException, for blocking rpc client, it is already
    // RemoteExcetion
    assertThat(cause,
      either(instanceOf(DecoderException.class)).or(instanceOf(RemoteException.class)));
    RemoteException rme;
    if (!(cause instanceof RemoteException)) {
      assertThat(cause.getCause(), instanceOf(RemoteException.class));
      rme = (RemoteException) cause.getCause();
    } else {
      rme = (RemoteException) cause;
    }
    assertEquals(SaslException.class.getName(), rme.getClassName());
    // the above failure will clear the remembered server principal, so this time we will get the
    // correct one. We use retry here just because a failure of sasl negotiation will trigger a
    // relogin and it may take some time, and for netty based implementation the relogin is async
    TEST_UTIL.waitFor(10000, () -> {
      try {
        echo("a");
      } catch (UndeclaredThrowableException e) {
        Throwable t = e.getCause().getCause();
        assertThat(t, instanceOf(IOException.class));
        if (!(t instanceof FailedServerException)) {
          // for netty rpc client
          assertThat(e.getCause().getMessage(),
            containsString(RpcConnectionConstants.RELOGIN_IS_IN_PROGRESS));
        }
        return false;
      }
      return true;
    });
  }
Tests for HBASE-28321, where we have multiple server principals candidates for a rpc service.
<p>
Put here just because we need to visit some package private classes under this package.
hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestMultipleServerPrincipalsIPC.java