@Test
  public void nioHandshakeValidatesHostName() throws Exception {
    ServerSocketChannel serverChannel = ServerSocketChannel.open();
    serverSocket = serverChannel.socket();

    InetSocketAddress addr = new InetSocketAddress(localHost, 0);
    serverSocket.bind(addr, 10);
    int serverPort = serverSocket.getLocalPort();

    serverThread = startServerNIO(serverSocket, 15000);

    await().until(() -> serverThread.isAlive());

    SocketChannel clientChannel = SocketChannel.open();
    await().until(
        () -> clientChannel.connect(new InetSocketAddress(localHost, serverPort)));

    clientSocket = clientChannel.socket();

    SSLEngine sslEngine =
        socketCreator.createSSLEngine(localHost.getHostName(), 1234, true);

    try {
      socketCreator.handshakeSSLSocketChannel(clientSocket.getChannel(),
          sslEngine, 0,
          ByteBuffer.allocate(sslEngine.getSession().getPacketBufferSize()),
          new BufferPool(mock(DMStats.class)));

      if (doEndPointIdentification && !addCertificateSAN) {
        fail("Failed to validate hostname in certificate SAN");
      } else {
        assertNull(serverException);
      }
    } catch (SSLHandshakeException sslException) {
      if (doEndPointIdentification && !addCertificateSAN) {
        assertThat(sslException).hasRootCauseInstanceOf(CertificateException.class)
            .hasStackTraceContaining("No name matching " + localHost.getHostName() + " found");
      } else {
        assertThat(sslException).doesNotHaveSameClassAs(new CertificateException(
            "No name matching " + localHost.getHostName() + " found"));
        throw sslException;
      }
    }
  }
Integration tests for SSL handshake specifically focusing on hostname validation.
geode-core/src/integrationTest/java/org/apache/geode/internal/net/SSLSocketHostNameVerificationIntegrationTest.java