private Thread startServerNIO(final ServerSocket serverSocket, int timeoutMillis) {
    Thread serverThread = new Thread(new MyThreadGroup(testName.getMethodName()), () -> {
      NioSslEngine engine = null;
      Socket socket = null;
      try {
        socket = serverSocket.accept();
        SocketCreator sc = SocketCreatorFactory.getSocketCreatorForComponent(CLUSTER);
        final SSLEngine sslEngine = sc.createSSLEngine(localHost.getHostName(), 1234, false);
        engine =
            sc.handshakeSSLSocketChannel(socket.getChannel(),
                sslEngine,
                timeoutMillis,
                ByteBuffer.allocate(sslEngine.getSession().getPacketBufferSize()),
                new BufferPool(mock(DMStats.class)));
      } catch (Throwable throwable) {
        serverException = throwable;
      } finally {
        if (engine != null) {
          final NioSslEngine nioSslEngine = engine;
          engine.close(socket.getChannel());
          assertThatThrownBy(() -> {
            try (final ByteBufferSharing unused =
                nioSslEngine.unwrap(ByteBuffer.wrap(new byte[0]))) {
            }
          }).isInstanceOf(IOException.class);
        }
      }
    }, testName.getMethodName() + "-server");

    serverThread.start();
    return serverThread;
  }
Integration tests for SSL handshake specifically focusing on hostname validation.
geode-core/src/integrationTest/java/org/apache/geode/internal/net/SSLSocketHostNameVerificationIntegrationTest.java