private ByteBuf tryDecodeChallenge(ByteBuf in, int offset, int readableBytes) throws IOException {
    if (readableBytes < 4) {
      return null;
    }
    int len = in.getInt(offset);
    if (len <= 0) {
      // fall back to simple
      in.readerIndex(offset + 4);
      return in.retainedSlice(offset, 4);
    }
    if (len > MAX_CHALLENGE_SIZE) {
      throw new IOException(
        "Sasl challenge too large(" + len + "), max allowed is " + MAX_CHALLENGE_SIZE);
    }
    int totalLen = 4 + len;
    if (readableBytes < totalLen) {
      return null;
    }
    in.readerIndex(offset + totalLen);
    return in.retainedSlice(offset, totalLen);
  }
Decode the sasl challenge sent by RpcServer.
hbase-client/src/main/java/org/apache/hadoop/hbase/security/SaslChallengeDecoder.java