private Token parseSeparatorToken(String candidate, Token lastAddedToken, boolean fillMode,
      int begin) {
    checkFillModeOff(fillMode, begin);
    // try to clump separator with immediately preceding separators (e.g. "---" counts as one
    // separator)
    if (lastAddedToken != null && lastAddedToken.type == TokenType.SEPARATOR) {
      lastAddedToken.string += candidate;
      lastAddedToken.length += 1;
    } else {
      lastAddedToken = new Token(TokenType.SEPARATOR, candidate);
      tokens.add(lastAddedToken);
    }
    return lastAddedToken;
  }
Parse pattern to list of tokens.
common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java