List<Token> tokenize() throws ScanException {
    List<Token> tokenList = new ArrayList<Token>();
    StringBuffer buf = new StringBuffer();

    while (pointer < patternLength) {
      char c = pattern.charAt(pointer);
      pointer++;

      switch (state) {
        case LITERAL_STATE:
          handleLiteralState(c, tokenList, buf);
          break;
        case FORMAT_MODIFIER_STATE:
          handleFormatModifierState(c, tokenList, buf);
          break;
        case OPTION_STATE:
          processOption(c, tokenList, buf);
          break;
        case KEYWORD_STATE:
          handleKeywordState(c, tokenList, buf);
          break;
        case RIGHT_PARENTHESIS_STATE:
          handleRightParenthesisState(c, tokenList, buf);
          break;

        default:
      }
    }

    // EOS
    switch (state) {
      case LITERAL_STATE:
        addValuedToken(Token.LITERAL, buf, tokenList);
        break;
      case KEYWORD_STATE:
        tokenList.add(new Token(Token.SIMPLE_KEYWORD, buf.toString()));
        break;
      case RIGHT_PARENTHESIS_STATE:
        tokenList.add(Token.RIGHT_PARENTHESIS_TOKEN);
        break;

      case FORMAT_MODIFIER_STATE:
      case OPTION_STATE:
        throw new ScanException("Unexpected end of pattern string");
    }

    return tokenList;
  }
@Test
  public void testEmptyP2() throws ScanException {
    List<Token> tl = new TokenStream("%()").tokenize();
    List<Token> witness = new ArrayList<Token>();
    witness.add(Token.PERCENT_TOKEN);
    witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
    witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
    assertEquals(witness, tl);
  }