private void process() {
		boolean justProcessedEquals = false;
		while (pos < max) {
			char ch = toProcess[pos];

			if (justProcessedEquals) {
				if (!isWhitespace(ch) && ch != 0) {
					// following an '=' we commence a variant of regular tokenization,
					// here we consume everything up to the next special char.
					// This allows SpEL expressions to be used without quoting in many
					// situations.
					lexArgValueIdentifier();
				}
				justProcessedEquals = false;
				continue;
			}

			if (isAlphabetic(ch) || isDigit(ch) || ch == '_') {
				lexIdentifier();
			}
			else {
				switch (ch) {
					case '-':
						if (!isTwoCharToken(TokenKind.DOUBLE_MINUS)) {
							throw new StreamDefinitionException(
									expressionString, pos,
									XDDSLMessages.MISSING_CHARACTER, "-");
						}
						pushPairToken(TokenKind.DOUBLE_MINUS);
						break;
					case '=':
						justProcessedEquals = true;
						pushCharToken(TokenKind.EQUALS);
						break;
					case '&':
						pushCharToken(TokenKind.AND);
						break;
					case '|':
						pushCharToken(TokenKind.PIPE);
						break;
					case ' ':
					case '\t':
					case '\r':
						// drift over white space
						pos++;
						break;
					case '\n':
						pushCharToken(TokenKind.NEWLINE);
						break;
					case '.':
						pushCharToken(TokenKind.DOT);
						break;
					case '>':
						pushCharToken(TokenKind.GT);
						break;
					case ':':
						pushCharToken(TokenKind.COLON);
						break;
					case ';':
						pushCharToken(TokenKind.SEMICOLON);
						break;
					case '\'':
						lexQuotedStringLiteral();
						break;
					case '"':
						lexDoubleQuotedStringLiteral();
						break;
					case '@':
						pushCharToken(TokenKind.REFERENCE);
						break;
					case 0:
						// hit sentinel at end of char data
						pos++; // will take us to the end
						break;
					case '\\':
						throw new StreamDefinitionException(
								expressionString, pos, XDDSLMessages.UNEXPECTED_ESCAPE_CHAR);
					default:
						throw new StreamDefinitionException(
								expressionString, pos, XDDSLMessages.UNEXPECTED_DATA,
								Character.valueOf(ch).toString());
				}
			}
		}
	}
Lex some input data into a stream of tokens that can then then be parsed.

@author Andy Clement
spring-xd-dirt/src/main/java/org/springframework/xd/dirt/stream/dsl/Tokenizer.java