private void checkTokens(Tokenizer tokenizer, Token... expectedTokens) {
		List<Token> tokens = tokenizer.getTokens();
		if (tokens.size() != expectedTokens.length) {
			fail("Expected stream\n" + stringify(expectedTokens) + "\n but found\n" + stringify(tokens));
		}
		for (int t = 0; t < expectedTokens.length; t++) {
			if (!expectedTokens[t].equals(tokens.get(t))) {
				fail("Token #" + t + " is not as expected. Expected " + expectedTokens[t] + " but was " + tokens.get(t));
			}
		}
	}
Check the tokenizer breaks strings up into correct token streams.

@author Andy Clement
spring-xd-dirt/src/test/java/org/springframework/xd/dirt/stream/dsl/TokenizerTests.java