public static List<Token> splitParamsSpan(final List<Token> tokens) {
        final List<TokenSpan> tokenSpans = getSpans(tokens);

        if (tokenSpans.isEmpty()) {
            return tokens;
        }

        final List<Token> out = new ArrayList<>();
        int lastTokenIndex = 0;
        int lastTokenChar = 0;
        for (final TokenSpan tokenSpan : tokenSpans) {
            final TokenIndex start = tokenSpan.start;
            final TokenIndex end = tokenSpan.end;

            // Copy whole tokens before span.
            if (start.tokenIndex > lastTokenIndex) {
                for (int i = lastTokenIndex; i < start.tokenIndex; i++) {
                    final Token token = tokens.get(i);
                    if (lastTokenChar > token.start) {
                        out.add(new Token(token.getTokenType(), token.getChars(), lastTokenChar, token.end));
                    } else {
                        out.add(token);
                    }
                }
            }

            // Break start spanned token if needed.
            final Token token = tokens.get(start.tokenIndex);
            if (token.start < start.charIndex) {
                if (lastTokenChar > token.start) {
                    out.add(new Token(token.getTokenType(), token.getChars(), lastTokenChar, start.charIndex - 1));
                } else {
                    out.add(new Token(token.getTokenType(), token.getChars(), token.start, start.charIndex - 1));
                }
            }

            // Output span.
            out.add(new ParamToken.Builder()
                    .tokenType(TokenType.PARAM)
                    .chars(token.getChars())
                    .start(start.charIndex)
                    .end(end.charIndex)
                    .build());

            lastTokenIndex = end.tokenIndex;
            lastTokenChar = end.charIndex + 1;
        }

        // Output final tokens.
        for (int i = lastTokenIndex; i < tokens.size(); i++) {
            final Token token = tokens.get(i);
            if (lastTokenChar > token.start) {
                out.add(new Token(token.getTokenType(), token.getChars(), lastTokenChar, token.end));
            } else {
                out.add(token);
            }
        }

        return out;
    }
This code is left here in case we decide we want to allow quoted param names.

@param tokens
@return
stroom-query/stroom-query-api/src/main/java/stroom/query/api/token/BasicTokeniser.java