public List<String> tokenize(String text)
    {
        List<String> tokens = new ArrayList<>();

        Matcher regexMatcher = regexExpression.matcher(text);

        int lastIndexOfPrevMatch = 0;

        while (regexMatcher.find(lastIndexOfPrevMatch)) // this is where the magic happens:
                                    // the regexp is used to find a matching pattern for substitution
        {
            int beginIndexOfNextMatch = regexMatcher.start();

            String prevToken = text.substring(lastIndexOfPrevMatch, beginIndexOfNextMatch);

            if (!prevToken.isEmpty())
            {
                tokens.add(prevToken);
            }

            String currentMatch = regexMatcher.group();

            tokens.add(currentMatch);

            lastIndexOfPrevMatch = regexMatcher.end();
            if (lastIndexOfPrevMatch < text.length() && text.charAt(lastIndexOfPrevMatch) != '_')
            {
                // beause it is sometimes positioned after the "_", but it should be positioned
                // before the "_"
                --lastIndexOfPrevMatch;
            }
        }

        String tail = text.substring(lastIndexOfPrevMatch);

        if (!tail.isEmpty())
        {
            tokens.add(tail);
        }

        return tokens;
    }
@Test
    void testTokenize_happyPath_8()
    {
        // given
        CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
                new HashSet<>(Arrays.asList("_100_101_102_", "_101_102_", "_103_104_")));
        String text = "_100_101_102_103_104_";

        // when
        List<String> tokens = tokenizer.tokenize(text);

        // then
        assertEquals(Arrays.asList("_100_101_102_", "_103_104_"), tokens);
    }