@Override
    public String[] split(String text) {
        boundary.setText(text);
        List<String> words = new ArrayList<>();
        int start = boundary.first();
        int end = boundary.next();

        while (end != BreakIterator.DONE) {
            String word = text.substring(start, end).trim();
            if (!word.isEmpty()) {
                words.add(word);
            }
            start = end;
            end = boundary.next();
        }

        return words.toArray(new String[0]);
    }
@Test
    public void testSplit() {
        System.out.println("tokenize");
        String text = "Good muffins cost $3.88\nin New York.  Please buy "
                + "me\ntwo of them.\n\nYou cannot eat them. I gonna eat them. "
                + "Thanks. Of course, I won't. ";

        String[] expResult = {"Good", "muffins", "cost", "$3.88", "in",
            "New", "York", ".", "Please", "buy", "me", "two", "of", "them", ".",
            "You", "cannot", "eat", "them", ".", "I", "gonna", "eat", "them", ".",
            "Thanks", ".", "Of", "course", ",", "I", "won't", "."};

        BreakIteratorTokenizer instance = new BreakIteratorTokenizer();
        String[] result = instance.split(text);

        assertEquals(expResult.length, result.length);
        for (int i = 0; i < result.length; i++) {
            assertEquals(expResult[i], result[i]);
        }
    }