public Optional<Example<T>> generateExample(ColumnarIterator.Row row, boolean outputRequired) {

        List<String> responseValues = responseProcessor.getFieldNames().stream()
                .map(f -> row.getRowData().getOrDefault(f, ""))
                .collect(Collectors.toList());
        Optional<T> labelOpt = responseProcessor.process(responseValues);
        if (!labelOpt.isPresent() && outputRequired) {
            return Optional.empty();
        }

        List<ColumnarFeature> features = generateFeatures(row.getRowData());

        if (features.isEmpty()) {
            logger.warning(String.format("Row %d empty of features, omitting", row.getIndex()));
            return Optional.empty();
        } else {
            T label = labelOpt.orElse(responseProcessor.getOutputFactory().getUnknownOutput());

            Map<String, Object> metadata = generateMetadata(row);

            Example<T> example;
            if (weightExtractor == null) {
                example = new ArrayExample<>(label, metadata);
            } else {
                example = new ArrayExample<>(label,
                        weightExtractor.extract(row).orElse(Example.DEFAULT_WEIGHT),
                        metadata);
            }
            example.addAll(features);
            return Optional.of(example);
        }
    }
@Test
    public void replaceNewlinesWithSpacesTest() {
        final Pattern BLANK_LINES = Pattern.compile("(\n[\\s-]*\n)+");

        final Function<CharSequence, CharSequence> newLiner = (CharSequence charSequence) -> {
            if (charSequence == null || charSequence.length() == 0) {
                return charSequence;
            }
            return BLANK_LINES.splitAsStream(charSequence).collect(Collectors.joining(" *\n\n"));
        };

        Tokenizer tokenizer = new MungingTokenizer(new BreakIteratorTokenizer(Locale.US), newLiner);
        TokenPipeline textPipeline = new TokenPipeline(tokenizer, 2, false);

        final Map<String, FieldProcessor> fieldProcessors = new HashMap<>();
        fieldProcessors.put("order_text", new TextFieldProcessor("order_text", textPipeline));

        MockResponseProcessor response = new MockResponseProcessor("Label");

        Map<String,String> row = new HashMap<>();
        row.put("order_text", "Jimmy\n\n\n\nHoffa");
        row.put("Label", "Sheep");

        RowProcessor<MockOutput> processor = new RowProcessor<>(Collections.emptyList(),null,response,fieldProcessors,Collections.emptyMap(),Collections.emptySet(), false);

        Example<MockOutput> example = processor.generateExample(row,true).get();

        // Check example is extracted correctly
        assertEquals(5, example.size());
        assertEquals("Sheep", example.getOutput().label);
        Iterator<Feature> featureIterator = example.iterator();
        Feature a = featureIterator.next();
        assertEquals("order_text@1-N=*", a.getName());
        assertEquals(1.0, a.getValue());
        a = featureIterator.next();
        assertEquals("order_text@1-N=Hoffa", a.getName());
        a = featureIterator.next();
        assertEquals("order_text@1-N=Jimmy", a.getName());
        a = featureIterator.next();
        assertEquals("order_text@2-N=*/Hoffa", a.getName());
        a = featureIterator.next();
        assertEquals("order_text@2-N=Jimmy/*", a.getName());
        assertFalse(featureIterator.hasNext());

        // same input with replaceNewlinesWithSpacesTest=true (the default) produces different features
        processor = new RowProcessor<>(Collections.emptyList(),null,response,fieldProcessors,Collections.emptyMap(),Collections.emptySet(), true);

        example = processor.generateExample(row,true).get();

        // Check example is extracted correctly
        assertEquals(3, example.size());
        assertEquals("Sheep", example.getOutput().label);
        featureIterator = example.iterator();
        a = featureIterator.next();
        assertEquals("order_text@1-N=Hoffa", a.getName());
        assertEquals(1.0, a.getValue());
        a = featureIterator.next();
        assertEquals("order_text@1-N=Jimmy", a.getName());
        a = featureIterator.next();
        assertEquals("order_text@2-N=Jimmy/Hoffa", a.getName());
        assertFalse(featureIterator.hasNext());
    }