private void processData(final String name,
                             final Path inputDir,
                             final Path outputDir,
                             final boolean reference,
                             final boolean compareOutput,
                             final List<Exception> exceptions) {
        // Create a stream processor for each pipeline.
        final List<DocRef> pipelines = pipelineStore.findByName(name);

        assertThat(pipelines)
                .hasSize(1);

        final DocRef pipelineRef = pipelines.get(0);

        final List<DocRef> feedRefs = feedStore.findByName(pipelineRef.getName());

        FeedDoc feed = null;
        if (feedRefs.size() > 0) {
            feed = feedStore.readDocument(feedRefs.get(0));
        }
        final FeedDoc feedDoc = feed;

        if (feedDoc != null && feedDoc.isReference() == reference) {
            int priority = 1;
            if (feed.isReference()) {
                priority++;
            }

            final String streamType = feed.isReference()
                    ? StreamTypeNames.RAW_REFERENCE
                    : StreamTypeNames.RAW_EVENTS;

            final QueryData findStreamQueryData = QueryData.builder()
                    .dataSource(MetaFields.STREAM_STORE_DOC_REF)
                    .expression(ExpressionOperator.builder()
                            .addTextTerm(MetaFields.FEED, ExpressionTerm.Condition.EQUALS, feedDoc.getName())
                            .addTextTerm(MetaFields.TYPE, ExpressionTerm.Condition.EQUALS, streamType)
                            .build())
                    .build();

            processorFilterService.create(
                    CreateProcessFilterRequest
                            .builder()
                            .pipeline(pipelineRef)
                            .queryData(findStreamQueryData)
                            .priority(priority)
                            .build());

            // Add data.
            final List<Path> files = new ArrayList<>();
            addFiles(inputDir, files, feed.getName(), "in");
            addFiles(inputDir, files, feed.getName(), "zip");

            files.sort(Comparator.naturalOrder());

            LOGGER.info("Testing {} files: {}",
                    files.size(),
                    files.stream()
                            .map(file ->
                                    file.getFileName().toString())
                            .collect(Collectors.joining(", ")));

            files.forEach(filePath -> {
                // Add and test each file.
                final String fileName = filePath.getFileName().toString();
                final int index = fileName.lastIndexOf(".");
                final String stem = fileName.substring(0, index);

                try {
                    test(filePath, feedDoc, outputDir, stem, compareOutput, exceptions);
                } catch (final IOException | RuntimeException e) {
                    fail(e.getMessage());
                }
            });
        }
    }
NOTE some of the input data for this test is buried in the following zip file so you will need
to crack it open to see what is being loaded.
stroom/stroom-core/src/test/resources/samples/input/ZIP_TEST-DATA_SPLITTER-EVENTS~1.zip
stroom-app/src/test/java/stroom/pipeline/task/TranslationTest.java