private boolean ensureRefStreamAvailability(final ReferenceDataResult result,
                                                final PipelineReference pipelineReference,
                                                final RefStreamDefinition refStreamDefinition,
                                                final RefDataStore offHeapRefDataStore) {
        final boolean isAvailableForLookups;

        // First check the pipeline scoped object to save us hitting the store for every lookup in a
        // pipeline process run.
        if (refDataLoaderHolder.isRefStreamAvailable(refStreamDefinition)) {
            // we have already loaded or confirmed the load state for this refStream in this
            // pipeline process so no need to try again
            LOGGER.trace("refStreamDefinition {} is available for use", refStreamDefinition);
            isAvailableForLookups = true;
        } else {
            // we don't know what the load state is for this refStreamDefinition so need to find out
            // by querying the store. This will also update the last accessed time so will prevent
            // (unless the purge age is very small) a purge from removing the data we are about to use
            final Optional<ProcessingState> optLoadState =
                    offHeapRefDataStore.getLoadState(refStreamDefinition);

            result.logLazyTemplate(Severity.INFO,
                    "Load status of stream {} is '{}', feed '{}', pipeline: '{}'",
                    () -> Arrays.asList(
                            refStreamDefinition.getStreamId(),
                            optLoadState.map(ProcessingState::toString).orElse("NOT_LOADED"),
                            pipelineReference.getFeed().getName(),
                            pipelineReference.getPipeline().getName()));

            final boolean isRefLoadRequired = optLoadState
                    .filter(loadState ->
                            loadState.equals(ProcessingState.COMPLETE))
                    .isEmpty();
            LOGGER.trace("optLoadState {}, isRefLoadRequired {}", optLoadState, isRefLoadRequired);

            if (optLoadState.isPresent() && optLoadState.get().equals(ProcessingState.FAILED)) {
                throw new RuntimeException(LogUtil.message(
                        "Reference stream {} has been loaded previously but failed, " +
                        "aborting lookup against this stream.",
                        refStreamDefinition.getStreamId()));
            } else {
                if (isRefLoadRequired) {
                    // we don't have the complete data so kick off a process to load it all
                    LOGGER.debug("Creating task to load reference data {}", refStreamDefinition);

                    // Initiate a load of the ref data for this stream
                    // It is possible for another thread, i.e. another pipeline process to be trying to
                    // load the same stream at the same time. There is concurrency protection further
                    // down to protect against this.
                    final StoredErrorReceiver storedErrorReceiver =
                            securityContext.asProcessingUserResult(() ->
                                    referenceDataLoader.load(refStreamDefinition));

                    // Replay any errors/warning from the load onto our result
                    if (storedErrorReceiver != null) {
                        storedErrorReceiver.replay(result);
                    }

                    LOGGER.debug(() -> LogUtil.message(
                            "Loaded {} refStreamDefinition", refStreamDefinition));

                    // No point in continuing if the load was interrupted
                    if (!isTerminated()
                        && (storedErrorReceiver == null
                            || storedErrorReceiver.getCount(Severity.FATAL_ERROR) == 0)) {
                        // mark this ref stream defs as available for future lookups within this
                        // pipeline process
                        refDataLoaderHolder.markRefStreamAsAvailable(refStreamDefinition);

                        isAvailableForLookups = true;
                    } else {
                        isAvailableForLookups = false;
                    }
                } else {
                    isAvailableForLookups = true;
                }

                if (isAvailableForLookups) {
                    // Now the data is loaded, query the DB to record the list of maps available
                    // in this stream so future lookups in this pipe scope can decide whether to
                    // perform a lookup on a stream or not.
                    refDataStoreHolder.addKnownMapNames(
                            offHeapRefDataStore,
                            refStreamDefinition);
                }
            }
        }
        return isAvailableForLookups;
    }
Ensures that the effective external ref stream is loaded, doing the load if it
is not. It then mutates result to set a value proxy on it so that downstream
code can use the proxy to get the value from the store.
The effective stream is determined based on the time used in the lookup() call
and the effective time of the various reference streams, i.e. picking the last ref
stream before the lookup time.
stroom-pipeline/src/main/java/stroom/pipeline/refdata/ReferenceData.java