@Override
    public KeyValueIterator<Windowed<K>, V> all() {
        if (!open) {
            throw new InvalidStateStoreException("Store is not open");
        }
        final List<KeyValue<Windowed<K>, V>> results = new ArrayList<>();
        for (final long now : data.keySet()) {
            final NavigableMap<K, V> kvMap = data.get(now);
            if (kvMap != null) {
                for (final Entry<K, V> entry : kvMap.entrySet()) {
                    results.add(new KeyValue<>(new Windowed<>(entry.getKey(), new TimeWindow(now, now + windowSize)), entry.getValue()));
                }
            }
        }
        final Iterator<KeyValue<Windowed<K>, V>> iterator = results.iterator();

        return new KeyValueIterator<Windowed<K>, V>() {
            @Override
            public void close() {
            }

            @Override
            public Windowed<K> peekNextKey() {
                throw new UnsupportedOperationException("peekNextKey() not supported in " + getClass().getName());
            }

            @Override
            public boolean hasNext() {
                return iterator.hasNext();
            }

            @Override
            public KeyValue<Windowed<K>, V> next() {
                return iterator.next();
            }

        };
    }
A very simple window store stub for testing purposes.
streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java