public void generate() throws IOException {

        Path currentWorkingDir = Paths.get("").toAbsolutePath();

        final InputStream rawDoc = Files.newInputStream(currentWorkingDir.resolve(clientParameters.inputFile));

        BufferedReader reader = new BufferedReader(new InputStreamReader(rawDoc));

        long i = 1;
        while (reader.ready()) {
            String line = reader.readLine();
            JsonReader jsonReader = new JsonReader(types, new ByteArrayInputStream(Utf8.toBytes(line)), parserFactory);
            String wikimediaId = "id:wikimedia:" + languageTag.languageCode() + "::" + i;

            ParsedDocumentOperation operation = jsonReader.readSingleDocumentStreaming(DocumentOperationType.PUT, wikimediaId);
            DocumentPut put = (DocumentPut) operation.operation();
            Document document = put.getDocument();
            FieldValue fieldValue = document.getFieldValue(clientParameters.field);
            this.handleTokenization(fieldValue.toString());
            if (i % 50000 == 0) {
                System.out.println("Documents processed: " + i + ", unique words: " + documentFrequency.size());
            }
            i++;
        }

        long pageCount = i - 1;
        System.out.println("Total documents processed: " + pageCount + ", unique words: " + documentFrequency.size());

        SignificanceModelFile modelFile;
        File outputFile = Paths.get(clientParameters.outputFile).toFile();
        String languagesKey = String.join(",", this.languages.stream().map(Language::languageCode).toList());
        if (outputFile.exists()) {

            InputStream in = outputFile.toString().endsWith(".zst") ?
                    new ZstdInputStream(new FileInputStream(outputFile)) :
                    new FileInputStream(outputFile);

            modelFile = objectMapper.readValue(in, SignificanceModelFile.class);

            modelFile.addLanguage(languagesKey, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency()));

        } else {
            HashMap<String, DocumentFrequencyFile> languages = new HashMap<>() {{
                put(languagesKey, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency()));
            }};

            modelFile = new SignificanceModelFile(VERSION, ID, SIGNIFICANCE_DESCRIPTION + clientParameters.inputFile, languages);
        }
        try {
            ObjectWriter writer = objectMapper.writerWithDefaultPrettyPrinter();

            OutputStream out = useZstCompression ?
                    new ZstdOutputStream(new FileOutputStream(clientParameters.outputFile)) :
                    new FileOutputStream(clientParameters.outputFile);

            writer.writeValue(out, modelFile);
        } catch (IOException e) {
            throw new IllegalStateException("Failed to write model to output file", e);
        }
    }
@Test
    void testGenerateFileWithMultipleLanguagesForSingleDocumentFrequency() throws IOException {
        String inputPath = "no.jsonl";
        String outputPath = "output.json";
        ClientParameters params = createParameters(inputPath, outputPath,  "text", "nb,un", "nb", "false").build();
        SignificanceModelGenerator generator = createSignificanceModelGenerator(params);
        generator.generate();

        File outputFile = new File(tempDir.resolve(outputPath).toString());
        assertTrue(outputFile.exists());

        SignificanceModelFile modelFile = objectMapper.readValue(outputFile, SignificanceModelFile.class);

        HashMap<String, DocumentFrequencyFile> languages = modelFile.languages();
        assertEquals(1, languages.size());

        assertTrue(languages.containsKey("nb,un"));

        DocumentFrequencyFile documentFrequencyFile = languages.get("nb,un");

        assertEquals(3, documentFrequencyFile.frequencies().get("fra"));
        assertEquals(3, documentFrequencyFile.frequencies().get("skriveform"));
        assertEquals(3, documentFrequencyFile.frequencies().get("kategori"));
        assertEquals(3, documentFrequencyFile.frequencies().get("eldr"));
    }