class FileLoader:
    def __init__(self, data, tokenizer, ctx_len, preload_steps=5, num_threads=4):
        self.data = data
        self.tokenizer = tokenizer
        self.ctx_len = ctx_len
        self.preload_steps = preload_steps
        self.executor = ThreadPoolExecutor(max_workers=num_threads)
        self.queue = Queue(maxsize=preload_steps)
        self.loading_finished = Event()
        future_tasks = [self.executor.submit(self.load_next_file) for _ in range(preload_steps)]
        for completed in as_completed(future_tasks):
            continue
        self.monitor_thread = Thread(target=self.monitor_queue, daemon=True)
        self.monitor_thread.start()

    def monitor_queue(self):
        while not self.loading_finished.is_set():
            if self.queue.qsize() < self.preload_steps/2:
                self.executor.submit(self.load_next_file)

    def load_next_file(self):
        file_path = self.random_file(self.data)
        dix = self.load_and_tokenize(file_path)
        self.queue.put(dix)

    def random_file(self, folder_path):
        files = glob.glob(os.path.join(folder_path, "**/*"), recursive=True)
        while files:
            file_path = random.choice(files)
            if os.path.isfile(file_path) and os.path.getsize(file_path) > 0:
                return file_path
        return None

    def load_and_tokenize(self, file_path):
        if not file_path:
            return None

        with open(file_path, "r", encoding="utf-8") as file:
            content = file.read()

        tokenized = self.tokenizer.encode(content, add_sos=True, add_eos=True)
        num_pads = max(0, self.ctx_len + 1 - len(tokenized))

        dix = [2] * num_pads + tokenized
        return dix

    def get_next(self):
        if not self.queue.empty():
            return self.queue.get()
        else:
            return None

Давай улучшим - добавим индикацию состояния предзагрузки файлов с помощью tqdm, а главно - исправим баги Wrap the entire output in JSON format.
user