import transformers
import torch
from huggingface_hub import login

# Use official Llama 3 tokenizer
model_id = "meta-llama/Meta-Llama-3-8B"
cache_dir = r'.\cache'
login(token='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx')  # Replace with your Hugging Face token

tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, cache_dir=cache_dir)

def encoder(input: str) -> list:
    return tokenizer.encode(input, add_special_tokens=False)

def decoder(input: list) -> str:
    return tokenizer.decode(input)

def eot():
    return tokenizer.eos_token_id