from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing
from transformers import AutoTokenizer
from datasets import Dataset
import pandas as pd

#load base tokenizer to train on dataset
tokenizer_base = AutoTokenizer.from_pretrained("bert-base-cased")
# convert pandas dataset to HF dataset
dataset = Dataset.from_pandas(df.rename(columns={"comment":'text'}))

# define iterator
training_corpus = (
    dataset[i : i + 1000]["text"]
    for i in range(0, len(dataset), 1000)
)

#train the new tokenizer for dataset
tokenizer = tokenizer_base.train_new_from_iterator(training_corpus, 5000)
#test trained tokenizer for sample text
text = dataset['text'][123]
print(text)