from transformers import AutoTokenizer # init tokenizer teacher_tokenizer = AutoTokenizer.from_pretrained(teacher_id) student_tokenizer = AutoTokenizer.from_pretrained(student_id) # sample input sample = "This is a basic example, with different words to test." # assert results assert teacher_tokenizer(sample) == student_tokenizer(sample), "Tokenizers haven't created the same output"