"""Playing around with the tokenizer
"""
sentence = "Here's a weird word: Withoutadoubticus."
print(f'original sentence: "{sentence}"')
demo_tokens = tokenizer([sentence])
print(f"token IDs: {demo_tokens['input_ids']}")
tokens = tokenizer.convert_ids_to_tokens(demo_tokens['input_ids'][0])
print(f'token values: {tokens}')