from tokenizers import Tokenizer

tokenizer = Tokenizer.from_pretrained("HuggingFaceTB/SmolLM3-3B")
text = "Hello world"
encodings = tokenizer.encode(text)

print(encodings.ids)
# [9906, 1917]
print(encodings.tokens)
# ['Hello', 'Ġworld']