def count_and_print(text, tokenizer, model_name="Claude 3 Family"): num_tokens = tokenizer.count_tokens(text) print(f"--- {model_name} ---") print(f"Text:\n'{text}'") print(f"Token Count: {num_tokens}\n") count_and_print(text_short, tokenizer_claude_3) count_and_print(text_sentence, tokenizer_claude_3) count_and_print(text_paragraph, tokenizer_claude_3) count_and_print(text_code, tokenizer_claude_3) count_and_print(text_special_chars, tokenizer_claude_3) count_and_print(text_english_chinese, tokenizer_claude_3)