def get_html_for_token_perplexity(tok, sentence, tok_ppx, model_ppx): tokens = tok.encode(sentence).tokens ids = tok.encode(sentence).ids cleaned_tokens = [] for word in tokens: m = list(map(ord, word)) m = list(map(lambda x: x if x != 288 else ord(' '), m)) m = list(map(chr, m)) m = ''.join(m) cleaned_tokens.append(m) # html = [ f"{cleaned_tokens[0]}", ] for ct, ppx in zip(cleaned_tokens[1:], tok_ppx): color = "black" if ppx.item() >= 0: if ppx.item() <= model_ppx * 1.1: color = "red" else: color = "green" # # html.append(f"{ct}") # return "".join(html) #