# Dedup by last 200 chars seen = set() deduped = [] for line in examples: key = line[-200:] if key not in seen: seen.add(key) deduped.append(line)