{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "markdown", "metadata": { "id": "L4odnI5t1oSm" }, "source": [ "# Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gs4Gzr7Avv-A" }, "outputs": [], "source": [ "!pip install evaluate bert_score unbabel-comet" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "background_save": true }, "id": "fbx03SxLBq8t" }, "outputs": [], "source": [ "from datasets import load_dataset, DownloadMode\n", "import random\n", "import sacrebleu\n", "import evaluate\n", "from comet import download_model, load_from_checkpoint\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import torch\n", "from transformers import pipeline\n", "\n", "def _suppress_logs():\n", " import logging\n", " import os\n", " loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]\n", " for logger in loggers:\n", " logger.setLevel(logging.ERROR)\n", "\n", " os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", " logging.getLogger(\"transformers\").setLevel(logging.ERROR)\n", " logging.getLogger(\"accelerate\").setLevel(logging.ERROR)\n", " logging.getLogger(\"tokenizers\").setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "6mpTAz6hsJNL" }, "source": [ "# Helper Functions" ] }, { "cell_type": "markdown", "metadata": { "id": "XkzPFvYa1oSp" }, "source": [ "## Dataset Processing\n", "\n", "These functions handle:\n", "- Downloading the OPUS Books EN-FR translation dataset\n", "- Converting examples to chat template format (required by instruction-tuned models)\n", "- Random sampling for testing" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "background_save": true }, "id": "qDawwnVbD8S9" }, "outputs": [], "source": [ "# Do Not change this\n", "DATASET_NAME = \"opus_books\"\n", "LANGUAGE_PAIR = \"en-fr\"\n", "\n", "def download_and_process_translation_dataset():\n", " \"\"\"\n", " Download the translation_dataset\n", " :return: HuggingFace dataset\n", " \"\"\"\n", " ds = load_dataset(DATASET_NAME, LANGUAGE_PAIR, download_mode=DownloadMode.FORCE_REDOWNLOAD)\n", " ds = ds[\"train\"]\n", " ds = ds.map(convert_english_to_chat_template_format)\n", " return ds\n", "\n", "def convert_english_to_chat_template_format(examples) -> dict:\n", " \"\"\"\n", " Given an example, map fn transforms it to the below output\n", " You may print an example to see how it looks!\n", " We want to later use the content for translation\n", " {\n", " 'id': '0', 'translation':\n", " {'en': 'The Wanderer', 'fr': 'Le grand Meaulnes'},\n", " 'role': 'user',\n", " 'en_content': 'The Wanderer',\n", " 'fr_content': 'Le grand Meaulnes'\n", " }\n", " :param examples: One dataset example\n", " :return: Dict\n", " \"\"\"\n", " # solution\n", " return {\"role\": \"user\",\n", " \"en_content\": f\"{examples[\"translation\"][\"en\"]}\",\n", " \"fr_content\": f\"{examples[\"translation\"][\"fr\"]}\"}\n", "\n", "def sample_from_dataset(dataset, num_samples=100):\n", " \"\"\"\n", " Randomly sample num_samples from dataset\n", " :param dataset: HuggingFace dataset\n", " :param num_samples: Number of random samples\n", " :return: Subset of Dataset with samples\n", " \"\"\"\n", " random.seed(42)\n", " random_indices = random.sample(range(len(dataset)), k=num_samples)\n", " return dataset.select(random_indices)\n", "\n", "def create_english_user_prompt(example) -> dict:\n", " \"\"\"\n", "\n", " :param example: One Training data example from huggingFace dataset (assume it's like a dict)\n", " :return: Dict which looks like\n", " {'role': 'user', 'content': 'The Wanderer'}\n", " \"\"\"\n", " # Solution\n", " return {'role': example['role'], 'content': example['en_content']}\n", "\n", "def create_french_user_prompt(example) -> dict:\n", " \"\"\"\n", "\n", " :param example: One Training data example from huggingFace dataset (assume it's like a dict)\n", " :return: Dict which looks like\n", " {'role': 'user', 'content': 'The Wanderer'}\n", " \"\"\"\n", " # Solution\n", " return {'role': example['role'], 'content': example['fr_content']}" ] }, { "cell_type": "markdown", "metadata": { "id": "ckW54vhVsRa7" }, "source": [ "## Inference helper functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rhFS0061sOed" }, "outputs": [], "source": [ "def batch_translate_using_hf(user_prompts: list[dict[str, str]],\n", " system_prompt: dict[str, str],\n", " model_name: str,\n", " max_new_tokens: int,\n", " do_sample: bool = False):\n", " \"\"\"\n", " :param user_prompts: List of user prompts for batch translation\n", " :param system_prompt: System prompt of text\n", " :param model_name: Name of HuggingFace model\n", " :param max_new_tokens: Max tokens to decode\n", " :param do_sample: sampling prompts\n", " :return: List of outputs\n", " \"\"\"\n", " pipe = pipeline(task=\"text-generation\",\n", " model=model_name,\n", " dtype=torch.bfloat16,\n", " device_map=\"auto\")\n", "\n", " # Prepare batch of prompts\n", " prompts = []\n", " for user_prompt in user_prompts:\n", " messages = [system_prompt, user_prompt]\n", " prompt = pipe.tokenizer.apply_chat_template(\n", " messages, tokenize=False, add_generation_prompt=True)\n", " prompts.append(prompt)\n", "\n", " # Batch processing with batches of 32\n", " outputs = pipe(\n", " prompts,\n", " max_new_tokens=max_new_tokens,\n", " do_sample=do_sample,\n", " return_full_text=False,\n", " batch_size=32)\n", " return outputs\n", "\n", "\n", "def batch_post_process_output(batch_outputs) -> list[str]:\n", " \"\"\"\n", " Filter out the translated text from batch outputs\n", " :param batch_outputs: Batch outputs from batch_translate_using_hf\n", " :return: List of translated texts\n", " \"\"\"\n", " translations = []\n", " for outputs in batch_outputs:\n", " raw_output = outputs[0][\"generated_text\"]\n", " translation = raw_output.split(\"Output:\")[-1].split(\"<|im_end|>\")[0].strip()\n", " translations.append(translation)\n", " return translations\n" ] }, { "cell_type": "markdown", "metadata": { "id": "1vHVfvWX1oSq" }, "source": [ "---\n", "\n", "## Evaluation Metrics\n", "\n", "This section contains implementations for the translation quality metrics you'll examine, as well as an evaluation function which calls them." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DJ83m7zUoC5f" }, "outputs": [], "source": [ "# You will implement this function for question 1.2.\n", "def ai_as_judge_scorer() -> float:\n", " return -1.0 # TODO: Add your implementation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "I8tKsg8n_nz-" }, "outputs": [], "source": [ "# Load BERTScore once\n", "_bertscore = evaluate.load(\"bertscore\")\n", "\n", "\n", "def compute_bleu_score(\n", " predicted_translation: str,\n", " ground_truth: str,\n", ") -> float:\n", " \"\"\"\n", " Compute sentence-level BLEU using SacreBLEU with smoothing.\n", "\n", " :param predicted_translation: Model translation\n", " :param ground_truth: Reference translation\n", " :return: BLEU score\n", " \"\"\"\n", " bleu = sacrebleu.sentence_bleu(\n", " predicted_translation,\n", " [ground_truth],\n", " )\n", " return bleu.score\n", "\n", "\n", "def compute_bert_score(\n", " predicted_translation: str,\n", " ground_truth: str,\n", " lang: str,\n", ") -> float:\n", " \"\"\"\n", " Compute BERTScore (F1).\n", "\n", " :param predicted_translation: Model translation\n", " :param ground_truth: Reference translation\n", " :param lang: Language code (e.g., 'en', 'fr')\n", " :return: BERTScore F1\n", " \"\"\"\n", " result = _bertscore.compute(\n", " predictions=[predicted_translation],\n", " references=[ground_truth],\n", " lang=lang,\n", " )\n", " return result[\"f1\"][0]\n", "\n", "\n", "class CometScorer:\n", " def __init__(self):\n", " # Do not change this!\n", " self.model_path = download_model(\"Unbabel/wmt22-comet-da\")\n", " self.model = load_from_checkpoint(self.model_path)\n", "\n", " def compute_comet_score(\n", " self,\n", " reference: str,\n", " predicted_translation: str,\n", " ground_truth: str,\n", " ) -> float:\n", " \"\"\"\n", " Compute COMET score.\n", "\n", " :param reference: Source sentence\n", " :param predicted_translation: Model output\n", " :param ground_truth: Reference translation\n", " :return: COMET score\n", " \"\"\"\n", " data = [\n", " {\n", " \"src\": reference,\n", " \"mt\": predicted_translation,\n", " \"ref\": ground_truth,\n", " }\n", " ]\n", " output = self.model.predict(\n", " data,\n", " batch_size=1,\n", " progress_bar=False,\n", " )\n", " return output[\"scores\"][0]\n", "\n", " def compute_comet_scores_batch(\n", " self,\n", " references: list[str],\n", " predicted_translations: list[str],\n", " ground_truths: list[str],\n", " ) -> list[float]:\n", " \"\"\"\n", " Compute COMET scores for a batch.\n", "\n", " :param references: List of source sentences\n", " :param predicted_translations: List of model outputs\n", " :param ground_truths: List of reference translations\n", " :return: List of COMET scores\n", " \"\"\"\n", " data = [\n", " {\n", " \"src\": ref,\n", " \"mt\": pred,\n", " \"ref\": gt,\n", " }\n", " for ref, pred, gt in zip(references, predicted_translations, ground_truths)\n", " ]\n", " output = self.model.predict(\n", " data,\n", " batch_size=32,\n", " progress_bar=False,\n", " )\n", " return output[\"scores\"]\n", "\n", "def evaluate_translation_batch(\n", " predictions: list[str],\n", " references: list[str],\n", " sources: list[str],\n", " comet_scorer: CometScorer,\n", " lang: str,\n", ") -> list[dict]:\n", " \"\"\"\n", " Batch evaluation of translations.\n", "\n", " :param predictions: List of predicted texts\n", " :param references: List of ground truths\n", " :param sources: List of source texts\n", " :param comet_scorer: CometScorer instance\n", " :param lang: Language code\n", " :return: List of score dictionaries\n", " \"\"\"\n", " # Compute BLEU scores (not batched, but fast)\n", " bleu_scores = [compute_bleu_score(pred, ref) for pred, ref in zip(predictions, references)]\n", "\n", " # Compute BERTScores in batch\n", " bert_results = _bertscore.compute(\n", " predictions=predictions,\n", " references=references,\n", " lang=lang,\n", " )\n", " bert_scores = bert_results[\"f1\"]\n", "\n", " # Compute COMET scores in batch\n", " comet_scores = comet_scorer.compute_comet_scores_batch(\n", " references=sources,\n", " predicted_translations=predictions,\n", " ground_truths=references,\n", " )\n", "\n", " # Combine results\n", " results = []\n", " for bleu, bert, comet in zip(bleu_scores, bert_scores, comet_scores):\n", " # TODO Complete AI as judge scorer\n", " ai_judge = ai_as_judge_scorer()\n", " results.append({\n", " \"bleu\": bleu,\n", " \"bert\": bert,\n", " \"comet\": comet,\n", " \"ai_judge\": ai_judge,\n", " })\n", "\n", " return results\n", "\n", "def batch_translate(\n", " user_prompts: list[dict[str, str]],\n", " system_prompt: dict[str, str],\n", ") -> list[str]:\n", " outputs = batch_translate_using_hf(\n", " user_prompts=user_prompts,\n", " system_prompt=system_prompt,\n", " model_name=MODEL_NAME,\n", " max_new_tokens=MAX_NEW_TOKENS,\n", " do_sample=False,\n", " )\n", " return batch_post_process_output(outputs)\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "5dd80prB1oSs" }, "source": [ "---\n", "\n", "## Part 1 and 2: Direct Translation (FR→EN)\n", "\n", "This section evaluates **direct translation** from French to English.\n", "\n", "### Configuration\n", "\n", "Define the system prompt and model parameters:\n", "- **System prompt**: Instructions telling the model what task to perform\n", "- **Model**: SmolLM2-1.7B-Instruct (a small instruction-tuned language model)\n", "- **MAX_NEW_TOKENS**: Maximum length of generated translation\n", "- **NUM_SAMPLES**: Number of examples to evaluate\n", "\n", "**TODO for students**:\n", "\n", "1. Experiment with different system prompts to improve translation quality!\n", "2. Implement `ai_as_judge_scorer` above." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1SI-5xro_nz-" }, "outputs": [], "source": [ "DIRECT_TRANSLATION_PROMPT = {\n", " \"role\":\"system\",\n", " # TODO Change this prompt to better one\n", " \"content\": \"Translate this text from French to English.\"\n", "}\n", "\n", "MODEL_NAME = \"HuggingFaceTB/SmolLM2-1.7B-Instruct\"\n", "MAX_NEW_TOKENS = 512\n", "NUM_SAMPLES = 100" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "D-GSyRc4_nz-" }, "outputs": [], "source": [ "# Suppress verbose logs (especially COMET)\n", "_suppress_logs()\n", "\n", "dataset = download_and_process_translation_dataset()\n", "examples = sample_from_dataset(dataset, num_samples=NUM_SAMPLES)\n", "\n", "comet_scorer = CometScorer()\n", "\n", "metrics_fr_en = {\"bleu\": 0.0, \"bert\": 0.0, \"comet\": 0.0}\n", "\n", "# Prepare batch data\n", "user_prompts = [create_french_user_prompt(example) for example in examples]\n", "\n", "# Batch translation\n", "print(\"Starting batch translation...\")\n", "# Use specific variable name for Part 3 compatibility\n", "direct_translations_part1 = batch_translate(\n", " user_prompts, DIRECT_TRANSLATION_PROMPT)\n", "print(f\"Completed batch translation of {len(direct_translations_part1)} examples\\n\")\n", "\n", "# Prepare data for batch evaluation\n", "en_texts = [example[\"translation\"][\"en\"] for example in examples]\n", "fr_texts = [example[\"translation\"][\"fr\"] for example in examples]\n", "\n", "# Batch evaluation\n", "print(\"Starting batch evaluation...\")\n", "all_scores = evaluate_translation_batch(\n", " predictions=direct_translations_part1,\n", " references=en_texts,\n", " sources=fr_texts,\n", " comet_scorer=comet_scorer,\n", " lang=\"en\",\n", ")\n", "print(f\"Completed batch evaluation\\n\")\n", "\n", "# Print individual results and accumulate metrics\n", "for i, (example, en_pred, scores_fr_en) in enumerate(zip(examples, direct_translations_part1, all_scores)):\n", " en_text = example[\"translation\"][\"en\"]\n", " fr_text = example[\"translation\"][\"fr\"]\n", "\n", " print(f\"Example {i+1}/{NUM_SAMPLES}:\")\n", " print(f\"FR Reference: {fr_text}\")\n", " print(f\"EN Ground Truth: {en_text}\")\n", " print(f\"EN Prediction: {en_pred}\")\n", " print(f\"Scores FR→EN: {scores_fr_en}\\n\")\n", "\n", " for key in metrics_fr_en:\n", " metrics_fr_en[key] += scores_fr_en[key] / NUM_SAMPLES\n", "\n", "print(\"====== Final Averages ======\")\n", "print(f\"FR → EN | BLEU: {metrics_fr_en['bleu']:.4f}, \"\n", " f\"BERTScore: {metrics_fr_en['bert']:.4f}, \"\n", " f\"COMET: {metrics_fr_en['comet']:.4f}\")" ] }, { "cell_type": "markdown", "metadata": { "id": "sluxu3zA1oSt" }, "source": [ "---\n", "\n", "## Part 2: Backtranslation (EN→FR→EN)\n", "\n", "Backtranslation tests **information preservation** through round-trip translation:\n", "1. Translate English → French (forward)\n", "2. Translate French → English (back)\n", "3. Compare final English with original English\n", "\n", "### Why Backtranslation?\n", "- Measures how much meaning survives the translation process\n", "- Lower scores indicate information loss or distortion\n", "- Useful for evaluating translation robustness\n", "\n", "### Configure System Prompts\n", "\n", "Implement a prompt for back translation." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lPbPzxVs91jM" }, "outputs": [], "source": [ "# TODO For students\n", "# Improve the prompt\n", "BACK_TRANSLATION_SYSTEM_PROMPT = DIRECT_TRANSLATION_PROMPT\n", "\n", "FORWARD_TRANSLATION_SYSTEM_PROMPT = {\n", " \"role\":\"system\",\n", " # TODO Change this prompt to better one\n", " \"content\": \"Translate this text from English to French.\"\n", "}\n", "\n", "\n", "def _convert_backtranslation_to_chat_format(fr_pred):\n", " return {\"role\": \"user\", \"content\": f\"{fr_pred}\"}\n" ] }, { "cell_type": "markdown", "metadata": { "id": "S7lXheYc1oSu" }, "source": [ "### Run Backtranslation Pipeline\n", "\n", "Execute the complete backtranslation workflow:\n", "1. **Forward translation**: English → French\n", "2. **Back translation**: French → English \n", "3. **Evaluation**: Compare back-translated English with original English" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dxqtGIEi_nz-" }, "outputs": [], "source": [ "# Suppress verbose logs (especially COMET)\n", "_suppress_logs()\n", "\n", "dataset = download_and_process_translation_dataset()\n", "examples = sample_from_dataset(dataset, num_samples=NUM_SAMPLES)\n", "\n", "comet_scorer = CometScorer()\n", "\n", "metrics_fr_en = {\"bleu\": 0.0, \"bert\": 0.0, \"comet\": 0.0}\n", "\n", "# Prepare batch data for forward translation (EN -> FR)\n", "user_prompts = [create_english_user_prompt(example) for example in examples]\n", "\n", "# Forward translation: EN -> FR\n", "print(\"Starting forward translation (EN -> FR)...\")\n", "fr_predictions = batch_translate(user_prompts, FORWARD_TRANSLATION_SYSTEM_PROMPT)\n", "print(f\"Completed forward translation of {len(fr_predictions)} examples\\n\")\n", "\n", "# Prepare batch data for back translation (FR -> EN)\n", "back_user_prompts = [\n", " _convert_backtranslation_to_chat_format(fr_pred) for fr_pred in fr_predictions]\n", "\n", "# Back translation: FR -> EN\n", "print(\"Starting back translation (FR -> EN)...\")\n", "# Use specific variable name for Part 3 compatibility\n", "backtranslations_part2 = batch_translate(back_user_prompts, BACK_TRANSLATION_SYSTEM_PROMPT)\n", "print(f\"Completed back translation of {len(backtranslations_part2)} examples\\n\")\n", "\n", "# Prepare data for batch evaluation\n", "en_texts = [example[\"translation\"][\"en\"] for example in examples]\n", "fr_texts = [example[\"translation\"][\"fr\"] for example in examples]\n", "\n", "# Batch evaluation\n", "print(\"Starting batch evaluation...\")\n", "all_scores = evaluate_translation_batch(\n", " predictions=backtranslations_part2,\n", " references=en_texts,\n", " sources=fr_texts,\n", " comet_scorer=comet_scorer,\n", " lang=\"en\",\n", ")\n", "print(f\"Completed batch evaluation\\n\")\n", "\n", "# Print individual results and accumulate metrics\n", "for i, (example, fr_pred, en_pred, scores_fr_en) in enumerate(zip(examples, fr_predictions, backtranslations_part2, all_scores)):\n", " en_text = example[\"translation\"][\"en\"]\n", " fr_text = example[\"translation\"][\"fr\"]\n", "\n", " print(f\"Example {i+1}/{NUM_SAMPLES}:\")\n", " print(f\"EN Source: {en_text}\")\n", " print(f\"FR Prediction (forward): {fr_pred}\")\n", " print(f\"EN Prediction (back): {en_pred}\")\n", " print(f\"Backtranslation Scores: {scores_fr_en}\\n\")\n", "\n", " for key in metrics_fr_en:\n", " metrics_fr_en[key] += scores_fr_en[key] / NUM_SAMPLES\n", "\n", "print(\"====== Final Averages ======\")\n", "print(f\"Backtranslation (EN → FR → EN) | BLEU: {metrics_fr_en['bleu']:.4f}, \"\n", " f\"BERTScore: {metrics_fr_en['bert']:.4f}, \"\n", " f\"COMET: {metrics_fr_en['comet']:.4f}\")" ] }, { "cell_type": "markdown", "metadata": { "id": "_s8a3AGcA0vd" }, "source": [ "---\n", "\n", "## Part 3: Embedding Analysis - Comparing Translation Approaches\n", "\n", "**Prerequisites**: Run Part 1 and Part 2 first to generate the translation outputs.\n", "\n", "In this section, we'll use **semantic embeddings** to analyze and compare:\n", "1. **Ground truth English** (original reference)\n", "2. **Direct translation** (FR→EN from Part 1)\n", "3. **Backtranslation** (EN→FR→EN from Part 2)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "DoYTK7-PA3HI" }, "source": [ "### Install Required Package\n", "\n", "We'll use `sentence-transformers` for creating high-quality semantic embeddings.\n", "\n", "**Note**: If you encounter any missing package errors, run this cell." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "r8MARBQxAxby" }, "outputs": [], "source": [ "!pip install sentence-transformers" ] }, { "cell_type": "markdown", "metadata": { "id": "ICymRjs2A7Nu" }, "source": [ "### Prepare Data from Previous Parts\n", "\n", "Collect the translation outputs from Part 1 and Part 2. Make sure you've run those sections first!\n", "\n", "**Expected variables from previous parts:**\n", "- From Part 1: `en_predictions` (direct FR→EN translations), `examples`, `en_texts` (ground truth)\n", "- From Part 2: `en_predictions` from backtranslation (this will be renamed to avoid conflicts)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EKvfcTRnAxXK" }, "outputs": [], "source": [ "# Check Part 1 outputs\n", "print(f\"Part 1 Direct Translations: {len(direct_translations_part1)} found\")\n", "# Check Part 2 outputs\n", "print(f\"Part 2 Backtranslations: {len(backtranslations_part2)} found\")\n", "\n", "# Store Ground Truth reliably\n", "ground_truth_texts = [example[\"translation\"][\"en\"] for example in examples]\n", "print(f\"Ground Truth Texts: {len(ground_truth_texts)} found\")\n", "\n", "# Safety check: Ensure they are not identical\n", "if direct_translations_part1 == backtranslations_part2:\n", " print(\"\\n⚠️ WARNING: Direct translations and Backtranslations appear to be identical!\")\n", " print(\"This usually happens if both parts accidentally wrote to the same variable.\")\n", " print(\"Please RE-RUN Part 1 and Part 2 cells to generate fresh data.\")\n", "else:\n", " print(\"\\n✓ Data looks correct (Direct and Backtranslation outputs are different)\")\n", "\n", "print(\"\\n\" + \"=\"*60)\n", "print(\"Ready for embedding analysis!\")\n", "print(\"=\"*60)" ] }, { "cell_type": "markdown", "metadata": { "id": "cz3RqHsVA8lE" }, "source": [ "### Compute Embeddings\n", "\n", "Load a sentence embedding model and compute embeddings for all three text sets.\n", "\n", "**Model**: We'll use `all-MiniLM-L6-v2`, a lightweight but effective embedding model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PsgV2x1LA-Sn" }, "outputs": [], "source": [ "from sentence_transformers import SentenceTransformer\n", "import numpy as np\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "\n", "# Load embedding model\n", "print(\"Loading embedding model...\")\n", "embedding_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n", "print(\"✓ Model loaded\\n\")\n", "\n", "# Compute embeddings using the stored translations\n", "print(\"Computing embeddings...\")\n", "ground_truth_embeddings = embedding_model.encode(ground_truth_texts, show_progress_bar=True, convert_to_numpy=True)\n", "direct_translation_embeddings = embedding_model.encode(direct_translations_part1, show_progress_bar=True, convert_to_numpy=True)\n", "backtranslation_embeddings = embedding_model.encode(backtranslations_part2, show_progress_bar=True, convert_to_numpy=True)\n", "\n", "print(f\"\\n✓ Computed embeddings with shape: {ground_truth_embeddings.shape}\")" ] }, { "cell_type": "markdown", "metadata": { "id": "5Nv9R2ReBFLO" }, "source": [ "### Compute Cosine Similarities\n", "\n", "Calculate pairwise cosine similarities to compare all three outputs:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "t-UvqhEMBFdI" }, "outputs": [], "source": [ "# Compute cosine similarities\n", "direct_similarities = []\n", "backtranslation_similarities = []\n", "consistency_similarities = []\n", "\n", "for i in range(len(ground_truth_texts)):\n", " # 1. Ground truth vs Direct translation (Accuracy)\n", " direct_sim = cosine_similarity(\n", " ground_truth_embeddings[i].reshape(1, -1),\n", " direct_translation_embeddings[i].reshape(1, -1)\n", " )[0][0]\n", " direct_similarities.append(direct_sim)\n", "\n", " # 2. Ground truth vs Backtranslation (Robustness/Information Preservation)\n", " back_sim = cosine_similarity(\n", " ground_truth_embeddings[i].reshape(1, -1),\n", " backtranslation_embeddings[i].reshape(1, -1)\n", " )[0][0]\n", " backtranslation_similarities.append(back_sim)\n", "\n", " # 3. Direct vs Backtranslation (Consistency between the two En* outputs)\n", " consistency_sim = cosine_similarity(\n", " direct_translation_embeddings[i].reshape(1, -1),\n", " backtranslation_embeddings[i].reshape(1, -1)\n", " )[0][0]\n", " consistency_similarities.append(consistency_sim)\n", "\n", "# Convert to numpy arrays for analysis\n", "direct_similarities = np.array(direct_similarities)\n", "backtranslation_similarities = np.array(backtranslation_similarities)\n", "consistency_similarities = np.array(consistency_similarities)\n", "\n", "print(\"✓ Computed all cosine similarities\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Edodv1u-BIw6" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "# Create figure with multiple subplots\n", "fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n", "\n", "# 1. Histogram comparison including Consistency\n", "axes[0, 0].hist(direct_similarities, bins=20, alpha=0.5, label='Direct (vs GT)', color='blue', edgecolor='black')\n", "axes[0, 0].hist(backtranslation_similarities, bins=20, alpha=0.5, label='Backtrans (vs GT)', color='orange', edgecolor='black')\n", "axes[0, 0].set_xlabel('Cosine Similarity', fontsize=12)\n", "axes[0, 0].set_ylabel('Frequency', fontsize=12)\n", "axes[0, 0].set_title('Distribution of Cosine Similarities', fontsize=14, fontweight='bold')\n", "axes[0, 0].legend(fontsize=10, loc='upper left')\n", "axes[0, 0].grid(alpha=0.3)\n", "\n", "# 2. Box plot comparison\n", "box_data = [direct_similarities, backtranslation_similarities]\n", "bp = axes[0, 1].boxplot(box_data, labels=['Direct\\n(vs GT)', 'Backtrans\\n(vs GT)'],\n", " patch_artist=True, showmeans=True)\n", "colors = ['lightblue', 'lightyellow']\n", "for patch, color in zip(bp['boxes'], colors):\n", " patch.set_facecolor(color)\n", "axes[0, 1].set_ylabel('Cosine Similarity', fontsize=12)\n", "axes[0, 1].set_title('Similarity Metrics Comparison', fontsize=14, fontweight='bold')\n", "axes[0, 1].grid(alpha=0.3, axis='y')\n", "\n", "# 3. Scatter plot: Direct (Accuracy) vs Consistency\n", "sc = axes[1, 0].scatter(direct_similarities, consistency_similarities, alpha=0.7, s=50,\n", " c=backtranslation_similarities, cmap='viridis', edgecolors='k')\n", "axes[1, 0].plot([0, 1], [0, 1], 'r--', linewidth=2, label='y=x')\n", "axes[1, 0].set_xlabel('Direct Translation Accuracy (vs GT)', fontsize=12)\n", "axes[1, 0].set_ylabel('Consistency (Direct vs Backtrans)', fontsize=12)\n", "axes[1, 0].set_title('Accuracy vs Consistency\\n(Color = Backtrans Accuracy)', fontsize=14, fontweight='bold')\n", "plt.colorbar(sc, ax=axes[1, 0], label='Backtranslation Accuracy')\n", "axes[1, 0].grid(alpha=0.3)\n", "axes[1, 0].set_xlim(0, 1)\n", "axes[1, 0].set_ylim(0, 1)\n", "\n", "# 4. Difference plot\n", "differences = direct_similarities - backtranslation_similarities\n", "axes[1, 1].hist(differences, bins=20, color='purple', alpha=0.7, edgecolor='black')\n", "axes[1, 1].axvline(x=0, color='red', linestyle='--', linewidth=2, label='No difference')\n", "axes[1, 1].axvline(x=differences.mean(), color='green', linestyle='-', linewidth=2, label=f'Mean diff: {differences.mean():.3f}')\n", "axes[1, 1].set_xlabel('Information Loss (Direct - Backtrans)', fontsize=12)\n", "axes[1, 1].set_ylabel('Frequency', fontsize=12)\n", "axes[1, 1].set_title('Information Loss Distribution', fontsize=14, fontweight='bold')\n", "axes[1, 1].legend(fontsize=10)\n", "axes[1, 1].grid(alpha=0.3)\n", "\n", "plt.tight_layout()\n", "plt.show()\n" ] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "provenance": [], "include_colab_link": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.11" } }, "nbformat": 4, "nbformat_minor": 0 }