{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Recitation 0: Losses\n","\n","Prepared by: Massa Baali (mbaali@andrew.cmu.edu)\n","\n","\n","**Where do we use the loss? **"],"metadata":{"id":"GAR2NBbvyhSN"}},{"cell_type":"code","source":["# Template for calling the loss\n","# Training loop\n","n_epochs = 5\n","for epoch in range(n_epochs):\n","    model.train()\n","    # Loop through your dataset\n","    for batch in dataloader:\n","        # Forward pass\n","        outputs = model(batch)\n","        optimizer.zero_grad()\n","\n","        # Forward pass\n","        loss = criterion(outputs, targets)\n","\n","        # Backward pass and optimization\n","        loss.backward()\n","        optimizer.step()\n"],"metadata":{"id":"r9yhTBKzj69U"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Mean Squared Error (MSE)\n"],"metadata":{"id":"wNRQYlVDkJ12"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","\n","y_true = torch.tensor([300.0, 400.0, 500.0])  # Ground truth (Real Prices)\n","y_pred = torch.tensor([280.0, 390.0, 520.0])  # Predictions  (Predicted Prices)\n","\n","mse_loss = nn.MSELoss()\n","loss = mse_loss(y_pred, y_true)\n","\n","print(\"Mean Squared Error:\", loss.item()) # the lower the value the better the model\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VCdeUvvnM1sp","outputId":"e9080084-84cd-473b-90a2-7f06181501b8"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mean Squared Error: 300.0\n"]}]},{"cell_type":"markdown","source":["### Cross Entropy (CE)\n"],"metadata":{"id":"00-__RzzQTkk"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","\n","# True labels (class indices)\n","y_true = torch.tensor([0, 1, 2])  # 3 examples: Cat, Dog, Bird\n","\n","# Predicted probabilities (logits for 3 classes: Cat, Dog, Bird)\n","y_pred = torch.tensor([[0.7, 0.2, 0.1],  # Probabilities for example 1\n","                       [0.1, 0.8, 0.1],  # Probabilities for example 2\n","                       [0.2, 0.3, 0.5]])  # Probabilities for example 3\n","\n","# Use CrossEntropyLoss\n","criterion = nn.CrossEntropyLoss()\n","loss = criterion(y_pred, y_true)\n","\n","print(\"Cross-Entropy Loss:\", loss.item())\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GaDUDvtGQQSg","outputId":"1bf4a668-5215-42a3-a886-5a4678e5dde8"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Cross-Entropy Loss: 0.7991690635681152\n"]}]},{"cell_type":"markdown","source":["### Binary Cross Entropy (BCE)\n"],"metadata":{"id":"TwDsWrfeev4Y"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","\n","# True labels (1: spam, 0: not spam)\n","y_true = torch.tensor([1.0, 0.0, 1.0])  # 3 examples\n","\n","# Predicted probabilities (model's confidence)\n","y_pred = torch.tensor([0.85, 0.1, 0.95])\n","\n","# Use Binary Cross-Entropy Loss\n","bce_loss = nn.BCELoss()\n","loss = bce_loss(y_pred, y_true)\n","\n","print(\"Binary Cross-Entropy Loss:\", loss.item())\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wZBZrZBfetnG","outputId":"4ffce41d-6d05-4aa1-f128-2576d387d9a4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Binary Cross-Entropy Loss: 0.10639091581106186\n"]}]},{"cell_type":"markdown","source":["### Centerloss\n"],"metadata":{"id":"cYlMt6c3slK_"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","\n","class CenterLoss(nn.Module):\n","    def __init__(self, num_classes=10, num_features=2):\n","        super(CenterLoss, self).__init__()\n","        self.num_class = num_classes\n","        self.num_feature = num_features\n","        self.centers = nn.Parameter(torch.randn(self.num_class, self.num_feature))\n","\n","    def forward(self, x, labels):\n","        # Get class centers for the input labels\n","        center = self.centers[labels]\n","        # Compute squared distance between features and centers\n","        dist = (x-center).pow(2).sum(dim=-1)\n","        # Distances are clamped to ensure they stay within a reasonable range, and the mean distance is computed to get the final loss.\n","        loss = torch.clamp(dist, min=1e-12, max=1e+12).mean(dim=-1)\n","        return loss\n","\n","embeddings = torch.tensor([[1.0, 1.0], [2.0, 2.0], [0.5, 0.5]])  # Embedding vectors\n","labels = torch.tensor([0, 1, 2])  # Corresponding class labels\n","CenterLoss  = CenterLoss(num_classes=3, num_features=2)\n","loss = CenterLoss(embeddings, labels)\n","print(\"Center Loss:\", loss.item())\n","\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eG4it1gAJYX2","outputId":"62f983a3-0667-40ff-d9a7-a290183a6cca"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Center Loss: 3.6008317470550537\n"]}]},{"cell_type":"markdown","source":["### Angular Softmax\n"],"metadata":{"id":"9DBpjlcljVbz"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","\n","\n","class AMSoftmax(nn.Module):\n","\n","    def __init__(self,\n","                 in_feats,\n","                 n_classes=10,\n","                 m=0.3,\n","                 s=15):\n","        super(AMSoftmax, self).__init__()\n","        self.m = m\n","        self.s = s\n","        self.in_feats = in_feats\n","        self.W = torch.nn.Parameter(torch.randn(in_feats, n_classes), requires_grad=True)\n","        self.ce = nn.CrossEntropyLoss()\n","        nn.init.xavier_normal_(self.W, gain=1)\n","\n","    def forward(self, x, lb):\n","        assert x.size()[0] == lb.size()[0]\n","        assert x.size()[1] == self.in_feats\n","        # Normalization\n","        x_norm = torch.norm(x, p=2, dim=1, keepdim=True).clamp(min=1e-9)\n","        x_norm = torch.div(x, x_norm)\n","        w_norm = torch.norm(self.W, p=2, dim=0, keepdim=True).clamp(min=1e-9)\n","        w_norm = torch.div(self.W, w_norm)\n","        # compute cos sim\n","        costh = torch.mm(x_norm, w_norm)\n","        delt_costh = torch.zeros_like(costh).scatter_(1, lb.unsqueeze(1), self.m)\n","        #subtracts the angular margin from the cos sim for the true clas\n","        costh_m = costh - delt_costh\n","        # scaled by the factor\n","        costh_m_s = self.s * costh_m\n","        loss = self.ce(costh_m_s, lb)\n","        return loss\n","\n","\n","# Inputs: embeddings and labels\n","embeddings = torch.tensor([[0.8, 0.6], [0.6, 0.8]])  # 2D embeddings\n","labels = torch.tensor([0, 1])  # Class labels\n","\n","loss = AMSoftmax(in_feats=2, n_classes=2, m=0.35, s=30)(embeddings, labels)\n","print(\"Angular SoftMax Loss:\", loss.item())\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QKVOPgl8-gGM","outputId":"c872daff-7b96-4f15-dffd-07c583e9692a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Angular SoftMax Loss: 18.93401336669922\n"]}]},{"cell_type":"markdown","source":["### Additive Margin Softmax"],"metadata":{"id":"GSaTv1cqPv1V"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","\n","\n","class AAMSoftmaxLoss(nn.Module):\n","\n","    def __init__(self, embedding_dim, n_classes, scale = 30.0, margin=0.4):\n","        super(AAMSoftmaxLoss, self).__init__()\n","        self.scale = scale\n","        self.margin = margin\n","        self.embedding_dim = embedding_dim\n","\n","        self.n_classes = n_classes\n","        self.W = torch.nn.Parameter(torch.randn(embedding_dim, n_classes), requires_grad=True)\n","\n","        self.ce = nn.CrossEntropyLoss()\n","\n","    def forward(self, x, labels):\n","        '''\n","        Input shape (N, embedding_dim)\n","        '''\n","        n, m = x.shape\n","        assert n == len(labels)\n","        assert m == self.embedding_dim\n","\n","        # Normalization\n","        x_norm = torch.norm(x, p=2, dim=1, keepdim=True).clamp(min=1e-9)\n","        x_norm = torch.div(x, x_norm)\n","        w_norm = torch.norm(self.W, p=2, dim=0, keepdim=True).clamp(min=1e-9)\n","        w_norm = torch.div(self.W, w_norm)\n","        # cos sim\n","        cos_theta = torch.matmul(w_norm, x_norm.T).T\n","        # subtracts the angular margin from the cosine similarity\n","        psi = cos_theta - self.margin\n","\n","        onehot = F.one_hot(labels, self.n_classes)\n","        # The modified cosine sim (psi) is scaled by the factor scale\n","        logits = self.scale * torch.where(onehot == 1, psi, cos_theta)\n","        err = self.ce(logits, labels)\n","\n","        return err\n","\n","# Inputs: embeddings and labels\n","embeddings = torch.tensor([[0.8, 0.6], [0.6, 0.8]])  # 2D embeddings\n","labels = torch.tensor([0, 1])  # Class labels\n","\n","loss = AAMSoftmaxLoss(embedding_dim=2, n_classes=2, scale=30, margin=0.35)(embeddings, labels)\n","print(\"Additive Angular Margin SoftMax Loss:\", loss.item())\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZS0syt5cAiud","outputId":"30c18975-b8c2-4370-8603-bfbaf081d5aa"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Additive Angular Margin SoftMax Loss: 29.59213638305664\n"]}]},{"cell_type":"markdown","source":["### Triplet Loss"],"metadata":{"id":"HSLEEZ4nUgc5"}},{"cell_type":"code","source":["import torch\n","import torch.nn.functional as F\n","\n","# Example embeddings for anchor, positive, and negative\n","anchor = torch.tensor([1.0, 2.0])\n","positive = torch.tensor([1.2, 2.1])  # Same class as anchor\n","negative = torch.tensor([3.0, 4.0])  # Different class from anchor\n","\n","margin = 1.0  # Triplet margin\n","def triplet(anchor, positive, negative, margin):\n","  # Compute distances\n","  pos_dist = F.pairwise_distance(anchor.unsqueeze(0), positive.unsqueeze(0))\n","  neg_dist = F.pairwise_distance(anchor.unsqueeze(0), negative.unsqueeze(0))\n","  # Compute triplet loss (apply relu to ensure that the loss is non-negative)\n","  loss = torch.relu(pos_dist - neg_dist + margin)\n","  return loss\n","\n","loss = triplet(anchor, positive, negative, margin)\n","\n","print(\"Triplet Loss:\", loss.item())\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wEYsx7wKIUM6","outputId":"e3204d40-2d87-4ae4-9ad0-2fff8b2ef665"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Triplet Loss: 0.0\n"]}]},{"cell_type":"markdown","source":["# References and further reading\n","\n","1.   https://github.com/tomastokar/Additive-Margin-Softmax/blob/main/AMSloss.py\n","2.   https://github.com/CoinCheung/pytorch-loss/tree/master/pytorch_loss\n","3.   https://pytorch.org/docs/stable/generated/torch.nn.TripletMarginLoss.html\n","4.   https://www.kaggle.com/code/farhanrenaldi/pytorch-faster-rcnn-w-center-loss-for-fmd#Center-Loss\n","5.   https://arxiv.org/abs/1801.05599\n","6.   https://arxiv.org/pdf/1806.03464\n","7.   https://kpzhang93.github.io/papers/eccv2016.pdf\n","8.   https://arxiv.org/pdf/1503.03832\n"],"metadata":{"id":"QlvVfdOnRWnn"}}]}