{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "Tree.ipynb", "provenance": [], "collapsed_sections": [], "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "metadata": { "id": "dENII4iGN3S4", "colab_type": "code", "outputId": "e87f5c44-3da0-4df4-8f8b-f91c5adbb93e", "colab": { "base_uri": "https://localhost:8080/", "height": 314 } }, "source": [ "!pip install -qqq torchtext wandb\n", "!pip install -qqq pytorch-transformers\n", "!pip install -qqqU git+https://github.com/harvardnlp/pytorch-struct\n", "!git clone -q http://github.com/srush/temp" ], "execution_count": 5, "outputs": [ { "output_type": "stream", "text": [ "\u001b[K |████████████████████████████████| 1.3MB 2.9MB/s \n", "\u001b[K |████████████████████████████████| 102kB 25.4MB/s \n", "\u001b[K |████████████████████████████████| 235kB 44.2MB/s \n", "\u001b[K |████████████████████████████████| 92kB 25.6MB/s \n", "\u001b[K |████████████████████████████████| 460kB 43.4MB/s \n", "\u001b[K |████████████████████████████████| 92kB 26.0MB/s \n", "\u001b[K |████████████████████████████████| 256kB 45.1MB/s \n", "\u001b[K |████████████████████████████████| 71kB 25.0MB/s \n", "\u001b[K |████████████████████████████████| 184kB 41.2MB/s \n", "\u001b[?25h Building wheel for gql (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for shortuuid (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for watchdog (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n", " Building wheel for torch-struct (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "fatal: destination path 'temp' already exists and is not an empty directory.\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "EghAJ7aJDl4V", "colab_type": "code", "outputId": "970f7f28-6f4a-4c80-f4c8-3b348bdbbfd7", "colab": { "base_uri": "https://localhost:8080/", "height": 121 } }, "source": [ "import torchtext\n", "import torch\n", "import torch.nn as nn\n", "from torch_struct import DependencyCRF\n", "import torch_struct.data\n", "import torchtext.data as data\n", "from pytorch_transformers import AdamW, WarmupLinearSchedule\n", "from pytorch_transformers import *\n", "\n", "config = {\"bert\": \"bert-base-cased\", \"H\" : 768, \"dropout\": 0.2\n", " }\n", "\n", "# Comment or add your wandb\n", "import wandb\n", "wandb.init(project=\"pytorch-struct-tagging\", config=config)" ], "execution_count": 6, "outputs": [ { "output_type": "display_data", "data": { "application/javascript": [ "\n", " window._wandbApiKey = new Promise((resolve, reject) => {\n", " function loadScript(url) {\n", " return new Promise(function(resolve, reject) {\n", " let newScript = document.createElement(\"script\");\n", " newScript.onerror = reject;\n", " newScript.onload = resolve;\n", " document.body.appendChild(newScript);\n", " newScript.src = url;\n", " });\n", " }\n", " loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n", " const iframe = document.createElement('iframe')\n", " iframe.style.cssText = \"width:0;height:0;border:none\"\n", " document.body.appendChild(iframe)\n", " const handshake = new Postmate({\n", " container: iframe,\n", " url: 'https://app.wandb.ai/authorize'\n", " });\n", " const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n", " handshake.then(function(child) {\n", " child.on('authorize', data => {\n", " clearTimeout(timeout)\n", " resolve(data)\n", " });\n", " });\n", " })\n", " });\n", " " ], "text/plain": [ "" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "wandb: ERROR Not authenticated. Copy a key from https://app.wandb.ai/authorize\n" ], "name": "stderr" }, { "output_type": "stream", "text": [ "API Key: ··········\n" ], "name": "stdout" }, { "output_type": "stream", "text": [ "wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n" ], "name": "stderr" }, { "output_type": "display_data", "data": { "text/html": [ "\n", " Notebook configured with W&B. You can open the run page, or call %%wandb\n", " in a cell containing your training loop to display live results. Learn more in our docs.\n", " " ], "text/plain": [ "" ] }, "metadata": { "tags": [] } }, { "output_type": "execute_result", "data": { "text/plain": [ "W&B Run: https://app.wandb.ai/srush/pytorch-struct-tagging/runs/j1dqjny2" ] }, "metadata": { "tags": [] }, "execution_count": 6 } ] }, { "cell_type": "markdown", "metadata": { "id": "l63Zy6ap95Ug", "colab_type": "text" }, "source": [ "Parse the conll dependency data." ] }, { "cell_type": "markdown", "metadata": { "id": "vQoaK1lv-CXc", "colab_type": "text" }, "source": [ "TorchText batching setup." ] }, { "cell_type": "code", "metadata": { "id": "YsKQXHflOmDe", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "outputId": "0d184e8e-a2a4-4c5a-830d-3d0bc96e3963" }, "source": [ "model_class, tokenizer_class, pretrained_weights = BertModel, BertTokenizer, config['bert']\n", "tokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n", "def batch_num(nums):\n", " lengths = torch.tensor([len(n) for n in nums]).long()\n", " n = lengths.max()\n", " out = torch.zeros(len(nums), n).long()\n", " for b, n in enumerate(nums):\n", " out[b, :len(n)] = torch.tensor(n)\n", " return out, lengths\n", "HEAD = data.RawField(preprocessing= lambda x: [int(i) for i in x],\n", " postprocessing=batch_num)\n", "WORD = torch_struct.data.SubTokenizedField(tokenizer)\n", "HEAD.is_target = True\n", "train = torch_struct.data.ConllXDataset(\"temp/wsj.train.conllx\", (('word', WORD), ('head', HEAD)),\n", " filter_pred=lambda x: 5 < len(x.word[0]) < 40)\n", "train_iter = torch_struct.data.TokenBucket(train, 750)\n", "val = torch_struct.data.ConllXDataset(\"temp/wsj.dev.conllx\", (('word', WORD), ('head', HEAD)),\n", " filter_pred=lambda x: 5 < len(x.word[0]) < 40)\n", "val_iter = torchtext.data.BucketIterator(val, \n", " batch_size=20,\n", " device=\"cuda:0\")" ], "execution_count": 7, "outputs": [ { "output_type": "stream", "text": [ "100%|██████████| 213450/213450 [00:00<00:00, 385689.56B/s]\n" ], "name": "stderr" } ] }, { "cell_type": "markdown", "metadata": { "id": "Z-40o2UA-Ha7", "colab_type": "text" }, "source": [ "Make a Bert model to compute the potentials" ] }, { "cell_type": "code", "metadata": { "id": "QUKVqGUsSmYY", "colab_type": "code", "outputId": "6317e23a-5ac7-45f2-9d01-2249d9675b2c", "colab": { "base_uri": "https://localhost:8080/", "height": 361 } }, "source": [ "H = config[\"H\"]\n", "class Model(nn.Module):\n", " def __init__(self, hidden):\n", " super().__init__()\n", " self.base_model = model_class.from_pretrained(pretrained_weights)\n", " self.linear = nn.Linear(H, H)\n", " self.bilinear = nn.Linear(H, H)\n", " self.root = nn.Parameter(torch.rand(H))\n", " self.dropout = nn.Dropout(config[\"dropout\"])\n", " \n", " def forward(self, words, mapper):\n", " out = self.dropout(self.base_model(words)[0])\n", " out = torch.einsum(\"bca,bch->bah\", mapper.float().cuda(), out)\n", " final2 = torch.einsum(\"bnh,hg->bng\", out, self.linear.weight)\n", " final = torch.einsum(\"bnh,hg,bmg->bnm\", out, self.bilinear.weight, final2)\n", " root_score = torch.einsum(\"bnh,h->bn\", out, self.root)\n", " final = final[:, 1:-1, 1:-1]\n", " N = final.shape[1]\n", " final[:, torch.arange(N), torch.arange(N)] += root_score[:, 1:-1]\n", " return final\n", "\n", "model = Model(H)\n", "wandb.watch(model)\n", "model.cuda()\n", "None" ], "execution_count": 18, "outputs": [ { "output_type": "error", "ename": "ValueError", "evalue": "ignored", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mH\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mwandb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/wandb/__init__.py\u001b[0m in \u001b[0;36mwatch\u001b[0;34m(models, criterion, log, log_freq)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mwatch_called\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 122\u001b[0m raise ValueError(\n\u001b[0;32m--> 123\u001b[0;31m \u001b[0;34m\"You can only call `wandb.watch` once per process. If you want to watch multiple models, pass them in as a tuple.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 124\u001b[0m )\n\u001b[1;32m 125\u001b[0m \u001b[0mwatch_called\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mValueError\u001b[0m: You can only call `wandb.watch` once per process. If you want to watch multiple models, pass them in as a tuple." ] } ] }, { "cell_type": "markdown", "metadata": { "id": "grxGRu2b-LdT", "colab_type": "text" }, "source": [ "Generic training loop. " ] }, { "cell_type": "code", "metadata": { "id": "HENkCYOKSwHr", "colab_type": "code", "colab": {} }, "source": [ "def validate():\n", " incorrect_edges = 0\n", " total_edges = 0\n", " model.eval()\n", " for i, ex in enumerate(val_iter):\n", " words, mapper, _ = ex.word\n", " label, lengths = ex.head\n", " batch, _ = label.shape\n", "\n", " final = model(words.cuda(), mapper)\n", " for b in range(batch):\n", " final[b, lengths[b]-1:, :] = 0\n", " final[b, :, lengths[b]-1:] = 0\n", " dist = DependencyCRF(final, lengths=lengths)\n", " argmax = dist.argmax\n", " gold = dist.struct.to_parts(label, lengths=lengths).type_as(argmax)\n", " incorrect_edges += (out[:, :].cpu() - gold[:, :].cpu()).abs().sum() / 2.0\n", " total_edges += gold.sum()\n", "\n", " print(total_edges, incorrect_edges) \n", " model.train()\n", "\n", "def train(train_iter, val_iter, model):\n", " opt = AdamW(model.parameters(), lr=1e-4, eps=1e-8)\n", " scheduler = WarmupLinearSchedule(opt, warmup_steps=20, t_total=2500)\n", " model.train()\n", " losses = []\n", " for i, ex in enumerate(train_iter):\n", " opt.zero_grad()\n", " words, mapper, _ = ex.word\n", " label, lengths = ex.head\n", " batch, _ = label.shape\n", " \n", " # Model\n", " final = model(words.cuda(), mapper)\n", " for b in range(batch):\n", " final[b, lengths[b]-1:, :] = 0\n", " final[b, :, lengths[b]-1:] = 0\n", " \n", " if not lengths.max() <= final.shape[1] + 1:\n", " print(\"fail\")\n", " continue\n", " dist = DependencyCRF(final, lengths=lengths)\n", "\n", " labels = dist.struct.to_parts(label, lengths=lengths).type_as(final)\n", " log_prob = dist.log_prob(labels)\n", "\n", " loss = log_prob.sum()\n", " (-loss).backward()\n", " torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n", "\n", " opt.step()\n", " scheduler.step()\n", " losses.append(loss.detach())\n", " if i % 50 == 1: \n", " print(-torch.tensor(losses).mean(), words.shape)\n", " losses = []\n", " if i % 600 == 500:\n", " validate(val_iter) " ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "D-HW3z1VT2MG", "colab_type": "code", "outputId": "0d7b9ef1-8c82-4c8b-c25b-4737237e02ce", "colab": { "base_uri": "https://localhost:8080/", "height": 335 } }, "source": [ "train(train_iter, val_iter, model)" ], "execution_count": 17, "outputs": [ { "output_type": "error", "ename": "RuntimeError", "evalue": "ignored", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(train_iter, val_iter, model)\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlog_prob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 50\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 91\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 92\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mRuntimeError\u001b[0m: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [24, 29, 29]], which is output 0 of SliceBackward, is at version 49; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True)." ] } ] } ] }