{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"image-captioning-classifier-v2.ipynb","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"0cf057a3ecd94722b81201e9d36d630f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_2c251121d2b6406ebbd110ff327f6ad3","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_387aad4a61dc4cc795901713c3a86489","IPY_MODEL_44b40b1f34bb497cadd4cef9237aa565"]}},"2c251121d2b6406ebbd110ff327f6ad3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"387aad4a61dc4cc795901713c3a86489":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_7f89cd01842849108f3e5d137948d610","_dom_classes":[],"description":"Processing 2661 examples on 2 cores: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":2661,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2661,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_384e57df336b41f5bfa230e665ef525c"}},"44b40b1f34bb497cadd4cef9237aa565":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_06542471442b46ed87b464e843d2ff0d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2661/2661 [38:22<00:00, 1.16it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f933a5664ac54e1a847a364167dd2d8c"}},"7f89cd01842849108f3e5d137948d610":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"384e57df336b41f5bfa230e665ef525c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"06542471442b46ed87b464e843d2ff0d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f933a5664ac54e1a847a364167dd2d8c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"344cfc793554492c91bb47d3ae88545f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f93f112dd57d4922936de1ce5c9664f2","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_631e066380c94dbab3e89c81c82129ad","IPY_MODEL_ebfbb066208f496f8e5514d8d8015aa1"]}},"f93f112dd57d4922936de1ce5c9664f2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"631e066380c94dbab3e89c81c82129ad":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_9e400c1f6cb94b2ab3ad5146e739fe3d","_dom_classes":[],"description":"Processing 766 examples on 2 cores: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":766,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":766,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ab730ce1c76d4bc6a99b8132309b7a20"}},"ebfbb066208f496f8e5514d8d8015aa1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_02fe49e507fe4ca5904fc45c5cf2f1aa","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 766/766 [38:21<00:00, 3.00s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_bca7c9d8443140dba85d3870c1cc8119"}},"9e400c1f6cb94b2ab3ad5146e739fe3d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"ab730ce1c76d4bc6a99b8132309b7a20":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"02fe49e507fe4ca5904fc45c5cf2f1aa":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"bca7c9d8443140dba85d3870c1cc8119":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2a9f7995a141443c90593e282a447b8b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_492966b1127e4510b314009654be4f57","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_094cefe6347247999014ebcc01afc02e","IPY_MODEL_ff4655a98ef745fb8719d970a611e8c8"]}},"492966b1127e4510b314009654be4f57":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"094cefe6347247999014ebcc01afc02e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d8fc34c04ecf468f938892d6e7cb77a6","_dom_classes":[],"description":"Epoch [1/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_39af62ecfcd14c73af8fd8c5bc046759"}},"ff4655a98ef745fb8719d970a611e8c8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c214b201553d48a3b075909e133aa9cf","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.542 [02:55<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_734bfe5687934b519c40f04cd335baf2"}},"d8fc34c04ecf468f938892d6e7cb77a6":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"39af62ecfcd14c73af8fd8c5bc046759":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c214b201553d48a3b075909e133aa9cf":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"734bfe5687934b519c40f04cd335baf2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"465f62e09237469d909093bb5a25cf38":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_94289f71172d42aea582ac7a15a5e12e","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_15710a5ad7184d6992ed49e31fd29a83","IPY_MODEL_bedbefd3e084432e88f31655853cdfe0"]}},"94289f71172d42aea582ac7a15a5e12e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"15710a5ad7184d6992ed49e31fd29a83":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_9d8d85841a884aa3883dd533f191756c","_dom_classes":[],"description":"Epoch [2/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e3b86cfa651c4d2782f67c032664e475"}},"bedbefd3e084432e88f31655853cdfe0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_7e5618afebb14f5fab65fcd94cd97316","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.521 [01:56<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_db9a658990524118a4ba4a2c9e43fcee"}},"9d8d85841a884aa3883dd533f191756c":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"e3b86cfa651c4d2782f67c032664e475":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7e5618afebb14f5fab65fcd94cd97316":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"db9a658990524118a4ba4a2c9e43fcee":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9d43bbddb3144931a78ac6060a6a0729":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_07819972bfbf4d35b209e2b3ac8cc9d1","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_9cb9f622b61748408876c2a6b3cb89e2","IPY_MODEL_d218c768ad204ed191b970f4b9f3ab1b"]}},"07819972bfbf4d35b209e2b3ac8cc9d1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9cb9f622b61748408876c2a6b3cb89e2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_93c87439598d4d10b7046e4032d10581","_dom_classes":[],"description":"Epoch [3/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e6bf040065734b799875569013b09c3d"}},"d218c768ad204ed191b970f4b9f3ab1b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_96d7f88e28474644bccfcc4fedf5be16","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.489 [35:44<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_9b3d06abbb784bb696d4fb949d58fbbb"}},"93c87439598d4d10b7046e4032d10581":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"e6bf040065734b799875569013b09c3d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"96d7f88e28474644bccfcc4fedf5be16":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"9b3d06abbb784bb696d4fb949d58fbbb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c83224cf0c244474a77ee12d3e8af730":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_548fea41927b444e937415eeb04f5b25","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_799cb0aa7e674094bb15cac18a29e7bf","IPY_MODEL_326ddb685bba41aeb500de29ad13488b"]}},"548fea41927b444e937415eeb04f5b25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"799cb0aa7e674094bb15cac18a29e7bf":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_a7ecda0068c841e5aa2b14ab731f26ac","_dom_classes":[],"description":"Epoch [4/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_78ba82c246eb45449b5b37c8987ecf6c"}},"326ddb685bba41aeb500de29ad13488b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ad9959f7a2fd49228e2291d721d27bb8","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.477 [32:54<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4e440371d54b4827bc3a47a1c06513bf"}},"a7ecda0068c841e5aa2b14ab731f26ac":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"78ba82c246eb45449b5b37c8987ecf6c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ad9959f7a2fd49228e2291d721d27bb8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"4e440371d54b4827bc3a47a1c06513bf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c7bf2a1ddab64f21ab2f8107365e2e05":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_fccecfa197fa4d62a0a558885255a76c","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_37a5c8db4cd34bef856e4b468187d91b","IPY_MODEL_6ff46092582c4b51b1c132c25e9eeb1d"]}},"fccecfa197fa4d62a0a558885255a76c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"37a5c8db4cd34bef856e4b468187d91b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_432a5a233c894f4f871af41646812247","_dom_classes":[],"description":"Epoch [5/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0c2074c9c2414d8ca919720850d4ab93"}},"6ff46092582c4b51b1c132c25e9eeb1d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_846716d075c84358a4639bf890493fbd","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.491 [31:55<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d96d59ff069343ebb902194880270d67"}},"432a5a233c894f4f871af41646812247":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"0c2074c9c2414d8ca919720850d4ab93":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"846716d075c84358a4639bf890493fbd":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"d96d59ff069343ebb902194880270d67":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ce73826afd7a4cf3bbb152d2c168b2a8":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_89de87775fda4643aa6985687da4bc9c","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_e5928ed938d743dcac577e427acf6942","IPY_MODEL_3f9f1d5600994f8e81151475098081a4"]}},"89de87775fda4643aa6985687da4bc9c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e5928ed938d743dcac577e427acf6942":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_cdb08e19110a4b399a55cddde941872b","_dom_classes":[],"description":"Epoch [6/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6c2229f17f364c79a418f53f75874c25"}},"3f9f1d5600994f8e81151475098081a4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_3e410112b02042ad896e36f820488e18","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.454 [30:56<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_b060cdfd706543c7a4fb3379d65b3ec3"}},"cdb08e19110a4b399a55cddde941872b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6c2229f17f364c79a418f53f75874c25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3e410112b02042ad896e36f820488e18":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"b060cdfd706543c7a4fb3379d65b3ec3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"34e3457f69d34255b9f8b2263a21765e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f3bb8d50c3384e249259f3cc43429781","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_83cefe6b831c44a9bffbf887519ac431","IPY_MODEL_c44612eed7fc4c19a1d8f646ccb552f7"]}},"f3bb8d50c3384e249259f3cc43429781":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"83cefe6b831c44a9bffbf887519ac431":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_b47b770bbe4f492fb47bde1af0cc1c0f","_dom_classes":[],"description":"Epoch [7/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_8832b8c8456046e5b804e2bf399285bf"}},"c44612eed7fc4c19a1d8f646ccb552f7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_deb24156ac2b40ba8ae4b9e8261f3121","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.446 [29:58<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_48c2137f94a7496f9b7d518c2506232d"}},"b47b770bbe4f492fb47bde1af0cc1c0f":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"8832b8c8456046e5b804e2bf399285bf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"deb24156ac2b40ba8ae4b9e8261f3121":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"48c2137f94a7496f9b7d518c2506232d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a8435e3254b74779bce08df4c8278d72":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_917d5d60ac184ad38d180d24e2fde79a","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a6fa4e9a37b141818d0b647949211bbc","IPY_MODEL_732f57dd940e4c48810bee9d390bae63"]}},"917d5d60ac184ad38d180d24e2fde79a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a6fa4e9a37b141818d0b647949211bbc":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d1ce78e18b2242238aa18d1e155879dd","_dom_classes":[],"description":"Epoch [8/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_9f564a3c830c4a5ab938a651fc80ed62"}},"732f57dd940e4c48810bee9d390bae63":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f5227c3bb518489d8ad962bba077de2a","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.466 [28:59<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d3614bac3ddc49349906efb3b46ec785"}},"d1ce78e18b2242238aa18d1e155879dd":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"9f564a3c830c4a5ab938a651fc80ed62":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f5227c3bb518489d8ad962bba077de2a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"d3614bac3ddc49349906efb3b46ec785":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"739d0f39fb4f41e88bba635f86c16984":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_b23cfe9588f24e60b3c0a0000595dca5","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_1f86c396f25b4c4d8e9416166a962217","IPY_MODEL_d5311479b4ba402b8ef4ed90e45991d3"]}},"b23cfe9588f24e60b3c0a0000595dca5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"1f86c396f25b4c4d8e9416166a962217":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_bcb1a72a01054d3f8cd0332ed8a9f310","_dom_classes":[],"description":"Epoch [9/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_144bf9b58b87474ea86bd385f9cc8d15"}},"d5311479b4ba402b8ef4ed90e45991d3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_589be329569c4c3b85987ef4e2ff710a","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.425 [28:00<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6ca9daef5e8744d2a0c6631649b97816"}},"bcb1a72a01054d3f8cd0332ed8a9f310":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"144bf9b58b87474ea86bd385f9cc8d15":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"589be329569c4c3b85987ef4e2ff710a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"6ca9daef5e8744d2a0c6631649b97816":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6da6be3065424e03b1fa24afeadf47dc":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_148a7107ed404619bf69dae28cab1bff","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a9ba25bd82554b99a742c77101fcb018","IPY_MODEL_ae000ff2e16448649987f9d72d9e5f5d"]}},"148a7107ed404619bf69dae28cab1bff":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a9ba25bd82554b99a742c77101fcb018":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_f176b4d20ca5452ab30f64952fdb0fca","_dom_classes":[],"description":"Epoch [10/10]: [84/84] 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6f6c7ceeb65a43ad9fe3f2a961b2783a"}},"ae000ff2e16448649987f9d72d9e5f5d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_77d48e2da1ac4555afd8f39a097b6b65","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":", loss=0.43 [27:01<00:00]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_bd9712584c6148a3804e098d84274bfc"}},"f176b4d20ca5452ab30f64952fdb0fca":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6f6c7ceeb65a43ad9fe3f2a961b2783a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"77d48e2da1ac4555afd8f39a097b6b65":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"bd9712584c6148a3804e098d84274bfc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ef85d36f743948aa969386638a17e2d1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_390d96f81861494e8fc13dcc153698e1","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_d20001c9b9134e86a5f9f82d1a93d7ea","IPY_MODEL_159ec926f06746198b32441468b49aeb"]}},"390d96f81861494e8fc13dcc153698e1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d20001c9b9134e86a5f9f82d1a93d7ea":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_9df8000b43f64195a492849d0815d1e7","_dom_classes":[],"description":"Processing 766 examples on 2 cores: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":766,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":766,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_70bcdef81fd54fe0ac1e4dd395da55f8"}},"159ec926f06746198b32441468b49aeb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c13224a053e345c393ad1e8180523658","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 766/766 [01:32<00:00, 8.32it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a17d61b8448b471c96c9d18cdb853b56"}},"9df8000b43f64195a492849d0815d1e7":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"70bcdef81fd54fe0ac1e4dd395da55f8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c13224a053e345c393ad1e8180523658":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a17d61b8448b471c96c9d18cdb853b56":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"8f1228ecc42346ad8b6dbb658673e092":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_379d1b7a433643b2a498157245618f0d","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_621bc7571ecc4493bf55a3778e3183e0","IPY_MODEL_dccdda5805474198821bc5b219f3ce49"]}},"379d1b7a433643b2a498157245618f0d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"621bc7571ecc4493bf55a3778e3183e0":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_78e91ec9134a4c62957b6e88b6e35c7e","_dom_classes":[],"description":"Processing 2661 examples on 2 cores: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":2661,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2661,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c88ca7fe7bcb4e299e257f64cba98d6a"}},"dccdda5805474198821bc5b219f3ce49":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_e03b4afdc54c4b3ca79b82ea16fe6739","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2661/2661 [23:12<00:00, 1.91it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_9086f42d12034cbd8f1c50dee4072ddd"}},"78e91ec9134a4c62957b6e88b6e35c7e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c88ca7fe7bcb4e299e257f64cba98d6a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e03b4afdc54c4b3ca79b82ea16fe6739":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"9086f42d12034cbd8f1c50dee4072ddd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"HpKpr6QRu07G","colab_type":"text"},"source":["Based on: https://medium.com/swlh/transformer-fine-tuning-for-sentiment-analysis-c000da034bb5"]},{"cell_type":"code","metadata":{"id":"CVEeWZpevLuL","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":768},"executionInfo":{"status":"ok","timestamp":1592535681915,"user_tz":420,"elapsed":8541,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"4221c88c-6080-4e06-ae9f-9f28e0e12f24"},"source":["!pip install pytorch-transformers\n","!pip install pytorch-ignite"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting pytorch-transformers\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/a3/b7/d3d18008a67e0b968d1ab93ad444fc05699403fa662f634b2f2c318a508b/pytorch_transformers-1.2.0-py3-none-any.whl (176kB)\n","\r\u001b[K |█▉ | 10kB 19.9MB/s eta 0:00:01\r\u001b[K |███▊ | 20kB 3.0MB/s eta 0:00:01\r\u001b[K |█████▋ | 30kB 4.0MB/s eta 0:00:01\r\u001b[K |███████▍ | 40kB 4.4MB/s eta 0:00:01\r\u001b[K |█████████▎ | 51kB 3.5MB/s eta 0:00:01\r\u001b[K |███████████▏ | 61kB 4.0MB/s eta 0:00:01\r\u001b[K |█████████████ | 71kB 4.2MB/s eta 0:00:01\r\u001b[K |██████████████▉ | 81kB 4.6MB/s eta 0:00:01\r\u001b[K |████████████████▊ | 92kB 5.0MB/s eta 0:00:01\r\u001b[K |██████████████████▋ | 102kB 4.7MB/s eta 0:00:01\r\u001b[K |████████████████████▍ | 112kB 4.7MB/s eta 0:00:01\r\u001b[K |██████████████████████▎ | 122kB 4.7MB/s eta 0:00:01\r\u001b[K |████████████████████████▏ | 133kB 4.7MB/s eta 0:00:01\r\u001b[K |██████████████████████████ | 143kB 4.7MB/s eta 0:00:01\r\u001b[K |███████████████████████████▉ | 153kB 4.7MB/s eta 0:00:01\r\u001b[K |█████████████████████████████▊ | 163kB 4.7MB/s eta 0:00:01\r\u001b[K |███████████████████████████████▋| 174kB 4.7MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 184kB 4.7MB/s \n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (2.23.0)\n","Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.5.0+cu101)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.18.5)\n","Collecting sacremoses\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n","\u001b[K |████████████████████████████████| 890kB 14.4MB/s \n","\u001b[?25hCollecting sentencepiece\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n","\u001b[K |████████████████████████████████| 1.1MB 26.9MB/s \n","\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (2019.12.20)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (4.41.1)\n","Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.14.2)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (2020.4.5.2)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (2.9)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (3.0.4)\n","Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.0.0->pytorch-transformers) (0.16.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (1.12.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (7.1.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (0.15.1)\n","Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (0.3.3)\n","Requirement already satisfied: botocore<1.18.0,>=1.17.2 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (1.17.2)\n","Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (0.10.0)\n","Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.2->boto3->pytorch-transformers) (2.8.1)\n","Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.2->boto3->pytorch-transformers) (0.15.2)\n","Building wheels for collected packages: sacremoses\n"," Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=8e4e2f0e6de88c2d1094549804401c811717cf4e5a50d8582c8a1960fd5f214a\n"," Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n","Successfully built sacremoses\n","Installing collected packages: sacremoses, sentencepiece, pytorch-transformers\n","Successfully installed pytorch-transformers-1.2.0 sacremoses-0.0.43 sentencepiece-0.1.91\n","Collecting pytorch-ignite\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/35/55/41e8a995876fd2ade29bdba0c3efefa38e7d605cb353c70f3173c04928b5/pytorch_ignite-0.3.0-py2.py3-none-any.whl (103kB)\n","\u001b[K |████████████████████████████████| 112kB 4.5MB/s \n","\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from pytorch-ignite) (1.5.0+cu101)\n","Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch->pytorch-ignite) (0.16.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch->pytorch-ignite) (1.18.5)\n","Installing collected packages: pytorch-ignite\n","Successfully installed pytorch-ignite-0.3.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"QefYvKIxHbPC","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":122},"executionInfo":{"status":"ok","timestamp":1592535883704,"user_tz":420,"elapsed":25730,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"75ae31fe-bd08-41e6-f029-a120d5245fbe"},"source":["from google.colab import drive\n","drive.mount('/content/drive')\n","\n","# !cp drive/'My Drive'/cs231n-project/train_captions_v2.pkl .\n","# !cp drive/'My Drive'/cs231n-project/val_captions_v2.pkl .\n","!cp drive/'My Drive'/cs231n-project/train_captions_pretrained_v1.pkl .\n","!cp drive/'My Drive'/cs231n-project/val_captions_pretrained_v1.pkl ."],"execution_count":null,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2UgcdZg2Himy","colab_type":"code","colab":{}},"source":["import pickle\n","with open(\"train_captions_pretrained_v1.pkl\", \"rb\") as f:\n"," train_captions_obj = pickle.load(f)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uTrYDIG7Hj9l","colab_type":"code","colab":{}},"source":["import pickle\n","with open(\"val_captions_pretrained_v1.pkl\", \"rb\") as f:\n"," val_captions_obj = pickle.load(f)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6AB7hn8lY7Ka","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":119},"executionInfo":{"status":"ok","timestamp":1592537645784,"user_tz":420,"elapsed":693,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"1f1f4aca-8062-454f-f3b7-0c3c99e1b454"},"source":["val_captions_obj[\"val_captions\"][0][0]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'caption': 'a group of people sitting around a table with laptops .',\n"," 'log_prob': -8.27091265593244},\n"," {'caption': 'a group of people sitting at a table with laptops .',\n"," 'log_prob': -8.779468823420707},\n"," {'caption': 'a group of people standing around a table with a laptop .',\n"," 'log_prob': -9.77529027415948}]"]},"metadata":{"tags":[]},"execution_count":38}]},{"cell_type":"code","metadata":{"id":"Ip5dgbelHnAC","colab_type":"code","colab":{}},"source":["import pandas as pd\n","\n","train_captions = []\n","for tc in train_captions_obj[\"train_captions\"]:\n"," # For each video\n"," sentences = []\n"," for t in tc:\n"," # For each frame, find the best caption (caption with the max log prob)\n"," best_caption = \"\"\n"," best_log_prob = -100\n","\n"," for candidate in t:\n"," if candidate[\"log_prob\"] > best_log_prob:\n"," best_log_prob = candidate[\"log_prob\"]\n"," best_caption = candidate[\"caption\"]\n","\n"," best_caption = best_caption.replace(\" .\", \"\")\n"," sentences.append(best_caption)\n"," break\n"," train_captions.append(\"\".join(sentences))\n","df_train = pd.DataFrame(list(zip(train_captions_obj[\"train_categories\"], train_captions, train_captions_obj[\"train_videos\"])), columns =['label', 'text', 'vid_name']) \n","\n","\n","val_captions = []\n","for tc in val_captions_obj[\"val_captions\"]:\n"," # For each video\n"," sentences = []\n"," for t in tc:\n"," # For each frame, find the best caption (caption with the max log prob)\n"," best_caption = \"\"\n"," best_log_prob = -100\n","\n"," for candidate in t:\n"," if candidate[\"log_prob\"] > best_log_prob:\n"," best_log_prob = candidate[\"log_prob\"]\n"," best_caption = candidate[\"caption\"]\n","\n"," best_caption = best_caption.replace(\" .\", \"\")\n"," sentences.append(best_caption)\n"," break\n"," val_captions.append(\"\".join(sentences))\n","df_val = pd.DataFrame(list(zip(val_captions_obj[\"val_categories\"], val_captions, val_captions_obj[\"val_videos\"])), columns =['label', 'text', 'vid_name']) \n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"LPktGfdkHwq9","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":419},"executionInfo":{"status":"ok","timestamp":1592537650753,"user_tz":420,"elapsed":867,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"888f2c4e-bd47-4d43-c197-bd4d903cddc0"},"source":["df_train.head(100)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
labeltextvid_name
02a man and a woman sitting at a table57_4
12a group of people cutting a cake together286_6
22a group of people sitting around a table276_3
32a couple of women standing next to each other61_10
42a group of people sitting around a table eatin...64_11
............
952a man and a woman standing next to each other168_4
962a group of people standing next to each other324_37
972a group of people standing around a table with...61_16
982a group of people standing next to each other286_4
992a group of women standing next to each other221_11
\n","

100 rows × 3 columns

\n","
"],"text/plain":[" label text vid_name\n","0 2 a man and a woman sitting at a table 57_4\n","1 2 a group of people cutting a cake together 286_6\n","2 2 a group of people sitting around a table 276_3\n","3 2 a couple of women standing next to each other 61_10\n","4 2 a group of people sitting around a table eatin... 64_11\n",".. ... ... ...\n","95 2 a man and a woman standing next to each other 168_4\n","96 2 a group of people standing next to each other 324_37\n","97 2 a group of people standing around a table with... 61_16\n","98 2 a group of people standing next to each other 286_4\n","99 2 a group of women standing next to each other 221_11\n","\n","[100 rows x 3 columns]"]},"metadata":{"tags":[]},"execution_count":40}]},{"cell_type":"code","metadata":{"id":"W5FMvpnUCj9A","colab_type":"code","colab":{}},"source":["import sys\n","import os\n","import logging\n","from tqdm import tqdm_notebook as tqdm\n","\n","logger = logging.getLogger()\n","\n","# text and label column names\n","TEXT_COL = \"text\"\n","LABEL_COL = \"label\"\n","\n","# path to data \n","DATA_DIR = os.path.abspath('./data')\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"LuSQDwLnF6Ld","colab_type":"code","colab":{}},"source":["# list of labels\n","labels = list(set(df_train[LABEL_COL].tolist()))\n","\n","# labels to integers mapping\n","label2int = {label: i for i, label in enumerate(labels)}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"D4HOfKP0IGjX","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592537659298,"user_tz":420,"elapsed":550,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"e117d1e6-86b3-4508-f995-191feae007d5"},"source":["label2int"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'1': 2, '2': 0, '3': 1}"]},"metadata":{"tags":[]},"execution_count":43}]},{"cell_type":"code","metadata":{"id":"h-V3MDbmF82r","colab_type":"code","colab":{}},"source":["import torch\n","from torch.utils.data import TensorDataset, random_split, DataLoader\n","import numpy as np\n","import warnings\n","from tqdm import tqdm_notebook as tqdm\n","from typing import List, Tuple\n","\n","NUM_MAX_POSITIONS = 256\n","BATCH_SIZE = 32\n","\n","class TextProcessor:\n"," \n"," # special tokens for classification and padding\n"," CLS = '[CLS]'\n"," PAD = '[PAD]'\n"," \n"," def __init__(self, tokenizer, label2id: dict, num_max_positions:int=512):\n"," self.tokenizer=tokenizer\n"," self.label2id = label2id\n"," self.num_labels = len(label2id)\n"," self.num_max_positions = num_max_positions\n"," \n"," \n"," def process_example(self, example: Tuple[str, str]):\n"," \"Convert text (example[0]) to sequence of IDs and label (example[1] to integer\"\n"," assert len(example) == 2\n"," label, text = example[0], example[1]\n"," assert isinstance(text, str)\n"," tokens = self.tokenizer.tokenize(text)\n","\n"," # truncate if too long\n"," if len(tokens) >= self.num_max_positions:\n"," tokens = tokens[:self.num_max_positions-1] \n"," ids = self.tokenizer.convert_tokens_to_ids(tokens) + [self.tokenizer.vocab[self.CLS]]\n"," # pad if too short\n"," else:\n"," pad = [self.tokenizer.vocab[self.PAD]] * (self.num_max_positions-len(tokens)-1)\n"," ids = self.tokenizer.convert_tokens_to_ids(tokens) + [self.tokenizer.vocab[self.CLS]] + pad\n"," \n"," return np.array(ids, dtype='int64'), self.label2id[label]\n"," \n","# download the 'bert-base-cased' tokenizer\n","from pytorch_transformers import BertTokenizer\n","tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)\n","\n","# initialize a TextProcessor\n","processor = TextProcessor(tokenizer, label2int, num_max_positions=NUM_MAX_POSITIONS)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OTKJVBUiF-eF","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":54},"executionInfo":{"status":"ok","timestamp":1592537665135,"user_tz":420,"elapsed":483,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"133ff92b-d217-4e32-96d5-cd0cab34dd8f"},"source":["from collections import namedtuple\n","import torch\n","\n","LOG_DIR = \"./logs/\"\n","CACHE_DIR = \"./cache/\"\n","\n","device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n","\n","FineTuningConfig = namedtuple('FineTuningConfig',\n"," field_names=\"num_classes, dropout, init_range, batch_size, lr, max_norm,\"\n"," \"n_warmup, valid_pct, gradient_acc_steps, device, log_dir\")\n","\n","finetuning_config = FineTuningConfig(\n"," 3, 0.05, 0.02, BATCH_SIZE, 6.5e-5, 1.0,\n"," 10, 0.1, 2, device, LOG_DIR)\n","\n","finetuning_config"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["FineTuningConfig(num_classes=3, dropout=0.05, init_range=0.02, batch_size=32, lr=6.5e-05, max_norm=1.0, n_warmup=10, valid_pct=0.1, gradient_acc_steps=2, device=device(type='cuda', index=0), log_dir='./logs/')"]},"metadata":{"tags":[]},"execution_count":45}]},{"cell_type":"code","metadata":{"id":"hNG0DqXbF_3a","colab_type":"code","colab":{}},"source":["from multiprocessing import cpu_count\n","from concurrent.futures import ProcessPoolExecutor\n","from itertools import repeat\n","\n","num_cores = cpu_count()\n","\n","def process_row(processor, row):\n"," return processor.process_example((row[1][LABEL_COL], row[1][TEXT_COL]))\n","\n","def create_dataloader(df: pd.DataFrame,\n"," processor: TextProcessor,\n"," batch_size: int = 32,\n"," shuffle: bool = True,\n"," valid_pct: float = None,\n"," text_col: str = \"text\",\n"," label_col: str = \"label\"):\n"," \"Process rows in `df` with `processor` and return a DataLoader\"\n","\n"," with ProcessPoolExecutor(max_workers=num_cores) as executor:\n"," result = list(\n"," tqdm(executor.map(process_row,\n"," repeat(processor),\n"," df.iterrows(),\n"," chunksize=len(df) // 10),\n"," desc=f\"Processing {len(df)} examples on {num_cores} cores\",\n"," total=len(df)))\n","\n"," features = [r[0] for r in result]\n"," labels = [r[1] for r in result]\n","\n"," dataset = TensorDataset(torch.tensor(features, dtype=torch.long),\n"," torch.tensor(labels, dtype=torch.long))\n","\n"," if valid_pct is not None:\n"," valid_size = int(valid_pct * len(df))\n"," train_size = len(df) - valid_size\n"," valid_dataset, train_dataset = random_split(dataset,\n"," [valid_size, train_size])\n"," valid_loader = DataLoader(valid_dataset,\n"," batch_size=batch_size,\n"," shuffle=False)\n"," train_loader = DataLoader(train_dataset,\n"," batch_size=batch_size,\n"," shuffle=True)\n"," return train_loader, valid_loader\n","\n"," data_loader = DataLoader(dataset,\n"," batch_size=batch_size,\n"," shuffle=shuffle)\n"," return data_loader"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"QfyN7C_tGBl7","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":169,"referenced_widgets":["0cf057a3ecd94722b81201e9d36d630f","2c251121d2b6406ebbd110ff327f6ad3","387aad4a61dc4cc795901713c3a86489","44b40b1f34bb497cadd4cef9237aa565","7f89cd01842849108f3e5d137948d610","384e57df336b41f5bfa230e665ef525c","06542471442b46ed87b464e843d2ff0d","f933a5664ac54e1a847a364167dd2d8c","344cfc793554492c91bb47d3ae88545f","f93f112dd57d4922936de1ce5c9664f2","631e066380c94dbab3e89c81c82129ad","ebfbb066208f496f8e5514d8d8015aa1","9e400c1f6cb94b2ab3ad5146e739fe3d","ab730ce1c76d4bc6a99b8132309b7a20","02fe49e507fe4ca5904fc45c5cf2f1aa","bca7c9d8443140dba85d3870c1cc8119"]},"executionInfo":{"status":"ok","timestamp":1592537715130,"user_tz":420,"elapsed":3206,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"2aafa2e9-9add-431f-c191-0fdb66386808"},"source":["train_dl = create_dataloader(df_train, processor, \n"," batch_size=finetuning_config.batch_size, \n"," valid_pct=None)\n","\n","valid_dl = create_dataloader(df_val, processor, \n"," batch_size=finetuning_config.batch_size, \n"," valid_pct=None)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:26: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n","Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0cf057a3ecd94722b81201e9d36d630f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Processing 2661 examples on 2 cores', max=2661.0, style=P…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"344cfc793554492c91bb47d3ae88545f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Processing 766 examples on 2 cores', max=766.0, style=Pro…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XZLzLMo8GCsB","colab_type":"code","colab":{}},"source":["import torch.nn as nn\n","\n","def get_num_params(model):\n"," mp = filter(lambda p: p.requires_grad, model.parameters())\n"," return sum(np.prod(p.size()) for p in mp)\n","\n","class Transformer(nn.Module):\n"," \"Adopted from https://github.com/huggingface/naacl_transfer_learning_tutorial\"\n","\n"," def __init__(self, embed_dim, hidden_dim, num_embeddings, num_max_positions, num_heads, num_layers, dropout, causal):\n"," super().__init__()\n"," self.causal = causal\n"," self.tokens_embeddings = nn.Embedding(num_embeddings, embed_dim)\n"," self.position_embeddings = nn.Embedding(num_max_positions, embed_dim)\n"," self.dropout = nn.Dropout(dropout)\n","\n"," self.attentions, self.feed_forwards = nn.ModuleList(), nn.ModuleList()\n"," self.layer_norms_1, self.layer_norms_2 = nn.ModuleList(), nn.ModuleList()\n"," for _ in range(num_layers):\n"," self.attentions.append(nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout))\n"," self.feed_forwards.append(nn.Sequential(nn.Linear(embed_dim, hidden_dim),\n"," nn.ReLU(),\n"," nn.Linear(hidden_dim, embed_dim)))\n"," self.layer_norms_1.append(nn.LayerNorm(embed_dim, eps=1e-12))\n"," self.layer_norms_2.append(nn.LayerNorm(embed_dim, eps=1e-12))\n","\n"," def forward(self, x, padding_mask=None):\n"," \"\"\" x has shape [seq length, batch], padding_mask has shape [batch, seq length] \"\"\"\n"," positions = torch.arange(len(x), device=x.device).unsqueeze(-1)\n"," h = self.tokens_embeddings(x)\n"," h = h + self.position_embeddings(positions).expand_as(h)\n"," h = self.dropout(h)\n","\n"," attn_mask = None\n"," if self.causal:\n"," attn_mask = torch.full((len(x), len(x)), -float('Inf'), device=h.device, dtype=h.dtype)\n"," attn_mask = torch.triu(attn_mask, diagonal=1)\n","\n"," for layer_norm_1, attention, layer_norm_2, feed_forward in zip(self.layer_norms_1, self.attentions,\n"," self.layer_norms_2, self.feed_forwards):\n"," h = layer_norm_1(h)\n"," x, _ = attention(h, h, h, attn_mask=attn_mask, need_weights=False, key_padding_mask=padding_mask)\n"," x = self.dropout(x)\n"," h = x + h\n","\n"," h = layer_norm_2(h)\n"," x = feed_forward(h)\n"," x = self.dropout(x)\n"," h = x + h\n"," return h\n","\n","\n","class TransformerWithClfHead(nn.Module):\n"," \"Adopted from https://github.com/huggingface/naacl_transfer_learning_tutorial\"\n"," def __init__(self, config, fine_tuning_config):\n"," super().__init__()\n"," self.config = fine_tuning_config\n"," self.transformer = Transformer(config.embed_dim, config.hidden_dim, config.num_embeddings,\n"," config.num_max_positions, config.num_heads, config.num_layers,\n"," fine_tuning_config.dropout, causal=not config.mlm)\n"," \n"," self.classification_head = nn.Linear(config.embed_dim, fine_tuning_config.num_classes)\n"," self.apply(self.init_weights)\n","\n"," def init_weights(self, module):\n"," if isinstance(module, (nn.Linear, nn.Embedding, nn.LayerNorm)):\n"," module.weight.data.normal_(mean=0.0, std=self.config.init_range)\n"," if isinstance(module, (nn.Linear, nn.LayerNorm)) and module.bias is not None:\n"," module.bias.data.zero_()\n","\n"," def forward(self, x, clf_tokens_mask, clf_labels=None, padding_mask=None):\n"," hidden_states = self.transformer(x, padding_mask)\n","\n"," clf_tokens_states = (hidden_states * clf_tokens_mask.unsqueeze(-1).float()).sum(dim=0)\n"," clf_logits = self.classification_head(clf_tokens_states)\n","\n"," if clf_labels is not None:\n"," loss_fct = nn.CrossEntropyLoss(ignore_index=-1)\n"," loss = loss_fct(clf_logits.view(-1, clf_logits.size(-1)), clf_labels.view(-1))\n"," return clf_logits, loss\n"," return clf_logits"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fqR6iLTCGLVq","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":71},"executionInfo":{"status":"ok","timestamp":1592537720217,"user_tz":420,"elapsed":2151,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"afbe50f8-b824-4577-9775-c5e1a1171a34"},"source":["from pytorch_transformers import cached_path\n","\n","# download pre-trained model and config\n","state_dict = torch.load(cached_path(\"https://s3.amazonaws.com/models.huggingface.co/\"\n"," \"naacl-2019-tutorial/model_checkpoint.pth\"), map_location='cpu')\n","\n","config = torch.load(cached_path(\"https://s3.amazonaws.com/models.huggingface.co/\"\n"," \"naacl-2019-tutorial/model_training_args.bin\"))\n","\n","# init model: Transformer base + classifier head\n","model = TransformerWithClfHead(config=config, fine_tuning_config=finetuning_config).to(finetuning_config.device)\n","\n","incompatible_keys = model.load_state_dict(state_dict, strict=False)\n","print(f\"Parameters discarded from the pretrained model: {incompatible_keys.unexpected_keys}\")\n","print(f\"Parameters added in the model: {incompatible_keys.missing_keys}\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Parameters discarded from the pretrained model: ['lm_head.weight']\n","Parameters added in the model: ['classification_head.weight', 'classification_head.bias']\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"TDCSxvtpJd7F","colab_type":"code","colab":{}},"source":["NUM_EPOCHS = 10"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"himuaQoxGNh5","colab_type":"code","colab":{}},"source":["from ignite.engine import Engine, Events\n","from ignite.metrics import RunningAverage, Accuracy \n","from ignite.handlers import ModelCheckpoint\n","from ignite.contrib.handlers import CosineAnnealingScheduler, PiecewiseLinear, create_lr_scheduler_with_warmup, ProgressBar\n","import torch.nn.functional as F\n","from pytorch_transformers.optimization import AdamW\n","\n","# Bert optimizer\n","optimizer = AdamW(model.parameters(), lr=finetuning_config.lr, correct_bias=False) \n","\n","def update(engine, batch):\n"," \"update function for training\"\n"," model.train()\n"," inputs, labels = (t.to(finetuning_config.device) for t in batch)\n"," inputs = inputs.transpose(0, 1).contiguous() # [S, B]\n"," _, loss = model(inputs, \n"," clf_tokens_mask = (inputs == tokenizer.vocab[processor.CLS]), \n"," clf_labels=labels)\n"," loss = loss / finetuning_config.gradient_acc_steps\n"," loss.backward()\n"," \n"," torch.nn.utils.clip_grad_norm_(model.parameters(), finetuning_config.max_norm)\n"," if engine.state.iteration % finetuning_config.gradient_acc_steps == 0:\n"," optimizer.step()\n"," optimizer.zero_grad()\n"," return loss.item()\n","\n","def inference(engine, batch):\n"," \"update function for evaluation\"\n"," model.eval()\n"," with torch.no_grad():\n"," batch, labels = (t.to(finetuning_config.device) for t in batch)\n"," inputs = batch.transpose(0, 1).contiguous()\n"," logits = model(inputs,\n"," clf_tokens_mask = (inputs == tokenizer.vocab[processor.CLS]),\n"," padding_mask = (batch == tokenizer.vocab[processor.PAD]))\n"," return logits, labels\n","\n","def predict(model, tokenizer, int2label, input=\"test\"):\n"," \"predict `input` with `model`\"\n"," tok = tokenizer.tokenize(input)\n"," ids = tokenizer.convert_tokens_to_ids(tok) + [tokenizer.vocab['[CLS]']]\n"," tensor = torch.tensor(ids, dtype=torch.long)\n"," tensor = tensor.to(device)\n"," tensor = tensor.reshape(1, -1)\n"," tensor_in = tensor.transpose(0, 1).contiguous() # [S, 1]\n"," logits = model(tensor_in,\n"," clf_tokens_mask = (tensor_in == tokenizer.vocab['[CLS]']),\n"," padding_mask = (tensor == tokenizer.vocab['[PAD]']))\n"," val, _ = torch.max(logits, 0)\n"," val = F.softmax(val, dim=0).detach().cpu().numpy() \n"," return {int2label[val.argmax()]: val.max(),\n"," int2label[val.argmin()]: val.min()}\n","\n","trainer = Engine(update)\n","evaluator = Engine(inference)\n","\n","# add metric to evaluator \n","Accuracy().attach(evaluator, \"accuracy\")\n","\n","# add evaluator to trainer: eval on valid set after each epoch\n","@trainer.on(Events.EPOCH_COMPLETED)\n","def log_validation_results(engine):\n"," evaluator.run(train_dl)\n"," print(f\"validation epoch: {engine.state.epoch} train acc: {100*evaluator.state.metrics['accuracy']}\")\n"," evaluator.run(valid_dl)\n"," print(f\"validation epoch: {engine.state.epoch} val acc: {100*evaluator.state.metrics['accuracy']}\")\n","\n","# lr schedule: linearly warm-up to lr and then to zero\n","scheduler = PiecewiseLinear(optimizer, 'lr', [(0, 0.0), (finetuning_config.n_warmup, finetuning_config.lr),\n"," (len(train_dl)*NUM_EPOCHS, 0.0)])\n","trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n","\n","\n","# add progressbar with loss\n","RunningAverage(output_transform=lambda x: x).attach(trainer, \"loss\")\n","ProgressBar(persist=True).attach(trainer, metric_names=['loss'])\n","\n","# save checkpoints and finetuning config\n","checkpoint_handler = ModelCheckpoint(finetuning_config.log_dir, 'sentiment-transformer', \n"," n_saved=None, require_empty=False)\n","trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {'sentiment-transformer': model})\n","\n","int2label = {i:label for label,i in label2int.items()}\n","\n","# save metadata\n","torch.save({\n"," \"config\": config,\n"," \"config_ft\": finetuning_config,\n"," \"int2label\": int2label\n","}, os.path.join(finetuning_config.log_dir, \"sentiment-transformer.metadata.bin\"))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RrGVP998GPdk","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["2a9f7995a141443c90593e282a447b8b","492966b1127e4510b314009654be4f57","094cefe6347247999014ebcc01afc02e","ff4655a98ef745fb8719d970a611e8c8","d8fc34c04ecf468f938892d6e7cb77a6","39af62ecfcd14c73af8fd8c5bc046759","c214b201553d48a3b075909e133aa9cf","734bfe5687934b519c40f04cd335baf2","465f62e09237469d909093bb5a25cf38","94289f71172d42aea582ac7a15a5e12e","15710a5ad7184d6992ed49e31fd29a83","bedbefd3e084432e88f31655853cdfe0","9d8d85841a884aa3883dd533f191756c","e3b86cfa651c4d2782f67c032664e475","7e5618afebb14f5fab65fcd94cd97316","db9a658990524118a4ba4a2c9e43fcee","9d43bbddb3144931a78ac6060a6a0729","07819972bfbf4d35b209e2b3ac8cc9d1","9cb9f622b61748408876c2a6b3cb89e2","d218c768ad204ed191b970f4b9f3ab1b","93c87439598d4d10b7046e4032d10581","e6bf040065734b799875569013b09c3d","96d7f88e28474644bccfcc4fedf5be16","9b3d06abbb784bb696d4fb949d58fbbb","c83224cf0c244474a77ee12d3e8af730","548fea41927b444e937415eeb04f5b25","799cb0aa7e674094bb15cac18a29e7bf","326ddb685bba41aeb500de29ad13488b","a7ecda0068c841e5aa2b14ab731f26ac","78ba82c246eb45449b5b37c8987ecf6c","ad9959f7a2fd49228e2291d721d27bb8","4e440371d54b4827bc3a47a1c06513bf","c7bf2a1ddab64f21ab2f8107365e2e05","fccecfa197fa4d62a0a558885255a76c","37a5c8db4cd34bef856e4b468187d91b","6ff46092582c4b51b1c132c25e9eeb1d","432a5a233c894f4f871af41646812247","0c2074c9c2414d8ca919720850d4ab93","846716d075c84358a4639bf890493fbd","d96d59ff069343ebb902194880270d67","ce73826afd7a4cf3bbb152d2c168b2a8","89de87775fda4643aa6985687da4bc9c","e5928ed938d743dcac577e427acf6942","3f9f1d5600994f8e81151475098081a4","cdb08e19110a4b399a55cddde941872b","6c2229f17f364c79a418f53f75874c25","3e410112b02042ad896e36f820488e18","b060cdfd706543c7a4fb3379d65b3ec3","34e3457f69d34255b9f8b2263a21765e","f3bb8d50c3384e249259f3cc43429781","83cefe6b831c44a9bffbf887519ac431","c44612eed7fc4c19a1d8f646ccb552f7","b47b770bbe4f492fb47bde1af0cc1c0f","8832b8c8456046e5b804e2bf399285bf","deb24156ac2b40ba8ae4b9e8261f3121","48c2137f94a7496f9b7d518c2506232d","a8435e3254b74779bce08df4c8278d72","917d5d60ac184ad38d180d24e2fde79a","a6fa4e9a37b141818d0b647949211bbc","732f57dd940e4c48810bee9d390bae63","d1ce78e18b2242238aa18d1e155879dd","9f564a3c830c4a5ab938a651fc80ed62","f5227c3bb518489d8ad962bba077de2a","d3614bac3ddc49349906efb3b46ec785","739d0f39fb4f41e88bba635f86c16984","b23cfe9588f24e60b3c0a0000595dca5","1f86c396f25b4c4d8e9416166a962217","d5311479b4ba402b8ef4ed90e45991d3","bcb1a72a01054d3f8cd0332ed8a9f310","144bf9b58b87474ea86bd385f9cc8d15","589be329569c4c3b85987ef4e2ff710a","6ca9daef5e8744d2a0c6631649b97816","6da6be3065424e03b1fa24afeadf47dc","148a7107ed404619bf69dae28cab1bff","a9ba25bd82554b99a742c77101fcb018","ae000ff2e16448649987f9d72d9e5f5d","f176b4d20ca5452ab30f64952fdb0fca","6f6c7ceeb65a43ad9fe3f2a961b2783a","77d48e2da1ac4555afd8f39a097b6b65","bd9712584c6148a3804e098d84274bfc"]},"executionInfo":{"status":"ok","timestamp":1592538340793,"user_tz":420,"elapsed":588140,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"79b396fa-d385-4fdf-9f82-9b5ca23aa6dc"},"source":["# fit the model on train_dl\n","trainer.run(train_dl, max_epochs=NUM_EPOCHS)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2a9f7995a141443c90593e282a447b8b","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 1 train acc: 49.3423524990605\n","validation epoch: 1 val acc: 41.38381201044387\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"465f62e09237469d909093bb5a25cf38","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 2 train acc: 50.84554678692221\n","validation epoch: 2 val acc: 46.344647519582246\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9d43bbddb3144931a78ac6060a6a0729","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 3 train acc: 53.10033821871477\n","validation epoch: 3 val acc: 46.47519582245431\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c83224cf0c244474a77ee12d3e8af730","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 4 train acc: 55.956407365652005\n","validation epoch: 4 val acc: 49.47780678851175\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c7bf2a1ddab64f21ab2f8107365e2e05","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 5 train acc: 57.19654265313792\n","validation epoch: 5 val acc: 49.21671018276763\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ce73826afd7a4cf3bbb152d2c168b2a8","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 6 train acc: 57.68508079669298\n","validation epoch: 6 val acc: 49.73890339425587\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"34e3457f69d34255b9f8b2263a21765e","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 7 train acc: 58.436677940623824\n","validation epoch: 7 val acc: 48.30287206266319\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a8435e3254b74779bce08df4c8278d72","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 8 train acc: 58.47425779782037\n","validation epoch: 8 val acc: 48.825065274151434\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"739d0f39fb4f41e88bba635f86c16984","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 9 train acc: 59.07553551296505\n","validation epoch: 9 val acc: 49.47780678851175\n","\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6da6be3065424e03b1fa24afeadf47dc","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["validation epoch: 10 train acc: 59.15069522735814\n","validation epoch: 10 val acc: 49.21671018276763\n","\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["State:\n","\titeration: 840\n","\tepoch: 10\n","\tepoch_length: 84\n","\tmax_epochs: 10\n","\toutput: 0.23070499300956726\n","\tbatch: \n","\tmetrics: \n","\tdataloader: \n","\tseed: 12"]},"metadata":{"tags":[]},"execution_count":53}]},{"cell_type":"code","metadata":{"id":"uKfXBSGTu0as","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":275},"executionInfo":{"status":"ok","timestamp":1592540554502,"user_tz":420,"elapsed":2084,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"25eba40b-e632-4407-e79a-369ff2a643e0"},"source":["!ls -lht logs"],"execution_count":null,"outputs":[{"output_type":"stream","text":["total 2.1G\n","-rw-r--r-- 1 root root 1.1K Jun 19 04:10 sentiment-transformer.metadata.bin\n","-rw------- 1 root root 193M Jun 19 03:45 sentiment-transformer_sentiment-transformer_840.pth\n","-rw------- 1 root root 193M Jun 19 03:44 sentiment-transformer_sentiment-transformer_756.pth\n","-rw------- 1 root root 193M Jun 19 03:43 sentiment-transformer_sentiment-transformer_672.pth\n","-rw------- 1 root root 193M Jun 19 03:42 sentiment-transformer_sentiment-transformer_588.pth\n","-rw------- 1 root root 193M Jun 19 03:41 sentiment-transformer_sentiment-transformer_504.pth\n","-rw------- 1 root root 193M Jun 19 03:40 sentiment-transformer_sentiment-transformer_420.pth\n","-rw------- 1 root root 193M Jun 19 03:39 sentiment-transformer_sentiment-transformer_336.pth\n","-rw------- 1 root root 193M Jun 19 03:38 sentiment-transformer_sentiment-transformer_252.pth\n","-rw------- 1 root root 193M Jun 19 03:37 sentiment-transformer_sentiment-transformer_168.pth\n","-rw------- 1 root root 193M Jun 19 03:36 sentiment-transformer_sentiment-transformer_84.pth\n","-rw-r--r-- 1 root root 1.1K Jun 19 03:22 metadata.bin\n","-rw------- 1 root root 193M Jun 19 03:14 finetuning_checkpoint_imdb_model_225.pth\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"sfyN2f9_xNxZ","colab_type":"code","colab":{}},"source":["\n","# save metadata\n","torch.save({\n"," \"config\": config,\n"," \"config_ft\": finetuning_config,\n"," \"int2label\": int2label\n","}, os.path.join(finetuning_config.log_dir, \"sentiment-transformer.metadata.bin\"))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"X9MabAWKvFFq","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1592539365142,"user_tz":420,"elapsed":350,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"54184ffc-c642-4783-9555-7565d6b43477"},"source":["print(model)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["TransformerWithClfHead(\n"," (transformer): Transformer(\n"," (tokens_embeddings): Embedding(28996, 410)\n"," (position_embeddings): Embedding(256, 410)\n"," (dropout): Dropout(p=0.05, inplace=False)\n"," (attentions): ModuleList(\n"," (0): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (1): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (2): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (3): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (4): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (5): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (6): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (7): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (8): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (9): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (10): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (11): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (12): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (13): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (14): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (15): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," )\n"," (feed_forwards): ModuleList(\n"," (0): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (1): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (2): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (3): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (4): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (5): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (6): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (7): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (8): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (9): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (10): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (11): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (12): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (13): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (14): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (15): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," )\n"," (layer_norms_1): ModuleList(\n"," (0): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (1): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (2): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (3): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (4): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (5): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (6): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (7): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (8): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (9): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (10): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (11): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (12): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (13): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (14): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (15): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," )\n"," (layer_norms_2): ModuleList(\n"," (0): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (1): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (2): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (3): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (4): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (5): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (6): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (7): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (8): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (9): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (10): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (11): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (12): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (13): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (14): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (15): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," )\n"," )\n"," (classification_head): Linear(in_features=410, out_features=3, bias=True)\n",")\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3QeQVRYyw5Cv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":71},"executionInfo":{"status":"ok","timestamp":1592542659391,"user_tz":420,"elapsed":1413,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"b77e9980-bbe8-447e-8e7f-93a36d0e7bf4"},"source":["### Restore the best checkpoint\n","best_metadata = torch.load(\"logs/sentiment-transformer.metadata.bin\")\n","print(best_metadata)\n","best_model = TransformerWithClfHead(best_metadata[\"config\"], best_metadata[\"config_ft\"]).to(best_metadata[\"config_ft\"].device)\n","int2label = best_metadata[\"int2label\"]\n","best_model.load_state_dict(torch.load(\"logs/sentiment-transformer_sentiment-transformer_504.pth\"))\n"],"execution_count":115,"outputs":[{"output_type":"stream","text":["{'config': Namespace(dataset_cache='./data/wikitext-103/dataset_cache', dataset_path='./data/wikitext-103/', device=device(type='cuda', index=0), distributed=True, dropout=0.1, embed_dim=410, eval_every=-1, gradient_accumulation_steps=1, hidden_dim=2100, initializer_range=0.02, local_rank=0, lr=0.00025, max_norm=0.25, mlm=False, mlm_probability=0.15, n_epochs=200, n_warmup=1000, num_embeddings=28996, num_heads=10, num_layers=16, num_max_positions=256, sinusoidal_embeddings=False, train_batch_size=16, valid_batch_size=8, weight_decay=0.0), 'config_ft': FineTuningConfig(num_classes=3, dropout=0.05, init_range=0.02, batch_size=32, lr=6.5e-05, max_norm=1.0, n_warmup=10, valid_pct=0.1, gradient_acc_steps=2, device=device(type='cuda', index=0), log_dir='./logs/'), 'int2label': {0: '2', 1: '3', 2: '1'}}\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":115}]},{"cell_type":"code","metadata":{"id":"LmRxT-Eg7XWe","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":68},"executionInfo":{"status":"ok","timestamp":1592542581557,"user_tz":420,"elapsed":17357,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"ec7d2122-1d72-4290-d8c3-926a5502105d"},"source":["from google.colab import auth\n","auth.authenticate_user()\n","project_id = 'calm-depot-274104'\n","!gcloud config set project {project_id}\n","!gsutil ls"],"execution_count":113,"outputs":[{"output_type":"stream","text":["Updated property [core/project].\n","gs://calm-depot-274104/\n","gs://cs231n-emotiw/\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Rw92aRPH71fl","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":272},"executionInfo":{"status":"ok","timestamp":1592542624020,"user_tz":420,"elapsed":4434,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"aff3ab22-da42-4f9e-acb5-48cd55030aed"},"source":["bucket_name = 'cs231n-emotiw'\n","!gsutil -m cp -r logs/sentiment-transformer_sentiment-transformer_504.pth gs://{bucket_name}/models/\n"],"execution_count":114,"outputs":[{"output_type":"stream","text":["Copying file://logs/sentiment-transformer_sentiment-transformer_504.pth [Content-Type=application/octet-stream]...\n","/ [0/1 files][ 0.0 B/192.3 MiB] 0% Done \r==> NOTE: You are uploading one or more large file(s), which would run\n","significantly faster if you enable parallel composite uploads. This\n","feature can be enabled by editing the\n","\"parallel_composite_upload_threshold\" value in your .boto\n","configuration file. However, note that if you do this large files will\n","be uploaded as `composite objects\n","`_,which\n","means that any user who downloads such objects will need to have a\n","compiled crcmod installed (see \"gsutil help crcmod\"). This is because\n","without a compiled crcmod, computing checksums on composite objects is\n","so slow that gsutil disables downloads of composite objects.\n","\n","| [1/1 files][192.3 MiB/192.3 MiB] 100% Done \n","Operation completed over 1 objects/192.3 MiB. \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"1nuoCVBiyOfE","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1592542668387,"user_tz":420,"elapsed":5403,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}}},"source":["### Make sure evaluation works on restored model\n","\n","actual_labels = []\n","pred_labels = []\n","\n","for batch in valid_dl:\n"," actual_labels.extend(batch[1].tolist())\n"," \n"," best_model.eval()\n"," with torch.no_grad():\n"," batch, labels = (t.to(finetuning_config.device) for t in batch)\n"," inputs = batch.transpose(0, 1).contiguous()\n"," logits = best_model(inputs,\n"," clf_tokens_mask = (inputs == tokenizer.vocab[processor.CLS]),\n"," padding_mask = (batch == tokenizer.vocab[processor.PAD]))\n"," pred_labels.extend(logits.argmax(axis=1).tolist())\n"],"execution_count":116,"outputs":[]},{"cell_type":"code","metadata":{"id":"i3s3o-qYykGv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592542668394,"user_tz":420,"elapsed":3784,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"72413b2a-a60a-4135-dd35-1999e3d29ef1"},"source":["correct = 0\n","for i in range(len(pred_labels)):\n"," if pred_labels[i] == actual_labels[i]:\n"," correct += 1\n","\n","print(f\"Accuracy = {correct/len(pred_labels)}\")"],"execution_count":117,"outputs":[{"output_type":"stream","text":["Accuracy = 0.49738903394255873\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"K7i-PNLd8L2f","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592542934153,"user_tz":420,"elapsed":469,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"3a1798b6-477e-4beb-a723-365e0aa7b245"},"source":["int2label"],"execution_count":122,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{0: '2', 1: '3', 2: '1'}"]},"metadata":{"tags":[]},"execution_count":122}]},{"cell_type":"code","metadata":{"id":"TzjasJTq8J9I","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":570},"executionInfo":{"status":"ok","timestamp":1592542994143,"user_tz":420,"elapsed":783,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"034b3bc0-53af-417d-c131-ca7ca1d765a0"},"source":["from sklearn import metrics \n","import pandas as pd\n","import seaborn as sn\n","import matplotlib.pyplot as plt\n","import tensorflow\n","\n","actual_labels_final = [int(int2label[x]) - 1 for x in actual_labels]\n","pred_labels_final = [int(int2label[x]) - 1 for x in pred_labels]\n","\n","# Note the specific ordering\n","classes=['Pos', 'Neu', 'Neg'] \n","con_mat = tensorflow.math.confusion_matrix(labels=actual_labels_final, predictions=pred_labels_final).numpy()\n","con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)\n","con_mat_df = pd.DataFrame(con_mat_norm,\n"," index = classes, \n"," columns = classes)\n","\n","figure = plt.figure(figsize=(11, 9))\n","plt.title(\"Image Captioning Confusion Matrix\")\n","sn.heatmap(con_mat_df, annot=True,cmap=plt.cm.Blues)\n"],"execution_count":124,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":124},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAlgAAAIYCAYAAACxPpKwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5zc493/8ddnNxs5EDmfExLiEII6hNLSKo3DjarqTbWOd/24S7VarVbr1JuipbRopepUVUVbh0aL0lRpkZQ4BxEqCXJEEJFk9/r9MZM1u9nDVzK7383s69nHPLrf71xzzfWdjOwn7+uaayKlhCRJksqnKu8BSJIkVRoLLEmSpDKzwJIkSSozCyxJkqQys8CSJEkqMwssSZKkMrPAktYSEfFORIwuQz9/jogjyjGm9hQRu0TEC8XX4TNr0M9aef2lImJk8XWoznsskppmgaUOJyJejog98h5HayKiV0RcHBGvFH/ZvVg87l+GvidHxP+UnksprZtSmrmmfaeU9k4pXbum/TSlLV8T4Gzg0uLrcOvqdtJW1x8R10REiogDGp3/SfH8kRn7afX9n1J6pfg61K7BkCW1IQssaTVERFfgXmALYC+gF/BRYCEwPseh5aYdXpMNgKfL0E9beh44fOVBRHQBPg+8WK4nKPYpqYOzwFKHFhFHRsSDxRTgzYiYGRE7F8/Pioh5pdM9EbFvRDwWEYuL95/ZqL/DI+I/EbEwIr5fmhZERFVEnFpMXRZGxE0R0beZoR0OjAQOTCk9k1KqSynNSyn9IKV0Z7G/lX29HRHPRMSBTVzXpRHxVkRMj4hPFe87B/g4cGkxBbq0eD5FxMbFn9ePiOsiYn7xer4XEVUlfT8QET+OiDci4qWI2LvkuevTsQxtR0XE/cVr+GtEXBYR16/Ba7J58fnfjIinI2L/kue6ptj/pOLzPRwRGxXvexEYDdxRfE3WaZz0RMSZK8cWEd0i4vrin+ObETElIgY1cf1VxdfuP8X30nURsX7xvg2Lr/kRxURuQUSc1sy1r3QH8LGI6FM83gt4Ani9ZJwbRcR9xbEtiIjfRETv4n2/Lr6GK6/zWyXjOCYiXgHuKznXJSL6RsTsiNiv2Me6ETEjIg5HUm4ssLQ22JHCL6l+wA3AjcAOwMbAFykUIusW275L4Rd9b2Bf4PgorteJiLHA5cBhwBBgfWBYyfOcCHwG2A0YCrwBXNbMmPYA/pJSeqeFcb9IoVBaHzgLuD4ihjS6rheB/sAZwB8iom9K6TTgH8AJxWmgE5ro+2fFfkcXx3s4cFSjvp8r9n0B8KuIiGbG2VLbG4BHKLz2ZwJfauF6W3xNIqKGQgFyNzCQwuv9m4jYtKTZIRReqz7ADOAcgJTSRsArwH7F1+T9FsYBcASF12dEcezHAe810e7I4u2TFF7LdYFLG7X5GLAp8Cng9IjYvIXnXQrcVrwOKPy5XNeoTQA/pPAe27w4xjMBUkpfouF1XlDyuN2K7SeUdpZSWgQcDfwyIgYCPwGmpZQaP6+kdmSBpbXBSymlq4vrTX5H4RfS2Sml91NKdwPLKBRbpJQmp5SeLKYnTwC/pfCLCeBzwB0ppQdSSsuA04HSL+M8DjgtpTS7+Av8TOBz0fSUTD/gtZYGnVK6OaX0anEsvwNeoOFU2Tzg4pTS8uL9z1EoClsUhYXNhwDfSSm9nVJ6GbiQhsXPf1JKvyy+ZtdSKCgHNdNlk20jYiSFQvb0lNKylNIDwO0tDK2112QnCgXMecX+7gP+BBxa0uaPKaVHUkorgN8A27TQX0uWF8ezcUqpNqX075TS4ibaHQZclFKaWSwMvwMc0ujP/KyU0nsppceBx4GtW3nu64DDi6nUbkCD9WIppRkppXuK79/5wEV88B5tyZkppXdTSqsUisX/Dm6mMEW7D/D/MvQnqQ1ZYGltMLfk5/cAUkqNz60LEBE7RsTfilNnb1EomlYusB4KzFr5oJTSEgrrg1baAPhjcUrpTeBZoJamC5OFFAqRZkVhOnJaSX9blowFYE5q+G3r/ymOsTX9gZpi+9LHlqZx9VNSxeuE4mvUhObaDgUWlZyDktevCa29JkOBWSmluizjBpa0MObW/Bq4C7gxIl6NiAuKCVpTY2r8Onah4Z/5hxpTsRAdAJwG/KlxQRQRgyLixoiYExGLgetp+L5oTkuvPcBECu+xa1JKC1tpK6mNWWCp0txAIWUZkVJaH/gFhSkZKKQrw1c2jIjuFFKOlWYBe6eUepfcuqWU5jTxPH8FJkREz6YGEREbAL8ETgD6pZR6A0+VjAVgWKNpu5HAq8WfSwuvxhZQSGg2aPTYpsa5Jl4D+kZEj5JzI1po3+JrQuHaRqxcK1a0JuN+Fygd2+CVPxRTwbNSSmOBnYH/omTxeaMxNX4dV9CwqF8d1wPfYNXpQYBzKfz5jksp9aIwzV36Pmjuz77Z90Qx1ZxYfL7/jeJaPUn5scBSpVmPQuqyNCLGA18oue8WYL8oLJLvSmEKsPQX2y+Ac4rFERExIBp95L7ErykUZL+PiM2Ki6X7RcR3I2IfoCeFX4jzi30dRSFdKDUQ+GpE1ETEwRTW19xZvG8uhTVBqyhO5d1UHOt6xfGeTOGXetmklP4DTAXOjIiuEfFRYL8WHtLaa/IwhQToW8Vr/kSxvxtXc4jTKEzn1UTE9hSmgAGIiE9GxLhi4bGYQkFa10QfvwW+HoXF/OtSKH5+V5yiXBM/BfYE7m/ivvWAd4C3ImIYcEqj+5v9s2/Bdym8344GfgRcF+6RJeXKAkuV5n+BsyPibQprrG5aeUdK6WkKC6tvpJDOvENhHdTKBdOXUEi/7i4+/iEKC8BXUVyjtQcwHbiHwi/xRyhM9TycUnqGwrqof1H4hTkOeLBRNw8DYygkUucAnyuZ2rmEwvqvNyLip00M4UQKCc5M4AEKyd1Vrbw2q+MwPthq4f8orIFrcoF5htdkGYWCam8K13w5cHhKafpqju37wEYUPoxwFoXXYKXBFArqxRSmev9OoQBs7Kri+fuBlygsUj9xNcdTL6W0KKV0b6Mp4JXOArYF3gImAX9odP8Pge8Vp5a/2dpzRcR2FArsw4vF9/kUiq1T1+QaJK2ZaPq/f6nyFROLN4ExKaWX2vm5jwT+J6X0sfZ83jUVEb8DpqeUzsh7LJLUkZlgqVOJiP0iokdxndCPgSeBl/MdVccVETsU922qioi9gANo9Kk4SdKqLLDU2RxAYWHzqxSm5w5pZhpHBYOByRSmU38KHJ9SeizXEUlSmUXEXhHxXHGT3lWm16Ow2fW04u354ifDW+7T3y2SJKmzKn4g5HkKH0yZDUwBDi2upW2q/YnAR1JKR7fUrwmWJEnqzMYDM4obDi+j8EGo5j5BDoXNkX/bWqcWWJIkqTMbRsONfGfTcAPkesVtcUYB97XWaZt/K/vB1zzqHKQ6hLMnbNp6I6md3Pbs6603ktrJqbtv1Nx3lba57h85oU3rhKXTLvt/wLElpyamlCauZneHALcUt0RpUZsXWJIkSXkpFlMtFVRzaPgtFcNp/hsmDgG+kuV5LbAkSVJ+IvfVSlOAMRExikJhdQgNvwUEgIjYDOhDYQPpVuV+VZIkSXkpfjXWCRS+IP5Z4KaU0tMRcXZE7F/S9BDgxqxb+5hgSZKk/ERuy7/qpZTu5IPvgl157vRGx2d+mD5NsCRJksrMBEuSJOUn/zVYbaIyr0qSJClHJliSJCk/HWANVlswwZIkSSozEyxJkpQf12BJkiQpCxMsSZKUnwpdg2WBJUmS8uMUoSRJkrIwwZIkSfmp0ClCEyxJkqQyM8GSJEn5cQ2WJEmSsjDBkiRJ+XENliRJkrIwwZIkSflxDZYkSZKyMMGSJEn5cQ2WJEmSsjDBkiRJ+XENliRJkrIwwZIkSfkxwZIkSVIWJliSJCk/VX6KUJIkSRmYYEmSpPy4BkuSJElZmGBJkqT8VOhO7hZYkiQpP04RSpIkKQsTLEmSlJ8KnSI0wZIkSSozEyxJkpQf12BJkiQpCxMsSZKUH9dgSZIkKQsTLEmSlB/XYEmSJCkLEyxJkpQf12BJkiQpCxMsSZKUH9dgSZIkKQsTLEmSlB/XYEmSJCkLEyxJkpQf12BJkiQpCxMsSZKUHxMsSZIkZWGCJUmS8lOhnyK0wJIkSflxilCSJElZmGBJkqT8VOgUoQmWJElSmZlgSZKk/LgGS5IkSVmYYEmSpPy4BkuSJElZmGBJkqTchAmWJEmSsjDBkiRJuTHBkiRJUiYmWJIkKT+VGWCZYEmSJJWbCZYkScqNa7AkSZKUiQmWJEnKjQmWJEmSMjHBkiRJuTHBkiRJUiYmWJIkKTcmWJIkScrEBKsD2GZYL44aP5yqgHtfWMitT85tst2OG/Tmm58czbfvmM7MhUvqz/fvWcNPPjOWm6a9xh1Pz2uvYavCPfrwg1x56Y+pq61lz30P5KDDjmpw/203Xc89k/5IdXU1vXr34cRvncHAwUNzGq0q0eynp/LwTVeQUh2b7DKBrSZ8vsH90++fxLN//xNVVdV0Wacbuxz2VXoPGQnAotkv8c8bfsbypUsggv1OvYQuNV3zuAy1pjIDLAusvFUFHLPjCH5w9wssWrKcH/7Xpkx95S1mv7W0QbtuXarYZ/MBPD//3VX6OGKH4Tw2Z3F7DVmdQG1tLVdccj5n/fhy+g0YxCnHfZHxu+zGiA1H17cZPWZTLrzietbp1p0/33Yz115xCaeccX6Oo1Ylqaur5aEbL2fCV8+hR5/+3HHe1xi51U71BRTA6B0+yWa77gvAK48/xCO3/JJPn/gD6mpruf+aH7Hrkd+k7/DRLH1nMVXV1XldilrREaYII2Iv4BKgGrgypXReE20+D5wJJODxlNIXWurTKcKcbdy/J6+//T7z3lnGirrEgy+9wfYj11+l3SHbDuW2p+ayvLauwfkdRq7PvHeWMevNpas8RlpdL0x/iiHDhjN46HBqamr42O4TePjByQ3ajPvIDqzTrTsAm44dx8L5pqcqnwUvP896A4ay3oAhVHepYfT2u/LK4/9q0KZr9x71P69Y9sHfgXOefZQ+w0bRd3jhHwTd1u1FVZUFlpoWEdXAZcDewFjg0IgY26jNGOA7wC4ppS2Ar7XWb6YCKyJ6RkRV8edNImL/iKj5kNegJvTtUcPCd5fVHy96dzn9ejR8aUf17U6/HjU8OrthStWtSxWf2XIQN097rV3Gqs5j0fz59B8wuP6434CBLGqhgPrrpFvZdvwu7TE0dRJL3lxIzz7964979OnPu28uXKXds5Pv4JbvH82UP17Fjv99HACL584hgLt++j1uO/dEnrz75vYatlZDRLTpLYPxwIyU0syU0jLgRuCARm2+DFyWUnoDIKXU6r8osyZY9wPdImIYcDfwJeCa5hpHxLERMTUips6c/IeMT6GmBHDE+OFcN3XOKvcdvM0Q/vTMPJauqFv1gVI7mXz3JGY89wwHHnJ43kNRJ7T5J/bjcz+4iu0/cxSP33kjUJhenPviM+x29Cns+80f8Z9p/+LV6dNyHqnyUlqTFG/HNmoyDJhVcjy7eK7UJsAmEfFgRDxUnFJsUdY1WJFSWhIRxwCXp5QuiIhm360ppYnARICDr3k0ZXyOTmnRkuX06/nBwsu+PWtYuGR5/XH3mipG9O7OmXuNAaB39xq+/anRnH/vTMYM6MlOG/bmi9sPo2fXalKC5bWJv0yf3+7XocrSd8AAFsx/vf544fx59B0wcJV2j099mFuu/xX/d8mV1HR1AbHKp0fvfrz7xoL64yVvLKBn737Nth+9/W7867eXAdCzd38Gbbwl3dYtLLcYvuX2LHxlBkM326ZtB63V0tZrsEprkjXQBRgDfAIYDtwfEeNSSm+29IAsIiI+ChwGHFM854R2GcxY8C5Deq3DwHW7smjJcnYZ1YdL7n+5/v4ly+s45sYn6o/P3GsM102Zw8yFSzj9z8/Xnz94myEsXV5rcaWyGLPpFrw2exZzX5tD3/4DeeC+uzj5e+c2aDPzhelcftE5nHHBpfTu0zenkapS9d9gExbPe5W3F7xOj979mDn1fnY7+lsN2rw1bw7rDywEDbOemkKvgYVPsQ4buy1P3nMLK5Ytpaq6hteff4otPvWZdr8GrTXmACNKjocXz5WaDTycUloOvBQRz1MouKY012nWAutrFBZ3/TGl9HREjAb+lnXkal5dgl89NIvT9tyYqgj+NmMhs99cyn9vM4QXFy5h6qy38h6iOqHqLl348knf5qxTvkJtXR177L0/I0dtxA1X/ZyNNx3L+F1245qfX8zS95ZwwRmFX3oDBg3mtHMvznnkqhRV1dXsdMjx3P2z75Hq6hiz86fpM3QDHr3j1/QfOYaRW+/Es5Pv4LXp06iq7kLXHuvy8SO+AcA6Pddjy08dyB3nfQ0Ihm+5PSPGjc/3gtSsDvApwinAmIgYRaGwOgRo/AnBW4FDgasjoj+FKcOZLXUaKWWfwYuIdQFSSu9kfYxThOoozp6wad5DkOrd9uzrrTeS2smpu2+UW5XT7/DftmmdsPC6Q1u9tojYB7iYwuzcVSmlcyLibGBqSun2KFSBFwJ7AbXAOSmlG1vqM1OCFRHjgOuAvoXDmA8cnlJ6OsvjJUmSmpR7gAUppTuBOxudO73k5wScXLxlkvVThFcAJ6eUNkgpjQS+Afwy65NIkiR1JlnXYPVMKdWvuUopTY6Inm00JkmS1El0gDVYbSJrgTUzIr4P/Lp4/EVaWdwlSZLUWWUtsI4GzgL+QOE7eP5RPCdJkrTaOmWCFRHdgOOAjYEngW8U94CQJElSM1pLsK4FllNIrPYGNifDFxxKkiRl0SkTLGBsSmkcQET8Cnik7YckSZK0dmutwKqfDkwprajUKlOSJOWkQkuL1gqsrSNicfHnALoXj4PCvlu92nR0kiRJa6EWC6yUkl/oLEmS2kylzo5l3cldkiRJGWXdB0uSJKnsKjXBssCSJEm5qdQCyylCSZKkMjPBkiRJuTHBkiRJUiYmWJIkKT+VGWCZYEmSJJWbCZYkScqNa7AkSZKUiQmWJEnKjQmWJEmSMjHBkiRJuTHBkiRJUiYmWJIkKT+VGWCZYEmSJJWbCZYkScqNa7AkSZKUiQmWJEnKjQmWJEmSMjHBkiRJuTHBkiRJUiYmWJIkKTeVmmBZYEmSpPxUZn3lFKEkSVK5mWBJkqTcVOoUoQmWJElSmZlgSZKk3JhgSZIkKRMTLEmSlJsKDbBMsCRJksrNBEuSJOXGNViSJEnKxARLkiTlpkIDLBMsSZKkcjPBkiRJuXENliRJkjIxwZIkSbmp0ADLBEuSJKncTLAkSVJuqqoqM8IywZIkSSozEyxJkpQb12BJkiQpExMsSZKUm0rdB8sCS5Ik5aZC6yunCCVJksrNBEuSJOWmUqcITbAkSZLKzARLkiTlxgRLkiRJmZhgSZKk3FRogGWCJUmSVG4mWJIkKTeuwZIkSVImJliSJCk3FRpgmWBJkiSVmwmWJEnKjWuwJEmSlIkJliRJyk2FBlgmWJIkSeVmgiVJknLjGixJkiRlYoElSZJyE9G2t2xjiL0i4rmImBERpzZx/5ERMT8iphVv/9Nan04RSpKkTisiqoHLgD2B2cCUiLg9pfRMo6a/SymdkLVfCyxJkpSbDrAGazwwI6U0EyAibgQOABoXWB9KmxdYn916UFs/hZTJOffOyHsIUr1jx4/IewiSCoYBs0qOZwM7NtHuoIjYFXge+HpKaVYTbeq5BkuSJOWmrddgRcSxETG15HbsagzzDmDDlNJWwD3Ata09wClCSZKUm7aeIkwpTQQmttBkDlAaKQ8vnivtY2HJ4ZXABa09rwmWJEnqzKYAYyJiVER0BQ4Bbi9tEBFDSg73B55trVMTLEmSlJu817inlFZExAnAXUA1cFVK6emIOBuYmlK6HfhqROwPrAAWAUe21q8FliRJ6tRSSncCdzY6d3rJz98BvvNh+rTAkiRJuekA2zS0CddgSZIklZkJliRJyk2FBlgmWJIkSeVmgiVJknLjGixJkiRlYoIlSZJyY4IlSZKkTEywJElSbio0wDLBkiRJKjcTLEmSlBvXYEmSJCkTEyxJkpSbCg2wTLAkSZLKzQRLkiTlxjVYkiRJysQES5Ik5aZCAywLLEmSlJ+qCq2wnCKUJEkqMxMsSZKUmwoNsEywJEmSys0ES5Ik5cZtGiRJkpSJCZYkScpNVWUGWCZYkiRJ5WaCJUmScuMaLEmSJGVigiVJknJToQGWCZYkSVK5mWBJkqTcBJUZYZlgSZIklZkJliRJyo37YEmSJCkTEyxJkpQb98GSJElSJiZYkiQpNxUaYJlgSZIklZsJliRJyk1VhUZYFliSJCk3FVpfOUUoSZJUbiZYkiQpN27TIEmSpExMsCRJUm4qNMAywZIkSSo3EyxJkpSbSt2mwQRLkiSpzEywJElSbiozvzLBkiRJKjsTLEmSlBv3wZIkSVImJliSJCk3VZUZYJlgSZIklZsJliRJyo1rsCRJkpSJCZYkScpNhQZYJliSJEnlZoIlSZJy4xosSZIkZWKCJUmSclOp+2BZYEmSpNw4RShJkqRMTLAkSVJuKjO/MsGSJEkqOxMsSZKUmyrXYEmSJCkLEyxJkpSbCg2wTLAkSZLKzQRLkiTlxn2wJEmSlIkJliRJyk2FBlgmWJIkSeVmgiVJknJTqftgWWB1AC9Me4S/XHspdXV1bLv7Pnz8gC80uH/KPbcz5e7biKoqunbrzn5fPpmBwzdkydtvcdNPzmLOi9PZZrcJ7Hv0STldgSrFVkPW40s7DKUqgskzFnHH0/OabLfDiPU5abcN+f6dz/PSovcY3a87x+w4onBnwB+feJ2psxa348hViZ7697+48ZcXU1dXy8f33J+9Dz68wf133/pbHrj7dqqqq1mvV2+OPOk0+g0cAsDCea9z3c9+yKIFc4kIvnrGRfQfNCSPy1AnZYGVs7q6Wu686hK+dNqP6NVvAL/87vFsut3ODBy+YX2bcbt8ih323B+A6VMf5K5f/5wvfed8utR05ZOfP4p5s15i3qyXcroCVYoIOGL8MM67dyaLlizn7L3H8O/Zb/HqW+83aNetSxUTNuvPjPnv1p+b/eZSvv/n56lL0Lt7F87ZdxMenf0Mdam9r0KVoq62lht+cSFf/8El9Ok3kHNOPpqtd/w4Q0eOqm8zcvQmnHbR1azTrRuT7/wDt1x9Gf/v2/8HwFU/OZt9P38kYz8ynqXvLSHCFTEdVUcIsCJiL+ASoBq4MqV0XjPtDgJuAXZIKU1tqU/fcTmbM2M6fQcPo++goXTpUsOWO+/Oc1P/2aBNtx49639e/v5SovjVmF27dWeDzcbRpaZru45ZlWmjfj2Y+/Yy5r+zjNq6xEMvv8l2w9dfpd3nth7Mn56Zx/KS6mlZbaovpmqqqsDCSmvopReeYcCQ4QwYPIwuNTXssOseTHv4/gZtNttqO9bp1g2A0ZtuwRsLC4nrq6+8RF1tLWM/Mh6Abt171LeTGouIauAyYG9gLHBoRIxtot16wEnAw1n6zZRgRcTIps6nlF7J8ng1b/GiBfTqN7D+uFff/sye8ewq7R6561b+Nelmales4IjvX9ieQ1Qn0adHDYuWLKs/XrRkORv179GgzYZ9u9O3Zw3T5rzNvmMHNrhvo349+PJHR9C/Zw2/+OcrpldaI28unE/f/h+8x/r0G8hLzz/dbPsH7rmDLbf7KABz57xC957rcvm5p7Jg7qtsvvUOHHTE/1JVXd3m49aH1wH2wRoPzEgpzQSIiBuBA4BnGrX7AXA+cEqWTrMmWJOAPxX//15gJvDn5hpHxLERMTUipt77++szPoVaMn7CZzjpp79hjy8cy/1/9DVV+wvgsO2GcsO/X23y/hcXLuHUPz3H6X9+gf22GERNVe5/aaqTeOhvf+HlGdOZ8NnDgMLSixnPPM7BR5/IaRddxYLXX+XBeyflPErlpbQmKd6ObdRkGDCr5Hh28VxpH9sCI1JKmd9ImRKslNK4Jp7of1toPxGYCPDbx+b479gW9Orbn8ULP1hIvHjRAnr1HdBs+y13/iSTfnVxewxNncwbS5bTt8cH0819e9TwxpLl9cfdaqoYvn43TttzYwDW796Fkz8xiosmv8RLi96rb/fq4vdZuqKW4b27NTgvfRi9+w1g0YIP/m58Y+E8evdb9e/GZ6Y9wqSbruGUH15OTXG5RO9+AxkxagwDBhd+R26z067MfO6p9hm4PrS2XqtUWpOsjigs4LsIOPLDPG61riul9Ciw4+o8Vg0N3WgzFr4+hzfmvcaKFct56p/3sWkx5l5p4Wuz639+4bGH6DtkWONupDU2c+ESBq/XlQE9u1JdFey0YW8enf1W/f3vLa/j+Fue5uu3PsvXb32WFxcsqS+uBvTsysrAql/PGob26sb8d5c180xS6zYcsznzXp3F/NdfZcXy5Uy5/69sPf7jDdq88uJzXH/ZBZzw/R/Rq3ff+vOjxmzOknff4e233gBg+hP/brA4XmpkDjCi5Hh48dxK6wFbApMj4mVgJ+D2iNi+pU6zrsE6ueSwCtgWaHqeQB9KdXU1+xx1Ir8+99ukulo+8sm9GThiFPfddDVDR2/CZtvvwiN33crMp/5NVXUXuvdcjwOP/3b9439ywqG8/94SalcsZ/rUB/nSdy9o8AlEKau6BNdOmcO3PjWaqoC/v7iIOW+9z0FbDeKlRe/x6Ozmt13YZGBP9ttiFLV1iUTimkdm8877te04elWa6uoufOG4b3DxGV8j1dWxyx7/xbANRnPb9RPZYMzmbLPjx7nl6ktZunQJvzjvNAD6DRjECd//EVXV1Rx89Ilc+L0TISVGbrQZH//0ATlfkZrTAdZgTQHGRMQoCoXVIUD9fkkppbeA/iuPI2Iy8M3WPkUYKbU+gxcRZ5QcrgBeBn6fUlra2mOdIlRHMenpBXkPQap37PgRrTeS2smum/TNrcr56q3T27RO+OlnNmv12iJiH+BiCts0XJVSOicizgamppRub9R2MhkKrKxrsM4qdtojpbQky2MkSZJa0xE+D5NSuhO4s9G505tp+4ksfWZagxURH42IZ4DpxYO/FRIAABJdSURBVOOtI+LyLI+VJElqTlW07S2368rY7mJgArAQIKX0OLBrWw1KkiRpbZb5q3JSSrMaLURzBaskSVojHWCRe5vIWmDNioidgRQRNRS2il91u3FJkiRlLrCOo/AliMMofITxbuArbTUoSZLUOXSERe5tIeunCBcAh7XxWCRJkipCiwVWRDT5EcWilFL6QZnHI0mSOpEKXYLVaoL1bhPnegLHAP0ofLO0JEmSSrRYYKWULlz5c0SsR2Fx+1HAjcCFzT1OkiQpi6oKjbBaXYMVEX2BkymswboW2Dal9EZbD0ySJGlt1doarB8BnwUmAuNSSu+0y6gkSVKnkHXH87VNa9f1DWAo8D3g1YhYXLy9HRGL2354kiRJa5/W1mBVamEpSZI6gApdglWxyZwkSVJuMn8XoSRJUrlV6qcITbAkSZLKzARLkiTlpkIDLBMsSZKkcjPBkiRJuakywZIkSVIWJliSJCk3lfopQgssSZKUmwqtr5wilCRJKjcTLEmSlBsXuUuSJCkTEyxJkpSboDIjLBMsSZKkMjPBkiRJuXENliRJkjIxwZIkSbkxwZIkSVImJliSJCk3UaFbuZtgSZIklZkJliRJyo1rsCRJkpSJCZYkScpNhS7BMsGSJEkqNxMsSZKUm6oKjbBMsCRJksrMBEuSJOXGTxFKkiQpExMsSZKUmwpdgmWBJUmS8lNFZVZYThFKkiSVmQmWJEnKTaVOEZpgSZIklZkJliRJyo3bNEiSJCkTEyxJkpQbvypHkiRJmZhgSZKk3FRogGWCJUmSVG4mWJIkKTeuwZIkSVImJliSJCk3FRpgmWBJkiSVmwmWJEnKTaUmPZV6XZIkSbkxwZIkSbmJCl2EZYIlSZJUZiZYkiQpN5WZX5lgSZIklZ0JliRJyk2l7uRugSVJknJTmeWVU4SSJEllZ4IlSZJyU6EzhCZYkiRJ5WaCJUmScuNGo5IkScrEBEuSJOWmUpOeSr0uSZKk3JhgSZKk3LgGS5IkqQJFxF4R8VxEzIiIU5u4/7iIeDIipkXEAxExtrU+LbAkSVJuoo1vrT5/RDVwGbA3MBY4tIkC6oaU0riU0jbABcBFrfVrgSVJkjqz8cCMlNLMlNIy4EbggNIGKaXFJYc9gdRap22+Bmubwb3b+imkTA4cNyzvIUj1+uxwQt5DkOq999iluT13B1iDNQyYVXI8G9ixcaOI+ApwMtAV2L21Tk2wJElSxYqIYyNiasnt2NXpJ6V0WUppI+DbwPdaa++nCCVJUm7aOulJKU0EJrbQZA4wouR4ePFcc24Eft7a85pgSZKkzmwKMCYiRkVEV+AQ4PbSBhExpuRwX+CF1jo1wZIkSbnJew1WSmlFRJwA3AVUA1ellJ6OiLOBqSml24ETImIPYDnwBnBEa/1aYEmSpE4tpXQncGejc6eX/HzSh+3TAkuSJOUm988QthHXYEmSJJWZCZYkScpN/ttgtQ0LLEmSlJuqCp0kdIpQkiSpzEywJElSbip1itAES5IkqcxMsCRJUm7CNViSJEnKwgRLkiTlxjVYkiRJysQES5Ik5cZ9sCRJkpSJCZYkScqNa7AkSZKUiQmWJEnKjQmWJEmSMjHBkiRJuXEnd0mSJGVigiVJknJTVZkBlgmWJElSuZlgSZKk3LgGS5IkSZmYYEmSpNy4D5YkSZIyMcGSJEm5qdQ1WBZYkiQpN27TIEmSpExMsCRJUm4qdYrQBEuSJKnMTLAkSVJu3KZBkiRJmZhgSZKk3FRogGWCJUmSVG4mWJIkKTdVFboIywRLkiSpzEywJElSbiozvzLBkiRJKjsTLEmSlJ8KjbBMsCRJksrMBEuSJOXG7yKUJElSJiZYkiQpNxW6DZYJliRJUrmZYEmSpNxUaIBlgiVJklRuJliSJCk/FRphWWBJkqTcuE2DJEmSMjHBkiRJuXGbBkmSJGVigiVJknJToQGWCZYkSVK5mWBJkqT8VGiEZYIlSZJUZiZYkiQpN+6DJUmSpExMsCRJUm7cB0uSJEmZmGBJkqTcVGiAZYIlSZJUbiZYkiQpPxUaYZlgSZIklZkJliRJyo37YEmSJCkTEyxJkpQb98GSJElSJiZYkiQpNxUaYFlgSZKkHFVoheUUoSRJUpmZYEmSpNy4TYMkSZIyMcHqYB59+EGuvPTH1NXWsue+B3LQYUc1uP+2m67nnkl/pLq6ml69+3Dit85g4OChOY1WlezBf9zP+eedQ11tHQcedDDHfPnYBvf/e+oULjjvXF54/jnO/9FF7Dlhr5xGqs5gz50358enfI7qqiquufWf/Pjqexrcf8E3PsuuO2wCQI9uXRnQd12G7PqtPIaqD6lSt2mwwOpAamtrueKS8znrx5fTb8AgTjnui4zfZTdGbDi6vs3oMZty4RXXs0637vz5tpu59opLOOWM83MctSpRbW0t555zNlf88moGDRrEF/77c3zik7uz0cYb17cZPGQIPzjnh1x7zVU5jlSdQVVVcPGpn2ff4y9lztw3eeA3p/Cnvz/J9Jmv17f51oV/qP/5+EN2Y+tNh+cxVK2lImIv4BKgGrgypXReo/tPBv4HWAHMB45OKf2npT6dIuxAXpj+FEOGDWfw0OHU1NTwsd0n8PCDkxu0GfeRHVinW3cANh07joXz5+UwUlW6p558ghEjNmD4iBHUdO3KXvvsy+S/3dugzbBhw9lk082oCv8aUdvaYcsNeXHWAl6es5DlK2q5+a5H+a9PbNVs+8/vtR03/eXf7ThCrYlo41urzx9RDVwG7A2MBQ6NiLGNmj0GbJ9S2gq4BbigtX4z/c0YEX2buNVkeayyWzR/Pv0HDK4/7jdgIItaKKD+OulWth2/S3sMTZ3MvLlzGTzkg/fiwEGDmDt3bo4jUmc2dOD6zJ77Rv3xnLlvMGzA+k22HTmkDxsM7cfkKc+11/C09hsPzEgpzUwpLQNuBA4obZBS+ltKaUnx8CGg1Yg06z89H6UQiT0PvFD8+eWIeDQitmvcOCKOjYipETH1puudPmgLk++exIznnuHAQw7PeyiS1GEcPGE7br13GnV1Ke+hKKs2jrBKa5LireGCUhgGzCo5nl0815xjgD+3dllZ12DdA9ySUroLICI+DRwEXA1cDuxY2jilNBGYCPDsa+/6Ls+o74ABLJj/wZqChfPn0XfAwFXaPT71YW65/lf83yVXUtO1a3sOUZ3EwEGDeP21D96L8+bOZdCgQTmOSJ3Zq/PeYvigPvXHwwb1Yc78t5ps+7kJ2/H1825qr6FpLVBak6ypiPgisD2wW2ttsyZYO60srgBSSncDH00pPQSss1qj1CrGbLoFr82exdzX5rB8+XIeuO8uxu/c8M9w5gvTufyic/juuRfTu0/fnEaqSrfFluN45ZWXmT17FsuXLeMvd05it0/unvew1ElNffo/bDxyABsM7UdNl2oOnrAtkyY/sUq7TTYcRJ9ePXjo8ZdyGKVWV7Tx/zKYA4woOR5ePNdwnBF7AKcB+6eU3m+t06wJ1msR8W0K85IA/w3MLS4Mq8vYh1pR3aULXz7p25x1yleoratjj733Z+Sojbjhqp+z8aZjGb/Lblzz84tZ+t4SLjij8PHjAYMGc9q5F+c8clWaLl268J3TTuf4Y/+HurpaPnPgQWy88Rgu+9klbLHFlnxi90/x1JNP8PWTTmDx4sX8ffLfuPyyn/HH2yflPXRVoNraOr5+/k3ccflXqK4Krr3tIZ6d+TrfP35fHn3mFSb9/UmgMD14810ubteHNgUYExGjKBRWhwBfKG0QER8BrgD2Sill+nRZpNT6DF5E9AfOAD4GJOBB4GzgLWBkSmlGc491ilAdxagBPfMeglSvzw4n5D0Eqd57j12a225Uz72+pE3rhE0H92j12iJiH+BiCts0XJVSOicizgamppRuj4i/AuOA14oPeSWltH9LfWZKsFJKC4ATI6JnSundRnc3W1xJkiR1dCmlO4E7G507veTnPT5sn1m3adg5Ip4Bni0ebx0Rl3/YJ5MkSSqV9z5YbSXrIvefABOAhQAppceBXdtqUJIkSWuzzF+Vk1KaFQ2/MKi2/MORJEmdSif/LsJZEbEzkIo7uJ9EcbpQkiRJDWUtsI6j8CWIwyh8hPFu4CttNShJktQ5ZNyraq3zYT5FeFgbj0WSJKkitFhgRcTpLdydUko/KPN4JElSJxKVGWC1mmA13vMKoCeFLzrsB1hgSZKk1Vah9VXLBVZK6cKVP0fEehQWtx9F4StzLmzucZIkSZ1Zq2uwIqIvcDKFNVjXAtumlN5o64FJkqROoEIjrNbWYP0I+CwwERiXUnqnXUYlSZK0FmstwfoG8D7wPeC0ko1Gg8Ii915tODZJklThOuU2DSmlrF+lI0mSpKLMX5UjSZJUbpW6TYMJlSRJUpmZYEmSpNxUaIBlgiVJklRuJliSJCk/FRphmWBJkiSVmQmWJEnKTaXug2WCJUmSVGYmWJIkKTfugyVJkqRMTLAkSVJuKjTAMsGSJEkqNxMsSZKUG9dgSZIkKRMTLEmSlKPKjLBMsCRJksrMBEuSJOWmUtdgWWBJkqTcVGh95RShJElSuZlgSZKk3FTqFKEJliRJUpmZYEmSpNxEha7CMsGSJEkqMxMsSZKUn8oMsEywJEmSys0ES5Ik5aZCAywTLEmSpHIzwZIkSblxHyxJkiRlYoIlSZJy4z5YkiRJysQES5Ik5acyAywTLEmSpHIzwZIkSbmp0ADLBEuSJKncTLAkSVJu3AdLkiRJmZhgSZKk3FTqPlgWWJIkKTdOEUqSJCkTCyxJkqQys8CSJEkqM9dgSZKk3LgGS5IkSZmYYEmSpNxU6jYNJliSJEllZoIlSZJy4xosSZIkZWKCJUmSclOhAZYJliRJUrmZYEmSpPxUaIRlgiVJklRmJliSJCk37oMlSZKkTEywJElSbtwHS5IkSZmYYEmSpNxUaIBlgiVJklRuJliSJCk/FRphmWBJkqTcRBv/L9MYIvaKiOciYkZEnNrE/btGxKMRsSIiPpelTwssSZLUaUVENXAZsDcwFjg0IsY2avYKcCRwQ9Z+nSKUJEm56QDbNIwHZqSUZgJExI3AAcAzKxuklF4u3leXtVMTLEmS1JkNA2aVHM8unlsjbZ5gbT6kZ/61aQWIiGNTShPzHocEvh/L4b3HLs17CBXB9+Lar1uXtl3mHhHHAseWnJrYHu8ZE6y1x7GtN5Haje9HdRS+F9WilNLElNL2JbfGxdUcYETJ8fDiuTVigSVJkjqzKcCYiBgVEV2BQ4Db17RTCyxJktRppZRWACcAdwHPAjellJ6OiLMjYn+AiNghImYDBwNXRMTTrfUbKaW2HLfKxHUG6kh8P6qj8L2ojsoCS5IkqcycIpQkSSozC6wOJCJqI2JaRDwVETdHRI+8x6TOJyJSRFxYcvzNiDgzxyGpk/K9qLWZBVbH8l5KaZuU0pbAMuC4vAekTul94LMR0T/vgajT872otZYFVsf1D2DjiOgbEbdGxBMR8VBEbAUQEbsV065pEfFYRKyX83hVOVYAE4GvN74jIgZExO8jYkrxtkvx/JkR8c2Sdk9FxIbtNWBVrNV5Lw6IiHsi4umIuDIi/mOBpjxYYHVAEdGFwpdOPgmcBTyWUtoK+C5wXbHZN4GvpJS2AT4OvJfHWFWxLgMOi4j1G52/BPhJSmkH4CDgynYfmTqbD/tePAO4L6W0BXALMLLdRiqV8MueO5buETGt+PM/gF8BD1P4y4OU0n0R0S8iegEPAhdFxG+AP6SUZucyYlWklNLiiLgO+CoNi/c9gLHxwbez9oqIddt7fOo8VuO9+DHgwOJj/xIRb7TneKWVLLA6lveKiVS9aOZrxlNK50XEJGAf4MGImJBSmt4OY1TncTHwKHB1ybkqYKeU0tLShhGxgoaJeLe2H546kQ/zXmzPcUnNcoqw4/sHcBhARHwCWFD8F91GKaUnU0rnU9jmf7Mcx6gKlFJaBNwEHFNy+m7gxJUHEbHyHwQvA9sWz20LjGqfUaoz+JDvxQeBzxfPfRro007DlBqwwOr4zgS2i4gngPOAI4rnv1ZcSPwEsBz4c07jU2W7EChdIPxVYPvihy6e4YNPuv4e6Fv8+ogTgOfbd5jqBLK+F88CPh0RT1H4WpPXgbfbdaQS7uQuSaogEbEOUJtSWhERHwV+3njphdQeXIMlSaokI4GbIqKKwn6CX855POqkTLAkSZLKzDVYkiRJZWaBJUmSVGYWWJIkSWVmgSVJklRmFliSJEllZoElSZJUZv8fCn5kp3+0BOQAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"id":"ZO8YGzMpmhLP","colab_type":"code","colab":{}},"source":["### Remove last classifier head\n","\n","class Identity(nn.Module):\n"," def __init__(self):\n"," super(Identity, self).__init__()\n"," \n"," def forward(self, x):\n"," return x\n","\n","best_model.classification_head = Identity()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aswT5XE8yPH3","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1592540677654,"user_tz":420,"elapsed":625,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"4f9f39d3-ef94-4cbe-8de6-7f0c8cb1c330"},"source":["print(best_model)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["TransformerWithClfHead(\n"," (transformer): Transformer(\n"," (tokens_embeddings): Embedding(28996, 410)\n"," (position_embeddings): Embedding(256, 410)\n"," (dropout): Dropout(p=0.05, inplace=False)\n"," (attentions): ModuleList(\n"," (0): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (1): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (2): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (3): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (4): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (5): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (6): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (7): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (8): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (9): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (10): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (11): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (12): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (13): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (14): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," (15): MultiheadAttention(\n"," (out_proj): Linear(in_features=410, out_features=410, bias=True)\n"," )\n"," )\n"," (feed_forwards): ModuleList(\n"," (0): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (1): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (2): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (3): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (4): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (5): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (6): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (7): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (8): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (9): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (10): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (11): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (12): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (13): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (14): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," (15): Sequential(\n"," (0): Linear(in_features=410, out_features=2100, bias=True)\n"," (1): ReLU()\n"," (2): Linear(in_features=2100, out_features=410, bias=True)\n"," )\n"," )\n"," (layer_norms_1): ModuleList(\n"," (0): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (1): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (2): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (3): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (4): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (5): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (6): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (7): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (8): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (9): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (10): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (11): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (12): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (13): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (14): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (15): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," )\n"," (layer_norms_2): ModuleList(\n"," (0): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (1): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (2): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (3): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (4): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (5): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (6): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (7): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (8): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (9): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (10): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (11): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (12): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (13): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (14): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," (15): LayerNorm((410,), eps=1e-12, elementwise_affine=True)\n"," )\n"," )\n"," (classification_head): Identity()\n",")\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Iiu3bb4AyohQ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592540607376,"user_tz":420,"elapsed":483,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"fd833c62-6a11-494b-afc5-acd27fae960f"},"source":["int2label"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{0: '2', 1: '3', 2: '1'}"]},"metadata":{"tags":[]},"execution_count":79}]},{"cell_type":"code","metadata":{"id":"AMmh61AOztF2","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":120,"referenced_widgets":["ef85d36f743948aa969386638a17e2d1","390d96f81861494e8fc13dcc153698e1","d20001c9b9134e86a5f9f82d1a93d7ea","159ec926f06746198b32441468b49aeb","9df8000b43f64195a492849d0815d1e7","70bcdef81fd54fe0ac1e4dd395da55f8","c13224a053e345c393ad1e8180523658","a17d61b8448b471c96c9d18cdb853b56"]},"executionInfo":{"status":"ok","timestamp":1592541395537,"user_tz":420,"elapsed":5347,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"868dc18f-77c3-4244-c29a-177eadff471d"},"source":["### Create the embeddings for the val\n","\n","\n","valid_dl_eval = create_dataloader(df_val, processor, \n"," batch_size=finetuning_config.batch_size, \n"," valid_pct=None,\n"," shuffle=False)\n","\n","pred_labels = torch.zeros((len(df_val), 410))\n","\n","\n","i = 0\n","for batch in valid_dl_eval:\n"," actual_labels.extend(batch[1].tolist())\n"," \n"," best_model.eval()\n"," with torch.no_grad():\n"," batch, labels = (t.to(finetuning_config.device) for t in batch)\n"," inputs = batch.transpose(0, 1).contiguous()\n"," logits = best_model(inputs,\n"," clf_tokens_mask = (inputs == tokenizer.vocab[processor.CLS]),\n"," padding_mask = (batch == tokenizer.vocab[processor.PAD]))\n"," begin_index = i * 32\n"," end_index = begin_index + len(labels)\n"," pred_labels[begin_index:end_index, :] = logits\n"," i += 1\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:26: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n","Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ef85d36f743948aa969386638a17e2d1","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Processing 766 examples on 2 cores', max=766.0, style=Pro…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"q-XDr6Lr3Q6Z","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592541395543,"user_tz":420,"elapsed":2064,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"6cb8f32a-8642-498a-a7ba-e30df02c5891"},"source":["pred_labels.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([766, 410])"]},"metadata":{"tags":[]},"execution_count":101}]},{"cell_type":"code","metadata":{"id":"3vUSB4A2229T","colab_type":"code","colab":{}},"source":["val_output = {\n"," \"embedding\": pred_labels.numpy(),\n"," \"vid_names\": df_val[\"vid_name\"].tolist()\n","}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"FXwxF9D-1vKo","colab_type":"code","colab":{}},"source":["import pickle\n","with open(\"transformer_val_embeddings.pkl\", \"wb\") as f:\n"," pickle.dump(val_output, f)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XI7_jC2f3JEv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":136},"executionInfo":{"status":"ok","timestamp":1592541423297,"user_tz":420,"elapsed":1737,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"49f2408a-73a8-4070-a3d8-bee207d1c7a7"},"source":["!ls -lh"],"execution_count":null,"outputs":[{"output_type":"stream","text":["total 13M\n","drwx------ 5 root root 4.0K Jun 19 03:04 drive\n","drwxr-xr-x 3 root root 4.0K Jun 19 04:10 logs\n","drwxr-xr-x 1 root root 4.0K Jun 17 16:18 sample_data\n","-rw------- 1 root root 9.2M Jun 19 03:04 train_captions_pretrained_v1.pkl\n","-rw-r--r-- 1 root root 1.3M Jun 19 04:36 transformer_train_processed.pkl\n","-rw------- 1 root root 2.6M Jun 19 03:04 val_captions_pretrained_v1.pkl\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Snss0UED3Z69","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":120,"referenced_widgets":["8f1228ecc42346ad8b6dbb658673e092","379d1b7a433643b2a498157245618f0d","621bc7571ecc4493bf55a3778e3183e0","dccdda5805474198821bc5b219f3ce49","78e91ec9134a4c62957b6e88b6e35c7e","c88ca7fe7bcb4e299e257f64cba98d6a","e03b4afdc54c4b3ca79b82ea16fe6739","9086f42d12034cbd8f1c50dee4072ddd"]},"executionInfo":{"status":"ok","timestamp":1592541499729,"user_tz":420,"elapsed":17586,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"1c13c0bd-8c16-4d0c-8e19-81c2371ef4d7"},"source":["### Create the embeddings for the train\n","\n","\n","train_dl_eval = create_dataloader(df_train, processor, \n"," batch_size=finetuning_config.batch_size, \n"," valid_pct=None,\n"," shuffle=False)\n","\n","pred_labels = torch.zeros((len(df_train), 410))\n","\n","\n","i = 0\n","for batch in train_dl_eval:\n"," actual_labels.extend(batch[1].tolist())\n"," \n"," best_model.eval()\n"," with torch.no_grad():\n"," batch, labels = (t.to(finetuning_config.device) for t in batch)\n"," inputs = batch.transpose(0, 1).contiguous()\n"," logits = best_model(inputs,\n"," clf_tokens_mask = (inputs == tokenizer.vocab[processor.CLS]),\n"," padding_mask = (batch == tokenizer.vocab[processor.PAD]))\n"," begin_index = i * 32\n"," end_index = begin_index + len(labels)\n"," pred_labels[begin_index:end_index, :] = logits\n"," i += 1\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:26: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n","Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8f1228ecc42346ad8b6dbb658673e092","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Processing 2661 examples on 2 cores', max=2661.0, style=P…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ZAUczWxn3oxA","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592541507893,"user_tz":420,"elapsed":343,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"b1a298b6-838b-4ad0-8d00-53a9de1b0768"},"source":["pred_labels.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([2661, 410])"]},"metadata":{"tags":[]},"execution_count":106}]},{"cell_type":"code","metadata":{"id":"6Bw7nDVa3st2","colab_type":"code","colab":{}},"source":["train_output = {\n"," \"embedding\": pred_labels.numpy(),\n"," \"vid_names\": df_train[\"vid_name\"].tolist()\n","}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RDbaWn7D3vyt","colab_type":"code","colab":{}},"source":["import pickle\n","with open(\"transformer_train_embeddings.pkl\", \"wb\") as f:\n"," pickle.dump(train_output, f)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gxHHGdJV38EQ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":153},"executionInfo":{"status":"ok","timestamp":1592541576289,"user_tz":420,"elapsed":1617,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"9db1d2b5-d7a7-442c-a65b-90f0d8af82cb"},"source":["!ls -lh"],"execution_count":null,"outputs":[{"output_type":"stream","text":["total 18M\n","drwx------ 5 root root 4.0K Jun 19 03:04 drive\n","drwxr-xr-x 3 root root 4.0K Jun 19 04:10 logs\n","drwxr-xr-x 1 root root 4.0K Jun 17 16:18 sample_data\n","-rw------- 1 root root 9.2M Jun 19 03:04 train_captions_pretrained_v1.pkl\n","-rw-r--r-- 1 root root 4.2M Jun 19 04:38 transformer_train_embeddings.pkl\n","-rw-r--r-- 1 root root 1.3M Jun 19 04:36 transformer_val_embeddings.pkl\n","-rw------- 1 root root 2.6M Jun 19 03:04 val_captions_pretrained_v1.pkl\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"8p1HuE873_R1","colab_type":"code","colab":{}},"source":["!cp transformer_train_embeddings.pkl drive/'My Drive'/cs231n-project/\n","!cp transformer_val_embeddings.pkl drive/'My Drive'/cs231n-project/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"h3LwnvQU4Glt","colab_type":"code","colab":{}},"source":[""],"execution_count":null,"outputs":[]}]}