Fine-tuning_XLNet_tokenizer.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "provenance": [
    {
     "file_id": "1fDa6mqXRySSq3cBtB-JXI0EI1dOmRAv3",
     "timestamp": 1669469254618
    }
   ],
   "authorship_tag": "ABX9TyPD3OO2+bBdNQFo070bWbeF"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  },
  "accelerator": "GPU",
  "gpuClass": "standard",
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "6033ebb4dd414a2d945164c3619c1a2b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7e0c2ce80082435a86007dd3fa7d6994",
       "IPY_MODEL_3d4c839dbd35464bb90cddd8f4f339c8",
       "IPY_MODEL_03dde8900cdb4ef49a0b271977d05aa2"
      ],
      "layout": "IPY_MODEL_e9802b38663c4acb9fac55c9d4338b72"
     }
    },
    "7e0c2ce80082435a86007dd3fa7d6994": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_021118be3912493e9a41823571bf34c0",
      "placeholder": "​",
      "style": "IPY_MODEL_96a225a9814e4163b984e31594f5b7db",
      "value": "Downloading: 100%"
     }
    },
    "3d4c839dbd35464bb90cddd8f4f339c8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_85b14bf03b464dafa635843772788d31",
      "max": 798011,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8d76864564da4c2c86e8541c1b13cb6c",
      "value": 798011
     }
    },
    "03dde8900cdb4ef49a0b271977d05aa2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3e4dbd03dc58418ab4eac9427881035c",
      "placeholder": "​",
      "style": "IPY_MODEL_056b74eceac343f8a56eae8638f792f5",
      "value": " 798k/798k [00:00&lt;00:00, 1.90MB/s]"
     }
    },
    "e9802b38663c4acb9fac55c9d4338b72": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "021118be3912493e9a41823571bf34c0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "96a225a9814e4163b984e31594f5b7db": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "85b14bf03b464dafa635843772788d31": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8d76864564da4c2c86e8541c1b13cb6c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3e4dbd03dc58418ab4eac9427881035c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "056b74eceac343f8a56eae8638f792f5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## 1 Importing the Pre-Processed Dataset\n",
    "\n",
    "As the dataset has been separated to 4 parts, we need to reread them from files:\n",
    "\n",
    "- X_train (training variables of the dataset)\n",
    "- X_val (validation variables of the dataset)\n",
    "- y_train (training labels of the dataset)\n",
    "- y_val (validation labels of the dataset)"
   ],
   "metadata": {
    "id": "gPJUvMopwS6U"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "import pandas as pd"
   ],
   "metadata": {
    "id": "lDJgCJwYwSYK"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "X_train = pd.read_csv('X_train.csv')\n",
    "X_val = pd.read_csv('X_val.csv')\n",
    "y_train = pd.read_csv('y_train.csv')\n",
    "y_val = pd.read_csv('y_val.csv')"
   ],
   "metadata": {
    "id": "fyWvX_PVwXe7"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "Because we are only attempting to classify the ```True``` from the ```False``` by the text. Then we should select the variable \"text\" from X_train and X_val, and select the variable \"target\" from y_train and y_val."
   ],
   "metadata": {
    "id": "m2EU3C9XDe9u"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "train_text = X_train['cleaned_text'].to_list()\n",
    "train_label = y_train['target'].to_list()\n",
    "val_text = X_val['cleaned_text'].to_list()\n",
    "val_label = y_val['target'].to_list()"
   ],
   "metadata": {
    "id": "IjMN9ZcYwYQ5"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "train_text[:5]"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vbPqjNZ5Dg0e",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469721701,
     "user_tz": -480,
     "elapsed": 3,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "05e39b06-b90e-4f67-92fa-53764b36b7a3"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['  jimmyfallon crush squirrel bone mortar pestl school  bio dept  realli sure whi worstsummerjob',\n",
       " ' mccainenl think spectacular look stonewal riot obliter white house ',\n",
       " 'can t bloodi wait   soni set date stephen king       the dark tower    stephenk thedarktow    bdisgust',\n",
       " 'protest ralli stone mountain  atleast they r burn build loot store like individu  protest ',\n",
       " ' rbcinsur quot websit   disaster  tri 3 browser  amp  3 machines  alway get  miss info  error due non exist drop down ']"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "train_label[:5]"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vpCQagIsDjeu",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469722422,
     "user_tz": -480,
     "elapsed": 4,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "0b8988ba-0bad-4a0d-b837-c1bb52dbf960"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[0, 1, 0, 0, 0]"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 2 Tokenize the text by the pre-trained model XLNet\n",
    "\n",
    "In this notebook, we choose a pre-trained model named XLNet-base to tokenize the text from the dataset.\n",
    "\n",
    "Then, we should download the pre-trained model first:"
   ],
   "metadata": {
    "id": "7FQblnxuDnyE"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "!pip install transformers\n",
    "!pip install sentencepiece"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hZcwBrl6Ew-Z",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469738703,
     "user_tz": -480,
     "elapsed": 13523,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "ee73260c-da8e-4983-a13c-e3633c4042ca"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.24.0)\n",
      "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.11.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.2)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.97)\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "from transformers import XLNetTokenizer\n",
    "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "6033ebb4dd414a2d945164c3619c1a2b",
      "7e0c2ce80082435a86007dd3fa7d6994",
      "3d4c839dbd35464bb90cddd8f4f339c8",
      "03dde8900cdb4ef49a0b271977d05aa2",
      "e9802b38663c4acb9fac55c9d4338b72",
      "021118be3912493e9a41823571bf34c0",
      "96a225a9814e4163b984e31594f5b7db",
      "85b14bf03b464dafa635843772788d31",
      "8d76864564da4c2c86e8541c1b13cb6c",
      "3e4dbd03dc58418ab4eac9427881035c",
      "056b74eceac343f8a56eae8638f792f5"
     ]
    },
    "id": "tYX0kbWwDkbu",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469743917,
     "user_tz": -480,
     "elapsed": 5221,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "5130a11b-20e1-4881-f77e-e2ab5cc15fda"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "Downloading:   0%|          | 0.00/798k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "6033ebb4dd414a2d945164c3619c1a2b"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Apply the pre-trained model to the text by the tokenizer."
   ],
   "metadata": {
    "id": "7yeTShTZE_Qo"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "train_encodings = tokenizer(train_text, truncation=True, padding=True)\n",
    "val_encodings = tokenizer(val_text, truncation=True, padding=True)"
   ],
   "metadata": {
    "id": "Lvnft0D1DyhE",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469747033,
     "user_tz": -480,
     "elapsed": 1297,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "9dee78fe-8cad-4de7-f316-85f7f4640c17"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 3 Defining the Dataset Class\n",
    "\n",
    "Before we input the dataset into Neural Network, we should define a Class then instantiate it to store the encodings and labels of the data."
   ],
   "metadata": {
    "id": "P6uBIqmeFFEX"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "import torch\n",
    "\n",
    "class DatasetClass(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = DatasetClass(train_encodings, train_label)\n",
    "val_dataset = DatasetClass(val_encodings, val_label)"
   ],
   "metadata": {
    "id": "FT-BXRI9FEuf"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "train_dataset.__getitem__(1)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "y5Q0LR0mFTGl",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469754967,
     "user_tz": -480,
     "elapsed": 4,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "c61fd313-b49c-412b-c776-be868d5d1ec1"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'input_ids': tensor([    5,     5,     5,     5,     5,     5,     5,     5,     5,     5,\n",
       "             5,     5,     5,     5,     5,     5,     5,     5,     5,     5,\n",
       "             5,     5,     5,     5,     5,     5,     5,     5,     5,     5,\n",
       "             5,     5,     5,     5,     5,     5,     5,     5,     5,     5,\n",
       "             5,     5,     5,     5,     5,     5,     5,     5,    17,    98,\n",
       "          8664,   153,   254,   368,   232,  8073,   338,  3085,  9760, 10666,\n",
       "          6837,  9803,   817,   480,     4,     3]),\n",
       " 'token_type_ids': tensor([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "         3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]),\n",
       " 'attention_mask': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]),\n",
       " 'labels': tensor(1)}"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 4 Defining Evaluation Matrics"
   ],
   "metadata": {
    "id": "RUpEerWJFYcm"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "\n",
    "def evaluation_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\n",
    "        'accuracy': acc,\n",
    "        'f1': f1,\n",
    "        'precision': precision,\n",
    "        'recall': recall\n",
    "    }"
   ],
   "metadata": {
    "id": "SxCSfNXCFYRO"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 5 Fine-tuning with Huggingface Trainer\n",
    "\n",
    "Huggingface Trainer is a highly packaged trainer, we need to define the following arguments before we train:\n",
    "\n",
    "- **args** -> TrainingArguments (Contains the definition of hyperparameters, which is also an important feature of the trainer, where most of the training-related parameters are set)\n",
    "- **model** -> Model (is a model that integrates *transformers.PreTrainedMode* or *torch.nn.module*, which is officially mentioned as being optimised by Trainer for transformers.PreTrainedModel)\n",
    "- **compute_metrics** -> Evaluation Metrics (to define how to evaluate the results of the fine-tuned model)\n",
    "- **train_dataset** -> Train Dataset\n",
    "- **eval_dataset** -> Validation Dataset"
   ],
   "metadata": {
    "id": "PQ0ABkeWFd8H"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'\n",
    "device"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "ttpZNmq3FVUt",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469761819,
     "user_tz": -480,
     "elapsed": 433,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "0baf8831-47ad-4a38-ba2d-885d7fabe48e"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'cuda'"
      ],
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      }
     },
     "metadata": {},
     "execution_count": 12
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "from transformers import XLNetForSequenceClassification, Trainer, TrainingArguments\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./XLNet-results',# output directory\n",
    "    num_train_epochs=3,              # total number of training epochs\n",
    "    per_device_train_batch_size=16,  # batch size per device during training\n",
    "    per_device_eval_batch_size=64,   # batch size for evaluation\n",
    "    warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
    "    weight_decay=0.01,               # strength of weight decay\n",
    "    logging_dir='./logs',            # directory for storing logs\n",
    "    logging_steps=10,\n",
    ")\n",
    "\n",
    "model = XLNetForSequenceClassification.from_pretrained(\"xlnet-base-cased\")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,                         # the instantiated pre-trained model to be trained\n",
    "    args=training_args,                  # training arguments, defined above\n",
    "    compute_metrics=evaluation_metrics,\n",
    "    train_dataset=train_dataset,         # training dataset\n",
    "    eval_dataset=val_dataset,             # validation dataset\n",
    ")"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "c5YphTjDFgZC",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669469776014,
     "user_tz": -480,
     "elapsed": 9938,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "c2b3695c-9563-474a-dceb-75a701e97555"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n",
      "Some weights of the model checkpoint at xlnet-base-cased were not used when initializing XLNetForSequenceClassification: ['lm_loss.weight', 'lm_loss.bias']\n",
      "- This IS expected if you are initializing XLNetForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLNetForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLNetForSequenceClassification were not initialized from the model checkpoint at xlnet-base-cased and are newly initialized: ['logits_proj.bias', 'sequence_summary.summary.weight', 'logits_proj.weight', 'sequence_summary.summary.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "trainer.train()"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "45zWFXIPF5m4",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669470100981,
     "user_tz": -480,
     "elapsed": 321095,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "a157f261-5169-4bb9-c87f-641ebf761098"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n",
      "***** Running training *****\n",
      "  Num examples = 6090\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 1143\n",
      "  Number of trainable parameters = 117310466\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1143' max='1143' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1143/1143 05:17, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.715400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.693400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.736400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.713200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.705400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.677200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>0.705800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.702500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>0.669000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.703800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>0.684600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.647900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>0.679100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.624700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.544200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.524300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>0.609700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.612700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>0.588000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.513300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>0.578900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.499500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>230</td>\n",
       "      <td>0.471400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>0.565600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>0.566600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>0.536200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>270</td>\n",
       "      <td>0.511700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>0.422800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>290</td>\n",
       "      <td>0.527800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.407700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>310</td>\n",
       "      <td>0.534700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320</td>\n",
       "      <td>0.507100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>330</td>\n",
       "      <td>0.452300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>340</td>\n",
       "      <td>0.633900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>0.600100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>360</td>\n",
       "      <td>0.486600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>370</td>\n",
       "      <td>0.452800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>380</td>\n",
       "      <td>0.469200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>390</td>\n",
       "      <td>0.428300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.448000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>410</td>\n",
       "      <td>0.547700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>420</td>\n",
       "      <td>0.479200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>430</td>\n",
       "      <td>0.465700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>440</td>\n",
       "      <td>0.426000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>0.432500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>460</td>\n",
       "      <td>0.451200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>470</td>\n",
       "      <td>0.465400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>480</td>\n",
       "      <td>0.506700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>490</td>\n",
       "      <td>0.516500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.489200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>510</td>\n",
       "      <td>0.461900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>520</td>\n",
       "      <td>0.495500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>530</td>\n",
       "      <td>0.457900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>540</td>\n",
       "      <td>0.401200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>0.513300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>560</td>\n",
       "      <td>0.402000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>570</td>\n",
       "      <td>0.493800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>580</td>\n",
       "      <td>0.545800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>590</td>\n",
       "      <td>0.472800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.569700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>610</td>\n",
       "      <td>0.541300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>620</td>\n",
       "      <td>0.523000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>630</td>\n",
       "      <td>0.447700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>640</td>\n",
       "      <td>0.463200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>0.400500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660</td>\n",
       "      <td>0.513000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>670</td>\n",
       "      <td>0.407200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>680</td>\n",
       "      <td>0.356700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>690</td>\n",
       "      <td>0.354200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>0.502400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>710</td>\n",
       "      <td>0.551600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>720</td>\n",
       "      <td>0.527300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>730</td>\n",
       "      <td>0.495400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>740</td>\n",
       "      <td>0.415000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>0.390100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>760</td>\n",
       "      <td>0.557000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>770</td>\n",
       "      <td>0.442900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>780</td>\n",
       "      <td>0.435800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>790</td>\n",
       "      <td>0.329000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.334700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>810</td>\n",
       "      <td>0.504900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>820</td>\n",
       "      <td>0.379100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>830</td>\n",
       "      <td>0.406900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>840</td>\n",
       "      <td>0.333400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>0.377400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>860</td>\n",
       "      <td>0.301500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>870</td>\n",
       "      <td>0.439100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>880</td>\n",
       "      <td>0.379000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>890</td>\n",
       "      <td>0.370200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.491800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>910</td>\n",
       "      <td>0.356500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>920</td>\n",
       "      <td>0.377900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>930</td>\n",
       "      <td>0.304500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>940</td>\n",
       "      <td>0.301500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>0.377800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>960</td>\n",
       "      <td>0.357100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>970</td>\n",
       "      <td>0.290100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>980</td>\n",
       "      <td>0.401300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>990</td>\n",
       "      <td>0.351700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.429600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1010</td>\n",
       "      <td>0.406000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1020</td>\n",
       "      <td>0.341100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1030</td>\n",
       "      <td>0.275500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1040</td>\n",
       "      <td>0.360000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>0.377600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1060</td>\n",
       "      <td>0.286100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1070</td>\n",
       "      <td>0.321800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1080</td>\n",
       "      <td>0.296400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1090</td>\n",
       "      <td>0.354000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>0.323900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1110</td>\n",
       "      <td>0.336700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1120</td>\n",
       "      <td>0.286400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1130</td>\n",
       "      <td>0.333700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1140</td>\n",
       "      <td>0.262200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ]
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Saving model checkpoint to ./RoBERTa-results/checkpoint-500\n",
      "Configuration saved in ./RoBERTa-results/checkpoint-500/config.json\n",
      "Model weights saved in ./RoBERTa-results/checkpoint-500/pytorch_model.bin\n",
      "Saving model checkpoint to ./RoBERTa-results/checkpoint-1000\n",
      "Configuration saved in ./RoBERTa-results/checkpoint-1000/config.json\n",
      "Model weights saved in ./RoBERTa-results/checkpoint-1000/pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1143, training_loss=0.4717025792296269, metrics={'train_runtime': 320.4815, 'train_samples_per_second': 57.008, 'train_steps_per_second': 3.567, 'total_flos': 670926442752720.0, 'train_loss': 0.4717025792296269, 'epoch': 3.0})"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 6 Results of Evaluating the Validation Dataset"
   ],
   "metadata": {
    "id": "HUeAOU5dGLUm"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "trainer.evaluate()"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 258
    },
    "id": "MpWOToqLF-y3",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1669470106348,
     "user_tz": -480,
     "elapsed": 5380,
     "user": {
      "displayName": "Trenton Wong",
      "userId": "06174541562312717890"
     }
    },
    "outputId": "2dfacf32-327f-4509-e071-baf2fe27d247"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 1523\n",
      "  Batch size = 64\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [24/24 00:05]\n",
       "    </div>\n",
       "    "
      ]
     },
     "metadata": {}
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'eval_loss': 0.5068272352218628,\n",
       " 'eval_accuracy': 0.7951411687458962,\n",
       " 'eval_f1': 0.7539432176656151,\n",
       " 'eval_precision': 0.7599364069952306,\n",
       " 'eval_recall': 0.7480438184663537,\n",
       " 'eval_runtime': 5.8328,\n",
       " 'eval_samples_per_second': 261.111,\n",
       " 'eval_steps_per_second': 4.115,\n",
       " 'epoch': 3.0}"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Some code references from:\n",
    "1. https://huggingface.co/transformers/v3.2.0/custom_datasets.html\n",
    "2. https://huggingface.co/docs/transformers/v4.24.0/en/model_doc/xlnet"
   ],
   "metadata": {
    "id": "chRI7VN-GQDG"
   }
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "4cTmRtzrGRBu"
   },
   "execution_count": null,
   "outputs": []
  }
 ]
}