phi3_classifier.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "499fc531",
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "papermill": {
          "duration": 0.008006,
          "end_time": "2024-04-22T23:11:46.414405",
          "exception": false,
          "start_time": "2024-04-22T23:11:46.406399",
          "status": "completed"
        },
        "tags": [],
        "id": "499fc531"
      },
      "source": [
        "# Pip"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece -q\n",
        "!pip install transformers -q\n",
        "!pip install accelerate -q\n",
        "!pip install bitsandbytes -q\n",
        "!pip install lightning -q\n",
        "!pip install flash-attn --no-build-isolation -q"
      ],
      "metadata": {
        "id": "jgUQbiVhI6iw"
      },
      "id": "jgUQbiVhI6iw",
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "924b602f",
      "metadata": {
        "papermill": {
          "duration": 0.00706,
          "end_time": "2024-04-22T23:12:30.270021",
          "exception": false,
          "start_time": "2024-04-22T23:12:30.262961",
          "status": "completed"
        },
        "tags": [],
        "id": "924b602f"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "04dd9cfd",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:12:30.286229Z",
          "iopub.status.busy": "2024-04-22T23:12:30.285840Z",
          "iopub.status.idle": "2024-04-22T23:12:39.353797Z",
          "shell.execute_reply": "2024-04-22T23:12:39.352992Z"
        },
        "papermill": {
          "duration": 9.079057,
          "end_time": "2024-04-22T23:12:39.356253",
          "exception": false,
          "start_time": "2024-04-22T23:12:30.277196",
          "status": "completed"
        },
        "tags": [],
        "id": "04dd9cfd"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from transformers import (\n",
        "    AutoConfig,\n",
        "    AutoTokenizer,\n",
        "    AutoModelForCausalLM,\n",
        "    get_linear_schedule_with_warmup,\n",
        "    get_cosine_schedule_with_warmup\n",
        ")\n",
        "\n",
        "import os\n",
        "import time\n",
        "import zipfile\n",
        "import urllib.request\n",
        "from pathlib import Path\n",
        "from tqdm.auto import tqdm\n",
        "\n",
        "import torch\n",
        "from torch import nn, optim\n",
        "import torch.nn.functional as F\n",
        "from torch.cuda.amp import GradScaler, autocast\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import log_loss, accuracy_score\n",
        "import lightning as L\n",
        "\n",
        "tqdm.pandas()\n",
        "os.environ['TOKENIZERS_PARALLELISM'] = 'false'"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Get data"
      ],
      "metadata": {
        "id": "4jU03Y0TrP8f"
      },
      "id": "4jU03Y0TrP8f"
    },
    {
      "cell_type": "code",
      "source": [
        "url = \"https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip\"\n",
        "zip_path = \"sms_spam_collection.zip\"\n",
        "extracted_path = \"sms_spam_collection\"\n",
        "data_file_path = Path(extracted_path) / \"SMSSpamCollection.tsv\"\n",
        "\n",
        "def download_and_unzip(url, zip_path, extracted_path, data_file_path):\n",
        "    if data_file_path.exists():\n",
        "        print(f\"{data_file_path} already exists. Skipping download and extraction.\")\n",
        "        return\n",
        "\n",
        "    # Downloading the file\n",
        "    with urllib.request.urlopen(url) as response:\n",
        "        with open(zip_path, \"wb\") as out_file:\n",
        "            out_file.write(response.read())\n",
        "\n",
        "    # Unzipping the file\n",
        "    with zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n",
        "        zip_ref.extractall(extracted_path)\n",
        "\n",
        "    # Add .tsv file extension\n",
        "    original_file_path = Path(extracted_path) / \"SMSSpamCollection\"\n",
        "    os.rename(original_file_path, data_file_path)\n",
        "    print(f\"File downloaded and saved as {data_file_path}\")\n",
        "\n",
        "download_and_unzip(url, zip_path, extracted_path, data_file_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WLvpzJU6kYCO",
        "outputId": "b478dade-3117-4f11-a0c9-bdcb8a6b46de"
      },
      "id": "WLvpzJU6kYCO",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sms_spam_collection/SMSSpamCollection.tsv already exists. Skipping download and extraction.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(data_file_path, sep=\"\\t\", header=None, names=[\"target\", \"text\"])\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "57_fyXXjkexm",
        "outputId": "359e664a-ce6b-442b-e47b-49d319dd9fb3"
      },
      "id": "57_fyXXjkexm",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     target                                               text\n",
              "0       ham  Go until jurong point, crazy.. Available only ...\n",
              "1       ham                      Ok lar... Joking wif u oni...\n",
              "2      spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3       ham  U dun say so early hor... U c already then say...\n",
              "4       ham  Nah I don't think he goes to usf, he lives aro...\n",
              "...     ...                                                ...\n",
              "5567   spam  This is the 2nd time we have tried 2 contact u...\n",
              "5568    ham               Will ü b going to esplanade fr home?\n",
              "5569    ham  Pity, * was in mood for that. So...any other s...\n",
              "5570    ham  The guy did some bitching but I acted like i'd...\n",
              "5571    ham                         Rofl. Its true to its name\n",
              "\n",
              "[5572 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4dd04800-0359-475c-9edc-91e57c6f3470\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ham</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ham</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>spam</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ham</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ham</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5567</th>\n",
              "      <td>spam</td>\n",
              "      <td>This is the 2nd time we have tried 2 contact u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5568</th>\n",
              "      <td>ham</td>\n",
              "      <td>Will ü b going to esplanade fr home?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5569</th>\n",
              "      <td>ham</td>\n",
              "      <td>Pity, * was in mood for that. So...any other s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5570</th>\n",
              "      <td>ham</td>\n",
              "      <td>The guy did some bitching but I acted like i'd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5571</th>\n",
              "      <td>ham</td>\n",
              "      <td>Rofl. Its true to its name</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5572 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4dd04800-0359-475c-9edc-91e57c6f3470')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4dd04800-0359-475c-9edc-91e57c6f3470 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4dd04800-0359-475c-9edc-91e57c6f3470');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-cfe410c4-aa87-4126-b303-0061af7f04de\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cfe410c4-aa87-4126-b303-0061af7f04de')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-cfe410c4-aa87-4126-b303-0061af7f04de button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_422ddb05-f9e1-4478-b9b9-4e2374740e5b\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_422ddb05-f9e1-4478-b9b9-4e2374740e5b button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 5572,\n  \"fields\": [\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"spam\",\n          \"ham\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5169,\n        \"samples\": [\n          \"K, makes sense, btw carlos is being difficult so you guys are gonna smoke while I go pick up the second batch and get gas\",\n          \"URGENT! Your mobile No *********** WON a \\u00a32,000 Bonus Caller Prize on 02/06/03! This is the 2nd attempt to reach YOU! Call 09066362220 ASAP! BOX97N7QP, 150ppm\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_balanced_dataset(df):\n",
        "\n",
        "    # Count the instances of \"spam\"\n",
        "    num_spam = df[df[\"target\"] == \"spam\"].shape[0]\n",
        "\n",
        "    # Randomly sample \"ham\" instances to match the number of \"spam\" instances\n",
        "    ham_subset = df[df[\"target\"] == \"ham\"].sample(num_spam, random_state=123)\n",
        "\n",
        "    # Combine ham \"subset\" with \"spam\"\n",
        "    balanced_df = pd.concat([ham_subset, df[df[\"target\"] == \"spam\"]])\n",
        "\n",
        "    return balanced_df\n",
        "\n",
        "balanced_df = create_balanced_dataset(df)\n",
        "print(balanced_df[\"target\"].value_counts())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kc_Sp9OZvBNP",
        "outputId": "ee420c45-2ec4-48fa-c6ab-46ff79fb66a3"
      },
      "id": "Kc_Sp9OZvBNP",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "target\n",
            "ham     747\n",
            "spam    747\n",
            "Name: count, dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "balanced_df['target'] = df.target.map({'spam': 1, 'ham': 0})"
      ],
      "metadata": {
        "id": "ieBD0VyYvZRm"
      },
      "id": "ieBD0VyYvZRm",
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def random_split(df, train_frac, validation_frac):\n",
        "    # Shuffle the entire DataFrame\n",
        "    df = df.sample(frac=1, random_state=123).reset_index(drop=True)\n",
        "\n",
        "    # Calculate split indices\n",
        "    train_end = int(len(df) * train_frac)\n",
        "    validation_end = train_end + int(len(df) * validation_frac)\n",
        "\n",
        "    # Split the DataFrame\n",
        "    train_df = df[:train_end]\n",
        "    validation_df = df[train_end:validation_end]\n",
        "    test_df = df[validation_end:]\n",
        "\n",
        "    return train_df, validation_df, test_df\n",
        "\n",
        "train_df, validation_df, test_df = random_split(balanced_df, 0.7, 0.1)\n",
        "# Test size is implied to be 0.2 as the remainder\n",
        "\n",
        "train_df.to_csv(\"train.csv\", index=None)\n",
        "validation_df.to_csv(\"validation.csv\", index=None)\n",
        "test_df.to_csv(\"test.csv\", index=None)"
      ],
      "metadata": {
        "id": "5ZSF2NpOvMs2"
      },
      "id": "5ZSF2NpOvMs2",
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "60deb920",
      "metadata": {
        "papermill": {
          "duration": 0.007691,
          "end_time": "2024-04-22T23:12:40.371231",
          "exception": false,
          "start_time": "2024-04-22T23:12:40.363540",
          "status": "completed"
        },
        "tags": [],
        "id": "60deb920"
      },
      "source": [
        "# Load tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "e2a5586a",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:12:40.389191Z",
          "iopub.status.busy": "2024-04-22T23:12:40.388389Z",
          "iopub.status.idle": "2024-04-22T23:12:40.525522Z",
          "shell.execute_reply": "2024-04-22T23:12:40.524408Z"
        },
        "papermill": {
          "duration": 0.149076,
          "end_time": "2024-04-22T23:12:40.528192",
          "exception": false,
          "start_time": "2024-04-22T23:12:40.379116",
          "status": "completed"
        },
        "tags": [],
        "id": "e2a5586a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d116d555-833c-4f52-faea-d7c5d05dae34"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        }
      ],
      "source": [
        "# Initialize tokenizer with model ID and authentication token\n",
        "model_id = 'microsoft/Phi-3-mini-128k-instruct'\n",
        "hf_token = 'hf_' # Replace your token here on huggingface\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token)\n",
        "\n",
        "# Set padding token to end-of-sequence token and configure padding side\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.padding_side = 'left'"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Split data"
      ],
      "metadata": {
        "id": "jWLj2R2Q4uiw"
      },
      "id": "jWLj2R2Q4uiw"
    },
    {
      "cell_type": "code",
      "source": [
        "train = pd.read_csv('train.csv')\n",
        "test = pd.read_csv('test.csv')\n",
        "val = pd.read_csv('validation.csv')"
      ],
      "metadata": {
        "id": "yOOWKsDvvqfF"
      },
      "id": "yOOWKsDvvqfF",
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# follow the phi 3 on huggingface:\n",
        "# \"Some applications/frameworks might not include a BOS token (<s>) at the start of the conversation.\n",
        "# Please ensure that it is included since it provides more reliable results.\"\n",
        "\n",
        "train['text'] = tokenizer.bos_token + train['text']\n",
        "test['text'] = tokenizer.bos_token + test['text']\n",
        "val['text'] = tokenizer.bos_token + val['text']"
      ],
      "metadata": {
        "id": "a232xwiyFMe4"
      },
      "id": "a232xwiyFMe4",
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sample = tokenizer(train.text[0], add_special_tokens=False).input_ids\n",
        "tokenizer.decode(sample)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "pvIUdJUCHXx_",
        "outputId": "362909b9-98e5-4683-d66d-677757943d34"
      },
      "id": "pvIUdJUCHXx_",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<s> Dude how do you like the buff wind.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c87eca07",
      "metadata": {
        "papermill": {
          "duration": 0.008283,
          "end_time": "2024-04-22T23:14:26.773511",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.765228",
          "status": "completed"
        },
        "tags": [],
        "id": "c87eca07"
      },
      "source": [
        "# Dataset and DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "id": "c652a192",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:14:26.791438Z",
          "iopub.status.busy": "2024-04-22T23:14:26.791103Z",
          "iopub.status.idle": "2024-04-22T23:14:26.797238Z",
          "shell.execute_reply": "2024-04-22T23:14:26.796383Z"
        },
        "papermill": {
          "duration": 0.017497,
          "end_time": "2024-04-22T23:14:26.799251",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.781754",
          "status": "completed"
        },
        "tags": [],
        "id": "c652a192"
      },
      "outputs": [],
      "source": [
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, texts, targets):\n",
        "        self.texts = texts\n",
        "        self.targets = torch.tensor(targets, dtype=torch.long)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        text = self.texts[idx]\n",
        "        target = self.targets[idx]\n",
        "        return text, target\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.targets)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set seed for reproducibility\n",
        "L.seed_everything(seed=252)\n",
        "\n",
        "# Create train dataset and dataloader\n",
        "train_dataset = CustomDataset(\n",
        "    texts=train['text'].values.tolist(),\n",
        "    targets=train['target'].values.tolist()\n",
        ")\n",
        "train_dataloader = DataLoader(\n",
        "    dataset=train_dataset,\n",
        "    batch_size=8,\n",
        "    shuffle=True,\n",
        "    drop_last=True\n",
        ")\n",
        "\n",
        "# Create test dataset and dataloader\n",
        "test_dataset = CustomDataset(\n",
        "    texts=test['text'].values.tolist(),\n",
        "    targets=test['target'].values.tolist()\n",
        ")\n",
        "test_dataloader = DataLoader(\n",
        "    dataset=test_dataset,\n",
        "    batch_size=16,\n",
        "    shuffle=False,\n",
        "    drop_last=False\n",
        ")\n",
        "\n",
        "# Create validation dataset and dataloader\n",
        "val_dataset = CustomDataset(\n",
        "    texts=val['text'].values.tolist(),\n",
        "    targets=val['target'].values.tolist()\n",
        ")\n",
        "val_dataloader = DataLoader(\n",
        "    dataset=val_dataset,\n",
        "    batch_size=16,\n",
        "    shuffle=False,\n",
        "    drop_last=False\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tLKsACAGw3b0",
        "outputId": "26c135e5-ec53-43ec-9be4-05d61cfa5e39"
      },
      "id": "tLKsACAGw3b0",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO: Seed set to 252\n",
            "INFO:lightning.fabric.utilities.seed:Seed set to 252\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7da5c45b",
      "metadata": {
        "papermill": {
          "duration": 0.008031,
          "end_time": "2024-04-22T23:14:26.815699",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.807668",
          "status": "completed"
        },
        "tags": [],
        "id": "7da5c45b"
      },
      "source": [
        "# Tokenization function"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_text(text):\n",
        "    \"\"\"\n",
        "    Tokenize the text and return PyTorch tensors with dynamic padding\n",
        "    \"\"\"\n",
        "    encodings = tokenizer(\n",
        "        text,\n",
        "        return_tensors='pt',\n",
        "        padding='longest',  # Dynamically pad each batch to the length of the longest sequence\n",
        "        add_special_tokens=False\n",
        "    )\n",
        "\n",
        "    return encodings\n"
      ],
      "metadata": {
        "id": "YRMQboHnJ7gJ"
      },
      "id": "YRMQboHnJ7gJ",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "a2a03f3a",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T17:22:30.925403Z",
          "iopub.status.busy": "2024-04-22T17:22:30.924941Z",
          "iopub.status.idle": "2024-04-22T17:22:30.933174Z",
          "shell.execute_reply": "2024-04-22T17:22:30.932089Z",
          "shell.execute_reply.started": "2024-04-22T17:22:30.925363Z"
        },
        "papermill": {
          "duration": 0.00841,
          "end_time": "2024-04-22T23:14:26.857109",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.848699",
          "status": "completed"
        },
        "tags": [],
        "id": "a2a03f3a"
      },
      "source": [
        "# Architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "id": "2aa40aea",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:14:26.875515Z",
          "iopub.status.busy": "2024-04-22T23:14:26.874796Z",
          "iopub.status.idle": "2024-04-22T23:14:26.883751Z",
          "shell.execute_reply": "2024-04-22T23:14:26.883062Z"
        },
        "papermill": {
          "duration": 0.020179,
          "end_time": "2024-04-22T23:14:26.885633",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.865454",
          "status": "completed"
        },
        "tags": [],
        "id": "2aa40aea"
      },
      "outputs": [],
      "source": [
        "def disable_dropout(model: torch.nn.Module):\n",
        "    \"\"\"Disable dropout in a model.\"\"\"\n",
        "    for module in model.modules():\n",
        "        if isinstance(module, torch.nn.Dropout):\n",
        "            module.p = 0\n",
        "\n",
        "# Define a neural network class\n",
        "class Net(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "\n",
        "        # Load configuration from a pre-trained model\n",
        "        config = AutoConfig.from_pretrained(model_id)\n",
        "\n",
        "        # Load pre-trained language model with specific configurations\n",
        "        self.llm = AutoModelForCausalLM.from_pretrained(\n",
        "            model_id,\n",
        "            device_map=\"cuda\",\n",
        "            low_cpu_mem_usage=True,\n",
        "            torch_dtype=torch.float16,\n",
        "            trust_remote_code=True,\n",
        "            token=hf_token,\n",
        "            # attn_implementation='flash_attention_2'\n",
        "        )\n",
        "\n",
        "        # Replace language model head with an identity function\n",
        "        self.llm.lm_head = nn.Identity()\n",
        "\n",
        "        # Freeze all parameters of the language model backbone\n",
        "        for name, param in self.llm.named_parameters():\n",
        "            param.requires_grad = False\n",
        "\n",
        "        # Define classification head\n",
        "        self.cls_head = nn.Sequential(\n",
        "            nn.Linear(config.hidden_size, 768),\n",
        "            nn.ReLU(),\n",
        "            nn.LayerNorm(768),\n",
        "            nn.Linear(768, 2)\n",
        "        )\n",
        "\n",
        "    # Define the forward pass\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        x = self.llm(input_ids, attention_mask).logits  # get last hidden state\n",
        "        logits = self.cls_head(x)[:, -1, :]  # Apply classification head to the last token's output\n",
        "        return logits\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7928c895",
      "metadata": {
        "papermill": {
          "duration": 0.008326,
          "end_time": "2024-04-22T23:14:26.902630",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.894304",
          "status": "completed"
        },
        "tags": [],
        "id": "7928c895"
      },
      "source": [
        "# Optimizer and Scheduler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "id": "faa1f254",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:14:26.920974Z",
          "iopub.status.busy": "2024-04-22T23:14:26.920665Z",
          "iopub.status.idle": "2024-04-22T23:14:26.933845Z",
          "shell.execute_reply": "2024-04-22T23:14:26.933046Z"
        },
        "papermill": {
          "duration": 0.024583,
          "end_time": "2024-04-22T23:14:26.935644",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.911061",
          "status": "completed"
        },
        "tags": [],
        "id": "faa1f254"
      },
      "outputs": [],
      "source": [
        "def get_optimizer(model, learning_rate=0.0001, diff_lr=0.00001, weight_decay=0.01):\n",
        "    \"\"\"\n",
        "    Get optimizer with different learning rates for specified layers.\n",
        "\n",
        "    Args:\n",
        "        model (torch.nn.Module): The neural network model.\n",
        "        learning_rate (float): Learning rate for non-differential layers.\n",
        "        diff_lr (float): Learning rate for differential layers.\n",
        "        weight_decay (float): Weight decay (decoupled from L2 penalty) for optimizer.\n",
        "\n",
        "    Returns:\n",
        "        torch.optim.AdamW: Optimizer for the model.\n",
        "    \"\"\"\n",
        "\n",
        "    # Define parameters with different learning rates and weight decay\n",
        "    no_decay = ['bias', 'LayerNorm.weight']\n",
        "    differential_layers = ['llm']\n",
        "\n",
        "    optimizer = torch.optim.AdamW(\n",
        "            [\n",
        "                {\n",
        "                    \"params\": [\n",
        "                        param\n",
        "                        for name, param in model.named_parameters()\n",
        "                        if (not any(layer in name for layer in differential_layers))\n",
        "                        and (not any(nd in name for nd in no_decay))\n",
        "                    ],\n",
        "                    \"lr\": learning_rate,\n",
        "                    \"weight_decay\": weight_decay,\n",
        "                },\n",
        "                {\n",
        "                    \"params\": [\n",
        "                        param\n",
        "                        for name, param in model.named_parameters()\n",
        "                        if (not any(layer in name for layer in differential_layers))\n",
        "                        and (any(nd in name for nd in no_decay))\n",
        "                    ],\n",
        "                    \"lr\": learning_rate,\n",
        "                    \"weight_decay\": 0,\n",
        "                },\n",
        "                {\n",
        "                    \"params\": [\n",
        "                        param\n",
        "                        for name, param in model.named_parameters()\n",
        "                        if (any(layer in name for layer in differential_layers))\n",
        "                        and (not any(nd in name for nd in no_decay))\n",
        "                    ],\n",
        "                    \"lr\": diff_lr,\n",
        "                    \"weight_decay\": weight_decay,\n",
        "                },\n",
        "                {\n",
        "                    \"params\": [\n",
        "                        param\n",
        "                        for name, param in model.named_parameters()\n",
        "                        if (any(layer in name for layer in differential_layers))\n",
        "                        and (any(nd in name for nd in no_decay))\n",
        "                    ],\n",
        "                    \"lr\": diff_lr,\n",
        "                    \"weight_decay\": 0,\n",
        "                },\n",
        "            ],\n",
        "            lr=learning_rate,\n",
        "            weight_decay=weight_decay,\n",
        "    )\n",
        "\n",
        "    return optimizer"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "45b5f796",
      "metadata": {
        "papermill": {
          "duration": 0.008311,
          "end_time": "2024-04-22T23:14:26.952420",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.944109",
          "status": "completed"
        },
        "tags": [],
        "id": "45b5f796"
      },
      "source": [
        "# Hyperameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "id": "12927cb6",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:14:26.970958Z",
          "iopub.status.busy": "2024-04-22T23:14:26.970321Z",
          "iopub.status.idle": "2024-04-22T23:14:26.974903Z",
          "shell.execute_reply": "2024-04-22T23:14:26.974056Z"
        },
        "papermill": {
          "duration": 0.01576,
          "end_time": "2024-04-22T23:14:26.976701",
          "exception": false,
          "start_time": "2024-04-22T23:14:26.960941",
          "status": "completed"
        },
        "tags": [],
        "id": "12927cb6"
      },
      "outputs": [],
      "source": [
        "num_epochs = 1\n",
        "learning_rate = 0.0002\n",
        "diff_lr = 0.00001 # not being used because I freeze the llm backbone\n",
        "warmup_steps = 0\n",
        "seed = 252\n",
        "weight_decay = 0.01"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8e96c270",
      "metadata": {
        "papermill": {
          "duration": 0.008328,
          "end_time": "2024-04-22T23:14:27.017463",
          "exception": false,
          "start_time": "2024-04-22T23:14:27.009135",
          "status": "completed"
        },
        "tags": [],
        "id": "8e96c270"
      },
      "source": [
        "# Fine-tuning"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print(f'Using device: {device}')"
      ],
      "metadata": {
        "id": "P3P8mxxg0bLM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "26e048cc-1ff8-4e84-f14d-93a3f20ec4e2"
      },
      "id": "P3P8mxxg0bLM",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using device: cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "id": "63bb4bb0",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-04-22T23:14:27.035820Z",
          "iopub.status.busy": "2024-04-22T23:14:27.035559Z",
          "iopub.status.idle": "2024-04-23T03:56:08.292048Z",
          "shell.execute_reply": "2024-04-23T03:56:08.290997Z"
        },
        "papermill": {
          "duration": 16901.267986,
          "end_time": "2024-04-23T03:56:08.294020",
          "exception": false,
          "start_time": "2024-04-22T23:14:27.026034",
          "status": "completed"
        },
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399,
          "referenced_widgets": [
            "09a88a2069344ee69310868baf0cea46",
            "e7af670b41174f109bca1481b128c503",
            "57b968301ee644d2a30d1f398c4e403e",
            "5f71f183467e48ae96f91a5be81bed45",
            "075584f0cabb404fa5bbc7f716a3c2ee",
            "0a363e83adc8405db251a703509f2f58",
            "33378970bc95441bb868c7fa8886336f",
            "c601d07f75204915adae1bdd61e5944b",
            "082fb7cea3cd482f952908e68ba53971",
            "4131e9456de040fe8ffe94a6e3a99f10",
            "21e6b3db0d284038819957aeadbfa44d"
          ]
        },
        "id": "63bb4bb0",
        "outputId": "b8a26e66-afa5-4527-d9e6-2701bb3884f5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO: Seed set to 252\n",
            "INFO:lightning.fabric.utilities.seed:Seed set to 252\n",
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The repository for microsoft/Phi-3-mini-128k-instruct contains custom code which must be executed to correctly load the model. You can inspect the repository content at https://hf.co/microsoft/Phi-3-mini-128k-instruct.\n",
            "You can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n",
            "\n",
            "Do you wish to run the custom code? [y/N] y\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "09a88a2069344ee69310868baf0cea46"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Here are the trainable parameters:\n",
            "cls_head.0.weight\n",
            "cls_head.0.bias\n",
            "cls_head.2.weight\n",
            "cls_head.2.bias\n",
            "cls_head.3.weight\n",
            "cls_head.3.bias\n"
          ]
        }
      ],
      "source": [
        "# Set seed for reproducibility\n",
        "L.seed_everything(seed=seed)\n",
        "\n",
        "# Instantiate the neural network model\n",
        "model = Net()\n",
        "model.to(device)  # Move model to the device\n",
        "\n",
        "# Display the names of trainable parameters\n",
        "print('Here are the trainable parameters:')\n",
        "for n, p in model.named_parameters():\n",
        "    if p.requires_grad:\n",
        "        print(n)\n",
        "\n",
        "# Get the optimizer\n",
        "optimizer = get_optimizer(\n",
        "    model,\n",
        "    learning_rate=learning_rate,\n",
        "    diff_lr=diff_lr,\n",
        "    weight_decay=weight_decay\n",
        ")\n",
        "\n",
        "# Set up the scheduler for learning rate adjustment\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "    optimizer=optimizer,\n",
        "    num_warmup_steps=warmup_steps,\n",
        "    num_training_steps=num_epochs*len(train_dataloader)\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scaler = GradScaler()\n",
        "\n",
        "start_time = time.time()\n",
        "for epoch in range(num_epochs):\n",
        "\n",
        "    for batch_idx, batch in enumerate(train_dataloader):\n",
        "\n",
        "        model.train()\n",
        "\n",
        "        prompt, targets = batch\n",
        "\n",
        "        encodings = tokenize_text(prompt)\n",
        "\n",
        "        input_ids = encodings['input_ids'].to(device)\n",
        "        attention_mask = encodings['attention_mask'].to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        # Perform forward pass with autocast for mixed precision training\n",
        "        with autocast():\n",
        "            logits = model(input_ids, attention_mask)\n",
        "            loss = F.cross_entropy(logits, targets)\n",
        "\n",
        "        # Backward pass, optimization step, and learning rate adjustment\n",
        "        optimizer.zero_grad()\n",
        "        scaler.scale(loss).backward()\n",
        "        scaler.step(optimizer)\n",
        "        scaler.update()\n",
        "        scheduler.step()\n",
        "\n",
        "        # Logging training progress\n",
        "        print(\n",
        "            f'Epoch: {epoch+1} / {num_epochs}'\n",
        "            f'| Batch: {batch_idx+1}/{len(train_dataloader)}'\n",
        "            f'| Loss: {loss.item():.4f}'\n",
        "        )\n",
        "\n",
        "end_time = time.time()\n",
        "training_time = (end_time - start_time) / 60\n",
        "print(f'Total training time: {training_time:.2f} min')"
      ],
      "metadata": {
        "id": "613alv-BrAql",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c7eecf09-fd01-40e5-eab0-a8c5e0369682"
      },
      "id": "613alv-BrAql",
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:transformers_modules.microsoft.Phi-3-mini-128k-instruct.8a362e755d2faf8cec2bf98850ce2216023d178a.modeling_phi3:You are not running the flash-attention implementation, expect numerical differences.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 / 1| Batch: 1/130| Loss: 0.9640\n",
            "Epoch: 1 / 1| Batch: 2/130| Loss: 0.5873\n",
            "Epoch: 1 / 1| Batch: 3/130| Loss: 0.1968\n",
            "Epoch: 1 / 1| Batch: 4/130| Loss: 0.2019\n",
            "Epoch: 1 / 1| Batch: 5/130| Loss: 0.0804\n",
            "Epoch: 1 / 1| Batch: 6/130| Loss: 0.7383\n",
            "Epoch: 1 / 1| Batch: 7/130| Loss: 0.0936\n",
            "Epoch: 1 / 1| Batch: 8/130| Loss: 0.0290\n",
            "Epoch: 1 / 1| Batch: 9/130| Loss: 0.0236\n",
            "Epoch: 1 / 1| Batch: 10/130| Loss: 0.4675\n",
            "Epoch: 1 / 1| Batch: 11/130| Loss: 0.1215\n",
            "Epoch: 1 / 1| Batch: 12/130| Loss: 0.0745\n",
            "Epoch: 1 / 1| Batch: 13/130| Loss: 0.0056\n",
            "Epoch: 1 / 1| Batch: 14/130| Loss: 0.0109\n",
            "Epoch: 1 / 1| Batch: 15/130| Loss: 0.0082\n",
            "Epoch: 1 / 1| Batch: 16/130| Loss: 0.7438\n",
            "Epoch: 1 / 1| Batch: 17/130| Loss: 0.0224\n",
            "Epoch: 1 / 1| Batch: 18/130| Loss: 0.0744\n",
            "Epoch: 1 / 1| Batch: 19/130| Loss: 0.2771\n",
            "Epoch: 1 / 1| Batch: 20/130| Loss: 0.0506\n",
            "Epoch: 1 / 1| Batch: 21/130| Loss: 0.7216\n",
            "Epoch: 1 / 1| Batch: 22/130| Loss: 0.0886\n",
            "Epoch: 1 / 1| Batch: 23/130| Loss: 0.0070\n",
            "Epoch: 1 / 1| Batch: 24/130| Loss: 0.0640\n",
            "Epoch: 1 / 1| Batch: 25/130| Loss: 0.0505\n",
            "Epoch: 1 / 1| Batch: 26/130| Loss: 0.0357\n",
            "Epoch: 1 / 1| Batch: 27/130| Loss: 0.0580\n",
            "Epoch: 1 / 1| Batch: 28/130| Loss: 0.2388\n",
            "Epoch: 1 / 1| Batch: 29/130| Loss: 0.0547\n",
            "Epoch: 1 / 1| Batch: 30/130| Loss: 0.3191\n",
            "Epoch: 1 / 1| Batch: 31/130| Loss: 0.0128\n",
            "Epoch: 1 / 1| Batch: 32/130| Loss: 0.0134\n",
            "Epoch: 1 / 1| Batch: 33/130| Loss: 0.0048\n",
            "Epoch: 1 / 1| Batch: 34/130| Loss: 0.3201\n",
            "Epoch: 1 / 1| Batch: 35/130| Loss: 0.0502\n",
            "Epoch: 1 / 1| Batch: 36/130| Loss: 0.0036\n",
            "Epoch: 1 / 1| Batch: 37/130| Loss: 0.0162\n",
            "Epoch: 1 / 1| Batch: 38/130| Loss: 0.0232\n",
            "Epoch: 1 / 1| Batch: 39/130| Loss: 0.0274\n",
            "Epoch: 1 / 1| Batch: 40/130| Loss: 0.0189\n",
            "Epoch: 1 / 1| Batch: 41/130| Loss: 0.0462\n",
            "Epoch: 1 / 1| Batch: 42/130| Loss: 0.1924\n",
            "Epoch: 1 / 1| Batch: 43/130| Loss: 0.0024\n",
            "Epoch: 1 / 1| Batch: 44/130| Loss: 0.5279\n",
            "Epoch: 1 / 1| Batch: 45/130| Loss: 0.0574\n",
            "Epoch: 1 / 1| Batch: 46/130| Loss: 0.4083\n",
            "Epoch: 1 / 1| Batch: 47/130| Loss: 0.0167\n",
            "Epoch: 1 / 1| Batch: 48/130| Loss: 0.0107\n",
            "Epoch: 1 / 1| Batch: 49/130| Loss: 0.2714\n",
            "Epoch: 1 / 1| Batch: 50/130| Loss: 0.0076\n",
            "Epoch: 1 / 1| Batch: 51/130| Loss: 0.0158\n",
            "Epoch: 1 / 1| Batch: 52/130| Loss: 0.0136\n",
            "Epoch: 1 / 1| Batch: 53/130| Loss: 0.0255\n",
            "Epoch: 1 / 1| Batch: 54/130| Loss: 0.0020\n",
            "Epoch: 1 / 1| Batch: 55/130| Loss: 0.0333\n",
            "Epoch: 1 / 1| Batch: 56/130| Loss: 0.0606\n",
            "Epoch: 1 / 1| Batch: 57/130| Loss: 0.0045\n",
            "Epoch: 1 / 1| Batch: 58/130| Loss: 0.0201\n",
            "Epoch: 1 / 1| Batch: 59/130| Loss: 0.0014\n",
            "Epoch: 1 / 1| Batch: 60/130| Loss: 0.0027\n",
            "Epoch: 1 / 1| Batch: 61/130| Loss: 0.2034\n",
            "Epoch: 1 / 1| Batch: 62/130| Loss: 0.2655\n",
            "Epoch: 1 / 1| Batch: 63/130| Loss: 0.0214\n",
            "Epoch: 1 / 1| Batch: 64/130| Loss: 0.1776\n",
            "Epoch: 1 / 1| Batch: 65/130| Loss: 0.0095\n",
            "Epoch: 1 / 1| Batch: 66/130| Loss: 0.0055\n",
            "Epoch: 1 / 1| Batch: 67/130| Loss: 0.0027\n",
            "Epoch: 1 / 1| Batch: 68/130| Loss: 0.0170\n",
            "Epoch: 1 / 1| Batch: 69/130| Loss: 0.0234\n",
            "Epoch: 1 / 1| Batch: 70/130| Loss: 0.0066\n",
            "Epoch: 1 / 1| Batch: 71/130| Loss: 0.1326\n",
            "Epoch: 1 / 1| Batch: 72/130| Loss: 0.0981\n",
            "Epoch: 1 / 1| Batch: 73/130| Loss: 0.0391\n",
            "Epoch: 1 / 1| Batch: 74/130| Loss: 0.0482\n",
            "Epoch: 1 / 1| Batch: 75/130| Loss: 0.0162\n",
            "Epoch: 1 / 1| Batch: 76/130| Loss: 0.0124\n",
            "Epoch: 1 / 1| Batch: 77/130| Loss: 0.0195\n",
            "Epoch: 1 / 1| Batch: 78/130| Loss: 0.0913\n",
            "Epoch: 1 / 1| Batch: 79/130| Loss: 0.0066\n",
            "Epoch: 1 / 1| Batch: 80/130| Loss: 0.0276\n",
            "Epoch: 1 / 1| Batch: 81/130| Loss: 0.0054\n",
            "Epoch: 1 / 1| Batch: 82/130| Loss: 0.0343\n",
            "Epoch: 1 / 1| Batch: 83/130| Loss: 0.0706\n",
            "Epoch: 1 / 1| Batch: 84/130| Loss: 0.0623\n",
            "Epoch: 1 / 1| Batch: 85/130| Loss: 0.1548\n",
            "Epoch: 1 / 1| Batch: 86/130| Loss: 0.0043\n",
            "Epoch: 1 / 1| Batch: 87/130| Loss: 0.3382\n",
            "Epoch: 1 / 1| Batch: 88/130| Loss: 0.0089\n",
            "Epoch: 1 / 1| Batch: 89/130| Loss: 0.0041\n",
            "Epoch: 1 / 1| Batch: 90/130| Loss: 0.0103\n",
            "Epoch: 1 / 1| Batch: 91/130| Loss: 0.0814\n",
            "Epoch: 1 / 1| Batch: 92/130| Loss: 0.0027\n",
            "Epoch: 1 / 1| Batch: 93/130| Loss: 0.0021\n",
            "Epoch: 1 / 1| Batch: 94/130| Loss: 0.0227\n",
            "Epoch: 1 / 1| Batch: 95/130| Loss: 0.0028\n",
            "Epoch: 1 / 1| Batch: 96/130| Loss: 0.0029\n",
            "Epoch: 1 / 1| Batch: 97/130| Loss: 0.0169\n",
            "Epoch: 1 / 1| Batch: 98/130| Loss: 0.0026\n",
            "Epoch: 1 / 1| Batch: 99/130| Loss: 0.3956\n",
            "Epoch: 1 / 1| Batch: 100/130| Loss: 0.0056\n",
            "Epoch: 1 / 1| Batch: 101/130| Loss: 0.0067\n",
            "Epoch: 1 / 1| Batch: 102/130| Loss: 0.1972\n",
            "Epoch: 1 / 1| Batch: 103/130| Loss: 0.0045\n",
            "Epoch: 1 / 1| Batch: 104/130| Loss: 0.0005\n",
            "Epoch: 1 / 1| Batch: 105/130| Loss: 0.0073\n",
            "Epoch: 1 / 1| Batch: 106/130| Loss: 0.0129\n",
            "Epoch: 1 / 1| Batch: 107/130| Loss: 0.1285\n",
            "Epoch: 1 / 1| Batch: 108/130| Loss: 0.2644\n",
            "Epoch: 1 / 1| Batch: 109/130| Loss: 0.0129\n",
            "Epoch: 1 / 1| Batch: 110/130| Loss: 0.1028\n",
            "Epoch: 1 / 1| Batch: 111/130| Loss: 0.0031\n",
            "Epoch: 1 / 1| Batch: 112/130| Loss: 0.0019\n",
            "Epoch: 1 / 1| Batch: 113/130| Loss: 0.0673\n",
            "Epoch: 1 / 1| Batch: 114/130| Loss: 0.3091\n",
            "Epoch: 1 / 1| Batch: 115/130| Loss: 0.0126\n",
            "Epoch: 1 / 1| Batch: 116/130| Loss: 0.0083\n",
            "Epoch: 1 / 1| Batch: 117/130| Loss: 0.0108\n",
            "Epoch: 1 / 1| Batch: 118/130| Loss: 0.0020\n",
            "Epoch: 1 / 1| Batch: 119/130| Loss: 0.1194\n",
            "Epoch: 1 / 1| Batch: 120/130| Loss: 0.0361\n",
            "Epoch: 1 / 1| Batch: 121/130| Loss: 0.0250\n",
            "Epoch: 1 / 1| Batch: 122/130| Loss: 0.0063\n",
            "Epoch: 1 / 1| Batch: 123/130| Loss: 0.0112\n",
            "Epoch: 1 / 1| Batch: 124/130| Loss: 0.0062\n",
            "Epoch: 1 / 1| Batch: 125/130| Loss: 0.0026\n",
            "Epoch: 1 / 1| Batch: 126/130| Loss: 0.0320\n",
            "Epoch: 1 / 1| Batch: 127/130| Loss: 0.0292\n",
            "Epoch: 1 / 1| Batch: 128/130| Loss: 0.0261\n",
            "Epoch: 1 / 1| Batch: 129/130| Loss: 0.0155\n",
            "Epoch: 1 / 1| Batch: 130/130| Loss: 0.0247\n",
            "Total training time: 0.63 min\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation"
      ],
      "metadata": {
        "id": "EC7DG0T26xXg"
      },
      "id": "EC7DG0T26xXg"
    },
    {
      "cell_type": "code",
      "source": [
        "def calc_accuracy(dataloader, type):\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        pred_scores = []\n",
        "        actual_scores = []\n",
        "        for batch in tqdm(dataloader, total=len(dataloader), desc=f'Calc {type} accuracy'):\n",
        "            prompt, targets = batch\n",
        "            encodings = tokenize_text(prompt)\n",
        "\n",
        "            input_ids = encodings['input_ids'].to(device)\n",
        "            attention_mask = encodings['attention_mask'].to(device)\n",
        "\n",
        "            with autocast():\n",
        "                logits = model(input_ids, attention_mask)\n",
        "                pred_score = F.softmax(logits, dim=-1).argmax(dim=-1).cpu().detach().numpy().tolist()\n",
        "                pred_scores.extend(pred_score)\n",
        "                actual_scores.extend(targets.numpy().tolist())\n",
        "\n",
        "        pred_scores = np.array(pred_scores)\n",
        "        accuracy = accuracy_score(actual_scores, pred_scores)\n",
        "\n",
        "        return accuracy"
      ],
      "metadata": {
        "id": "lKYSEfOC6yvf"
      },
      "id": "lKYSEfOC6yvf",
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_acc = calc_accuracy(train_dataloader, type='train')\n",
        "test_acc = calc_accuracy(test_dataloader, type='test')\n",
        "val_acc = calc_accuracy(val_dataloader, type='val')\n",
        "\n",
        "print('Train accuracy:', train_acc)\n",
        "print('Test accuracy:', test_acc)\n",
        "print('Val accuracy:', val_acc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165,
          "referenced_widgets": [
            "ba4a1925c8a24cfdad396045703b0683",
            "55ff75b34b854173828df59eff723a9b",
            "5bdd9a7c10134ed89397ff73adf3e72b",
            "22d03b3e180c4b109066f305e7dfdee3",
            "c4d224c1544b40eebd4df8bb7e9fcdc2",
            "810b0f9023d54ac0a477f5f40baecc61",
            "19cbce82023542bf8863bc57ddc63b2f",
            "daa79014f63546bb96370ff71d073576",
            "db0274877eca4445aec841a425dfd853",
            "6448c362704a464f8303f24876484b0e",
            "0dfff314366e48a0b2242788cffd111d",
            "496def211437485dac55865538d959f3",
            "14d9efab60c14c9ca6631e593a3a8231",
            "297ac08d4f854811971b068d615adcb0",
            "c05583241a114eec8af2ca55a2f5a12e",
            "3aabb2bc4ac2404abe8ca1345e6f1b48",
            "7a8b76900f084b5692930d7cc0cc500e",
            "6d8f3e54008740ecb3ce90bc85989e3b",
            "3977320e46574eb58b4f1df03ecb41c8",
            "16b6c99bd1ec437b8ebf16d93b2be550",
            "22efaf54298a47b98b66ce2b259f00c1",
            "9ad24930d36b43cdaff8f2337e58a0a4",
            "449d12edba0c418ab38cb83d722641c2",
            "51e3d66820c5402cafefd4bd7ce295a7",
            "2f83fb81d8424eafa585728d7685812a",
            "a3db1047148f4a5eb88cb70fe1e982b1",
            "1b8dbc1f967e46f58e275368cc9e2183",
            "33f296bff5ce4944b8415aa78181890f",
            "9ec75aa1671c4b07bd098d4753ad9a81",
            "e1fb4c173b0542638a4a7cd41c654451",
            "510c5b6386604fdd8a0ab66b691c9ca9",
            "7b4f28ef606f4bd296200fd88faa4935",
            "bc02123a32914b519d3d1cacd66d986f"
          ]
        },
        "id": "Z84KNZQF5OvX",
        "outputId": "51f5c55f-599c-4079-b3d9-7342cd449f52"
      },
      "id": "Z84KNZQF5OvX",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Calc train accuracy:   0%|          | 0/130 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ba4a1925c8a24cfdad396045703b0683"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Calc test accuracy:   0%|          | 0/19 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "496def211437485dac55865538d959f3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Calc val accuracy:   0%|          | 0/10 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "449d12edba0c418ab38cb83d722641c2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train accuracy: 0.9951923076923077\n",
            "Test accuracy: 0.9666666666666667\n",
            "Val accuracy: 0.9932885906040269\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "| Model        | Weights   | Trainable token | Trainable layers        | Context length                            | CPU/GPU | Training time | Training acc | Validation acc | Test acc |\n",
        "|--------------|-----------|-----------------|-------------------------|-------------------------------------------|---------|---------------|--------------|----------------|-----------|\n",
        "| Phi-3 (3.8 B)| instruct  | last            | head                    | dynamic padding (batch-wise)        | T4 (Colab free)    | 0.63 min      | 99.51%       | 99.32%         | 96.66%    |"
      ],
      "metadata": {
        "id": "TKXcWqnW4-Ox"
      },
      "id": "TKXcWqnW4-Ox"
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "uBJ0mI_D_QVC"
      },
      "id": "uBJ0mI_D_QVC",
      "execution_count": 22,
      "outputs": []
    }
  ],
  "metadata": {
    "kaggle": {
      "accelerator": "gpu",
      "dataSources": [
        {
          "databundleVersionId": 8059942,
          "sourceId": 71485,
          "sourceType": "competition"
        },
        {
          "sourceId": 159748558,
          "sourceType": "kernelVersion"
        },
        {
          "sourceId": 165914575,
          "sourceType": "kernelVersion"
        },
        {
          "sourceId": 165914677,
          "sourceType": "kernelVersion"
        },
        {
          "sourceId": 166361201,
          "sourceType": "kernelVersion"
        },
        {
          "isSourceIdPinned": true,
          "modelInstanceId": 3900,
          "sourceId": 5112,
          "sourceType": "modelInstanceVersion"
        }
      ],
      "dockerImageVersionId": 30699,
      "isGpuEnabled": true,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    },
    "papermill": {
      "default_parameters": {},
      "duration": 17079.705434,
      "end_time": "2024-04-23T03:56:23.258603",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2024-04-22T23:11:43.553169",
      "version": "2.5.0"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "09a88a2069344ee69310868baf0cea46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e7af670b41174f109bca1481b128c503",
              "IPY_MODEL_57b968301ee644d2a30d1f398c4e403e",
              "IPY_MODEL_5f71f183467e48ae96f91a5be81bed45"
            ],
            "layout": "IPY_MODEL_075584f0cabb404fa5bbc7f716a3c2ee"
          }
        },
        "e7af670b41174f109bca1481b128c503": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a363e83adc8405db251a703509f2f58",
            "placeholder": "​",
            "style": "IPY_MODEL_33378970bc95441bb868c7fa8886336f",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "57b968301ee644d2a30d1f398c4e403e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c601d07f75204915adae1bdd61e5944b",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_082fb7cea3cd482f952908e68ba53971",
            "value": 2
          }
        },
        "5f71f183467e48ae96f91a5be81bed45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4131e9456de040fe8ffe94a6e3a99f10",
            "placeholder": "​",
            "style": "IPY_MODEL_21e6b3db0d284038819957aeadbfa44d",
            "value": " 2/2 [00:32&lt;00:00, 15.47s/it]"
          }
        },
        "075584f0cabb404fa5bbc7f716a3c2ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a363e83adc8405db251a703509f2f58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33378970bc95441bb868c7fa8886336f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c601d07f75204915adae1bdd61e5944b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "082fb7cea3cd482f952908e68ba53971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4131e9456de040fe8ffe94a6e3a99f10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21e6b3db0d284038819957aeadbfa44d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba4a1925c8a24cfdad396045703b0683": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_55ff75b34b854173828df59eff723a9b",
              "IPY_MODEL_5bdd9a7c10134ed89397ff73adf3e72b",
              "IPY_MODEL_22d03b3e180c4b109066f305e7dfdee3"
            ],
            "layout": "IPY_MODEL_c4d224c1544b40eebd4df8bb7e9fcdc2"
          }
        },
        "55ff75b34b854173828df59eff723a9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_810b0f9023d54ac0a477f5f40baecc61",
            "placeholder": "​",
            "style": "IPY_MODEL_19cbce82023542bf8863bc57ddc63b2f",
            "value": "Calc train accuracy: 100%"
          }
        },
        "5bdd9a7c10134ed89397ff73adf3e72b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_daa79014f63546bb96370ff71d073576",
            "max": 130,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_db0274877eca4445aec841a425dfd853",
            "value": 130
          }
        },
        "22d03b3e180c4b109066f305e7dfdee3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6448c362704a464f8303f24876484b0e",
            "placeholder": "​",
            "style": "IPY_MODEL_0dfff314366e48a0b2242788cffd111d",
            "value": " 130/130 [00:38&lt;00:00,  3.63it/s]"
          }
        },
        "c4d224c1544b40eebd4df8bb7e9fcdc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "810b0f9023d54ac0a477f5f40baecc61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "19cbce82023542bf8863bc57ddc63b2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "daa79014f63546bb96370ff71d073576": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db0274877eca4445aec841a425dfd853": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6448c362704a464f8303f24876484b0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0dfff314366e48a0b2242788cffd111d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "496def211437485dac55865538d959f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_14d9efab60c14c9ca6631e593a3a8231",
              "IPY_MODEL_297ac08d4f854811971b068d615adcb0",
              "IPY_MODEL_c05583241a114eec8af2ca55a2f5a12e"
            ],
            "layout": "IPY_MODEL_3aabb2bc4ac2404abe8ca1345e6f1b48"
          }
        },
        "14d9efab60c14c9ca6631e593a3a8231": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a8b76900f084b5692930d7cc0cc500e",
            "placeholder": "​",
            "style": "IPY_MODEL_6d8f3e54008740ecb3ce90bc85989e3b",
            "value": "Calc test accuracy: 100%"
          }
        },
        "297ac08d4f854811971b068d615adcb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3977320e46574eb58b4f1df03ecb41c8",
            "max": 19,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_16b6c99bd1ec437b8ebf16d93b2be550",
            "value": 19
          }
        },
        "c05583241a114eec8af2ca55a2f5a12e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_22efaf54298a47b98b66ce2b259f00c1",
            "placeholder": "​",
            "style": "IPY_MODEL_9ad24930d36b43cdaff8f2337e58a0a4",
            "value": " 19/19 [00:09&lt;00:00,  2.21it/s]"
          }
        },
        "3aabb2bc4ac2404abe8ca1345e6f1b48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a8b76900f084b5692930d7cc0cc500e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d8f3e54008740ecb3ce90bc85989e3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3977320e46574eb58b4f1df03ecb41c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16b6c99bd1ec437b8ebf16d93b2be550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "22efaf54298a47b98b66ce2b259f00c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ad24930d36b43cdaff8f2337e58a0a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "449d12edba0c418ab38cb83d722641c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_51e3d66820c5402cafefd4bd7ce295a7",
              "IPY_MODEL_2f83fb81d8424eafa585728d7685812a",
              "IPY_MODEL_a3db1047148f4a5eb88cb70fe1e982b1"
            ],
            "layout": "IPY_MODEL_1b8dbc1f967e46f58e275368cc9e2183"
          }
        },
        "51e3d66820c5402cafefd4bd7ce295a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33f296bff5ce4944b8415aa78181890f",
            "placeholder": "​",
            "style": "IPY_MODEL_9ec75aa1671c4b07bd098d4753ad9a81",
            "value": "Calc val accuracy: 100%"
          }
        },
        "2f83fb81d8424eafa585728d7685812a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e1fb4c173b0542638a4a7cd41c654451",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_510c5b6386604fdd8a0ab66b691c9ca9",
            "value": 10
          }
        },
        "a3db1047148f4a5eb88cb70fe1e982b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7b4f28ef606f4bd296200fd88faa4935",
            "placeholder": "​",
            "style": "IPY_MODEL_bc02123a32914b519d3d1cacd66d986f",
            "value": " 10/10 [00:04&lt;00:00,  2.54it/s]"
          }
        },
        "1b8dbc1f967e46f58e275368cc9e2183": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33f296bff5ce4944b8415aa78181890f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ec75aa1671c4b07bd098d4753ad9a81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e1fb4c173b0542638a4a7cd41c654451": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "510c5b6386604fdd8a0ab66b691c9ca9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7b4f28ef606f4bd296200fd88faa4935": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc02123a32914b519d3d1cacd66d986f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}