model_training_basic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7738d28f-c084-46ef-80ac-8d3f1a4e4a9a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "\n",
    "from transformers import BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "756802ca-8540-4bf6-be2a-a33508e866d9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from models import model_1 as mm\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e619561-7675-4beb-b57f-7dc80b09fa8f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Initalizing Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bd953104-e9e9-401c-a8d6-ed523f2efd40",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "d_model = 768\n",
    "max_token = 512\n",
    "nhead = 12\n",
    "nlayer = 6\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a70de0e-7fa9-499b-b968-77bc1789b532",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"bert_embedding.pkl\", \"rb\") as file:\n",
    "    embd = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce49df46-89db-4028-bd39-72943065fdfd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = mm.ad_transformer(d_model,nhead, nlayer, max_token, embd, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "42210a5e-7699-4eb4-82de-70f167be0ec3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1538f4b-d2ec-4fa6-b38b-81e7e2cd6fd4",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Initializing Default Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b1d838c8-78df-4296-9654-9e69ba751a33",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for p in model.parameters():\n",
    "    if p.dim() > 1:\n",
    "        nn.init.xavier_uniform_(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8269a6b-b0ed-480b-a3b1-91da395a9d5f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Loading Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7ce35864-2c1f-497d-80f1-120465690752",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"dataset/spam_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d03e325c-53fb-4114-9b42-80586f86eaf7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.drop(\"len\", axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "324b35c5-8ed9-44f0-9f40-85ecfb3a325f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_type</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spam</td>\n",
       "      <td>perfect visual solution for your business now ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>spam</td>\n",
       "      <td>an innovative plan for today s market give you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>all graphics software available cheap oem vers...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>spam</td>\n",
       "      <td>perfect logo charset koi 8 r thinking of breat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>spam</td>\n",
       "      <td>back to happy and healthy life we ve created a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  text_type                                               text\n",
       "0      spam  perfect visual solution for your business now ...\n",
       "1      spam  an innovative plan for today s market give you...\n",
       "2      spam  all graphics software available cheap oem vers...\n",
       "3      spam  perfect logo charset koi 8 r thinking of breat...\n",
       "4      spam  back to happy and healthy life we ve created a..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b273d18a-0f37-4f48-a573-768ae966cf60",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Removing Sequence more than 500 tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5f1e575e-73ad-44a4-9312-840608555e45",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (965 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "tok_threshold = []\n",
    "for i in range(len(df) - 1, -1, -1):\n",
    "    tok_len = len(tokenizer(df.iloc[i,1], return_tensors=\"pt\", padding=True)[\"input_ids\"][0])\n",
    "    if tok_len > 500:\n",
    "        tok_threshold.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ee9cc363-8bbc-45f7-9c77-defeb9421eb5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[859]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tok_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eb735ff9-0a2c-4d47-ab7f-dce34b02a627",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(859, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "863c7d5d-56d8-4ef6-989a-3d87113f981d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.iloc[:10000, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8102632c-6661-42ea-8f2d-d8f8ece3f360",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa9b4386-eba1-4b20-884f-e839e2b183fa",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Initializng Loss and Update Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4e6515b6-c058-42ac-a0f1-716ff9d51b5b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "optim = torch.optim.Adam(model.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "39c436f8-f4fd-4274-867a-51526617b3a6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "criterian = nn.CrossEntropyLoss(reduction='none')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ba0e956-0d7c-4125-a0fe-e21a2ad07e18",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "28f71e7e-a575-4317-8cec-b9c009253a61",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Assuming [spam, ham]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9b95071e-1c39-4741-b436-386b383ceeb7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "batch_size = 50\n",
    "total_datapoint = 10000\n",
    "num_epoch = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "63e29e71-1918-4185-ad6e-0e1cc495c90f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "### Epoch 0 Completed. ###\n",
      "Avg Loss : 0.0\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "### Epoch 1 Completed. ###\n",
      "Avg Loss : tensor(0.0033, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n",
      "1000 Datapoint Completed\n"
     ]
    }
   ],
   "source": [
    "total_loss = 0\n",
    "for epoch in range(num_epoch):\n",
    "\n",
    "    \n",
    "    print(\"### Epoch \" + str(epoch) + \" Completed. ###\")\n",
    "    print(\"Avg Loss : \" + str(batch_size*total_loss/total_datapoint))\n",
    "    total_loss = 0\n",
    "    for index in range(0, total_datapoint, batch_size):\n",
    "        if index % 1000 == 0:\n",
    "            print(\"1000 Datapoint Completed\")\n",
    "        \n",
    "        optim.zero_grad()\n",
    "        \n",
    "        input_list = df.iloc[index:index+batch_size, 1].tolist()\n",
    "        tok_input = tokenizer(input_list, return_tensors=\"pt\", padding=True).to(device)\n",
    "        \n",
    "        out = model(tok_input)\n",
    "        \n",
    "        tar_list = []\n",
    "\n",
    "        for i in range(index, index + batch_size):\n",
    "            if df.iloc[i,0] == \"spam\":\n",
    "                tar_list.append([1,0])\n",
    "            else:\n",
    "                tar_list.append([0,1])\n",
    "\n",
    "        target = torch.tensor(tar_list, dtype = torch.float).to(device)\n",
    "        \n",
    "        loss = criterian(out, target)\n",
    "        avg_loss = loss.sum()/batch_size\n",
    "        \n",
    "        total_loss = max(total_loss, avg_loss)   ## recording\n",
    "        \n",
    "        avg_loss.backward()\n",
    "        optim.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c963b924-7cd5-46b5-ac27-f5ec42109930",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_type</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>harvard business school pub order confirmation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>spam</td>\n",
       "      <td>hello you can make upto $11000 worth of btc di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>100 free hardcore megasite 100 free porn what ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>spam</td>\n",
       "      <td>𝑰 𝒔𝒂𝒘 𝒕𝒆𝒔𝒕𝒊𝒎𝒐𝒏𝒊𝒆𝒔 𝒐𝒇 𝒉𝒐𝒘 mrs heatherjgilbert 𝒉...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>spam</td>\n",
       "      <td>the most comprehensive adult match making serv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>ham</td>\n",
       "      <td>new resume dear vince i am so grateful for you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>ham</td>\n",
       "      <td>use perl daily headline mailer announcing url ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>ham</td>\n",
       "      <td>tony hamilton chris e hired tony to support gl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>ham</td>\n",
       "      <td>fw fea announces the release of energy 2 1 chr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>ham</td>\n",
       "      <td>invoice for ordered papers shirley the invoice...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     text_type                                               text\n",
       "0          ham  harvard business school pub order confirmation...\n",
       "1         spam  hello you can make upto $11000 worth of btc di...\n",
       "2         spam  100 free hardcore megasite 100 free porn what ...\n",
       "3         spam  𝑰 𝒔𝒂𝒘 𝒕𝒆𝒔𝒕𝒊𝒎𝒐𝒏𝒊𝒆𝒔 𝒐𝒇 𝒉𝒐𝒘 mrs heatherjgilbert 𝒉...\n",
       "4         spam  the most comprehensive adult match making serv...\n",
       "...        ...                                                ...\n",
       "9995       ham  new resume dear vince i am so grateful for you...\n",
       "9996       ham  use perl daily headline mailer announcing url ...\n",
       "9997       ham  tony hamilton chris e hired tony to support gl...\n",
       "9998       ham  fw fea announces the release of energy 2 1 chr...\n",
       "9999       ham  invoice for ordered papers shirley the invoice...\n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b7cdb93-ce6b-40a4-b4f3-d30f5258a1e4",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c31e487b-47af-41ec-acb5-a7252f60c97f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\"dataset/spam_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "5f588196-5adc-4753-b1a9-6655010ed2e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Actual Value (SPAM, HAM)  ## Y - Predicted Value (SPAM, HAM)\n",
    "conf_matrix = np.zeros((2,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "95288219-160b-47d8-bf20-1a06bf90e836",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for i in range(len(df_test)):\n",
    "    input_list = df_test.iloc[i,1]\n",
    "    tok_input = tokenizer(input_list, return_tensors=\"pt\", padding=True).to(device)\n",
    "    out = model(tok_input)\n",
    "    \n",
    "    index_1 = torch.argmax(out[0]).tolist()\n",
    "    \n",
    "    if df_test.iloc[i,0] == \"spam\":\n",
    "        index_2 = 0\n",
    "    else:\n",
    "        index_2 = 1\n",
    "        \n",
    "    conf_matrix[index_1][index_2] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f1656fc9-6d30-4082-96c2-70449fe90d25",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[446.,   8.],\n",
       "       [ 54., 492.]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "921ffdcb-2871-42e8-9bfd-65ce8b3a517d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "acfdcb76-7fb3-4074-9bf9-9ca11383b3c3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), \"weight_trained_1.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c3aadc8-24f2-4909-a914-4658d9312e39",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Local)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}