recommendation-system-using-matrix-factorization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "sporting-packet",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.034241,
     "end_time": "2021-06-07T13:57:30.577210",
     "exception": false,
     "start_time": "2021-06-07T13:57:30.542969",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Recommender System for E-commerce Use Case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "tracked-richmond",
   "metadata": {
    "_kg_hide-input": false,
    "_kg_hide-output": true,
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:57:30.649314Z",
     "iopub.status.busy": "2021-06-07T13:57:30.648775Z",
     "iopub.status.idle": "2021-06-07T13:58:00.742769Z",
     "shell.execute_reply": "2021-06-07T13:58:00.743515Z",
     "shell.execute_reply.started": "2021-06-07T13:18:41.644894Z"
    },
    "papermill": {
     "duration": 30.135169,
     "end_time": "2021-06-07T13:58:00.743847",
     "exception": false,
     "start_time": "2021-06-07T13:57:30.608678",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting h2o4gpu\r\n",
      "  Downloading h2o4gpu-0.4.1-cp37-cp37m-manylinux1_x86_64.whl (346.3 MB)\r\n",
      "\u001b[K     |████████████████████████████████| 346.3 MB 9.2 kB/s \r\n",
      "\u001b[?25hCollecting scikit-learn==0.21.2\r\n",
      "  Downloading scikit_learn-0.21.2-cp37-cp37m-manylinux1_x86_64.whl (6.7 MB)\r\n",
      "\u001b[K     |████████████████████████████████| 6.7 MB 59.2 MB/s \r\n",
      "\u001b[?25hRequirement already satisfied: tabulate>=0.8.2 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (0.8.9)\r\n",
      "Requirement already satisfied: numpy>=1.16.4 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (1.19.5)\r\n",
      "Requirement already satisfied: pytz>=2018.4 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (2021.1)\r\n",
      "Requirement already satisfied: scipy>=1.3.1 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (1.5.4)\r\n",
      "Requirement already satisfied: future>=0.16.0 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (0.18.2)\r\n",
      "Requirement already satisfied: python-dateutil>=2.7.2 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (2.8.1)\r\n",
      "Requirement already satisfied: pandas>=0.24.2 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (1.1.5)\r\n",
      "Requirement already satisfied: joblib>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (1.0.1)\r\n",
      "Requirement already satisfied: psutil>=5.6.3 in /opt/conda/lib/python3.7/site-packages (from h2o4gpu) (5.8.0)\r\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7.2->h2o4gpu) (1.15.0)\r\n",
      "Installing collected packages: scikit-learn, h2o4gpu\r\n",
      "  Attempting uninstall: scikit-learn\r\n",
      "    Found existing installation: scikit-learn 0.24.1\r\n",
      "    Uninstalling scikit-learn-0.24.1:\r\n",
      "      Successfully uninstalled scikit-learn-0.24.1\r\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "umap-learn 0.5.1 requires scikit-learn>=0.22, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "tsfresh 0.18.0 requires scikit-learn>=0.22.0, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "tpot 0.11.7 requires scikit-learn>=0.22.0, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "sklearn-pandas 2.1.0 requires scikit-learn>=0.23.0, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "pyldavis 3.3.1 requires numpy>=1.20.0, but you have numpy 1.19.5 which is incompatible.\r\n",
      "pyldavis 3.3.1 requires pandas>=1.2.0, but you have pandas 1.1.5 which is incompatible.\r\n",
      "pdpbox 0.2.1 requires matplotlib==3.1.1, but you have matplotlib 3.4.1 which is incompatible.\r\n",
      "kmodes 0.11.0 requires scikit-learn>=0.22.0, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "imbalanced-learn 0.8.0 requires scikit-learn>=0.24, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "cesium 0.9.12 requires scikit-learn>=0.22.1, but you have scikit-learn 0.21.2 which is incompatible.\r\n",
      "autogluon-core 0.1.0 requires scikit-learn<0.25,>=0.22.0, but you have scikit-learn 0.21.2 which is incompatible.\u001b[0m\r\n",
      "Successfully installed h2o4gpu-0.4.1 scikit-learn-0.21.2\r\n"
     ]
    }
   ],
   "source": [
    "!pip install h2o4gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "amended-studio",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:01.333335Z",
     "iopub.status.busy": "2021-06-07T13:58:01.332553Z",
     "iopub.status.idle": "2021-06-07T13:58:02.649417Z",
     "shell.execute_reply": "2021-06-07T13:58:02.648907Z",
     "shell.execute_reply.started": "2021-06-07T13:18:47.085469Z"
    },
    "papermill": {
     "duration": 1.523342,
     "end_time": "2021-06-07T13:58:02.649559",
     "exception": false,
     "start_time": "2021-06-07T13:58:01.126217",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Import required libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import h2o4gpu\n",
    "import itertools\n",
    "import pickle\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from math import sqrt\n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bearing-mambo",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:02.927291Z",
     "iopub.status.busy": "2021-06-07T13:58:02.926725Z",
     "iopub.status.idle": "2021-06-07T13:58:12.467400Z",
     "shell.execute_reply": "2021-06-07T13:58:12.466746Z",
     "shell.execute_reply.started": "2021-06-07T13:18:47.614279Z"
    },
    "papermill": {
     "duration": 9.684176,
     "end_time": "2021-06-07T13:58:12.467551",
     "exception": false,
     "start_time": "2021-06-07T13:58:02.783375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3166: DtypeWarning: Columns (10) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "#Reading data\n",
    "review_data=pd.read_csv('../input/amazon-reviews-dataset/review_data.csv')\n",
    "#metadata=pd.read_csv('./Data/meta_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "unlimited-privilege",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:12.969697Z",
     "iopub.status.busy": "2021-06-07T13:58:12.968319Z",
     "iopub.status.idle": "2021-06-07T13:58:12.972711Z",
     "shell.execute_reply": "2021-06-07T13:58:12.973386Z",
     "shell.execute_reply.started": "2021-06-07T13:18:53.964766Z"
    },
    "papermill": {
     "duration": 0.370779,
     "end_time": "2021-06-07T13:58:12.973540",
     "exception": false,
     "start_time": "2021-06-07T13:58:12.602761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Dropping unwanted column\n",
    "review_data=review_data.drop(['Unnamed: 0'],axis=1)\n",
    "#metadata=metadata.drop(['Unnamed: 0'],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "electric-meditation",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.135508,
     "end_time": "2021-06-07T13:58:13.243983",
     "exception": false,
     "start_time": "2021-06-07T13:58:13.108475",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We have two dataframes-review_data and metadata. review_data is information about the product reviews posted by each user. It has following columns:\n",
    "\n",
    "    reviewerID - ID of the reviewer\n",
    "    asin - ID of the product\n",
    "    reviewerName - name of the reviewer\n",
    "    vote - helpful votes of the review\n",
    "    style - a dictionary of the product metadata\n",
    "    reviewText - text of the review\n",
    "    overall - rating of the product\n",
    "    summary - summary of the review\n",
    "    unixReviewTime - time of the review (unix time)\n",
    "    reviewTime - time of the review (raw)\n",
    "    image - images that users post after they have received the product\n",
    "\n",
    "metadata includes information about the product:\n",
    "\n",
    "    asin - ID of the product\n",
    "    title - name of the product\n",
    "    feature - bullet-point format features of the product\n",
    "    description - description of the product\n",
    "    price - price in US dollars (at time of crawl)\n",
    "    image - url of the product image\n",
    "    related - related products (also bought, also viewed, bought together, buy after viewing)\n",
    "    salesRank - sales rank information\n",
    "    brand - brand name\n",
    "    categories - list of categories the product belongs to\n",
    "    tech1 - the first technical detail table of the product\n",
    "    tech2 - the second technical detail table of the product\n",
    "    similar - similar product table\n",
    "    \n",
    "Citation: \n",
    "Justifying recommendations using distantly-labeled reviews and fined-grained aspects\n",
    "Jianmo Ni, Jiacheng Li, Julian McAuley\n",
    "Empirical Methods in Natural Language Processing (EMNLP), 2019\n",
    "pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "optional-david",
   "metadata": {
    "_kg_hide-output": true,
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:13.530401Z",
     "iopub.status.busy": "2021-06-07T13:58:13.529521Z",
     "iopub.status.idle": "2021-06-07T13:58:13.543409Z",
     "shell.execute_reply": "2021-06-07T13:58:13.542897Z",
     "shell.execute_reply.started": "2021-06-07T13:18:54.192875Z"
    },
    "papermill": {
     "duration": 0.163469,
     "end_time": "2021-06-07T13:58:13.543522",
     "exception": false,
     "start_time": "2021-06-07T13:58:13.380053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>vote</th>\n",
       "      <th>style</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>10 20, 2014</td>\n",
       "      <td>A1D4G1SNUZWQOT</td>\n",
       "      <td>7106116521</td>\n",
       "      <td>Tracy</td>\n",
       "      <td>Exactly what I needed.</td>\n",
       "      <td>perfect replacements!!</td>\n",
       "      <td>1413763200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>09 28, 2014</td>\n",
       "      <td>A3DDWDH9PX2YX2</td>\n",
       "      <td>7106116521</td>\n",
       "      <td>Sonja Lau</td>\n",
       "      <td>I agree with the other review, the opening is ...</td>\n",
       "      <td>I agree with the other review, the opening is ...</td>\n",
       "      <td>1411862400</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>False</td>\n",
       "      <td>08 25, 2014</td>\n",
       "      <td>A2MWC41EW7XL15</td>\n",
       "      <td>7106116521</td>\n",
       "      <td>Kathleen</td>\n",
       "      <td>Love these... I am going to order another pack...</td>\n",
       "      <td>My New 'Friends' !!</td>\n",
       "      <td>1408924800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>08 24, 2014</td>\n",
       "      <td>A2UH2QQ275NV45</td>\n",
       "      <td>7106116521</td>\n",
       "      <td>Jodi Stoner</td>\n",
       "      <td>too tiny an opening</td>\n",
       "      <td>Two Stars</td>\n",
       "      <td>1408838400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "      <td>False</td>\n",
       "      <td>07 27, 2014</td>\n",
       "      <td>A89F3LQADZBS5</td>\n",
       "      <td>7106116521</td>\n",
       "      <td>Alexander D.</td>\n",
       "      <td>Okay</td>\n",
       "      <td>Three Stars</td>\n",
       "      <td>1406419200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall  verified   reviewTime      reviewerID        asin  reviewerName  \\\n",
       "0      5.0      True  10 20, 2014  A1D4G1SNUZWQOT  7106116521         Tracy   \n",
       "1      2.0      True  09 28, 2014  A3DDWDH9PX2YX2  7106116521     Sonja Lau   \n",
       "2      4.0     False  08 25, 2014  A2MWC41EW7XL15  7106116521      Kathleen   \n",
       "3      2.0      True  08 24, 2014  A2UH2QQ275NV45  7106116521   Jodi Stoner   \n",
       "4      3.0     False  07 27, 2014   A89F3LQADZBS5  7106116521  Alexander D.   \n",
       "\n",
       "                                          reviewText  \\\n",
       "0                             Exactly what I needed.   \n",
       "1  I agree with the other review, the opening is ...   \n",
       "2  Love these... I am going to order another pack...   \n",
       "3                                too tiny an opening   \n",
       "4                                               Okay   \n",
       "\n",
       "                                             summary  unixReviewTime vote  \\\n",
       "0                             perfect replacements!!      1413763200  NaN   \n",
       "1  I agree with the other review, the opening is ...      1411862400    3   \n",
       "2                                My New 'Friends' !!      1408924800  NaN   \n",
       "3                                          Two Stars      1408838400  NaN   \n",
       "4                                        Three Stars      1406419200  NaN   \n",
       "\n",
       "  style image  \n",
       "0   NaN   NaN  \n",
       "1   NaN   NaN  \n",
       "2   NaN   NaN  \n",
       "3   NaN   NaN  \n",
       "4   NaN   NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "listed-original",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:13.822171Z",
     "iopub.status.busy": "2021-06-07T13:58:13.821586Z",
     "iopub.status.idle": "2021-06-07T13:58:13.825185Z",
     "shell.execute_reply": "2021-06-07T13:58:13.825622Z",
     "shell.execute_reply.started": "2021-06-07T13:18:54.218581Z"
    },
    "papermill": {
     "duration": 0.146062,
     "end_time": "2021-06-07T13:58:13.825779",
     "exception": false,
     "start_time": "2021-06-07T13:58:13.679717",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#metadata.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "chief-preference",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.139191,
     "end_time": "2021-06-07T13:58:14.100662",
     "exception": false,
     "start_time": "2021-06-07T13:58:13.961471",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Taking a subset of the large review dataset as per our requirement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "supreme-brooklyn",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:14.411407Z",
     "iopub.status.busy": "2021-06-07T13:58:14.410423Z",
     "iopub.status.idle": "2021-06-07T13:58:15.055435Z",
     "shell.execute_reply": "2021-06-07T13:58:15.056614Z",
     "shell.execute_reply.started": "2021-06-07T13:18:54.225044Z"
    },
    "papermill": {
     "duration": 0.820863,
     "end_time": "2021-06-07T13:58:15.056849",
     "exception": false,
     "start_time": "2021-06-07T13:58:14.235986",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sub_matrix = review_data[['reviewerID', 'asin', 'overall']].copy().drop_duplicates()\n",
    "#sub_matrix.set_index('reviewerID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "small-twins",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:15.537268Z",
     "iopub.status.busy": "2021-06-07T13:58:15.534695Z",
     "iopub.status.idle": "2021-06-07T13:58:15.539422Z",
     "shell.execute_reply": "2021-06-07T13:58:15.538956Z",
     "shell.execute_reply.started": "2021-06-07T13:18:54.957618Z"
    },
    "papermill": {
     "duration": 0.248719,
     "end_time": "2021-06-07T13:58:15.539539",
     "exception": false,
     "start_time": "2021-06-07T13:58:15.290820",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1466296, 3)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "registered-hamburg",
   "metadata": {
    "_kg_hide-input": false,
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T13:58:15.818819Z",
     "iopub.status.busy": "2021-06-07T13:58:15.817670Z",
     "iopub.status.idle": "2021-06-07T14:00:14.594521Z",
     "shell.execute_reply": "2021-06-07T14:00:14.593991Z",
     "shell.execute_reply.started": "2021-06-07T13:18:54.964345Z"
    },
    "papermill": {
     "duration": 118.919056,
     "end_time": "2021-06-07T14:00:14.594667",
     "exception": false,
     "start_time": "2021-06-07T13:58:15.675611",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Saving user data into pickle files\n",
    "u=np.unique(sub_matrix.reviewerID)\n",
    "user_dict = { k : [] for k in u }\n",
    "for row in sub_matrix.iterrows():\n",
    "    new_lst=(row[1]['asin'],row[1]['overall'])\n",
    "    user_dict[row[1]['reviewerID']].append(new_lst)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "recognized-webmaster",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.134315,
     "end_time": "2021-06-07T14:00:14.862701",
     "exception": false,
     "start_time": "2021-06-07T14:00:14.728386",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "loved-citation",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:15.139636Z",
     "iopub.status.busy": "2021-06-07T14:00:15.138826Z",
     "iopub.status.idle": "2021-06-07T14:00:15.141587Z",
     "shell.execute_reply": "2021-06-07T14:00:15.141180Z",
     "shell.execute_reply.started": "2021-06-07T13:22:07.238373Z"
    },
    "papermill": {
     "duration": 0.145425,
     "end_time": "2021-06-07T14:00:15.141701",
     "exception": false,
     "start_time": "2021-06-07T14:00:14.996276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Taking a sub-set of data for training\n",
    "train=sub_matrix[0:500000]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "careful-resistance",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.133146,
     "end_time": "2021-06-07T14:00:15.406571",
     "exception": false,
     "start_time": "2021-06-07T14:00:15.273425",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We have to filter the test set to have rows for only those products and users present in the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "spare-father",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:15.913706Z",
     "iopub.status.busy": "2021-06-07T14:00:15.912915Z",
     "iopub.status.idle": "2021-06-07T14:00:15.952070Z",
     "shell.execute_reply": "2021-06-07T14:00:15.951579Z",
     "shell.execute_reply.started": "2021-06-07T13:22:08.146057Z"
    },
    "papermill": {
     "duration": 0.402955,
     "end_time": "2021-06-07T14:00:15.952202",
     "exception": false,
     "start_time": "2021-06-07T14:00:15.549247",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Preparing the test set\n",
    "test=sub_matrix[500000:]\n",
    "#test=remaining[sub_matrix['reviewerID'].isin(user_train) & sub_matrix['asin'].isin(item_train)]\n",
    "#test=test[test.reviewerID.isin(train.reviewerID) & test.asin.isin(train.asin)]\n",
    "test = test[test.reviewerID.isin(train.reviewerID)]\n",
    "test = test[test.asin.isin(train.asin)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "august-novelty",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:16.746758Z",
     "iopub.status.busy": "2021-06-07T14:00:16.746010Z",
     "iopub.status.idle": "2021-06-07T14:00:17.496050Z",
     "shell.execute_reply": "2021-06-07T14:00:17.495576Z",
     "shell.execute_reply.started": "2021-06-07T13:22:08.558104Z"
    },
    "papermill": {
     "duration": 1.373756,
     "end_time": "2021-06-07T14:00:17.496208",
     "exception": false,
     "start_time": "2021-06-07T14:00:16.122452",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/generic.py:5170: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[name] = value\n"
     ]
    }
   ],
   "source": [
    "user_le = LabelEncoder()\n",
    "item_le = LabelEncoder()\n",
    "\n",
    "train.reviewerID = user_le.fit_transform(train.reviewerID)\n",
    "test.reviewerID = user_le.transform(test.reviewerID)\n",
    "\n",
    "train.asin = item_le.fit_transform(train.asin)\n",
    "test.asin = item_le.transform(test.asin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "indian-bench",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:17.768433Z",
     "iopub.status.busy": "2021-06-07T14:00:17.767827Z",
     "iopub.status.idle": "2021-06-07T14:00:17.770919Z",
     "shell.execute_reply": "2021-06-07T14:00:17.771363Z",
     "shell.execute_reply.started": "2021-06-07T13:22:09.987648Z"
    },
    "papermill": {
     "duration": 0.142137,
     "end_time": "2021-06-07T14:00:17.771500",
     "exception": false,
     "start_time": "2021-06-07T14:00:17.629363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 3)\n",
      "(10816, 3)\n"
     ]
    }
   ],
   "source": [
    "print(train.shape)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "australian-enough",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:18.044075Z",
     "iopub.status.busy": "2021-06-07T14:00:18.043247Z",
     "iopub.status.idle": "2021-06-07T14:00:18.119500Z",
     "shell.execute_reply": "2021-06-07T14:00:18.119048Z",
     "shell.execute_reply.started": "2021-06-07T13:22:10.100570Z"
    },
    "papermill": {
     "duration": 0.21565,
     "end_time": "2021-06-07T14:00:18.119626",
     "exception": false,
     "start_time": "2021-06-07T14:00:17.903976",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_train, user_train_indices = np.unique(np.array(train['reviewerID'], dtype=\"int32\"), return_inverse=True)\n",
    "item_train, item_train_indices = np.unique(np.array(train['asin'], dtype=\"int32\"), return_inverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "higher-investment",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:18.390539Z",
     "iopub.status.busy": "2021-06-07T14:00:18.389718Z",
     "iopub.status.idle": "2021-06-07T14:00:18.393588Z",
     "shell.execute_reply": "2021-06-07T14:00:18.393959Z",
     "shell.execute_reply.started": "2021-06-07T13:22:11.420146Z"
    },
    "papermill": {
     "duration": 0.141999,
     "end_time": "2021-06-07T14:00:18.394096",
     "exception": false,
     "start_time": "2021-06-07T14:00:18.252097",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Now for test set\n",
    "user_test,user_test_indices= np.unique(np.array(test['reviewerID'], dtype=\"int32\"), return_inverse=True)\n",
    "item_test,item_test_indices=np.unique(np.array(test['asin'], dtype=\"int32\"),return_inverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "liable-seller",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:18.665481Z",
     "iopub.status.busy": "2021-06-07T14:00:18.664475Z",
     "iopub.status.idle": "2021-06-07T14:00:18.667436Z",
     "shell.execute_reply": "2021-06-07T14:00:18.666935Z",
     "shell.execute_reply.started": "2021-06-07T13:22:20.033774Z"
    },
    "papermill": {
     "duration": 0.142251,
     "end_time": "2021-06-07T14:00:18.667562",
     "exception": false,
     "start_time": "2021-06-07T14:00:18.525311",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test=test.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "hollow-highlight",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:18.940030Z",
     "iopub.status.busy": "2021-06-07T14:00:18.939341Z",
     "iopub.status.idle": "2021-06-07T14:00:18.942651Z",
     "shell.execute_reply": "2021-06-07T14:00:18.943155Z",
     "shell.execute_reply.started": "2021-06-07T13:22:20.434782Z"
    },
    "papermill": {
     "duration": 0.141485,
     "end_time": "2021-06-07T14:00:18.943306",
     "exception": false,
     "start_time": "2021-06-07T14:00:18.801821",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "444449\n",
      "47580\n",
      "9820\n",
      "5639\n"
     ]
    }
   ],
   "source": [
    "print(len(user_train))\n",
    "print(len(item_train))\n",
    "print(len(user_test))\n",
    "print(len(item_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "awful-potter",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.13673,
     "end_time": "2021-06-07T14:00:19.238678",
     "exception": false,
     "start_time": "2021-06-07T14:00:19.101948",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Creating and computing error values for simple baseline models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "severe-basin",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:19.510712Z",
     "iopub.status.busy": "2021-06-07T14:00:19.509796Z",
     "iopub.status.idle": "2021-06-07T14:00:19.514271Z",
     "shell.execute_reply": "2021-06-07T14:00:19.513832Z",
     "shell.execute_reply.started": "2021-06-07T13:22:21.652675Z"
    },
    "papermill": {
     "duration": 0.143839,
     "end_time": "2021-06-07T14:00:19.514395",
     "exception": false,
     "start_time": "2021-06-07T14:00:19.370556",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pred_global_avg = [train.overall.mean()] * test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "colored-optics",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:19.800544Z",
     "iopub.status.busy": "2021-06-07T14:00:19.799707Z",
     "iopub.status.idle": "2021-06-07T14:00:19.819008Z",
     "shell.execute_reply": "2021-06-07T14:00:19.818568Z",
     "shell.execute_reply.started": "2021-06-07T13:22:22.543886Z"
    },
    "papermill": {
     "duration": 0.167074,
     "end_time": "2021-06-07T14:00:19.819134",
     "exception": false,
     "start_time": "2021-06-07T14:00:19.652060",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "product_avg_preds = train.groupby('asin')['overall'].mean().reset_index().rename(\n",
    "    columns={'overall': 'pred_product_avg'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "responsible-cache",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:20.102840Z",
     "iopub.status.busy": "2021-06-07T14:00:20.102079Z",
     "iopub.status.idle": "2021-06-07T14:00:20.231657Z",
     "shell.execute_reply": "2021-06-07T14:00:20.231198Z",
     "shell.execute_reply.started": "2021-06-07T13:22:22.963775Z"
    },
    "papermill": {
     "duration": 0.26867,
     "end_time": "2021-06-07T14:00:20.231785",
     "exception": false,
     "start_time": "2021-06-07T14:00:19.963115",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_avg_preds = train.groupby('reviewerID')['overall'].mean().reset_index().rename(\n",
    "    columns={'overall': 'pred_user_avg'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "experimental-medication",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:20.507470Z",
     "iopub.status.busy": "2021-06-07T14:00:20.505386Z",
     "iopub.status.idle": "2021-06-07T14:00:20.561954Z",
     "shell.execute_reply": "2021-06-07T14:00:20.561402Z",
     "shell.execute_reply.started": "2021-06-07T13:22:23.623877Z"
    },
    "papermill": {
     "duration": 0.196321,
     "end_time": "2021-06-07T14:00:20.562079",
     "exception": false,
     "start_time": "2021-06-07T14:00:20.365758",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Merging with test\n",
    "test = test.merge(product_avg_preds, on='asin')\n",
    "test = test.merge(user_avg_preds, on='reviewerID')\n",
    "test['pred_global_avg'] = pred_global_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "professional-package",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:20.833489Z",
     "iopub.status.busy": "2021-06-07T14:00:20.832501Z",
     "iopub.status.idle": "2021-06-07T14:00:20.839399Z",
     "shell.execute_reply": "2021-06-07T14:00:20.838851Z",
     "shell.execute_reply.started": "2021-06-07T13:22:24.182105Z"
    },
    "papermill": {
     "duration": 0.146253,
     "end_time": "2021-06-07T14:00:20.839523",
     "exception": false,
     "start_time": "2021-06-07T14:00:20.693270",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sparse_train=sparse.coo_matrix((np.array(train.overall.values, dtype='float32'),(user_train_indices,item_train_indices)),shape=(len(user_train), len(item_train)))\n",
    "sparse_test=sparse.coo_matrix((np.ones(len(test), dtype=\"float32\"),(user_test_indices,item_test_indices)),shape=(sparse_train.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "regional-merchant",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:21.114760Z",
     "iopub.status.busy": "2021-06-07T14:00:21.114164Z",
     "iopub.status.idle": "2021-06-07T14:00:21.117334Z",
     "shell.execute_reply": "2021-06-07T14:00:21.117914Z",
     "shell.execute_reply.started": "2021-06-07T13:22:24.862801Z"
    },
    "papermill": {
     "duration": 0.142359,
     "end_time": "2021-06-07T14:00:21.118103",
     "exception": false,
     "start_time": "2021-06-07T14:00:20.975744",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(444449, 47580)\n",
      "(444449, 47580)\n"
     ]
    }
   ],
   "source": [
    "print(sparse_train.shape)\n",
    "print(sparse_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "significant-march",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.132656,
     "end_time": "2021-06-07T14:00:21.384174",
     "exception": false,
     "start_time": "2021-06-07T14:00:21.251518",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Using h2o4GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tough-harris",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.132132,
     "end_time": "2021-06-07T14:00:21.650638",
     "exception": false,
     "start_time": "2021-06-07T14:00:21.518506",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "coated-consent",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:21.923205Z",
     "iopub.status.busy": "2021-06-07T14:00:21.922342Z",
     "iopub.status.idle": "2021-06-07T14:00:21.925016Z",
     "shell.execute_reply": "2021-06-07T14:00:21.924519Z",
     "shell.execute_reply.started": "2021-06-07T13:22:27.544477Z"
    },
    "papermill": {
     "duration": 0.142645,
     "end_time": "2021-06-07T14:00:21.925130",
     "exception": false,
     "start_time": "2021-06-07T14:00:21.782485",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "components=[10, 20, 30, 50, 70]\n",
    "lambda_values=[0.01, 0.05, 0.1, 0.5]\n",
    "max_iterations=[50, 100, 200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cosmetic-equivalent",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:22.196120Z",
     "iopub.status.busy": "2021-06-07T14:00:22.193721Z",
     "iopub.status.idle": "2021-06-07T14:00:22.198335Z",
     "shell.execute_reply": "2021-06-07T14:00:22.198703Z",
     "shell.execute_reply.started": "2021-06-07T13:22:27.892065Z"
    },
    "papermill": {
     "duration": 0.141044,
     "end_time": "2021-06-07T14:00:22.198848",
     "exception": false,
     "start_time": "2021-06-07T14:00:22.057804",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "parameter_values=list(itertools.product(components,lambda_values,max_iterations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "weighted-savage",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:22.477752Z",
     "iopub.status.busy": "2021-06-07T14:00:22.476938Z",
     "iopub.status.idle": "2021-06-07T14:00:22.479798Z",
     "shell.execute_reply": "2021-06-07T14:00:22.479366Z",
     "shell.execute_reply.started": "2021-06-07T13:22:28.283205Z"
    },
    "papermill": {
     "duration": 0.146095,
     "end_time": "2021-06-07T14:00:22.479910",
     "exception": false,
     "start_time": "2021-06-07T14:00:22.333815",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Factorization function\n",
    "def execute_mf(parameters,X,X_test,model=False):\n",
    "    factorization = h2o4gpu.solvers.FactorizationH2O(parameters[0], parameters[1], parameters[2])\n",
    "    factorization.fit(X, X_BATCHES=10, THETA_BATCHES=10)\n",
    "    predictions=factorization.predict(X_test).data\n",
    "    if model:\n",
    "        return predictions,factorization\n",
    "    else:\n",
    "        return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cloudy-selling",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:00:22.753437Z",
     "iopub.status.busy": "2021-06-07T14:00:22.752556Z",
     "iopub.status.idle": "2021-06-07T14:20:29.841992Z",
     "shell.execute_reply": "2021-06-07T14:20:29.842522Z",
     "shell.execute_reply.started": "2021-06-07T13:22:29.502227Z"
    },
    "papermill": {
     "duration": 1207.229837,
     "end_time": "2021-06-07T14:20:29.842692",
     "exception": false,
     "start_time": "2021-06-07T14:00:22.612855",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing done. combinations left :  55\n",
      "Processing done. combinations left :  50\n",
      "Processing done. combinations left :  45\n",
      "Processing done. combinations left :  40\n",
      "Processing done. combinations left :  35\n",
      "Processing done. combinations left :  30\n",
      "Processing done. combinations left :  25\n",
      "Processing done. combinations left :  20\n",
      "Processing done. combinations left :  15\n",
      "Processing done. combinations left :  10\n",
      "Processing done. combinations left :  5\n",
      "Processing done. combinations left :  0\n"
     ]
    }
   ],
   "source": [
    "results=dict()\n",
    "c=0\n",
    "for i in parameter_values:#parameter_values[0:math.floor(len(parameter_values)/2)]:\n",
    "    c=c+1\n",
    "    preds=execute_mf(i,sparse_train,sparse_test)\n",
    "    if not np.isnan(preds).any():\n",
    "        results[i]=np.sqrt(mean_squared_error(test.overall, preds))\n",
    "    if c%5==0:\n",
    "        print('Processing done. combinations left : ',len(parameter_values)-c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "coordinated-legislation",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:30.162669Z",
     "iopub.status.busy": "2021-06-07T14:20:30.160998Z",
     "iopub.status.idle": "2021-06-07T14:20:30.165238Z",
     "shell.execute_reply": "2021-06-07T14:20:30.164571Z",
     "shell.execute_reply.started": "2021-06-07T13:43:14.695215Z"
    },
    "papermill": {
     "duration": 0.186305,
     "end_time": "2021-06-07T14:20:30.165401",
     "exception": false,
     "start_time": "2021-06-07T14:20:29.979096",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70, 0.01, 200)\n"
     ]
    }
   ],
   "source": [
    "#Fetching the parameter combination giving least RMSE value\n",
    "print(min(results, key=results.get))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "worth-punch",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.133814,
     "end_time": "2021-06-07T14:20:30.436179",
     "exception": false,
     "start_time": "2021-06-07T14:20:30.302365",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Thus, we infer that (70, 0.01, 200) is the best combination of parameter values. Hence we run our algorithm using that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "geological-person",
   "metadata": {
    "_kg_hide-input": true,
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:30.714418Z",
     "iopub.status.busy": "2021-06-07T14:20:30.713834Z",
     "iopub.status.idle": "2021-06-07T14:20:30.717850Z",
     "shell.execute_reply": "2021-06-07T14:20:30.717430Z",
     "shell.execute_reply.started": "2021-06-07T13:47:50.223738Z"
    },
    "papermill": {
     "duration": 0.14516,
     "end_time": "2021-06-07T14:20:30.717965",
     "exception": false,
     "start_time": "2021-06-07T14:20:30.572805",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#pred_mf,model=execute_mf([70,0.01,200],sparse_train,sparse_test,True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "adolescent-dispute",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:30.998530Z",
     "iopub.status.busy": "2021-06-07T14:20:30.996741Z",
     "iopub.status.idle": "2021-06-07T14:20:31.001484Z",
     "shell.execute_reply": "2021-06-07T14:20:31.001896Z",
     "shell.execute_reply.started": "2021-06-07T13:47:50.908120Z"
    },
    "papermill": {
     "duration": 0.149849,
     "end_time": "2021-06-07T14:20:31.002032",
     "exception": false,
     "start_time": "2021-06-07T14:20:30.852183",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Product Avg: 1.4675429714630877\n",
      "RMSE User Avg: 1.6234457217469518\n",
      "RMSE Global Avg: 1.4346134925714396\n",
      "RMSE H2O-MF:  2.0201818047054627\n"
     ]
    }
   ],
   "source": [
    "print(f'RMSE Product Avg: {np.sqrt(mean_squared_error(test.overall, test.pred_product_avg))}')\n",
    "print(f'RMSE User Avg: {np.sqrt(mean_squared_error(test.overall, test.pred_user_avg))}')\n",
    "print(f'RMSE Global Avg: {np.sqrt(mean_squared_error(test.overall, test.pred_global_avg))}')\n",
    "print(f'RMSE H2O-MF: ',results[(70, 0.01, 200)])#np.sqrt(mean_squared_error(test.overall, test.pred_mf))}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "thirty-table",
   "metadata": {
    "editable": false,
    "papermill": {
     "duration": 0.136584,
     "end_time": "2021-06-07T14:20:31.275674",
     "exception": false,
     "start_time": "2021-06-07T14:20:31.139090",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Saving our final model alongwith required data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "civil-secretariat",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:32.745859Z",
     "iopub.status.busy": "2021-06-07T14:20:32.745013Z",
     "iopub.status.idle": "2021-06-07T14:20:34.995846Z",
     "shell.execute_reply": "2021-06-07T14:20:34.995374Z",
     "shell.execute_reply.started": "2021-06-07T13:48:10.319179Z"
    },
    "papermill": {
     "duration": 3.585781,
     "end_time": "2021-06-07T14:20:34.995980",
     "exception": false,
     "start_time": "2021-06-07T14:20:31.410199",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_encode = LabelEncoder()\n",
    "item_encode = LabelEncoder()\n",
    "\n",
    "sub_matrix.reviewerID = user_encode.fit_transform(sub_matrix.reviewerID)\n",
    "sub_matrix.asin = item_encode.fit_transform(sub_matrix.asin)\n",
    "\n",
    "user, user_indices = np.unique(np.array(sub_matrix['reviewerID'], dtype=\"int32\"), return_inverse=True)\n",
    "item, item_indices = np.unique(np.array(sub_matrix['asin'], dtype=\"int32\"), return_inverse=True)\n",
    "\n",
    "sparse_mat=sparse.coo_matrix((np.array(sub_matrix.overall.values, dtype='float32'),(user_indices,item_indices)),shape=(len(user), len(item)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "bulgarian-roman",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:35.274383Z",
     "iopub.status.busy": "2021-06-07T14:20:35.273576Z",
     "iopub.status.idle": "2021-06-07T14:20:35.342108Z",
     "shell.execute_reply": "2021-06-07T14:20:35.341578Z",
     "shell.execute_reply.started": "2021-06-07T13:50:26.132338Z"
    },
    "papermill": {
     "duration": 0.208486,
     "end_time": "2021-06-07T14:20:35.342283",
     "exception": false,
     "start_time": "2021-06-07T14:20:35.133797",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "i=item_encode.inverse_transform(item)\n",
    "#Preparing Item-Index mapping dictionary\n",
    "items=dict(zip(i,item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "modern-custom",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:35.615640Z",
     "iopub.status.busy": "2021-06-07T14:20:35.614629Z",
     "iopub.status.idle": "2021-06-07T14:20:36.263120Z",
     "shell.execute_reply": "2021-06-07T14:20:36.262529Z",
     "shell.execute_reply.started": "2021-06-07T13:51:14.454423Z"
    },
    "papermill": {
     "duration": 0.787445,
     "end_time": "2021-06-07T14:20:36.263257",
     "exception": false,
     "start_time": "2021-06-07T14:20:35.475812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "u=user_encode.inverse_transform(user)\n",
    "#Preparing User-Index mapping dictionary\n",
    "users=dict(zip(u,user))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "together-assessment",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:36.536556Z",
     "iopub.status.busy": "2021-06-07T14:20:36.535769Z",
     "iopub.status.idle": "2021-06-07T14:20:36.539898Z",
     "shell.execute_reply": "2021-06-07T14:20:36.540528Z",
     "shell.execute_reply.started": "2021-06-07T13:52:18.156113Z"
    },
    "papermill": {
     "duration": 0.144121,
     "end_time": "2021-06-07T14:20:36.540703",
     "exception": false,
     "start_time": "2021-06-07T14:20:36.396582",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1239946\n",
      "216441\n",
      "(1239946, 216441)\n"
     ]
    }
   ],
   "source": [
    "print(len(users))\n",
    "print(len(items))\n",
    "print(sparse_mat.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ideal-aspect",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:36.812976Z",
     "iopub.status.busy": "2021-06-07T14:20:36.812125Z",
     "iopub.status.idle": "2021-06-07T14:20:36.814898Z",
     "shell.execute_reply": "2021-06-07T14:20:36.814486Z",
     "shell.execute_reply.started": "2021-06-07T13:53:06.966936Z"
    },
    "papermill": {
     "duration": 0.140947,
     "end_time": "2021-06-07T14:20:36.815010",
     "exception": false,
     "start_time": "2021-06-07T14:20:36.674063",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "rec = h2o4gpu.solvers.FactorizationH2O(70, 0.01, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "still-harbor",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:20:37.121622Z",
     "iopub.status.busy": "2021-06-07T14:20:37.119886Z",
     "iopub.status.idle": "2021-06-07T14:23:59.988056Z",
     "shell.execute_reply": "2021-06-07T14:23:59.988794Z",
     "shell.execute_reply.started": "2021-06-07T13:53:08.012286Z"
    },
    "papermill": {
     "duration": 203.041606,
     "end_time": "2021-06-07T14:23:59.988960",
     "exception": false,
     "start_time": "2021-06-07T14:20:36.947354",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h2o4gpu.solvers.factorization.FactorizationH2O at 0x7f6c2c506f50>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec.fit(sparse_mat, X_BATCHES=10, THETA_BATCHES=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "mighty-serve",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-07T14:24:00.260703Z",
     "iopub.status.busy": "2021-06-07T14:24:00.259900Z",
     "iopub.status.idle": "2021-06-07T14:24:09.028962Z",
     "shell.execute_reply": "2021-06-07T14:24:09.029698Z",
     "shell.execute_reply.started": "2021-06-07T13:56:35.976621Z"
    },
    "papermill": {
     "duration": 8.908174,
     "end_time": "2021-06-07T14:24:09.029935",
     "exception": false,
     "start_time": "2021-06-07T14:24:00.121761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Saving user data into pickle files\n",
    "#items=item_encode.inverse_transform(item)\n",
    "d_dict={'user':user_dict,'user_indices':users,'item_indices':items}\n",
    "with open('review_data.pickle','wb') as f:\n",
    "    pickle.dump(d_dict,f,pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "rental-carrier",
   "metadata": {
    "_kg_hide-input": true,
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:24:09.305400Z",
     "iopub.status.busy": "2021-06-07T14:24:09.304681Z",
     "iopub.status.idle": "2021-06-07T14:24:09.307643Z",
     "shell.execute_reply": "2021-06-07T14:24:09.307236Z"
    },
    "papermill": {
     "duration": 0.139217,
     "end_time": "2021-06-07T14:24:09.307757",
     "exception": false,
     "start_time": "2021-06-07T14:24:09.168540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Extracting User's features and Item's features from model\n",
    "#user_matrix=rec.XT\n",
    "#item_matrix=rec.thetaT\n",
    "#Getting users and items\n",
    "#users=user_encode.inverse_transform(user)\n",
    "#items=item_encode.inverse_transform(item)\n",
    "#users=list(map(str,users))\n",
    "#items=list(map(str,items))\n",
    "#Serializing predictions and users and items\n",
    "#recommend={'user_matrix':user_matrix,'item_matrix':item_matrix,'users':users,'items':items}\n",
    "#with open('recommend_amazon.pickle', 'wb') as f:\n",
    "#    # Pickle the 'data' dictionary using the highest protocol available.\n",
    "#    pickle.dump(recommend, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "extended-robin",
   "metadata": {
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:24:09.622105Z",
     "iopub.status.busy": "2021-06-07T14:24:09.580046Z",
     "iopub.status.idle": "2021-06-07T14:24:10.181436Z",
     "shell.execute_reply": "2021-06-07T14:24:10.180885Z",
     "shell.execute_reply.started": "2021-06-07T13:56:54.438147Z"
    },
    "papermill": {
     "duration": 0.740214,
     "end_time": "2021-06-07T14:24:10.181568",
     "exception": false,
     "start_time": "2021-06-07T14:24:09.441354",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Saving model file as pickle\n",
    "with open('recommend_model.pickle','wb') as f:\n",
    "    pickle.dump(rec,f,pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "familiar-variation",
   "metadata": {
    "_kg_hide-input": true,
    "editable": false,
    "execution": {
     "iopub.execute_input": "2021-06-07T14:24:10.455016Z",
     "iopub.status.busy": "2021-06-07T14:24:10.454465Z",
     "iopub.status.idle": "2021-06-07T14:24:10.457861Z",
     "shell.execute_reply": "2021-06-07T14:24:10.458375Z"
    },
    "papermill": {
     "duration": 0.14289,
     "end_time": "2021-06-07T14:24:10.458604",
     "exception": false,
     "start_time": "2021-06-07T14:24:10.315714",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Create dictionary of mini dataframes\n",
    "#user_data=dict()\n",
    "#user_data={row[1]['reviewerID']:sub_matrix[sub_matrix['reviewerID']==row[1]['reviewerID']] for row in sub_matrix.iterrows()}\n",
    "#user_data={key['reviewerID']:pd.DataFrame.from_records([l for l in df_dict if l['reviewerID']==key['reviewerID']]) for key in df_dict}\n",
    "\n",
    "#u=user_encode.inverse_transform(user)\n",
    "#user_data={val:sub_matrix[sub_matrix['reviewerID']==val][['asin','overall']] for val in u}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1610.397738,
   "end_time": "2021-06-07T14:24:13.332068",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-06-07T13:57:22.934330",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}