run_all_lw.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, install and import all neccessary python libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in ./venv/lib/python3.7/site-packages (3.8.1)\n",
      "Requirement already satisfied: scipy>=0.18.1 in ./venv/lib/python3.7/site-packages (from gensim) (1.3.2)\n",
      "Requirement already satisfied: six>=1.5.0 in ./venv/lib/python3.7/site-packages (from gensim) (1.13.0)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in ./venv/lib/python3.7/site-packages (from gensim) (1.9.0)\n",
      "Requirement already satisfied: numpy>=1.11.3 in ./venv/lib/python3.7/site-packages (from gensim) (1.17.4)\n",
      "Requirement already satisfied: boto>=2.32 in ./venv/lib/python3.7/site-packages (from smart-open>=1.8.1->gensim) (2.49.0)\n",
      "Requirement already satisfied: requests in ./venv/lib/python3.7/site-packages (from smart-open>=1.8.1->gensim) (2.22.0)\n",
      "Requirement already satisfied: boto3 in ./venv/lib/python3.7/site-packages (from smart-open>=1.8.1->gensim) (1.10.19)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in ./venv/lib/python3.7/site-packages (from requests->smart-open>=1.8.1->gensim) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in ./venv/lib/python3.7/site-packages (from requests->smart-open>=1.8.1->gensim) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.7/site-packages (from requests->smart-open>=1.8.1->gensim) (2019.9.11)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in ./venv/lib/python3.7/site-packages (from requests->smart-open>=1.8.1->gensim) (1.25.7)\n",
      "Requirement already satisfied: botocore<1.14.0,>=1.13.19 in ./venv/lib/python3.7/site-packages (from boto3->smart-open>=1.8.1->gensim) (1.13.19)\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in ./venv/lib/python3.7/site-packages (from boto3->smart-open>=1.8.1->gensim) (0.2.1)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in ./venv/lib/python3.7/site-packages (from boto3->smart-open>=1.8.1->gensim) (0.9.4)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in ./venv/lib/python3.7/site-packages (from botocore<1.14.0,>=1.13.19->boto3->smart-open>=1.8.1->gensim) (0.15.2)\n",
      "Requirement already satisfied: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in ./venv/lib/python3.7/site-packages (from botocore<1.14.0,>=1.13.19->boto3->smart-open>=1.8.1->gensim) (2.8.0)\n",
      "Requirement already satisfied: keras in ./venv/lib/python3.7/site-packages (2.3.1)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in ./venv/lib/python3.7/site-packages (from keras) (1.0.8)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in ./venv/lib/python3.7/site-packages (from keras) (1.1.0)\n",
      "Requirement already satisfied: numpy>=1.9.1 in ./venv/lib/python3.7/site-packages (from keras) (1.17.4)\n",
      "Requirement already satisfied: pyyaml in ./venv/lib/python3.7/site-packages (from keras) (5.1.2)\n",
      "Requirement already satisfied: h5py in ./venv/lib/python3.7/site-packages (from keras) (2.10.0)\n",
      "Requirement already satisfied: six>=1.9.0 in ./venv/lib/python3.7/site-packages (from keras) (1.13.0)\n",
      "Requirement already satisfied: scipy>=0.14 in ./venv/lib/python3.7/site-packages (from keras) (1.3.2)\n",
      "Requirement already satisfied: nltk in ./venv/lib/python3.7/site-packages (3.4.5)\n",
      "Requirement already satisfied: six in ./venv/lib/python3.7/site-packages (from nltk) (1.13.0)\n",
      "Requirement already satisfied: numpy in ./venv/lib/python3.7/site-packages (1.17.4)\n",
      "Requirement already satisfied: pandas in ./venv/lib/python3.7/site-packages (0.25.3)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in ./venv/lib/python3.7/site-packages (from pandas) (2.8.0)\n",
      "Requirement already satisfied: pytz>=2017.2 in ./venv/lib/python3.7/site-packages (from pandas) (2019.3)\n",
      "Requirement already satisfied: numpy>=1.13.3 in ./venv/lib/python3.7/site-packages (from pandas) (1.17.4)\n",
      "Requirement already satisfied: six>=1.5 in ./venv/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas) (1.13.0)\n",
      "Requirement already satisfied: sklearn in ./venv/lib/python3.7/site-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in ./venv/lib/python3.7/site-packages (from sklearn) (0.21.3)\n",
      "Requirement already satisfied: joblib>=0.11 in ./venv/lib/python3.7/site-packages (from scikit-learn->sklearn) (0.14.0)\n",
      "Requirement already satisfied: scipy>=0.17.0 in ./venv/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.3.2)\n",
      "Requirement already satisfied: numpy>=1.11.0 in ./venv/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.17.4)\n",
      "Requirement already satisfied: tensorflow==1.15 in ./venv/lib/python3.7/site-packages (1.15.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (0.8.0)\n",
      "Requirement already satisfied: wheel>=0.26 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (0.33.6)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (0.8.1)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.17.4)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (0.1.8)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.25.0)\n",
      "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.15.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (3.1.0)\n",
      "Requirement already satisfied: gast==0.2.2 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (0.2.2)\n",
      "Requirement already satisfied: six>=1.10.0 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.13.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (3.10.0)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /Users/lindawong/.local/lib/python3.7/site-packages (from tensorflow==1.15) (1.11.2)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: tensorflow-estimator==1.15.1 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.15.1)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in ./venv/lib/python3.7/site-packages (from tensorflow==1.15) (1.0.8)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in ./venv/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (41.6.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in ./venv/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (0.16.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in ./venv/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (3.1.1)\n",
      "Requirement already satisfied: h5py in ./venv/lib/python3.7/site-packages (from keras-applications>=1.0.8->tensorflow==1.15) (2.10.0)\n",
      "Requirement already satisfied: wordcloud in ./venv/lib/python3.7/site-packages (1.6.0)\n",
      "Requirement already satisfied: pillow in ./venv/lib/python3.7/site-packages (from wordcloud) (6.2.1)\n",
      "Requirement already satisfied: numpy>=1.6.1 in ./venv/lib/python3.7/site-packages (from wordcloud) (1.17.4)\n",
      "Requirement already satisfied: matplotlib in ./venv/lib/python3.7/site-packages (from wordcloud) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in ./venv/lib/python3.7/site-packages (from matplotlib->wordcloud) (2.8.0)\n",
      "Requirement already satisfied: cycler>=0.10 in ./venv/lib/python3.7/site-packages (from matplotlib->wordcloud) (0.10.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in ./venv/lib/python3.7/site-packages (from matplotlib->wordcloud) (2.4.5)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in ./venv/lib/python3.7/site-packages (from matplotlib->wordcloud) (1.1.0)\n",
      "Requirement already satisfied: six>=1.5 in ./venv/lib/python3.7/site-packages (from python-dateutil>=2.1->matplotlib->wordcloud) (1.13.0)\n",
      "Requirement already satisfied: setuptools in ./venv/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib->wordcloud) (41.6.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "[nltk_data] Downloading collection 'all'\n",
      "[nltk_data]    | \n",
      "[nltk_data]    | Downloading package abc to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package abc is already up-to-date!\n",
      "[nltk_data]    | Downloading package alpino to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package alpino is already up-to-date!\n",
      "[nltk_data]    | Downloading package biocreative_ppi to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package biocreative_ppi is already up-to-date!\n",
      "[nltk_data]    | Downloading package brown to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package brown is already up-to-date!\n",
      "[nltk_data]    | Downloading package brown_tei to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package brown_tei is already up-to-date!\n",
      "[nltk_data]    | Downloading package cess_cat to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package cess_cat is already up-to-date!\n",
      "[nltk_data]    | Downloading package cess_esp to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package cess_esp is already up-to-date!\n",
      "[nltk_data]    | Downloading package chat80 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package chat80 is already up-to-date!\n",
      "[nltk_data]    | Downloading package city_database to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package city_database is already up-to-date!\n",
      "[nltk_data]    | Downloading package cmudict to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package cmudict is already up-to-date!\n",
      "[nltk_data]    | Downloading package comparative_sentences to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package comparative_sentences is already up-to-\n",
      "[nltk_data]    |       date!\n",
      "[nltk_data]    | Downloading package comtrans to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package comtrans is already up-to-date!\n",
      "[nltk_data]    | Downloading package conll2000 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package conll2000 is already up-to-date!\n",
      "[nltk_data]    | Downloading package conll2002 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package conll2002 is already up-to-date!\n",
      "[nltk_data]    | Downloading package conll2007 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package conll2007 is already up-to-date!\n",
      "[nltk_data]    | Downloading package crubadan to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package crubadan is already up-to-date!\n",
      "[nltk_data]    | Downloading package dependency_treebank to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package dependency_treebank is already up-to-date!\n",
      "[nltk_data]    | Downloading package dolch to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package dolch is already up-to-date!\n",
      "[nltk_data]    | Downloading package europarl_raw to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package europarl_raw is already up-to-date!\n",
      "[nltk_data]    | Downloading package floresta to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package floresta is already up-to-date!\n",
      "[nltk_data]    | Downloading package framenet_v15 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package framenet_v15 is already up-to-date!\n",
      "[nltk_data]    | Downloading package framenet_v17 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package framenet_v17 is already up-to-date!\n",
      "[nltk_data]    | Downloading package gazetteers to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package gazetteers is already up-to-date!\n",
      "[nltk_data]    | Downloading package genesis to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package genesis is already up-to-date!\n",
      "[nltk_data]    | Downloading package gutenberg to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package gutenberg is already up-to-date!\n",
      "[nltk_data]    | Downloading package ieer to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package ieer is already up-to-date!\n",
      "[nltk_data]    | Downloading package inaugural to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package inaugural is already up-to-date!\n",
      "[nltk_data]    | Downloading package indian to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package indian is already up-to-date!\n",
      "[nltk_data]    | Downloading package jeita to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package jeita is already up-to-date!\n",
      "[nltk_data]    | Downloading package kimmo to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package kimmo is already up-to-date!\n",
      "[nltk_data]    | Downloading package knbc to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package knbc is already up-to-date!\n",
      "[nltk_data]    | Downloading package lin_thesaurus to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package lin_thesaurus is already up-to-date!\n",
      "[nltk_data]    | Downloading package mac_morpho to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package mac_morpho is already up-to-date!\n",
      "[nltk_data]    | Downloading package machado to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package machado is already up-to-date!\n",
      "[nltk_data]    | Downloading package masc_tagged to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package masc_tagged is already up-to-date!\n",
      "[nltk_data]    | Downloading package moses_sample to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package moses_sample is already up-to-date!\n",
      "[nltk_data]    | Downloading package movie_reviews to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package movie_reviews is already up-to-date!\n",
      "[nltk_data]    | Downloading package names to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package names is already up-to-date!\n",
      "[nltk_data]    | Downloading package nombank.1.0 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package nombank.1.0 is already up-to-date!\n",
      "[nltk_data]    | Downloading package nps_chat to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package nps_chat is already up-to-date!\n",
      "[nltk_data]    | Downloading package omw to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package omw is already up-to-date!\n",
      "[nltk_data]    | Downloading package opinion_lexicon to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package opinion_lexicon is already up-to-date!\n",
      "[nltk_data]    | Downloading package paradigms to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package paradigms is already up-to-date!\n",
      "[nltk_data]    | Downloading package pil to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package pil is already up-to-date!\n",
      "[nltk_data]    | Downloading package pl196x to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package pl196x is already up-to-date!\n",
      "[nltk_data]    | Downloading package ppattach to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package ppattach is already up-to-date!\n",
      "[nltk_data]    | Downloading package problem_reports to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package problem_reports is already up-to-date!\n",
      "[nltk_data]    | Downloading package propbank to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package propbank is already up-to-date!\n",
      "[nltk_data]    | Downloading package ptb to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package ptb is already up-to-date!\n",
      "[nltk_data]    | Downloading package product_reviews_1 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package product_reviews_1 is already up-to-date!\n",
      "[nltk_data]    | Downloading package product_reviews_2 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package product_reviews_2 is already up-to-date!\n",
      "[nltk_data]    | Downloading package pros_cons to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package pros_cons is already up-to-date!\n",
      "[nltk_data]    | Downloading package qc to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package qc is already up-to-date!\n",
      "[nltk_data]    | Downloading package reuters to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package reuters is already up-to-date!\n",
      "[nltk_data]    | Downloading package rte to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package rte is already up-to-date!\n",
      "[nltk_data]    | Downloading package semcor to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package semcor is already up-to-date!\n",
      "[nltk_data]    | Downloading package senseval to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package senseval is already up-to-date!\n",
      "[nltk_data]    | Downloading package sentiwordnet to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package sentiwordnet is already up-to-date!\n",
      "[nltk_data]    | Downloading package sentence_polarity to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package sentence_polarity is already up-to-date!\n",
      "[nltk_data]    | Downloading package shakespeare to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package shakespeare is already up-to-date!\n",
      "[nltk_data]    | Downloading package sinica_treebank to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package sinica_treebank is already up-to-date!\n",
      "[nltk_data]    | Downloading package smultron to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package smultron is already up-to-date!\n",
      "[nltk_data]    | Downloading package state_union to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package state_union is already up-to-date!\n",
      "[nltk_data]    | Downloading package stopwords to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package stopwords is already up-to-date!\n",
      "[nltk_data]    | Downloading package subjectivity to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package subjectivity is already up-to-date!\n",
      "[nltk_data]    | Downloading package swadesh to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package swadesh is already up-to-date!\n",
      "[nltk_data]    | Downloading package switchboard to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package switchboard is already up-to-date!\n",
      "[nltk_data]    | Downloading package timit to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data]    |   Package timit is already up-to-date!\n",
      "[nltk_data]    | Downloading package toolbox to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package toolbox is already up-to-date!\n",
      "[nltk_data]    | Downloading package treebank to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package treebank is already up-to-date!\n",
      "[nltk_data]    | Downloading package twitter_samples to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package twitter_samples is already up-to-date!\n",
      "[nltk_data]    | Downloading package udhr to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package udhr is already up-to-date!\n",
      "[nltk_data]    | Downloading package udhr2 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package udhr2 is already up-to-date!\n",
      "[nltk_data]    | Downloading package unicode_samples to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package unicode_samples is already up-to-date!\n",
      "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package universal_treebanks_v20 is already up-to-\n",
      "[nltk_data]    |       date!\n",
      "[nltk_data]    | Downloading package verbnet to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package verbnet is already up-to-date!\n",
      "[nltk_data]    | Downloading package verbnet3 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package verbnet3 is already up-to-date!\n",
      "[nltk_data]    | Downloading package webtext to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package webtext is already up-to-date!\n",
      "[nltk_data]    | Downloading package wordnet to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package wordnet is already up-to-date!\n",
      "[nltk_data]    | Downloading package wordnet_ic to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package wordnet_ic is already up-to-date!\n",
      "[nltk_data]    | Downloading package words to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package words is already up-to-date!\n",
      "[nltk_data]    | Downloading package ycoe to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package ycoe is already up-to-date!\n",
      "[nltk_data]    | Downloading package rslp to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package rslp is already up-to-date!\n",
      "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package maxent_treebank_pos_tagger is already up-\n",
      "[nltk_data]    |       to-date!\n",
      "[nltk_data]    | Downloading package universal_tagset to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package universal_tagset is already up-to-date!\n",
      "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package maxent_ne_chunker is already up-to-date!\n",
      "[nltk_data]    | Downloading package punkt to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package punkt is already up-to-date!\n",
      "[nltk_data]    | Downloading package book_grammars to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package book_grammars is already up-to-date!\n",
      "[nltk_data]    | Downloading package sample_grammars to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package sample_grammars is already up-to-date!\n",
      "[nltk_data]    | Downloading package spanish_grammars to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package spanish_grammars is already up-to-date!\n",
      "[nltk_data]    | Downloading package basque_grammars to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package basque_grammars is already up-to-date!\n",
      "[nltk_data]    | Downloading package large_grammars to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package large_grammars is already up-to-date!\n",
      "[nltk_data]    | Downloading package tagsets to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package tagsets is already up-to-date!\n",
      "[nltk_data]    | Downloading package snowball_data to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package snowball_data is already up-to-date!\n",
      "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package bllip_wsj_no_aux is already up-to-date!\n",
      "[nltk_data]    | Downloading package word2vec_sample to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package word2vec_sample is already up-to-date!\n",
      "[nltk_data]    | Downloading package panlex_swadesh to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package panlex_swadesh is already up-to-date!\n",
      "[nltk_data]    | Downloading package mte_teip5 to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package mte_teip5 is already up-to-date!\n",
      "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package averaged_perceptron_tagger is already up-\n",
      "[nltk_data]    |       to-date!\n",
      "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package averaged_perceptron_tagger_ru is already\n",
      "[nltk_data]    |       up-to-date!\n",
      "[nltk_data]    | Downloading package perluniprops to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package perluniprops is already up-to-date!\n",
      "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package nonbreaking_prefixes is already up-to-date!\n",
      "[nltk_data]    | Downloading package vader_lexicon to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package vader_lexicon is already up-to-date!\n",
      "[nltk_data]    | Downloading package porter_test to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package porter_test is already up-to-date!\n",
      "[nltk_data]    | Downloading package wmt15_eval to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package wmt15_eval is already up-to-date!\n",
      "[nltk_data]    | Downloading package mwa_ppdb to\n",
      "[nltk_data]    |     /Users/lindawong/nltk_data...\n",
      "[nltk_data]    |   Package mwa_ppdb is already up-to-date!\n",
      "[nltk_data]    | \n",
      "[nltk_data]  Done downloading collection all\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim\n",
    "!pip install keras\n",
    "!pip install nltk\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "!pip install sklearn\n",
    "!pip install tensorflow==1.15\n",
    "!pip install wordcloud\n",
    "\n",
    "import gensim\n",
    "from gensim.models import word2vec\n",
    "from itertools import cycle\n",
    "import json\n",
    "from keras.utils import np_utils\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn import preprocessing\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KDTree\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Bidirectional, Dense, Embedding, SimpleRNN, LSTM\n",
    "from wordcloud import WordCloud, ImageColorGenerator\n",
    "\n",
    "nltk.download('all')\n",
    "logging.basicConfig(level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter path of the Yelp dataset folder: ./yelp_dataset\n"
     ]
    }
   ],
   "source": [
    "dataset_folder = input(\"Enter path of the Yelp dataset folder: \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the original Yelp dataset, we will extract data from review.json and business.json."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions for data extraction\n",
    "def extract_data(business_data_folder, review_data_folder):\n",
    "    output_data = {}\n",
    "    with open(business_data_folder) as bd_handle:\n",
    "        business = bd_handle.readline()\n",
    "\n",
    "        while business:\n",
    "            bd_data = json.loads(business)\n",
    "\n",
    "            if bd_data['categories']:\n",
    "                output_data[bd_data['business_id']] = {\n",
    "                    'categories': [cat.strip() for cat in bd_data['categories'].split(',')],\n",
    "                    'reviews': []\n",
    "                }\n",
    "\n",
    "            business = bd_handle.readline()\n",
    "\n",
    "        bd_handle.close()\n",
    "\n",
    "    with open(review_data_folder) as rd_handle:\n",
    "        review = rd_handle.readline()\n",
    "\n",
    "        while review:\n",
    "            rv_data = json.loads(review)\n",
    "\n",
    "            if rv_data['business_id'] in output_data:\n",
    "                review = {}\n",
    "                review['review_id'] = rv_data['review_id']\n",
    "                review['text'] = rv_data['text']\n",
    "                review['useful'] = rv_data['useful']\n",
    "                review['funny'] = rv_data['funny']\n",
    "                review['cool'] = rv_data['cool']\n",
    "                output_data[rv_data['business_id']]['reviews'].append(review)\n",
    "\n",
    "            review = rd_handle.readline()\n",
    "\n",
    "        bd_handle.close()\n",
    "        \n",
    "    return output_data\n",
    "\n",
    "def write_extracted_data(output_data, output_file):\n",
    "    data_to_write = []\n",
    "    for business_id in output_data:\n",
    "        categories = output_data[business_id]['categories']\n",
    "\n",
    "        if 'Restaurants' in categories and len(output_data[business_id]['reviews']) > 0:\n",
    "            cat_to_write = ':'.join(categories)\n",
    "            for review in output_data[business_id]['reviews']:\n",
    "                review_id = review['review_id']\n",
    "                text = review['text']\n",
    "                useful = str(review['useful'])\n",
    "                funny = str(review['funny'])\n",
    "                cool = str(review['cool'])\n",
    "                data_to_write.append({\n",
    "                    'business_id': business_id,\n",
    "                    'categories': cat_to_write,\n",
    "                    'review_id': review_id,\n",
    "                    'text': text,\n",
    "                    'useful': useful,\n",
    "                    'funny': funny,\n",
    "                    'cool': cool\n",
    "                })           \n",
    "\n",
    "    with open(output_file, 'w') as of_handle:\n",
    "        json.dump(data_to_write, of_handle)\n",
    "        of_handle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "business_data_folder = os.path.join(dataset_folder, 'business.json')\n",
    "review_data_folder = os.path.join(dataset_folder, 'review.json')\n",
    "output_data = extract_data(business_data_folder, review_data_folder)\n",
    "dataset_file = './dataset.json'\n",
    "write_extracted_data(output_data, dataset_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, the data we need from review.json and business.json are stored in the file \"dateset.json\". Next, we will clean up the data and write the preprocessed datasets to \"normalised_data.json\".\n",
    "\n",
    "!!!TODO: explain clipping, making sure dataset has even ranges of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions for data preprocessing\n",
    "def preprocess_data(data_file):\n",
    "\n",
    "    df = pd.read_json(data_file)\n",
    "    df['useful'] = df['useful'].astype(float)\n",
    "    df['funny'] = df['funny'].astype(float)\n",
    "    df['cool'] = df['cool'].astype(float)\n",
    "    logging.info(\"Dataset size: {0:d} rows x {1:d} columns\".format(df.shape[0], df.shape[1]))\n",
    "\n",
    "    vote_labels = ['useful', 'funny', 'cool']\n",
    "    useful_min, useful_max = df['useful'].min(), df['useful'].max()\n",
    "    funny_min, funny_max = df['funny'].min(), df['funny'].max()\n",
    "    cool_min, cool_max = df['cool'].min(), df['cool'].max()\n",
    "\n",
    "    # print out distribution of reviews by their vote counts\n",
    "    for i in range(len(vote_labels)):\n",
    "        ranges = np.arange(-1, 100, 10)\n",
    "        counts = df.groupby(pd.cut(df[vote_labels[i]], ranges)).count()[vote_labels[i]]\n",
    "        logging.info(counts)\n",
    "        logging.info(\"Out of range counts: \", df.shape[0]-counts.sum())\n",
    "\n",
    "    # save raw vote counts\n",
    "    df['useful_raw'] = df['useful']\n",
    "    df['funny_raw'] = df['funny']\n",
    "    df['cool_raw'] = df['cool']\n",
    "    # clip vote counts to specific ranges\n",
    "    df['useful'] = df['useful'].clip(3.0, 27.0)\n",
    "    df['funny'] = df['funny'].clip(3.0, 20.0)\n",
    "    df['cool'] = df['cool'].clip(3.0, 24.0)\n",
    "\n",
    "    # normalize data with min-max scaling\n",
    "    min_max_scaler = preprocessing.MinMaxScaler()\n",
    "    useful_scaled = min_max_scaler.fit_transform(np.array(df['useful']).reshape(-1,1))\n",
    "    df['useful'] = pd.DataFrame(useful_scaled)\n",
    "    funny_scaled = min_max_scaler.fit_transform(np.array(df['funny']).reshape(-1,1))\n",
    "    df['funny'] = pd.DataFrame(funny_scaled)\n",
    "    cool_scaled = min_max_scaler.fit_transform(np.array(df['cool']).reshape(-1,1))\n",
    "    df['cool'] = pd.DataFrame(cool_scaled)\n",
    "\n",
    "    ranges = [-0.1, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n",
    "    processed_data = []\n",
    "\n",
    "    # balance dataset for each of vote category\n",
    "    # make sure there is an even number of reviews in each increment of 0.1\n",
    "    # from range 0.0 to 1.0\n",
    "    for i in range(len(vote_labels)):\n",
    "        counts = df.groupby(pd.cut(df[vote_labels[i]], ranges)).count()[vote_labels[i]]\n",
    "        min_count = counts.min()\n",
    "        new_df = pd.DataFrame()\n",
    "        for j in range(10):\n",
    "            lower = j/10\n",
    "            upper = (j+1)/10 if j < 9 else 1.1\n",
    "            new_data = df[(df[vote_labels[i]] >= lower) & (df[vote_labels[i]] < upper)]\n",
    "            new_data = new_data.sample(frac=1)\n",
    "            new_data = new_data.head(min_count)\n",
    "            new_df = new_df.append(new_data)\n",
    "\n",
    "        processed_data.append(new_df)\n",
    "    \n",
    "    return processed_data\n",
    "\n",
    "\n",
    "def write_extracted_data(useful, funny, cool):\n",
    "    output_dict = {\n",
    "        'useful': useful.to_json(orient = 'records'),\n",
    "        'funny': funny.to_json(orient = 'records'),\n",
    "        'cool': cool.to_json(orient = 'records')\n",
    "    }\n",
    "\n",
    "    with open('normalised_data.json', 'w') as op_handle:\n",
    "        json.dump(output_dict, op_handle)\n",
    "        op_handle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset size: 4201684 rows x 7 columns\n",
      "useful\n",
      "(-1, 9]     4148278\n",
      "(9, 19]       40402\n",
      "(19, 29]       7225\n",
      "(29, 39]       2674\n",
      "(39, 49]       1197\n",
      "(49, 59]        623\n",
      "(59, 69]        415\n",
      "(69, 79]        245\n",
      "(79, 89]        153\n",
      "(89, 99]        119\n",
      "Name: useful, dtype: int64\n",
      "Out of range counts:  353\n",
      "funny\n",
      "(-1, 9]     4180159\n",
      "(9, 19]       15647\n",
      "(19, 29]       3281\n",
      "(29, 39]       1236\n",
      "(39, 49]        564\n",
      "(49, 59]        282\n",
      "(59, 69]        159\n",
      "(69, 79]         89\n",
      "(79, 89]         52\n",
      "(89, 99]         41\n",
      "Name: funny, dtype: int64\n",
      "Out of range counts:  174\n",
      "cool\n",
      "(-1, 9]     4173089\n",
      "(9, 19]       20001\n",
      "(19, 29]       4550\n",
      "(29, 39]       1778\n",
      "(39, 49]        913\n",
      "(49, 59]        477\n",
      "(59, 69]        316\n",
      "(69, 79]        179\n",
      "(79, 89]        134\n",
      "(89, 99]         95\n",
      "Name: cool, dtype: int64\n",
      "Out of range counts:  152\n"
     ]
    }
   ],
   "source": [
    "processed_data = preprocess_data(dataset_file)\n",
    "write_extracted_data(processed_data[0], processed_data[1], processed_data[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a word2vec model using vocabulary of reviews in \"dataset.json\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions for creating the word2vec model\n",
    "def read_data(dataset_file):\n",
    "    logging.info(\"Reading data...\")\n",
    "    with open(dataset_file, 'r') as fh:\n",
    "        full_data = json.load(fh)\n",
    "        useful_data = pd.DataFrame(json.loads(full_data['useful']))\n",
    "        funny_data = pd.DataFrame(json.loads(full_data['funny']))\n",
    "        cool_data = pd.DataFrame(json.loads(full_data['cool']))\n",
    "        all_data = useful_data.append([funny_data, cool_data])\n",
    "    \n",
    "    return useful_data, funny_data, cool_data, all_data\n",
    "\n",
    "\n",
    "def clean_data(tokenized_data_file, all_data, tokenizer, stop_words):\n",
    "    logging.info(\"Cleaning data...\")\n",
    "    with open(tokenized_data_file, 'w') as fh:\n",
    "        reviews = all_data['text'].tolist()\n",
    "\n",
    "        for index in range(len(reviews)):\n",
    "            review = reviews[index]\n",
    "            no_tabs = str(review).replace('\\t', ' ').replace('\\n', '')\n",
    "            alphas_only = re.sub(\"[^a-zA-Z\\.]\", \" \", no_tabs)\n",
    "            multi_spaces = re.sub(\" +\", \" \", alphas_only)\n",
    "            no_spaces = multi_spaces.strip()\n",
    "            clean_text = no_spaces.lower()\n",
    "            sentences = tokenizer.tokenize(clean_text)\n",
    "            sentences = [re.sub(\"[\\.]\", \"\", sentence) for sentence in sentences]\n",
    "\n",
    "            if len(clean_text) > 0 and clean_text.count(' ') > 0:\n",
    "                for sentence in sentences:\n",
    "                    sentence = sentence.split(' ')\n",
    "                    pruned_sentence = [word for word in sentence if word not in stop_words]\n",
    "                    sentence = ' '.join(pruned_sentence)\n",
    "                    if sentence: \n",
    "                        fh.write(\"%s\\n\" % sentence)\n",
    "\n",
    "            if (index % 5000) == 0:\n",
    "                fh.flush()\n",
    "        fh.close()\n",
    "\n",
    "\n",
    "def create_wve(tokenized_data_file):\n",
    "    embedding_size = 250\n",
    "    min_word_count = 50\n",
    "    context = 30\n",
    "    downsampling = 1e-2\n",
    "    logging.info(\"Creating word vectors...\")\n",
    "    model = word2vec.Word2Vec(\n",
    "        word2vec.LineSentence(tokenized_data_file),\n",
    "        size=embedding_size,\n",
    "        min_count=min_word_count,\n",
    "        window=context,\n",
    "        sample=downsampling)\n",
    "    model.init_sims(replace=True)\n",
    "    model.save('./wve.model')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Reading data...\n",
      "INFO:root:Cleaning data...\n",
      "INFO:root:Creating word vectors...\n",
      "WARNING:gensim.models.base_any2vec:consider setting layer size to a multiple of 4 for greater performance\n",
      "INFO:gensim.models.word2vec:collecting all words and their counts\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #10000, processed 78521 words, keeping 10014 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #20000, processed 169787 words, keeping 16503 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #30000, processed 262543 words, keeping 21719 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #40000, processed 355778 words, keeping 26363 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #50000, processed 452363 words, keeping 30621 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #60000, processed 548970 words, keeping 34371 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #70000, processed 644108 words, keeping 37909 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #80000, processed 742612 words, keeping 41729 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #90000, processed 839034 words, keeping 44906 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #100000, processed 937094 words, keeping 48148 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #110000, processed 1034425 words, keeping 51277 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #120000, processed 1129977 words, keeping 54283 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #130000, processed 1224131 words, keeping 57013 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #140000, processed 1319779 words, keeping 59608 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #150000, processed 1413618 words, keeping 61970 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #160000, processed 1509069 words, keeping 64496 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #170000, processed 1602461 words, keeping 66654 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #180000, processed 1697221 words, keeping 69081 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #190000, processed 1791572 words, keeping 71555 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #200000, processed 1878707 words, keeping 73518 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #210000, processed 1966515 words, keeping 75495 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #220000, processed 2059437 words, keeping 77754 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #230000, processed 2155139 words, keeping 79967 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #240000, processed 2250536 words, keeping 82335 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #250000, processed 2349739 words, keeping 84779 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #260000, processed 2447587 words, keeping 86755 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #270000, processed 2543551 words, keeping 88852 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #280000, processed 2638055 words, keeping 90740 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #290000, processed 2734738 words, keeping 92797 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #300000, processed 2831380 words, keeping 94508 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #310000, processed 2926529 words, keeping 96274 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #320000, processed 3019422 words, keeping 98103 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #330000, processed 3116037 words, keeping 99941 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #340000, processed 3209013 words, keeping 101836 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #350000, processed 3305719 words, keeping 103540 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #360000, processed 3397028 words, keeping 105104 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #370000, processed 3491227 words, keeping 106978 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #380000, processed 3576499 words, keeping 108243 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #390000, processed 3669244 words, keeping 109964 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #400000, processed 3763063 words, keeping 111461 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #410000, processed 3861314 words, keeping 113146 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #420000, processed 3959940 words, keeping 114858 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #430000, processed 4060509 words, keeping 116217 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #440000, processed 4161693 words, keeping 117533 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #450000, processed 4261072 words, keeping 118698 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #460000, processed 4359578 words, keeping 119754 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #470000, processed 4457091 words, keeping 120710 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #480000, processed 4553999 words, keeping 121436 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #490000, processed 4649916 words, keeping 122004 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #500000, processed 4744987 words, keeping 122518 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #510000, processed 4843008 words, keeping 123082 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #520000, processed 4934414 words, keeping 123585 word types\n",
      "INFO:gensim.models.word2vec:PROGRESS: at sentence #530000, processed 5028203 words, keeping 124391 word types\n",
      "INFO:gensim.models.word2vec:collected 125346 word types from a corpus of 5115224 raw words and 539317 sentences\n",
      "INFO:gensim.models.word2vec:Loading a fresh vocabulary\n",
      "INFO:gensim.models.word2vec:effective_min_count=50 retains 7467 unique words (5% of original 125346, drops 117879)\n",
      "INFO:gensim.models.word2vec:effective_min_count=50 leaves 4650036 word corpus (90% of original 5115224, drops 465188)\n",
      "INFO:gensim.models.word2vec:deleting the raw counts dictionary of 125346 items\n",
      "INFO:gensim.models.word2vec:sample=0.01 downsamples 0 most-common words\n",
      "INFO:gensim.models.word2vec:downsampling leaves estimated 4650036 word corpus (100.0% of prior 4650036)\n",
      "INFO:gensim.models.base_any2vec:estimated required memory for 7467 words and 250 dimensions: 18667500 bytes\n",
      "INFO:gensim.models.word2vec:resetting layer weights\n",
      "INFO:gensim.models.base_any2vec:training model with 3 workers on 7467 vocabulary and 250 features, using sg=0 hs=0 sample=0.01 negative=5 window=30\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 15.17% examples, 688404 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 31.27% examples, 719980 words/s, in_qsize 4, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 46.93% examples, 718697 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 52.12% examples, 599042 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 67.57% examples, 619498 words/s, in_qsize 6, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 82.95% examples, 636596 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 1 - PROGRESS: at 98.19% examples, 594022 words/s, in_qsize 4, out_qsize 1\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 2 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 1 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 0 more threads\n",
      "INFO:gensim.models.base_any2vec:EPOCH - 1 : training on 5115224 raw words (4650036 effective words) took 7.8s, 595352 effective words/s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 15.17% examples, 690505 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 30.07% examples, 693660 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 45.77% examples, 701335 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 52.48% examples, 602933 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 68.89% examples, 632476 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 84.84% examples, 652809 words/s, in_qsize 6, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 2 - PROGRESS: at 95.23% examples, 610412 words/s, in_qsize 5, out_qsize 2\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 2 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 1 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 0 more threads\n",
      "INFO:gensim.models.base_any2vec:EPOCH - 2 : training on 5115224 raw words (4650036 effective words) took 7.5s, 619232 effective words/s\n",
      "INFO:gensim.models.base_any2vec:EPOCH 3 - PROGRESS: at 16.69% examples, 766800 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 3 - PROGRESS: at 33.22% examples, 769294 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 3 - PROGRESS: at 45.22% examples, 640979 words/s, in_qsize 6, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 3 - PROGRESS: at 61.03% examples, 662140 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 3 - PROGRESS: at 78.00% examples, 682975 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 3 - PROGRESS: at 94.42% examples, 699574 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 2 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 1 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 0 more threads\n",
      "INFO:gensim.models.base_any2vec:EPOCH - 3 : training on 5115224 raw words (4650036 effective words) took 7.1s, 655083 effective words/s\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 15.94% examples, 731953 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 32.42% examples, 752674 words/s, in_qsize 4, out_qsize 1\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 45.19% examples, 643126 words/s, in_qsize 6, out_qsize 1\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 60.83% examples, 661040 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 77.44% examples, 677414 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 93.08% examples, 688058 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 4 - PROGRESS: at 97.23% examples, 619409 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 2 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 1 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 0 more threads\n",
      "INFO:gensim.models.base_any2vec:EPOCH - 4 : training on 5115224 raw words (4650036 effective words) took 7.4s, 624175 effective words/s\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 16.30% examples, 749235 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 32.03% examples, 739414 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 41.16% examples, 585309 words/s, in_qsize 6, out_qsize 1\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 56.90% examples, 620268 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 73.26% examples, 639860 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 89.03% examples, 656870 words/s, in_qsize 5, out_qsize 1\n",
      "INFO:gensim.models.base_any2vec:EPOCH 5 - PROGRESS: at 95.40% examples, 607740 words/s, in_qsize 5, out_qsize 0\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 2 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 1 more threads\n",
      "INFO:gensim.models.base_any2vec:worker thread finished; awaiting finish of 0 more threads\n",
      "INFO:gensim.models.base_any2vec:EPOCH - 5 : training on 5115224 raw words (4650036 effective words) took 7.6s, 614345 effective words/s\n",
      "INFO:gensim.models.base_any2vec:training on a 25576120 raw words (23250180 effective words) took 37.5s, 620732 effective words/s\n",
      "INFO:gensim.models.keyedvectors:precomputing L2-norms of word weight vectors\n",
      "INFO:gensim.utils:saving Word2Vec object under ./wve.model, separately None\n",
      "INFO:gensim.utils:not storing attribute vectors_norm\n",
      "INFO:gensim.utils:not storing attribute cum_table\n",
      "INFO:gensim.utils:saved ./wve.model\n"
     ]
    }
   ],
   "source": [
    "useful_data, funny_data, cool_data, all_data = read_data('normalised_data.json')\n",
    "tokenized_data_file = './tokennized.txt'\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "stop_words = nltk.corpus.stopwords.words()\n",
    "clean_data(tokenized_data_file, all_data, tokenizer, stop_words)\n",
    "wve_model = create_wve(tokenized_data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class BiLSTMModel:\n",
    "    def __init__(self, data_file, label):\n",
    "        self.data_file = data_file\n",
    "        self.label = label\n",
    "        self.df = []\n",
    "        self.text = []\n",
    "        self.num_reviews = 0\n",
    "        self.max_length = 0\n",
    "        self.embedding_dim = 250\n",
    "        self.batch_size = 128\n",
    "        self.lstm_dim = 100\n",
    "        self.text_embedding = None\n",
    "        self.model = None\n",
    "        self.train_text = None\n",
    "        self.test_text = None\n",
    "        self.train_labels = None\n",
    "        self.test_labels = None\n",
    "\n",
    "    def read_data(self):        \n",
    "        # read normalized datatsets\n",
    "        with open(self.data_file, 'r') as f:\n",
    "            full_data = json.load(f)\n",
    "            self.df = pd.DataFrame(json.loads(full_data[self.label]))\n",
    "            self.df = self.df.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "    def create_input_embedding(self):\n",
    "        # preprocess text\n",
    "        # remove newlines, spaces, and punctuations\n",
    "        # lowercase words and separate by commas\n",
    "        for i in range(self.df.shape[0]):\n",
    "            line = gensim.utils.simple_preprocess(self.df['text'].iloc[i])\n",
    "            self.text.append(line)\n",
    "\n",
    "        length = max([len(s) for s in self.text])\n",
    "        self.num_reviews = len(self.text)\n",
    "        \n",
    "        # convert text to word vector embedding\n",
    "        text_embedding = np.zeros((self.num_reviews, length, self.embedding_dim))\n",
    "        for i in range(len(self.text)):\n",
    "            idx = 0\n",
    "            for j in range(len(self.text[i])):\n",
    "                # get jth word of ith review\n",
    "                word = self.text[i][j]\n",
    "                # if word is in vocab, add its vector embedding, else skip\n",
    "                if word in wve_model.wv.vocab:\n",
    "                    vec = wve_model[word]\n",
    "                    text_embedding[i,idx,:] = vec\n",
    "                    idx += 1\n",
    "            if idx > self.max_length:\n",
    "                self.max_length = idx\n",
    "        \n",
    "        self.text_embedding = text_embedding[:,:self.max_length,:]\n",
    "        logging.info(\"Text embedding size: \", text_embedding.shape)\n",
    "        \n",
    "    def build_regression_model(self):\n",
    "        # construct model\n",
    "        self.model = Sequential()\n",
    "        self.model.add(Dense(self.batch_size, input_shape=(self.max_length, self.embedding_dim)))\n",
    "        self.model.add(Dense(300, activation='relu'))\n",
    "        self.model.add(Dense(150, activation='relu'))\n",
    "        self.model.add(Bidirectional(LSTM(self.lstm_dim, return_sequences=False)))\n",
    "        self.model.add(Dense(1, activation='linear'))\n",
    "\n",
    "        self.model.compile(loss='mae', \n",
    "                           optimizer='adam', \n",
    "                           metrics=['mae', 'acc'])\n",
    "        \n",
    "    def train_regression_model(self):\n",
    "        labels = self.df[self.label].values\n",
    "        self.train_text, self.test_text, self.train_labels, self.test_labels = \\\n",
    "            train_test_split(self.text_embedding, labels, test_size=0.3)\n",
    "\n",
    "        logging.info(\"Train data size: \", self.train_text.shape, self.train_labels.shape)\n",
    "        logging.info(\"Test data size: \", self.test_text.shape, self.test_labels.shape)\n",
    "\n",
    "        i = self.num_reviews // self.batch_size\n",
    "        i *= self.batch_size\n",
    "        self.train_text = self.train_text[:i,:,:]\n",
    "        self.train_labels = self.train_labels[:i]\n",
    "\n",
    "        self.model.fit(self.train_text,self.train_labels, \n",
    "                  batch_size=self.batch_size, shuffle=True, epochs=5)\n",
    "        \n",
    "    def test_regression_model(self):\n",
    "        self.model.evaluate(self.test_text, self.test_labels)\n",
    "        \n",
    "    def build_classification_model(self):\n",
    "        logging.info(\"max_length = \", self.max_length)\n",
    "        # construct model\n",
    "        self.model = Sequential()\n",
    "        self.model.add(Dense(self.batch_size, input_shape=(self.max_length, self.embedding_dim)))\n",
    "        self.model.add(Dense(300, activation='relu'))\n",
    "        self.model.add(Dense(150, activation='relu'))\n",
    "        #self.model.add(Bidirectional(LSTM(self.lstm_dim, return_sequences=False)))\n",
    "        self.model.add(LSTM(self.lstm_dim, return_sequences=False))\n",
    "        self.model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "        self.model.compile(loss='binary_crossentropy', \n",
    "                           #optimizer=tf.keras.optimizers.Adam(0.003),\n",
    "                           optimizer='adam',\n",
    "                           metrics=['acc'])\n",
    "        \n",
    "    def train_classification_model(self):\n",
    "        # convert labels to 0 and 1\n",
    "        # 0 = not useful, 1 = useful\n",
    "        binary_labels = []\n",
    "        raw_labels = self.df[self.label].values\n",
    "        for l in raw_labels:\n",
    "            if l < 0.5:\n",
    "                binary_labels.append(0)\n",
    "            else:\n",
    "                binary_labels.append(1)\n",
    "        encoder = LabelEncoder()\n",
    "        encoder.fit(binary_labels)\n",
    "        labels = encoder.transform(binary_labels)\n",
    "\n",
    "        self.train_text, self.test_text, self.train_labels, self.test_labels = \\\n",
    "            train_test_split(self.text_embedding, labels, test_size=0.3)\n",
    "\n",
    "        logging.info(\"Train data size: \", self.train_text.shape, self.train_labels.shape)\n",
    "        logging.info(\"Test data size: \", self.test_text.shape, self.test_labels.shape)\n",
    "\n",
    "        i = self.num_reviews // self.batch_size\n",
    "        i *= self.batch_size\n",
    "        self.train_text = self.train_text[:i,:,:]\n",
    "        self.train_labels = self.train_labels[:i]\n",
    "\n",
    "        self.model.fit(self.train_text,self.train_labels, \n",
    "                  batch_size=self.batch_size, shuffle=True, epochs=2)\n",
    "        \n",
    "    def test_classification_model(self):\n",
    "        logging.info(\"Predict...\")\n",
    "        self.model.evaluate(self.test_text, self.test_labels)\n",
    "        pred = self.model.predict(self.test_text)\n",
    "        pred[pred < 0.5] = 0\n",
    "        pred[pred >= 0.5] = 1\n",
    "        \n",
    "        logging.info(classification_report(self.test_labels, pred))        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/ipykernel_launcher.py:45: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text embedding size:  (15190, 939, 250)\n",
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data size:  (10633, 479, 250) (10633,)\n",
      "Test data size:  (4557, 479, 250) (4557,)\n",
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10633 samples\n",
      "Epoch 1/5\n",
      "10633/10633 [==============================] - 468s 44ms/sample - loss: 0.2620 - mean_absolute_error: 0.2620 - acc: 0.1130\n",
      "Epoch 2/5\n",
      "10633/10633 [==============================] - 428s 40ms/sample - loss: 0.2534 - mean_absolute_error: 0.2534 - acc: 0.1193\n",
      "Epoch 3/5\n",
      "10633/10633 [==============================] - 442s 42ms/sample - loss: 0.2480 - mean_absolute_error: 0.2480 - acc: 0.1243\n",
      "Epoch 4/5\n",
      "10633/10633 [==============================] - 383s 36ms/sample - loss: 0.2431 - mean_absolute_error: 0.2431 - acc: 0.1298\n",
      "Epoch 5/5\n",
      "10633/10633 [==============================] - 423s 40ms/sample - loss: 0.2418 - mean_absolute_error: 0.2418 - acc: 0.1313\n"
     ]
    }
   ],
   "source": [
    "BiLSTM_reg_model = BiLSTMModel('normalised_data.json', 'useful')\n",
    "BiLSTM_reg_model.read_data()\n",
    "BiLSTM_reg_model.create_input_embedding()\n",
    "BiLSTM_reg_model.build_regression_model()\n",
    "BiLSTM_reg_model.train_regression_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4557/4557 [==============================] - 39s 9ms/sample - loss: 0.2457 - mean_absolute_error: 0.2457 - acc: 0.1150\n"
     ]
    }
   ],
   "source": [
    "BiLSTM_reg_model.test_regression_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/ipykernel_launcher.py:46: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text embedding size:  (15190, 939, 250)\n",
      "max_length =  479\n",
      "Train data size:  (10633, 479, 250) (10633,)\n",
      "Test data size:  (4557, 479, 250) (4557,)\n",
      "Train on 10633 samples\n",
      "Epoch 1/2\n",
      "10633/10633 [==============================] - 263s 25ms/sample - loss: 0.6934 - acc: 0.4968\n",
      "Epoch 2/2\n",
      "10633/10633 [==============================] - 228s 21ms/sample - loss: 0.6933 - acc: 0.4955\n"
     ]
    }
   ],
   "source": [
    "BiLSTM_class_model = BiLSTMModel('normalised_data.json', 'useful')\n",
    "BiLSTM_class_model.read_data()\n",
    "BiLSTM_class_model.create_input_embedding()\n",
    "BiLSTM_class_model.build_classification_model()\n",
    "BiLSTM_class_model.train_classification_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict...\n",
      "4557/4557 [==============================] - 29s 6ms/sample - loss: 0.6932 - acc: 0.4924\n",
      "[[1.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "F1 score:  0.4924292297564187\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00      2313\n",
      "           1       0.49      1.00      0.66      2244\n",
      "\n",
      "    accuracy                           0.49      4557\n",
      "   macro avg       0.25      0.50      0.33      4557\n",
      "weighted avg       0.24      0.49      0.32      4557\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lindawong/Documents/Github/yelp_review_prediction/venv/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "BiLSTM_class_model.test_classification_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}