separate_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Separate dataset\n",
    "\n",
    "Since \"new_train.csv\" has labels and \"test.csv\" does not have any, we have to separate \"new_train.csv\" again to get training set and validation dataset for our own evaluation of later machine learning work."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "   id keyword location                                               text  \\\n0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n\n   target                                       cleaned_text  \n0       1      deed reason thi earthquak may allah forgiv us  \n1       1              forest fire near la rong sask  canada  \n2       1  resid ask  shelter place  notifi officers  eva...  \n3       1  13 000 peopl receiv wildfir evacu order califo...  \n4       1  got sent thi photo rubi alaska smoke wildfir p...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>text</th>\n      <th>target</th>\n      <th>cleaned_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Our Deeds are the Reason of this #earthquake M...</td>\n      <td>1</td>\n      <td>deed reason thi earthquak may allah forgiv us</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Forest fire near La Ronge Sask. Canada</td>\n      <td>1</td>\n      <td>forest fire near la rong sask  canada</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>All residents asked to 'shelter in place' are ...</td>\n      <td>1</td>\n      <td>resid ask  shelter place  notifi officers  eva...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>13,000 people receive #wildfires evacuation or...</td>\n      <td>1</td>\n      <td>13 000 peopl receiv wildfir evacu order califo...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n      <td>1</td>\n      <td>got sent thi photo rubi alaska smoke wildfir p...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_train = pd.read_csv('pre-processing/new_train.csv')\n",
    "new_train.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "         id keyword location  \\\n0         1     NaN      NaN   \n1         4     NaN      NaN   \n2         5     NaN      NaN   \n3         6     NaN      NaN   \n4         7     NaN      NaN   \n...     ...     ...      ...   \n7608  10869     NaN      NaN   \n7609  10870     NaN      NaN   \n7610  10871     NaN      NaN   \n7611  10872     NaN      NaN   \n7612  10873     NaN      NaN   \n\n                                           cleaned_text  \n0         deed reason thi earthquak may allah forgiv us  \n1                 forest fire near la rong sask  canada  \n2     resid ask  shelter place  notifi officers  eva...  \n3     13 000 peopl receiv wildfir evacu order califo...  \n4     got sent thi photo rubi alaska smoke wildfir p...  \n...                                                 ...  \n7608    two giant crane hold bridg collaps nearbi home   \n7609   aria ahrari  thetawniest control wild fire ca...  \n7610             m1 94  01 04 utc  5km volcano hawaii    \n7611  polic investig e bik collid car littl portugal...  \n7612  latest  home raze northern california wildfir ...  \n\n[7613 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>cleaned_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>deed reason thi earthquak may allah forgiv us</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>forest fire near la rong sask  canada</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>resid ask  shelter place  notifi officers  eva...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>13 000 peopl receiv wildfir evacu order califo...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>got sent thi photo rubi alaska smoke wildfir p...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7608</th>\n      <td>10869</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>two giant crane hold bridg collaps nearbi home</td>\n    </tr>\n    <tr>\n      <th>7609</th>\n      <td>10870</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>aria ahrari  thetawniest control wild fire ca...</td>\n    </tr>\n    <tr>\n      <th>7610</th>\n      <td>10871</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>m1 94  01 04 utc  5km volcano hawaii</td>\n    </tr>\n    <tr>\n      <th>7611</th>\n      <td>10872</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>polic investig e bik collid car littl portugal...</td>\n    </tr>\n    <tr>\n      <th>7612</th>\n      <td>10873</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>latest  home raze northern california wildfir ...</td>\n    </tr>\n  </tbody>\n</table>\n<p>7613 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = new_train.loc[:, ['id', 'keyword', 'location', 'cleaned_text']]\n",
    "X"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "         id  target\n0         1       1\n1         4       1\n2         5       1\n3         6       1\n4         7       1\n...     ...     ...\n7608  10869       1\n7609  10870       1\n7610  10871       1\n7611  10872       1\n7612  10873       1\n\n[7613 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7608</th>\n      <td>10869</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7609</th>\n      <td>10870</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7610</th>\n      <td>10871</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7611</th>\n      <td>10872</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7612</th>\n      <td>10873</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>7613 rows × 2 columns</p>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = new_train.loc[:, ['id', 'target']]\n",
    "y"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "from sklearn import model_selection\n",
    "# separate dataset\n",
    "X_train, X_val, y_train, y_val = model_selection.train_test_split(X, y, test_size=0.2, random_state=888)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "        id              keyword            location  \\\n1888  2714              crushed         Chicago, IL   \n5162  7361           obliterate         Cymru araul   \n892   1290               bloody             Chicago   \n1336  1932  burning%20buildings                 NaN   \n2809  4041             disaster         Calgary, AB   \n...    ...                  ...                 ...   \n5270  7532          oil%20spill    Street of Dallas   \n1263  1821               burned  toronto Û¢ dallas   \n6637  9504            terrorist                 NaN   \n1946  2797               curfew            IM LOST    \n2558  3667              destroy                 NaN   \n\n                                           cleaned_text  \n1888    jimmyfallon crush squirrel bone mortar pestl...  \n5162   mccainenl think spectacular look stonewal rio...  \n892   can t bloodi wait   soni set date stephen king...  \n1336  protest ralli stone mountain  atleast they r b...  \n2809   rbcinsur quot websit   disaster  tri 3 browse...  \n...                                                 ...  \n5270     refugio oil spill may costlier bigger project   \n1263  hate im tri straighten hair brother come swoop...  \n6637               pull fight hoe call zayn terrorist    \n1946             da judg gave di girl 5pm curfew         \n2558  get   150 free bet regist boylesport casino lu...  \n\n[6090 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>cleaned_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1888</th>\n      <td>2714</td>\n      <td>crushed</td>\n      <td>Chicago, IL</td>\n      <td>jimmyfallon crush squirrel bone mortar pestl...</td>\n    </tr>\n    <tr>\n      <th>5162</th>\n      <td>7361</td>\n      <td>obliterate</td>\n      <td>Cymru araul</td>\n      <td>mccainenl think spectacular look stonewal rio...</td>\n    </tr>\n    <tr>\n      <th>892</th>\n      <td>1290</td>\n      <td>bloody</td>\n      <td>Chicago</td>\n      <td>can t bloodi wait   soni set date stephen king...</td>\n    </tr>\n    <tr>\n      <th>1336</th>\n      <td>1932</td>\n      <td>burning%20buildings</td>\n      <td>NaN</td>\n      <td>protest ralli stone mountain  atleast they r b...</td>\n    </tr>\n    <tr>\n      <th>2809</th>\n      <td>4041</td>\n      <td>disaster</td>\n      <td>Calgary, AB</td>\n      <td>rbcinsur quot websit   disaster  tri 3 browse...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5270</th>\n      <td>7532</td>\n      <td>oil%20spill</td>\n      <td>Street of Dallas</td>\n      <td>refugio oil spill may costlier bigger project</td>\n    </tr>\n    <tr>\n      <th>1263</th>\n      <td>1821</td>\n      <td>burned</td>\n      <td>toronto Û¢ dallas</td>\n      <td>hate im tri straighten hair brother come swoop...</td>\n    </tr>\n    <tr>\n      <th>6637</th>\n      <td>9504</td>\n      <td>terrorist</td>\n      <td>NaN</td>\n      <td>pull fight hoe call zayn terrorist</td>\n    </tr>\n    <tr>\n      <th>1946</th>\n      <td>2797</td>\n      <td>curfew</td>\n      <td>IM LOST</td>\n      <td>da judg gave di girl 5pm curfew</td>\n    </tr>\n    <tr>\n      <th>2558</th>\n      <td>3667</td>\n      <td>destroy</td>\n      <td>NaN</td>\n      <td>get   150 free bet regist boylesport casino lu...</td>\n    </tr>\n  </tbody>\n</table>\n<p>6090 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "        id         keyword                       location  \\\n4887  6959        massacre                       Colorado   \n4539  6453         injured                      Worldwide   \n4057  5764  forest%20fires                            NaN   \n2679  3841        detonate                  Morioh, Japan   \n1621  2341        collapse                         Europe   \n...    ...             ...                            ...   \n5308  7581        outbreak                    FCT, Abuja    \n3346  4790       evacuated                     Manchester   \n955   1382      body%20bag  sitting on Eddie Vedders lap,   \n2830  4068       displaced       Pedophile hunting ground   \n764   1105       blew%20up                            NaN   \n\n                                           cleaned_text  \n4887  colorado movi massacr trial juror reach verdic...  \n4539  top stori   googl 4 dead dozen injur gaza blas...  \n4057  group florida forest servic firefight could de...  \n2679    tinyjecht anoth stand user  deton killer queen   \n1621  greece  tax revenu collaps debt crisi continu ...  \n...                                                 ...  \n5308  8th person die ny legionnaires  diseas outbrea...  \n3346  trafford centr film fan angri odeon cinema eva...  \n955    questergirl ditto had  way feel drank vodka i...  \n2830   potu strategicpati strategi genocide  refugee...  \n764   queen gambit went well anakin blew droid contr...  \n\n[1523 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>cleaned_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>4887</th>\n      <td>6959</td>\n      <td>massacre</td>\n      <td>Colorado</td>\n      <td>colorado movi massacr trial juror reach verdic...</td>\n    </tr>\n    <tr>\n      <th>4539</th>\n      <td>6453</td>\n      <td>injured</td>\n      <td>Worldwide</td>\n      <td>top stori   googl 4 dead dozen injur gaza blas...</td>\n    </tr>\n    <tr>\n      <th>4057</th>\n      <td>5764</td>\n      <td>forest%20fires</td>\n      <td>NaN</td>\n      <td>group florida forest servic firefight could de...</td>\n    </tr>\n    <tr>\n      <th>2679</th>\n      <td>3841</td>\n      <td>detonate</td>\n      <td>Morioh, Japan</td>\n      <td>tinyjecht anoth stand user  deton killer queen</td>\n    </tr>\n    <tr>\n      <th>1621</th>\n      <td>2341</td>\n      <td>collapse</td>\n      <td>Europe</td>\n      <td>greece  tax revenu collaps debt crisi continu ...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5308</th>\n      <td>7581</td>\n      <td>outbreak</td>\n      <td>FCT, Abuja</td>\n      <td>8th person die ny legionnaires  diseas outbrea...</td>\n    </tr>\n    <tr>\n      <th>3346</th>\n      <td>4790</td>\n      <td>evacuated</td>\n      <td>Manchester</td>\n      <td>trafford centr film fan angri odeon cinema eva...</td>\n    </tr>\n    <tr>\n      <th>955</th>\n      <td>1382</td>\n      <td>body%20bag</td>\n      <td>sitting on Eddie Vedders lap,</td>\n      <td>questergirl ditto had  way feel drank vodka i...</td>\n    </tr>\n    <tr>\n      <th>2830</th>\n      <td>4068</td>\n      <td>displaced</td>\n      <td>Pedophile hunting ground</td>\n      <td>potu strategicpati strategi genocide  refugee...</td>\n    </tr>\n    <tr>\n      <th>764</th>\n      <td>1105</td>\n      <td>blew%20up</td>\n      <td>NaN</td>\n      <td>queen gambit went well anakin blew droid contr...</td>\n    </tr>\n  </tbody>\n</table>\n<p>1523 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "# export\n",
    "X_train.to_csv('X_train.csv', index=False)\n",
    "X_val.to_csv('X_val.csv', index=False)\n",
    "y_train.to_csv('y_train.csv', index=False)\n",
    "y_val.to_csv('y_val.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}