Tutorial4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.linear_model as skl\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report \n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tuning the hyper-parameters of your estimator\n",
    "- Train models with different combinations of hyper parameters\n",
    "- Use cross validation to evaluate the performance of models\n",
    "- Choose the best one\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading training/testing data\n",
    "all_data = pd.read_csv(\"preprocessed_data.csv\")\n",
    "X = all_data.drop([\"Survived\"], axis=1)\n",
    "y = all_data[\"Survived\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "712\n",
      "179\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=1)\n",
    "print(len(X_train))\n",
    "print(len(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9691011235955056\n",
      "testing accuracy: 0.7541899441340782\n"
     ]
    }
   ],
   "source": [
    "model = RandomForestClassifier(max_depth=100, n_estimators=10)\n",
    "model.fit(X_train, y_train)\n",
    "predict_y = model.predict(X_train)\n",
    "print(\"training accuracy:\", (predict_y == y_train).sum()/ len(predict_y))\n",
    "\n",
    "\n",
    "predict_y = model.predict(X_test)\n",
    "print(\"testing accuracy:\", (predict_y == y_test).sum()/ len(predict_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78212291, 0.80337079, 0.83707865, 0.7752809 , 0.83146067])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(model, X, y, cv=5, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can see more classification metics here: https://scikit-learn.org/stable/modules/model_evaluation.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': 100,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 10,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters:  {'max_depth': 10, 'n_estimators': 50}\n",
      "best score: 0.8371515808135527\n",
      "[0.81325717 0.81467547 0.81754161 0.81327686 0.81467547 0.81463607\n",
      " 0.83297548 0.82452477 0.83148823 0.83152763 0.81045011 0.82729243\n",
      " 0.83434453 0.83715158 0.82734167 0.80905151 0.80763321 0.82034867\n",
      " 0.82451492 0.81328671]\n",
      "[0.02416039 0.03089037 0.0372761  0.02704686 0.02671994 0.02226176\n",
      " 0.03506637 0.03053389 0.01808125 0.03432513 0.02026991 0.02208604\n",
      " 0.02815192 0.02671326 0.03288566 0.02672945 0.02158184 0.04261412\n",
      " 0.03334605 0.02978482]\n",
      "[{'max_depth': 3, 'n_estimators': 5}, {'max_depth': 3, 'n_estimators': 10}, {'max_depth': 3, 'n_estimators': 30}, {'max_depth': 3, 'n_estimators': 50}, {'max_depth': 3, 'n_estimators': 100}, {'max_depth': 5, 'n_estimators': 5}, {'max_depth': 5, 'n_estimators': 10}, {'max_depth': 5, 'n_estimators': 30}, {'max_depth': 5, 'n_estimators': 50}, {'max_depth': 5, 'n_estimators': 100}, {'max_depth': 10, 'n_estimators': 5}, {'max_depth': 10, 'n_estimators': 10}, {'max_depth': 10, 'n_estimators': 30}, {'max_depth': 10, 'n_estimators': 50}, {'max_depth': 10, 'n_estimators': 100}, {'max_depth': 20, 'n_estimators': 5}, {'max_depth': 20, 'n_estimators': 10}, {'max_depth': 20, 'n_estimators': 30}, {'max_depth': 20, 'n_estimators': 50}, {'max_depth': 20, 'n_estimators': 100}]\n"
     ]
    }
   ],
   "source": [
    "# Set the parameters by cross-validation\n",
    "tuned_parameters = [{'n_estimators': [5,10,30,50,100], 'max_depth': [3,5,10,20]}]\n",
    "\n",
    "\n",
    "clf = GridSearchCV(RandomForestClassifier(), tuned_parameters, cv=5,\n",
    "                   scoring='accuracy')\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"best parameters: \", clf.best_params_)\n",
    "print(\"best score:\", clf.best_score_ )\n",
    "print(clf.cv_results_['mean_test_score'])\n",
    "print(clf.cv_results_['std_test_score'])\n",
    "print(clf.cv_results_[\"params\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81005587, 0.80337079, 0.85393258, 0.80337079, 0.85393258])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = RandomForestClassifier(max_depth=10, n_estimators=50)\n",
    "cross_val_score(model, X, y, cv=5, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Detailed classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.90      0.82       106\n",
      "           1       0.79      0.58      0.67        73\n",
      "\n",
      "    accuracy                           0.77       179\n",
      "   macro avg       0.77      0.74      0.74       179\n",
      "weighted avg       0.77      0.77      0.76       179\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Detailed classification report:\") \n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(classification_report(y_true, y_pred))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}