example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from cross_val_custom import cross_validation_score_statement, cross_validation_score_fit_subset\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import mean_absolute_error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's load Boston dataset and add 20 outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X,y = load_boston(return_X_y=True)\n",
    "np.random.seed(0)\n",
    "for i in range(20):\n",
    "    y[np.random.randint(0,y.shape[0]+1)] += 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we're making out-of-fold predictions of target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0xc990550>]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPX18PHPSQiYIBIVHoUABS3Cr7iAxJbioxWppdYF\n6oJYrWClWIuiiFGsPlVbrTQJqyiKiCsqCogoKipqtS78DAREKhRFUEIUKsQKBMhynj9mJk4md7Zk\nZu4s5/168TK5M5l7boR77nc7X1FVjDHGmEBZbgdgjDEmOVmCMMYY48gShDHGGEeWIIwxxjiyBGGM\nMcaRJQhjjDGOLEEYY4xxZAnCGGOMI0sQxhhjHLVyO4CW6NChg3bv3t3tMIwxJqWsXLnyP6raMdz7\nUjpBdO/enbKyMrfDMMaYlCIiWyJ5n3UxGWOMcWQJwhhjjCNLEMYYYxxZgjDGGOPIEoQxxhhHliCM\nMcY4sgRhjDHGkSUIY4xJIarKSy+9xHvvvRf3c6X0QjljjMkklZWVlJSUUF5ezqBBgxg4cGBcz2cJ\nwhhjklx9fT2LFi1izpw5ZGdnc/3113P22WfH/byWIIwxJolt3ryZ4uJiPvnkEwYMGMD1119Px45h\nyyjFhCUIY4xJQrW1tTz55JM8/vjjtG3blltvvZXTTz8dEUlYDJYgjDEmyaxfv56SkhI2bdrE4MGD\nufrqq8nPz094HJYgjDEmSezfv5+HH36YZ555hsMPP5y77ror7gPRoViCMMaYJLB69WpKS0upqKjg\n7LPP5g9/+ANt27Z1NSZLEMYY46I9e/bwwAMP8MILL9C5c2emTJlCv3793A4LsARhjDGuef/995ky\nZQrffPMNF110EZdffjlt2rRxO6wGliCMMSbBqqqqmDlzJsuXL6dHjx789a9/pXfv3m6H1YQlCGOM\nSRBV5Y033uCee+5hz549XH755fzmN7+hVavkvBUnZ1TGGJNmduzYwdSpU3n//ffp3bs3N954Iz16\n9HA7rJDiliBEZC5wNrBdVY/1HjsMmA90BzYDw1V1l/e1m4ErgDpgnKoui1dsxhiTKKrK0qVLmTVr\nFnV1dfzxj3/k/PPPJysr+WulxrMF8QgwE3jM79hEYLmqThKRid7vbxKRHwEjgD5AZ+B1ETlGVevi\nGJ/JYIvLKyhZtoFtVdV0zs+laEgvhvUrcDssk2YqKiooLS1l9erV9OvXj6KiIjp16uR2WBGLW4JQ\n1bdFpHvA4aHAad6vHwXeAm7yHn9aVfcDn4vIp8CPgffjFZ/JXIvLK7h50VqqazzPHxVV1dy8aC2A\nJQkTE3V1dSxYsIC5c+fSqlUrbrjhBn71q18ltExGLCR6DOIIVa30fv0VcIT36wLgA7/3bfUeMybm\nSpZtaEgOPtU1dZQs22AJwrTYpk2bKCkpYf369QwcOJDx48fToUMHt8NqFtcGqVVVRUSj/TkRGQOM\nAejWrVvM4zLpb1tVdVTHjYlETU0NTzzxBPPmzaNdu3b8+c9/5rTTTku5VoO/RCeIr0Wkk6pWikgn\nYLv3eAXQ1e99XbzHmlDV2cBsgMLCwqgTjDGd83OpcEgGnfNzXYjGpINPPvmE4uJiNm/ezBlnnMHY\nsWNp376922G1WKKH0ZcAI71fjwSe9zs+QkTaiEgPoCfwvwmOzWSIoiG9yM3JbnQsNyeboiG9XIrI\npKp9+/Zx3333MXbsWPbs2cOkSZP405/+lBbJAeI7zfUpPAPSHURkK3AbMAl4RkSuALYAwwFUdZ2I\nPAP8C6gFxtoMJhMvvnEGm8VkWqK8vJySkhIqKysZOnQoY8aMIS8vz+2wYkpUU7eXprCwUMvKytwO\nwxiTQXbv3s3999/P0qVL6dKlC0VFRRx//PFuhxUVEVmpqoXh3mcrqY0xJkLvvvsuU6dOZefOnYwY\nMYJRo0YlVXG9WLMEYYwxYVRVVTFjxgzefPNNjjrqKO666y569Ur/MStLEMYYE4Sq8vrrr3PPPfdQ\nXV3N7373Oy6++OKkLa4Xa5lxlcYYE6Xt27czZcoUVqxYQZ8+fSgqKuIHP/iB22EllCUIY4zxo6os\nWbKEBx54gPr6eq655hqGDRuWEsX1Ys0ShDHGeG3dupWSkhI++ugj+vfvz4QJE1KquF6sWYIwxmS8\nuro6nn32WR5++GFat27NjTfeyC9/+cuULpMRC5YgjDEZ7bPPPuPvf/87Gzdu5JRTTuHaa6/l8MMP\ndzuspGAJwhiTkWpqanj88cd58sknadeuHbfffjunnnpqxrca/FmCMMZknHXr1lFSUsKWLVsYMmQI\nY8eOpV27dm6HlXQsQRhjMkZ1dTVz5szhueeeo2PHjhQXF3PSSSe5HVbSsgRhjMkIZWVlTJ48ma++\n+opf//rXjB49Ou2K68WaJQhjTFr77rvvmDVrFi+//DJdu3ZlxowZHHfccW6HlRIsQRhj0tY777zD\ntGnTqKqq4pJLLuGyyy6jdevWboeVMixBGGPSzs6dO5kxYwb/+Mc/+OEPf8ikSZPo2bOn22GlHEsQ\nxpi0oaq8+uqr3Hvvvezbt4/Ro0dz0UUXZUxxvViz35pJaovLK2znNxORr7/+msmTJ/Phhx9y7LHH\nUlRURLdu3dwOK6VZgjBJa3F5BTcvWkt1jWf32Yqqam5etBbAkoRpoKosXryY2bNnAzBu3DiGDRtm\nC95iwBKESVolyzY0JAef6po6SpZtsARhAPjyyy8pLi7m448/5qSTTmLChAkcccQRboeVNixBmKS1\nrao6quMmc9TW1jJ//nweffRRDjroICZOnMgvfvELazXEmCUIk7Q65+dS4ZAMOufnuhCNSRYbN26k\nuLiYTz/9lJ/97GeMGzeOww47zO2w0pIlCJO0iob0ajQGAZCbk03RkPTfC9g0deDAAR599FGefvpp\n8vPz+ctf/sIpp5zidlhpzRKESQqhZivZLCazdu1aiouL2bp1K2eeeSZXXXWVFddLAEsQxnXhZitZ\nQshce/fu5cEHH2Tx4sUceeSRlJSUUFhY6HZYGcMShHGdzVYyTj788ENKS0vZsWMH5513HqNHjyY3\n18afEskShHGdzVYy/v773/9y77338uqrr9KtWzfuuece+vTp43ZYGckShIlIPFc022wlA54Fb2+/\n/TbTpk3ju+++47e//S2XXnqpFddzkSsJQkTGA6MBBdYClwN5wHygO7AZGK6qu9yIzzQW7xXNNlvJ\nfPPNN0yfPp133nmHY445htLSUo4++mi3w8p4WYk+oYgUAOOAQlU9FsgGRgATgeWq2hNY7v3eJIFQ\nYwSxMKxfAXefdxwF+bkIUJCfy93nHWfjDxlAVXn55ZcZNWoUK1as4Morr+S+++6z5JAk3OpiagXk\nikgNnpbDNuBm4DTv648CbwE3uRGcaSwRYwQ2WynzVFZWMnnyZFauXMnxxx9PUVERXbp0cTss4yfh\nCUJVK0SkFPgCqAZeVdVXReQIVa30vu0rwAqqJAkbIzCxVF9f31BcLysri+uuu45zzz3XymQkITe6\nmA4FhgI9gM5AWxG51P89qqp4xiecfn6MiJSJSNmOHTviHq/xjBHk5mQ3OmZjBKY5tmzZwrhx47jn\nnnvo27cvjzzyCEOHDrXkkKTc6GL6OfC5qu4AEJFFwEDgaxHppKqVItIJ2O70w6o6G5gNUFhY6JhE\nTOz4Zi9V19SRLUKdKgW2otlEqba2lqeeeorHHnuMvLw8brnlFgYPHmyJIcm5kSC+AAaISB6eLqbB\nQBmwBxgJTPL+93kXYjN+Amcv1ak2tBwsOZhI/fvf/+bvf/87mzZtYtCgQYwbN478/Hy3wzIRcGMM\nYoWILABWAbVAOZ4WwcHAMyJyBbAFGJ7o2ExjtsLZtMT+/ft55JFHmD9/Pocddhh33nknJ598stth\nmSi4MotJVW8Dbgs4vB9Pa8IkCVvhbJrro48+oqSkhK1bt3LWWWfxhz/8gYMPPtjtsEyUbCW1Ccpm\nL5lo7d27l9mzZ/P888/TqVMnJk+ezIknnuh2WKaZEj6LyaQOm71korFixQpGjRrFkiVLuPDCC5k7\nd64lhxRnLQgTlO3HYCLx5Hsb+dsLH7O7Poc23S5g3Mju/PGsH7sdlokBSxAZLJ4F+Ez6U1XufvI1\nHlyzF81qDQL7W7Vl5gc7KehcYX+X0oAliAwVSQG+UO8Ba1lksm+++YYpU6bw9N4+aJv2jV6zmW7p\nwxJEhrp9ybqQBfhKlm1wHKCurqnjjhfWsa+mPm7VXU3yUlVeeuklZs2aRU1NDXX9fur4Ppvplh4s\nQaSBcF1Fga8P6t2Rquoax8/y3ewDk4e/XXub/qw9Naa/bdu2UVpaSnl5OSeccAJFRUUMf9z5QcJm\nuqUHSxApLlxXkdPr8z74IujnZYuETA6h2FNjeqqvr2fhwoU89NBDZGdnM2HCBM466yxEhKIh2F4e\nacwSRIoLt9rZ6fVQBazqtPnlreypMf18/vnnFBcXs379en76058yfvx4Onbs2PC6zXRLb5YgUly4\n1c7RPNUfmpdDXutWjl0G4dhTY3qpra1l3rx5PPHEE7Rt25Zbb72V008/3bG4nu3lkb4sQaS4cKud\ng70uNG5J5OZkc9s5no3hw41BAOTn5tC2TSt7akxD69evp7i4mM8//5zBgwdz9dVXW3G9DGUJIsWF\n28852Ovn9y/gzfU7gt7gfV0G+Xk57N5XS029Nvr528/tYwkhzezfv5+5c+fy7LPPcvjhh3PXXXcx\ncOBAt8MyLhJtQZ+z2woLC7WsrMztMFznP0spPy8HVfi2uqbhxg+EfD3cjT4eC+pskV5yKS8vp7S0\nlG3btnHOOedw5ZVX0rZtW7fDMnEiIitVtTDs+yxBpI/F5RUUPbum0dN+TpZQcuEJjjOawNMauPu8\n4xJ6c06WOAzs2bOH+++/nxdffJHOnTtTVFRE37593Q7LxFmkCcKK9aWR25esa5QcAGrqlduXrANC\nz3hKpGSJI9O99957jBw5kqVLl3LRRRcxd+5cSw6mERuDSCPBFr/5jifL/g7JEkemqqqqYubMmSxf\nvpyjjjqKO++8k969e7sdlklCliAySLLs75AscWQaVeWNN95gxowZ7N27l8svv5zf/OY3tGpltwHj\nLKIuJhEZISK3eL/uKiL94xuWaY5D83JCHi8a0ouc7Mbz2HOyJeHrF2yficTbsWMHt9xyC3feeScF\nBQU8+OCDXHbZZZYcTEhhE4SIzAQGAZd6D+0B7o9nUKZ5bjunj2MC8K1vAJouo3ZhjsKwfgXcfd5x\nFOTnIkBBfq4NUMeJqvLCCy8watQoVq1axdixY5k5cybdu3d3OzSTAiJ5fBioqieKSDmAqu4UkdZx\njss0Q7iyByXLNjgOYrtRZM9W38bf1q1bKS0tZc2aNZx44onccMMNdOrUye2wTAqJJEHUiEgW3mdN\nETkcqI9rVKbZQt14bXA4M9TV1bFgwQLmzp1LTk4ORUVFnHnmmY5lMowJJZIEcS+wEOgoIncAw4E7\n4hqViQsbHE5/mzZtori4mA0bNnDyySczfvx4Dj/8cLfDMikqbIJQ1cdEZCXwczwlfC5U1Y/jHpmJ\nuXBlOUzqqqmp4YknnmDevHm0a9eO2267jZ/97GfWajAtEjZBiEhnYBfwrP8xVd0Wz8BM7Flp5vT0\nr3/9i+LiYrZs2cIZZ5zB1VdfzSGHHOJ2WCYNRNLFtJzv57rkAl2BzwB77EyAWNcsssHh9LFv3z4e\neughFi5cSMeOHZk0aRI/+clP3A7LpJFIupj+x/97EfkxMDpuEZkG4XaLM5lr1apVlJaWUllZydCh\nQxkzZgx5eXluh2XSTNSrZFT1f0VkTjyCMY2F2y3OZJ7du3cza9YsXnrpJbp06cL06dM5/vjj3Q7L\npKlIxiDG+X2bBfQHvo5bRKaBTUs1/t59912mTp3Krl27uPjiixk5ciRt2rRxOyyTxiJpQXT0+7oW\neB2/AevmEJF8YA5wLJ7xjd8BG4D5QHdgMzBcVXe15DypzqalGoBdu3YxY8YM3nrrLY4++mj+9re/\nccwxx7gdlskAkYxB/L84nHc68IqqXuBdlZ0H/AlYrqqTRGQiMBG4KQ7nThk2LTWzqSqvvfYaM2fO\npLq6miuuuIIRI0ZY/SSTMEH/ponIc4So1KOq5zXnhCLSHjgVGOX9nAPAAREZCpzmfdujwFtkeIKw\naamZa/v27UyZMoUVK1bQp08fioqK+MEPfuB2WCbDhHoUmRmnc/YAdgAPi8gJwErgWuAIVa30vucr\n4Ig4nT+l2LTUzKKqLFmyhAceeABV5ZprrmHYsGFkZdneXibxgiYIVV0ex3OeCFyjqitEZDqe7iT/\nc6uIOLZeRGQMMAagW7ducQrRmMTbunUrJSUlfPTRRxQWFjJhwgSOPPJIt8MyGSySWUxHA3cBPwIO\n8h1X1eaOkm0FtqrqCu/3C/AkiK9FpJOqVopIJ2C70w+r6mxgNnj2pG5mDMYkjbq6Op555hkefvhh\n2rRpw0033cSQIUOsTIZxXSSjXY8AdwKlwJnA5bRgFwFV/UpEvhSRXqq6ARgM/Mv7ZyQwyfvf55t7\nDmNSxaeffkpxcTEbN27klFNO4dprr7XieiZpRJIg8lR1mYiUqupnwK0iUga0ZHbTNcA87wymTXiS\nThbwjIhcAWzBUzXWmLR04MABHnvsMZ566inat2/PHXfcwamnnup2WMY0EkmC2O/dD+IzEfkDUAG0\na8lJVXU1UOjw0uCWfK4xqWDdunUUFxfzxRdfMGTIEMaOHUu7di36J2VMXESSIMYDbYFxeMYiDsGz\nsM0YE4Xq6mrmzJnDc889R8eOHSkuLuakk05yOyxjgookQVSr6nfAd8Bv4xyPMWmprKyM0tJStm/f\nzrBhwxg9erQV1zNJL6Id5UTkUDzlNear6vo4x2RM2vjuu++47777eOWVV+jatSvTp0/nuOOOczss\nYyISSamNU0SkALgIeNQ7sDxfVSfFPTpjUtg777zDtGnTqKqq4pJLLuGyyy6jdevWbodlTMQiKuqi\nqhXAFBF5GbgZ+Cue6ajGmAA7d+5k+vTpvP322/Ts2ZNJkybRs2dPt8MyJmqRLJTriaf1cAGwG0/F\n1YyukWSME1Vl2bJl3Hfffezbt4/f//73DB8+3IrrmZQVyd/cJ4GngXNV9Ys4x2NMSvrqq6+YPHky\nZWVlHHfccRQVFdG1a1e3wzKmRSIZg7B5eMYEoao899xzPPjgg4gI1157LUOHDrUyGSYtWNvXmGb6\n4osvKCkp4eOPP+akk05iwoQJHHGEFSE26cMShDFRqq2tZf78+TzyyCPk5uZy8803c8YZZ1irwaSd\niBOEiLRR1f3xDMa03OLyCm5fso6q6hoADs3L4bZz+sR0T4nF5RUx28SoJfH64qioqiZbhDrVhv8W\n+MXlH2/73BxEoGpvDZ3zcxnUuyMvrqlsOH+4GDZu3EhxcTGffvopp512GuPGjePQQw8NG6PT7yqW\nv0dj4kFUQxdmFZEfAw8B7VW1m3eTn9Gqek0iAgylsLBQy8rK3A4jaSwur6Do2TXU1Df+f5qTLZRc\ncEJMbj6Lyysct0G9+7zjov78lsTrFEeg3Jxszu9fwMKVFSHf5yQwhv379/PYY4/x9NNPk5+fz3XX\nXccpp5wS8jNC/a6AmP0ejYmWiKxUVad6eI1Esk3VDOBs4BsAVV0DDGpZeCYeSpZtaHKzBaipU0qW\nbYjZOQJvttU1dc36/JbE6xRHoOqaOp744Iuok0NgDGvXrmX06NE8+eSTDBkyhEceeSRscggWo+93\nFcvfozHxEkkXU5aqbgnoX43+X5yJu21V1c16LRbnaM7nNzfexeUVVMToekLZVlXN9OnTWbx4MUce\neSSlpaX0798/op8NFWMi/j8ZEwuRtCC+9HYzqYhki8h1wL/jHJdphs75uc16LRbnaM7nNydeX7dN\nIuTU7Ob555/n/PPPZ+7cuVElh1Axds7Pjenv0Zh4iSRBXAVcD3QDvgYGeI+ZJFM0pBc5WU1n0uRk\nC0VDesXsHLk52Y2O5eZkN+vzmxNvJF1LMVFfy9F7Puaee+7h6quvJjc38ht3qBh9v6tY/h6NiZdI\nFsptB0YkIBbTQr7BzXjOYvJ9Tixm3wSL96zjO1GybAPj569u8vnN7YLJFqFeNfwsJlWyaqsZ1rWG\nv9/1Z3JycqI+V6gYAwehbRaTSWaRzGJ6EIc9qFV1TLyCipTNYko/4WZJnTzpDce+/YL8XN6deHqz\nZll98803TJs2jX/+858cc8wx3HjjjRx99NGNYormRh4uRmPcFukspkgGqV/3+/og4NfAl80NzJhQ\nQs3uGdavgKIhvRwTgK9rJpoWjqryyiuvcO+991JTU8OVV17JhRdeSHZ2dqM1FsL3T0gVVdUN4wvB\nkkS4GI1JFWFbEE1+wLM/9T9VdWB8QoqctSDST4+JS5s2V70K8nMdF7s1p2umsrKSyZMns3LlSo4/\n/niKioro0qULENkai3CtAVsEZ5JZLFsQgXoAVnDGxEXn/FzH7hmBhuNV1TXk5mQz9aK+Ud906+vr\nWbRoEXPmzCE7O5vx48dzzjnnNCqTEclAeLixkGH9CiwhmJQXyX4Qu/i+hZ0F7AQmxjMok7mcumf8\nu3h8/LudIjXntTVMfu3fVEsueX1/z8Qz/4dzf/ajJu+LZCDcpqOaTBAyQYjnseoEoMJ7qF6j7ZMy\nJgpOYwjNWXDmr7a2lpvuW8CiLw9Cs/MA2MtB3P36FxxySPsmSSbUOSH8tGHrXjLpImSCUFUVkZdU\n9dhEBWRMYPdMsFlBTk/xgTfnS49vR9nC+3nrkMFom8ZTVoO1QpxaMf7atm4V9IYfOH4RyaC2ST6W\n5D0iWSi3WkT6xT0SY4KIdFGZ7+ZcUVWN4rk5//0fX7O8w6+pa9Pe8bOdWiHD+hU0FNRz8q1f5ddA\nVmMp9Tn9Pbp50VoWl1eE/dl0EzRBiIivddEP+FBENojIKhEpF5FViQnPmO9v2AX5uQieGURO6xoc\nB5dFPH+CCDaWMKxfAQVBXsvPy+HkSW/QY+JSTp70RqMbRyxrVRl3WJL/Xqgupv8FTgTOTVAsJgNF\n2pSPZFZQtDfhcGsTnLqacrKF3ftq2bXX04oI7EIKNn5hg9qpw5L890J1MQmAqn7m9CdB8Zk0Fsum\n/AcffEBOze6I3huqFeLPqeXStnWrJiXK/Z8urcZS6rNCit8L1YLoKCLXB3tRVae05MQikg2UARWq\neraIHAbMB7oDm4HhqrqrJecwyS3cqulIfPvtt8ycOZPXX3+do3ufwqbDBrC/LvhEu2jLXQS2XHpM\nXOr4Pt/TZSxrVRl32Er474VKENnAwXhbEnFwLfAJcIj3+4nAclWdJCITvd/fFKdzmyQQqikfrutJ\nVXnzzTeZMWMGe/bsYeTIkVx66aW8uPZrxxIZEJt/5JF0IdkiudRmSf57QUttiMgqVT0xLicV6QI8\nCtwFXO9tQWwATlPVShHpBLylqiH/NVupjdQWbPpqfm4O+2vrgxbc+89//sPUqVN577336N27N0VF\nRRx11FFNPiceUxVjueWqMW6JRamNeLUcAKYBNwLt/I4doaqV3q+/Ikg5DxEZA4wB6NatWxxDNPEW\nrCkvQpCup/W0qihn1qxZ1NXVcdVVV3HBBReQleU8lBaPJ3nf593xwrqGgeo2rSKZLW5M6gn1N3tw\nPE4oImcD21V1ZbD3eFdrOzZtVHW2qhaqamHHjh3jEaJJkGDTV6v2Oq8zqNhVzeTJkznmmGN46KGH\nGD58OEvWVAadchpP+2rqG76uqq7J2HnyJr1FXc21xScUuRv4LVCLp3z4IcAi4CSsi8kA/f7yasPT\nub+cA/9l6untOOussxCRmHb3RNMdFc1+D7Yi1ySjSLuYEt42VtWbVbWLqnbHs1PdG6p6KbAEGOl9\n20jg+UTHZkJbXF4R96f1xeUV7N5X2+R4FvX8eVg/zj777IbKq7Fa0BTtdNtI58nbilyT6ppT7jte\nJgHPiMgVwBZguMvxZCynp14g4hpDi8srmr3tacmyDU3WGQAcktua357au9GxYDfqiqpqekxcGvET\ne7TTbSNdDBeLabzGuMnV0TVVfUtVz/Z+/Y2qDlbVnqr6c1Xd6WZsmSrYU+8dL6yL6Gl9cXkFRc+u\n+X6PZ2DX3hqKFqyJ6Mk52E3/2+qmrYpQC5eieWKPduVspIvhbEWuSXXJ1IIwSSDYU2+wyqb+N7vF\n5RVMeGYNdQ7jWjV12uTJ2b+l0qn9QZyY/SXZ+1tT61BYL0ukSaugaEgvip5d49ji8I893BN7qBZB\nqDGEcGMLVnbDpDpLEKaRaJ9ufTe7xeUVFC1wTg4+FVXVnDzpjYZtQ/ccqKXGu+p527f7qKzL59hD\ndrOxPqvRLCGg4XMDu7b8p5uGuqZQN/pg020H9e4YslstXDdRJq/ItcH59GATuE0jwZ5u83NzQnar\n3PHCuoabfTC+bUMVz9TQwPdrdg47czsz6bzjG6a+ZjtUYvXv2go2JbZR7Hk5IQeLg023fXP9jhYN\ngkdahTbd2OB8+rAWhGkk2FPv7ef2AYJ3q4R7iocgC1sCVFRVN5TKyBYJ2iLxtXTC7f6Wm5ONarCF\nd993PTm1CMbPXx3y3JFIVNmNZHpit8H59GEJwjQSrn/d6R94rJ8MfTf8UN1VvpZOqD2sC7yxN/dG\nnypjCMm2i50NzqcPSxCmiWieen2zlhLJv2srkgFjX4skkP+N3ukJPFXGEJLtiT1VEqsJzxKECStU\n98XtS9aFnEUUqfzcnEZTY4NxWlMRLqGFu9EHewI/v38BB+VkNRzPz83h9nMjW8+RSMn2xJ4qidWE\nZwnChBSu+yKSm3osVe2toWzLzqhu0k6tjEG9O1KybAPj568my2Gso7qmjnkffNFo3GR/beOZVcki\n2Z7YrVx2+kh4LaZYslpM8Reu7lD3IBvoxJMAUy/q67iCO5KbklMNp0hFu+FQIlgJchOtWJT7NiZk\nOYuQyaHuACLZaFZ28Pc0k0KT/vVbF69t9MQfaqDWqc8+Usk40GpP7CZeLEGYkMJNI3VUX8fJrTZz\n5q/O5I42HactAAASIUlEQVSXP6MmDj0zgSu4A7uDIPhAbUtu8sk60Gq72KWeZJqaHIwlCNPA6S/s\noN4dHW++obTKzuaCCy/k1yd2Yda726JPMBHwv1GXLNsQND6nZNCspEfqDrSmwo0o0yTb1ORgLEEY\nwPkv7Pj5q6NKDD61Cjc8+xEiEpcuGQEG9f5+s6hQ5wh84l9cXsHeA00L/+XmZIfsdipI0RtrqtyI\nMk2yTU0OxkptpJh47cng9Be2JdMX6lS5edFa8vNyWhaYAwUWrqxouPZg3T4CjZ74fTfLwFXf+bk5\nDSUxnPgGppPpH26kYrVnhomtZJuaHIy1IJKcf/dAfl4Ou/fVNqw78D0Nlm3ZyZvrd7SoCyEe3UCe\nG5OGfToPpSBId5D/01aw1dSXDOjWZMGcUxxt27RqeF+6zd9PlRtRpkm2qcnBWAsiiQUWPdu1t6bJ\nojTffH3/wmjXzV9N3ztejbh1Ec8iatU19Zzfv6Dh6bxp6b3gfE/uwX7Gd5NzKoo39aK+3DnsOMf3\nR/M5qT5VNNgNJ9luRJkm0j1F3GYtiCQW6XRMp66gquoax75mpwHLeHc3PPHBFxTk5zLtor7A96Uv\nfMX48gNKf0PjfyyRPG2FmsXju+ZgXWbRfk4qDfbaqubklCpTk22hXBLrMXFpi8YBoPHCrmALqprb\n/eOvbets9hwI/TmhFm+Fuvm2ZCFYuEVxLfmcVFmMloqJzcSXLZRLA5FMx/RVLg3Gv1sl2IBlSKrg\nsCdDoD0H6sjLyWJviEUPoWZphHpyb8nTVqhWWDQzk1Jl1okTWyNhmssSRBJz6h7IyRbatm7Ft9U1\nDTWFFq6sCHoTbJ+b07CLW7zbiqGSg09FVXWTrUMj0dybXLBxB4GoSmbYYK/JRJYgklikT86FPzjM\ncevNLGDPgdqWFdSLoPUQLf9dxiC+8/GDtcIUT52pSJNUqsw6MSaWLEEkkWB9xeFuYA17Ky9Y02ig\ntx6oD7MNqJuqa+qY8IxnL4l4JQmnVphPNEnKBntNJrJB6iQRbBD0/P4FIdc4+JJKPNYxJJLTPg+x\nEu53FGmFVhvsNeki0kFqSxBJIlhZ7cBBaP+ZMy0pW52McrKFkgtOiNtNN9isMAE+n3RWXM5pTDKK\nNEHYQrkkEWywM1iFUmhZ2epkVFOn3PHCurh89uLyCrKCjKfYOIIxzmwMIklEU2HU9750nEETONDu\nJNquHl9LK3DXOLBxBGNCsRZEknBaeh9Kv7+8Gvdpq8kosPyIb6A5VLmQYC2tbJGUWOhmjFsS3oIQ\nka7AY8AReHpQZqvqdBE5DJgPdAc2A8NVdVei43OL05TWUC2KSJ60U1F+bujqr8EWrN3xwrqgrYpg\nLa16VUsOxoTgRhdTLTBBVVeJSDtgpYi8BowClqvqJBGZCEwEbnIhPtcETmkNNnCdrnKyhNvP7RPy\nPcFu9rv21jQkzcDpq7aGwZjmSXgXk6pWquoq79ffAZ8ABcBQ4FHv2x4FhiU6tmSyuLyCPfubbmyT\nDrJFuHRAN6Zd1Lehymu2CDX1SsmyDU26i/z3wAg20BzIfzA/VSpnGpNsXB2DEJHuQD9gBXCEqlZ6\nX/oKTxdURvL1s7doBXQSq1Nl4UpPEvDdvH0DyIFjCoFjDk4DzcGkcxlvYxLBtVlMInIwsBC4TlX/\nK35PhqqqIuJ4JxCRMcAYgG7duiUi1IRLt+mrTvyf8EMVwWvJ7yLSMt7GGGeuJAgRycGTHOap6iLv\n4a9FpJOqVopIJ2C708+q6mxgNngWyiUk4BBivbp2cXlFxow7hJqmu62FU3mtC8mYlnNjFpMADwGf\nqOoUv5eWACOBSd7/Pp/o2KIV7YbwgduHqtJQldV3M/P9fCbwPeGHGkCOdH3IoXk55LVuZWUwjIkh\nN1oQJwO/BdaKyGrvsT/hSQzPiMgVwBZguAuxRSWaPQICk4n/NFVfYmnTKivtu5Z8/J/wQxXBC1Vs\nz//98arjZEwmS3iCUNV/Enxr4sGJjKWlgnV/VFRVNyklHa4vvbqmLq2TQ15OFoe2bRP0CT9YN53T\n+pBBvTuGLGBojIkNK7XRAqG6PwK7m9KxLEakcnOy+VuIWUPhBpBtgNkYd1ipjRYIVx7Df6ZOJIuy\n4rA3T1xEE6ZNKTUmdVmCaAH/+fXB+FoO4ZJJbk42qVJ5PdIwffssWHIwJjVZgmihYf0KeHfi6UGT\nhK/lMKxfAef3L3B8+s4SOL9/Abk56fW/Y1DvjkDjldAnT3ojZGE9Y0zySK87kosiKefw5vodjk/f\n9QpPfPAF1TX1cY4ysd5cv6NZ1VeNMcnBBqljxGm2TeDsmkwbqN5WVR31VGDb0tOY5GEJIoYCk4Rv\ngNp3PJpNgZJd4FaoTjrn5wZNioHHo110aIyJP+tiiqFw3SmDeneMagZQsirIz+WSAaHrYPm614LN\n3go8HqqlYYxxh7UgYijcTW7hyoqU3wXONzMJYOlHlY4bFwXu1BZqpbRPpC0NY0ziWIJogcA+81CL\n5sY/szplprH6BHYjBd7Ybzunj+PN3z85RDI2A8G732xTH2PcYwmimZz6zEP1y8c7OeRkeTbcCSVb\nhDpVCsKMhVw6oBt3Djsu7KBxpDf/SFZCO9VcsoqsxrhLNNUea/0UFhZqWVmZK+cOth1oJIO38eCr\nZuoUk3+3kE+w+J3emyg2i8mYxBCRlapaGO591oJopmBP4G6l26q9NUG7fJyewpPxid1qLhmTXCxB\nNMPi8oqEtxTats6mXjXoYrrO+bkRd/lA5N1DxpjMZQmiGUqWbUhocrh0QDcWrqwIudLaV9Yimqdw\ne2I3xoRi6yCaIZFTLwVPyYpwe0W8uX5HYgIyxmQMSxDNkMipl0pkCcnWCxhjYs0SRDMkeiA3koRk\n6wWMMbFmCaIZhvULXppbJLoNdcLJksj2krD1AsaYWLME0QwaYjaRamyf5uu18cZEAuTn5nBoXg6C\n7dhmjIkfm8UUpR07djB16lSy9x9LXZv2TV4Xgq+RaI4Cvw2HLAkYYxLJWhARUlVefPFFRo0axapV\nqxje66Am3UyxXhthXUfGGDdZCyICFRUVlJaWsnr1avr168cNN9xA586d+UmExfpCOTQvh3019U2m\nsR6al8Nt5/SxVoMxxjWWIEKor69nwYIFzJ07l+zsbCZMmMBZZ52FiGcYOrDbJ1h9o2Byc7K57Zw+\ngK1oNsYkH0sQQWzatImSkhLWr1/PwIEDGT9+PB06dAj5M071jQL5uqEKAhKBJQRjTLKxBBGgpqaG\nJ554gnnz5tGuXTv+/Oc/c9pppzW0GkJxqm80qHdH3ly/w1oHxpiUYwnCzyeffEJxcTGbN2/m5z//\nOVdffTXt2zedqRSKzTYyxqQLSxDAvn37mDt3LgsWLKBDhw7cfffdDBgwwO2wjDHGVUmXIETkl8B0\nIBuYo6qT4nm+8vJySkpKqKys5Nxzz+XKK68kLy8vnqc0xpiUkFQJQkSygXuBM4CtwIciskRV/xXr\nc+3evZv777+fpUuXUlBQwLRp0zjhhBNifRpjjElZSZUggB8Dn6rqJgAReRoYCsQ0QWzYsIFbbrmF\nnTt3MmLECEaNGkWbNm1ieQpjjEl5yZYgCoAv/b7fCvzE/w0iMgYYA9CtW7dmnaRz58706NGDu+66\ni169bKWyMcY4SbYEEZaqzgZmAxQWFjarskW7du0oKSmJaVzGGJNukq0WUwXQ1e/7Lt5jxhhjEizZ\nEsSHQE8R6SEirYERwBKXYzLGmIyUVF1MqlorIlcDy/BMc52rqutcDssYYzJSUiUIAFV9CXjJ7TiM\nMSbTJVsXkzHGmCRhCcIYY4wjSxDGGGMcWYIwxhjjSFRjuYtyYonIDmBLM3+8A/CfGIaTbOz6Ul+6\nX6Ndn3t+oKodw70ppRNES4hImaoWuh1HvNj1pb50v0a7vuRnXUzGGGMcWYIwxhjjKJMTxGy3A4gz\nu77Ul+7XaNeX5DJ2DMIYY0xomdyCMMYYE0LGJQgR+aWIbBCRT0VkotvxxIKIdBWRN0XkXyKyTkSu\n9R4/TEReE5GN3v8e6nasLSEi2SJSLiIver9Pm+sTkXwRWSAi60XkExH5aZpd33jv382PReQpETko\n1a9PROaKyHYR+djvWNBrEpGbvfedDSIyxJ2oo5NRCcJvz+szgR8BF4vIj9yNKiZqgQmq+iNgADDW\ne10TgeWq2hNY7v0+lV0LfOL3fTpd33TgFVXtDZyA5zrT4vpEpAAYBxSq6rF4KjWPIPWv7xHglwHH\nHK/J++9xBNDH+zP3ee9HSS2jEgR+e16r6gHAt+d1SlPVSlVd5f36Ozw3lwI81/ao922PAsPcibDl\nRKQLcBYwx+9wWlyfiLQHTgUeAlDVA6paRZpcn1crIFdEWgF5wDZS/PpU9W1gZ8DhYNc0FHhaVfer\n6ufAp3juR0kt0xKE057XBS7FEhci0h3oB6wAjlDVSu9LXwFHuBRWLEwDbgTq/Y6ly/X1AHYAD3u7\n0OaISFvS5PpUtQIoBb4AKoFvVfVV0uT6AgS7ppS892RagkhrInIwsBC4TlX/6/+aeqarpeSUNRE5\nG9iuqiuDvSeVrw/P0/WJwCxV7QfsIaC7JZWvz9sPPxRPIuwMtBWRS/3fk8rXF0w6XFOmJYi03fNa\nRHLwJId5qrrIe/hrEenkfb0TsN2t+FroZOBcEdmMp1vwdBF5gvS5vq3AVlVd4f1+AZ6EkS7X93Pg\nc1Xdoao1wCJgIOlzff6CXVNK3nsyLUGk5Z7XIiJ4+q8/UdUpfi8tAUZ6vx4JPJ/o2GJBVW9W1S6q\n2h3P/7M3VPVS0uf6vgK+FJFe3kODgX+RJteHp2tpgIjkef+uDsYzTpYu1+cv2DUtAUaISBsR6QH0\nBP7Xhfiio6oZ9Qf4FfBv4DPgFrfjidE1/V88TdmPgNXeP78CDsczk2Ij8DpwmNuxxuBaTwNe9H6d\nNtcH9AXKvP8PFwOHptn13QGsBz4GHgfapPr1AU/hGVOpwdMKvCLUNQG3eO87G4Az3Y4/kj+2ktoY\nY4yjTOtiMsYYEyFLEMYYYxxZgjDGGOPIEoQxxhhHliCMMcY4sgRhUp6I1InIam+l0GdFJK8Fn3Wa\nX7XYc0NV/PVWYP1jM85xu4jc0NwYY/05xgRjCcKkg2pV7aueSqEHgD/4vygeUf9dV9UlqjopxFvy\ngagThDGpwhKESTfvAD8Uke7euvuP4Vmc1VVEfiEi74vIKm9L42Bo2CNkvYisAs7zfZCIjBKRmd6v\njxCR50RkjffPQGAScLS39VLifV+RiHwoIh+JyB1+n3WLiPxbRP4J9CKAiLQXkS2+RCYibUXkSxHJ\nEZHfez9zjYgsdGohichbIlLo/bqDtyyJbw+NEr+YrozNr9lkAksQJm14S0mfCaz1HuoJ3KeqffAU\nwLsV+Lmqnohn1fL1InIQ8CBwDtAfODLIx88A/qGqJ+Cpk7QOT0G9z7ytlyIR+YX3nD/GszK6v4ic\nKiL98ZQI6YtnhftJgR+uqt/iWQH/M++hs4Fl6q1dpKonec/9CZ4Vu5G6Ak/11JO85/29t9SDMWG1\ncjsAY2IgV0RWe79+B09dqs7AFlX9wHt8AJ5Not71lAOiNfA+0BtPIbmNAN4igGMcznE6cBmAqtYB\n30rTHdB+4f1T7v3+YDwJox3wnKru9Z4jWP2v+cBFwJt4Esp93uPHisideLq0DgaWhfplOMR0vIhc\n4P2+vTemz6P4DJOhLEGYdFCtqn39D3iTwB7/Q8BrqnpxwPsa/VwLCXC3qj4QcI7rIvz5JcDfROQw\nPK2ZN7zHHwGGqeoaERmFpx5VoFq+7xE4KCCma1Q1mqRiDGBdTCZzfACcLCI/hIY+/mPwFJDrLiJH\ne993cZCfXw5c5f3ZbO8ucN/haR34LAN+5ze2USAi/wd4GxgmIrki0g5Pd1YTqrobT8Xh6XgKEtZ5\nX2oHVHpLul8SJL7NeJIKwAV+x5cBV3l/FhE5RjybERkTliUIkxFUdQcwCnhKRD7C272kqvvwdCkt\n9Q5SB9uT4FpgkIisBVYCP1LVb/B0WX0sIiXq2SXtSeB97/sWAO3Usx3sfGAN8DKeJBDMfOBS7399\n/h+eHQLfxZPQnJTiSQTlQAe/43PwlA5fJSIfAw9gPQcmQlbN1RhjjCNrQRhjjHFkCcIYY4wjSxDG\nGGMcWYIwxhjjyBKEMcYYR5YgjDHGOLIEYYwxxpElCGOMMY7+P/HQQxbqv70IAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xcbbb550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%pylab inline\n",
    "y_pred = cross_val_predict(estimator=GradientBoostingRegressor(random_state=0,n_estimators=100),X=X,y=y,cv=5)\n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(y=y,x=y_pred)\n",
    "ax.set_xlabel('Predicted value')\n",
    "ax.set_ylabel('True value')\n",
    "lims = [\n",
    "    np.min([ax.get_xlim(), ax.get_ylim()]),  # min of both axes\n",
    "    np.max([ax.get_xlim(), ax.get_ylim()]),  # max of both axes\n",
    "]\n",
    "ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The majority of points is not far from the y=x line. However we see our outliers in the left top corner (since we added 50 to some objects' targets). In the right bottom corner we see the abnormal high predictions caused by the presence of the outliers in the dataset.\n",
    "If we will use cross-validation now for tuning the parameters of our model - we will be mostly trying to fix these 'bad' points instead of improving quality for the majority of 'normal' objects. This is especially the case when the metric is sensitive to outliers (MSE, R2, ...)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It might be better to improve accuracy on the 'normal' objects situated not so far from y=x. These objects are the majority. We can't just drop outliers since we won't be able to detect them in the test dataset when target is unknown.\n",
    "The approach can be like this. We say: \"OK, we can't improve outliers much, let's try to improve quality on the 'normal' objects\".\n",
    "During cross-validation on each iteration we use all objects to perform fit, but use only normal objects for scoring.\n",
    "We define objects with absolute error > 25 as outliers and do not calculate score for them (statement=~outliers_mask)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.69233287,  3.61079391,  4.13051729,  3.83593642,  3.92674495])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outliers_mask = np.absolute(y-y_pred)>25\n",
    "cross_validation_score_statement(estimator=GradientBoostingRegressor(random_state=0),X=pd.DataFrame(X),y=pd.Series(y),n_splits=5,\n",
    "                                 statement=~outliers_mask,scoring=mean_absolute_error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "In some cases dropping outliers from the train can increase accuracy on test. Now are oing this: we fit only 'normal' objects, but predict and score all objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 4.21334503,  4.06242579,  4.96166443,  4.31248986,  4.16140199])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validation_score_fit_subset(estimator=GradientBoostingRegressor(random_state=0),X=pd.DataFrame(X),y=pd.Series(y),n_splits=5,\n",
    "                                  statement=~outliers_mask,scoring=mean_absolute_error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note, that the output of cross_validation_score_statement is not an assessment of your score on real test data. It is just a measure of how good is the particular estimator predicting 'normal' objects. The only use of this function is for comparing different machine learning pipelines.\n",
    "The cross_validation_score_fit_subset is an assessment of the score on real test data while it scores every object including outliers. By this function you can check if there is a sense to drop outliers from train."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}