xgboost-endpoint-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "be293ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize variables\n",
    "\n",
    "endpoint_name = f\"mwaa-sm-endpoint-27d3a1452377414b81b8a5fd43763b4d\"\n",
    "bucket = 'shimingr-data'\n",
    "key=\"customer-churn.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "833de1b4",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "Install s3fs to access S3",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mImportError\u001B[0m                               Traceback (most recent call last)",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/registry.py\u001B[0m in \u001B[0;36mget_filesystem_class\u001B[0;34m(protocol)\u001B[0m\n\u001B[1;32m    218\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 219\u001B[0;31m             \u001B[0mregister_implementation\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mprotocol\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0m_import_class\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mbit\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"class\"\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    220\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mImportError\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/registry.py\u001B[0m in \u001B[0;36m_import_class\u001B[0;34m(cls, minv)\u001B[0m\n\u001B[1;32m    241\u001B[0m         \u001B[0mmod\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mname\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcls\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrsplit\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\".\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 242\u001B[0;31m         \u001B[0mmod\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mimportlib\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mimport_module\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmod\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    243\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mgetattr\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmod\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mname\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/__init__.py\u001B[0m in \u001B[0;36mimport_module\u001B[0;34m(name, package)\u001B[0m\n\u001B[1;32m    126\u001B[0m             \u001B[0mlevel\u001B[0m \u001B[0;34m+=\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 127\u001B[0;31m     \u001B[0;32mreturn\u001B[0m \u001B[0m_bootstrap\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_gcd_import\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mname\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mlevel\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mpackage\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlevel\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    128\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/_bootstrap.py\u001B[0m in \u001B[0;36m_gcd_import\u001B[0;34m(name, package, level)\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/_bootstrap.py\u001B[0m in \u001B[0;36m_find_and_load\u001B[0;34m(name, import_)\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/_bootstrap.py\u001B[0m in \u001B[0;36m_find_and_load_unlocked\u001B[0;34m(name, import_)\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/_bootstrap.py\u001B[0m in \u001B[0;36m_load_unlocked\u001B[0;34m(spec)\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/_bootstrap_external.py\u001B[0m in \u001B[0;36mexec_module\u001B[0;34m(self, module)\u001B[0m\n",
      "\u001B[0;32m/usr/local/Cellar/python@3.9/3.9.7_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/_bootstrap.py\u001B[0m in \u001B[0;36m_call_with_frames_removed\u001B[0;34m(f, *args, **kwds)\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/s3fs/__init__.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0;34m.\u001B[0m\u001B[0mcore\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mS3FileSystem\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mS3File\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      2\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;34m.\u001B[0m\u001B[0mmapping\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mS3Map\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/s3fs/core.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m     11\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mfsspec\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mutils\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0minfer_storage_options\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtokenize\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 12\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0mfsspec\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0masyn\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mAsyncFileSystem\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0msync\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0msync_wrapper\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmaybe_sync\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     13\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;31mImportError\u001B[0m: cannot import name 'maybe_sync' from 'fsspec.asyn' (/Users/shimingr/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/asyn.py)",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001B[0;31mImportError\u001B[0m                               Traceback (most recent call last)",
      "\u001B[0;32m/var/folders/m8/kcmtvg3x5639v8tv9jcxd6700000gn/T/ipykernel_16199/2408855099.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      2\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[0;31m# read the csv file from S3\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m \u001B[0mresults\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mread_csv\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"s3://{}/{}\"\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mformat\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mbucket\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkey\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mheader\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mNone\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdelimiter\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m\",\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      5\u001B[0m \u001B[0mdf\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mDataFrame\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mresults\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mindex\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      6\u001B[0m \u001B[0mdf\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mdf\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0miloc\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstr\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msplit\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m','\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mexpand\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/util/_decorators.py\u001B[0m in \u001B[0;36mwrapper\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    309\u001B[0m                     \u001B[0mstacklevel\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mstacklevel\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    310\u001B[0m                 )\n\u001B[0;32m--> 311\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mfunc\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    312\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    313\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mwrapper\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001B[0m in \u001B[0;36mread_csv\u001B[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001B[0m\n\u001B[1;32m    584\u001B[0m     \u001B[0mkwds\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mupdate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mkwds_defaults\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    585\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 586\u001B[0;31m     \u001B[0;32mreturn\u001B[0m \u001B[0m_read\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfilepath_or_buffer\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkwds\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    587\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    588\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001B[0m in \u001B[0;36m_read\u001B[0;34m(filepath_or_buffer, kwds)\u001B[0m\n\u001B[1;32m    480\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    481\u001B[0m     \u001B[0;31m# Create the parser.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 482\u001B[0;31m     \u001B[0mparser\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mTextFileReader\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfilepath_or_buffer\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwds\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    483\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    484\u001B[0m     \u001B[0;32mif\u001B[0m \u001B[0mchunksize\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0miterator\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001B[0m in \u001B[0;36m__init__\u001B[0;34m(self, f, engine, **kwds)\u001B[0m\n\u001B[1;32m    809\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0moptions\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"has_index_names\"\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mkwds\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"has_index_names\"\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    810\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 811\u001B[0;31m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_engine\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_make_engine\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mengine\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    812\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    813\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mclose\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001B[0m in \u001B[0;36m_make_engine\u001B[0;34m(self, engine)\u001B[0m\n\u001B[1;32m   1038\u001B[0m             )\n\u001B[1;32m   1039\u001B[0m         \u001B[0;31m# error: Too many arguments for \"ParserBase\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1040\u001B[0;31m         \u001B[0;32mreturn\u001B[0m \u001B[0mmapping\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mengine\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mf\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0moptions\u001B[0m\u001B[0;34m)\u001B[0m  \u001B[0;31m# type: ignore[call-arg]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1041\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1042\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0m_failover_to_python\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/parsers/c_parser_wrapper.py\u001B[0m in \u001B[0;36m__init__\u001B[0;34m(self, src, **kwds)\u001B[0m\n\u001B[1;32m     49\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     50\u001B[0m         \u001B[0;31m# open handles\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 51\u001B[0;31m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_open_handles\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0msrc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkwds\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     52\u001B[0m         \u001B[0;32massert\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mhandles\u001B[0m \u001B[0;32mis\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     53\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/parsers/base_parser.py\u001B[0m in \u001B[0;36m_open_handles\u001B[0;34m(self, src, kwds)\u001B[0m\n\u001B[1;32m    220\u001B[0m         \u001B[0mLet\u001B[0m \u001B[0mthe\u001B[0m \u001B[0mreaders\u001B[0m \u001B[0mopen\u001B[0m \u001B[0mIOHandles\u001B[0m \u001B[0mafter\u001B[0m \u001B[0mthey\u001B[0m \u001B[0mare\u001B[0m \u001B[0mdone\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mtheir\u001B[0m \u001B[0mpotential\u001B[0m \u001B[0mraises\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    221\u001B[0m         \"\"\"\n\u001B[0;32m--> 222\u001B[0;31m         self.handles = get_handle(\n\u001B[0m\u001B[1;32m    223\u001B[0m             \u001B[0msrc\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    224\u001B[0m             \u001B[0;34m\"r\"\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/common.py\u001B[0m in \u001B[0;36mget_handle\u001B[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001B[0m\n\u001B[1;32m    607\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    608\u001B[0m     \u001B[0;31m# open URLs\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 609\u001B[0;31m     ioargs = _get_filepath_or_buffer(\n\u001B[0m\u001B[1;32m    610\u001B[0m         \u001B[0mpath_or_buf\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    611\u001B[0m         \u001B[0mencoding\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mencoding\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/pandas/io/common.py\u001B[0m in \u001B[0;36m_get_filepath_or_buffer\u001B[0;34m(filepath_or_buffer, encoding, compression, mode, storage_options)\u001B[0m\n\u001B[1;32m    356\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    357\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 358\u001B[0;31m             file_obj = fsspec.open(\n\u001B[0m\u001B[1;32m    359\u001B[0m                 \u001B[0mfilepath_or_buffer\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmode\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mfsspec_mode\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mstorage_options\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0;34m{\u001B[0m\u001B[0;34m}\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    360\u001B[0m             ).open()\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/core.py\u001B[0m in \u001B[0;36mopen\u001B[0;34m(urlpath, mode, compression, encoding, errors, protocol, newline, **kwargs)\u001B[0m\n\u001B[1;32m    450\u001B[0m     \u001B[0;31m`\u001B[0m\u001B[0;31m`\u001B[0m\u001B[0mOpenFile\u001B[0m\u001B[0;31m`\u001B[0m\u001B[0;31m`\u001B[0m \u001B[0mobject\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    451\u001B[0m     \"\"\"\n\u001B[0;32m--> 452\u001B[0;31m     return open_files(\n\u001B[0m\u001B[1;32m    453\u001B[0m         \u001B[0murlpath\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0murlpath\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    454\u001B[0m         \u001B[0mmode\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mmode\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/core.py\u001B[0m in \u001B[0;36mopen_files\u001B[0;34m(urlpath, mode, compression, encoding, errors, name_function, num, protocol, newline, auto_mkdir, expand, **kwargs)\u001B[0m\n\u001B[1;32m    284\u001B[0m     \u001B[0mbe\u001B[0m \u001B[0mused\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0ma\u001B[0m \u001B[0msingle\u001B[0m \u001B[0mcontext\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    285\u001B[0m     \"\"\"\n\u001B[0;32m--> 286\u001B[0;31m     fs, fs_token, paths = get_fs_token_paths(\n\u001B[0m\u001B[1;32m    287\u001B[0m         \u001B[0murlpath\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    288\u001B[0m         \u001B[0mmode\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/core.py\u001B[0m in \u001B[0;36mget_fs_token_paths\u001B[0;34m(urlpath, mode, num, name_function, storage_options, protocol, expand)\u001B[0m\n\u001B[1;32m    620\u001B[0m                         \u001B[0;34m\"share the same protocol\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    621\u001B[0m                     )\n\u001B[0;32m--> 622\u001B[0;31m             \u001B[0mcls\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mget_filesystem_class\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mprotocol\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    623\u001B[0m             \u001B[0moptionss\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlist\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmap\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcls\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_get_kwargs_from_urls\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0murlpath\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    624\u001B[0m             \u001B[0mpaths\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0mcls\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_strip_protocol\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mu\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mu\u001B[0m \u001B[0;32min\u001B[0m \u001B[0murlpath\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/mlops/airflow/venv2/lib/python3.9/site-packages/fsspec/registry.py\u001B[0m in \u001B[0;36mget_filesystem_class\u001B[0;34m(protocol)\u001B[0m\n\u001B[1;32m    219\u001B[0m             \u001B[0mregister_implementation\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mprotocol\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0m_import_class\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mbit\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"class\"\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    220\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mImportError\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 221\u001B[0;31m             \u001B[0;32mraise\u001B[0m \u001B[0mImportError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mbit\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"err\"\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    222\u001B[0m     \u001B[0mcls\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mregistry\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mprotocol\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    223\u001B[0m     \u001B[0;32mif\u001B[0m \u001B[0mgetattr\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcls\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m\"protocol\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32min\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0;34m\"abstract\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;31mImportError\u001B[0m: Install s3fs to access S3"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# read the csv file from S3\n",
    "results = pd.read_csv(\"s3://{}/{}\".format(bucket, key), header=None, delimiter=\",\")\n",
    "df = pd.DataFrame(results, index = None)\n",
    "df = df.iloc[:,0].str.split(',', expand=True)\n",
    "\n",
    "# Need to drop the label column\n",
    "df = df.drop(df.columns[0], axis=1)\n",
    "\n",
    "# Take a random sample\n",
    "df = df.sample(120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "889a9a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import json\n",
    "import time\n",
    "\n",
    "sm_client = boto3.client('sagemaker-runtime')\n",
    "\n",
    "df_to_list = df.to_string(header=False,\n",
    "                  index=False,\n",
    "                  index_names=False).split('\\n')\n",
    "\n",
    "df_to_csv = [','.join(v.split()) for v in df_to_list]\n",
    "\n",
    "for row in df_to_csv:\n",
    "    payload = row\n",
    "    response = sm_client.invoke_endpoint(\n",
    "        EndpointName=endpoint_name,\n",
    "        ContentType = \"text/csv\",\n",
    "        Body= payload)\n",
    "    \n",
    "    output_body = json.loads(response[\"Body\"].read().decode())\n",
    "    print(output_body)\n",
    "    time.sleep(0.5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}