Skip to content

Commit

Permalink
notebook: ecperimentations
Browse files Browse the repository at this point in the history
  • Loading branch information
@dev-abuke
dev-abuke committed Jun 27, 2024
1 parent e2d4f45 commit ec4992a
Show file tree
Hide file tree
Showing 3 changed files with 751 additions and 35 deletions.
263 changes: 245 additions & 18 deletions notebooks/backtesting.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,40 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.chdir('..')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"!pip install -r requirements.txt -q"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'backtesting'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[2], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# import pandas_ta as ta\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbacktesting\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Backtest\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbacktesting\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Strategy\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbacktesting\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m crossover\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'backtesting'"
]
}
],
"source": [
"import datetime\n",
"# import pandas_ta as ta\n",
Expand Down Expand Up @@ -340,7 +371,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -353,7 +384,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -363,29 +394,29 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(1, 15, datetime.date(2023, 9, 1), datetime.date(2024, 4, 1), 13, 3)\n",
"(2, 15, datetime.date(2024, 1, 19), datetime.date(2024, 6, 26), 7, 3)\n",
"(6, 15, datetime.date(2023, 10, 20), datetime.date(2024, 6, 26), 8, 3)\n",
"(13, 15, datetime.date(2023, 11, 10), datetime.date(2024, 6, 26), 3, 3)\n",
"(14, 15, datetime.date(2023, 11, 10), datetime.date(2024, 6, 26), 6, 3)\n",
"(15, 15, datetime.date(2024, 5, 26), datetime.date(2024, 6, 26), 4, 3)\n",
"(16, 15, datetime.date(2024, 3, 7), datetime.date(2024, 6, 26), 13, 3)\n",
"(19, 15, datetime.date(2024, 1, 3), datetime.date(2024, 6, 26), 9, 5)\n",
"(20, 15, datetime.date(2023, 2, 9), datetime.date(2024, 6, 25), 13, 3)\n",
"(22, 15, datetime.date(2023, 12, 27), datetime.date(2024, 6, 25), 13, 3)\n"
"(datetime.datetime(2023, 1, 3, 0, 0), 130.27999877929688, 130.89999389648438, 124.16999816894531, 125.06999969482422, 124.04804229736328, 112117500, 'AAPL')\n",
"(datetime.datetime(2023, 1, 4, 0, 0), 126.88999938964844, 128.66000366210938, 125.08000183105469, 126.36000061035156, 125.32750701904297, 89113600, 'AAPL')\n",
"(datetime.datetime(2023, 1, 5, 0, 0), 127.12999725341797, 127.7699966430664, 124.76000213623047, 125.0199966430664, 123.99845886230469, 80962700, 'AAPL')\n",
"(datetime.datetime(2023, 1, 6, 0, 0), 126.01000213623047, 130.2899932861328, 124.88999938964844, 129.6199951171875, 128.56085205078125, 87754700, 'AAPL')\n",
"(datetime.datetime(2023, 1, 9, 0, 0), 130.47000122070312, 133.41000366210938, 129.88999938964844, 130.14999389648438, 129.08651733398438, 70790800, 'AAPL')\n",
"(datetime.datetime(2023, 1, 10, 0, 0), 130.25999450683594, 131.25999450683594, 128.1199951171875, 130.72999572753906, 129.6617889404297, 63896200, 'AAPL')\n",
"(datetime.datetime(2023, 1, 11, 0, 0), 131.25, 133.50999450683594, 130.4600067138672, 133.49000549316406, 132.39926147460938, 69458900, 'AAPL')\n",
"(datetime.datetime(2023, 1, 12, 0, 0), 133.8800048828125, 134.25999450683594, 131.44000244140625, 133.41000366210938, 132.31991577148438, 71379600, 'AAPL')\n",
"(datetime.datetime(2023, 1, 13, 0, 0), 132.02999877929688, 134.9199981689453, 131.66000366210938, 134.75999450683594, 133.65887451171875, 57809700, 'AAPL')\n",
"(datetime.datetime(2023, 1, 17, 0, 0), 134.8300018310547, 137.2899932861328, 134.1300048828125, 135.94000244140625, 134.82920837402344, 63646600, 'AAPL')\n"
]
}
],
"source": [
"# Reflect the 'scenes' table\n",
"scenes = Table('scenes', metadata, autoload_with=engine)\n",
"scenes = Table('stock_data', metadata, autoload_with=engine)\n",
"\n",
"# Build the query\n",
"query = scenes.select().limit(10).offset(0)\n",
Expand All @@ -398,7 +429,35 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Date TIMESTAMP\n",
"Open DOUBLE_PRECISION\n",
"High DOUBLE_PRECISION\n",
"Low DOUBLE_PRECISION\n",
"Close DOUBLE_PRECISION\n",
"Adj Close DOUBLE_PRECISION\n",
"Volume BIGINT\n",
"symbol TEXT\n"
]
}
],
"source": [
"# get the column names\n",
"inspector = inspect(engine)\n",
"columns = inspector.get_columns('stock_data')\n",
"for column in columns:\n",
" print(column['name'], column['type'])"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -467,7 +526,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"[]\n"
"['users', 'stocks', 'scenes', 'indicators', 'backtest_results', 'stock_data']\n"
]
}
],
Expand Down Expand Up @@ -943,6 +1002,174 @@
"import yfinance as yf\n",
"yf.download(\"FB\", start=\"2023-01-01\", end=\"2024-01-01\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import os, pandas as pd\n",
"from sqlalchemy import create_engine"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'/home/abubeker_shamil/Scalable_Backtesting_Infrastructure_for_Crypto_Trading/notebooks'"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"os.getcwd()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_806104/3694470526.py:5: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
" stocks_df = pd.read_sql(\"SELECT * FROM stocks\", engine)\n"
]
},
{
"ename": "AttributeError",
"evalue": "'Engine' object has no attribute 'cursor'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[5], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m engine \u001b[38;5;241m=\u001b[39m create_engine(DATABASE_URL)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# Fetch stocks from database\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m stocks_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_sql\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSELECT * FROM stocks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/Scalable_Backtesting_Infrastructure_for_Crypto_Trading/.conda/lib/python3.12/site-packages/pandas/io/sql.py:706\u001b[0m, in \u001b[0;36mread_sql\u001b[0;34m(sql, con, index_col, coerce_float, params, parse_dates, columns, chunksize, dtype_backend, dtype)\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m pandasSQL_builder(con) \u001b[38;5;28;01mas\u001b[39;00m pandas_sql:\n\u001b[1;32m 705\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(pandas_sql, SQLiteDatabase):\n\u001b[0;32m--> 706\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpandas_sql\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_query\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 707\u001b[0m \u001b[43m \u001b[49m\u001b[43msql\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 708\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 709\u001b[0m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 710\u001b[0m \u001b[43m \u001b[49m\u001b[43mcoerce_float\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcoerce_float\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 711\u001b[0m \u001b[43m \u001b[49m\u001b[43mparse_dates\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_dates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 712\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 713\u001b[0m \u001b[43m \u001b[49m\u001b[43mdtype_backend\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype_backend\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 714\u001b[0m \u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 715\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 717\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 718\u001b[0m _is_table_name \u001b[38;5;241m=\u001b[39m pandas_sql\u001b[38;5;241m.\u001b[39mhas_table(sql)\n",
"File \u001b[0;32m~/Scalable_Backtesting_Infrastructure_for_Crypto_Trading/.conda/lib/python3.12/site-packages/pandas/io/sql.py:2738\u001b[0m, in \u001b[0;36mSQLiteDatabase.read_query\u001b[0;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize, dtype, dtype_backend)\u001b[0m\n\u001b[1;32m 2727\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread_query\u001b[39m(\n\u001b[1;32m 2728\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 2729\u001b[0m sql,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 2736\u001b[0m dtype_backend: DtypeBackend \u001b[38;5;241m|\u001b[39m Literal[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnumpy\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnumpy\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 2737\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataFrame \u001b[38;5;241m|\u001b[39m Iterator[DataFrame]:\n\u001b[0;32m-> 2738\u001b[0m cursor \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2739\u001b[0m columns \u001b[38;5;241m=\u001b[39m [col_desc[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m col_desc \u001b[38;5;129;01min\u001b[39;00m cursor\u001b[38;5;241m.\u001b[39mdescription]\n\u001b[1;32m 2741\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
"File \u001b[0;32m~/Scalable_Backtesting_Infrastructure_for_Crypto_Trading/.conda/lib/python3.12/site-packages/pandas/io/sql.py:2672\u001b[0m, in \u001b[0;36mSQLiteDatabase.execute\u001b[0;34m(self, sql, params)\u001b[0m\n\u001b[1;32m 2670\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mQuery must be a string unless using sqlalchemy.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 2671\u001b[0m args \u001b[38;5;241m=\u001b[39m [] \u001b[38;5;28;01mif\u001b[39;00m params \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m [params]\n\u001b[0;32m-> 2672\u001b[0m cur \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcon\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcursor\u001b[49m()\n\u001b[1;32m 2673\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 2674\u001b[0m cur\u001b[38;5;241m.\u001b[39mexecute(sql, \u001b[38;5;241m*\u001b[39margs)\n",
"\u001b[0;31mAttributeError\u001b[0m: 'Engine' object has no attribute 'cursor'"
]
}
],
"source": [
"DATABASE_URL = os.getenv(\"PYCOPG_DATABASE_URL\", \"postgresql+psycopg2://trading_db_av2v_user:210M6MA9QKEEgVdiasnUdMQDBNN417oy@dpg-cpqojbqj1k6c73bkqq3g-a.oregon-postgres.render.com/trading_db_av2v\")\n",
"engine = create_engine(DATABASE_URL)\n",
"\n",
"# Fetch stocks from database\n",
"stocks_df = pd.read_sql(\"SELECT * FROM stocks\", engine)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>name</th>\n",
" <th>symbol</th>\n",
" <th>description</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>Apple Inc</td>\n",
" <td>AAPL</td>\n",
" <td>Apple is a trillion-dollar company</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>Microsoft Corporation</td>\n",
" <td>MSFT</td>\n",
" <td>Microsoft is a multinational technology company</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" id name symbol \\\n",
"0 1 Apple Inc AAPL \n",
"1 2 Microsoft Corporation MSFT \n",
"\n",
" description \n",
"0 Apple is a trillion-dollar company \n",
"1 Microsoft is a multinational technology company "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"stocks_df.head(2)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"AAPL\n",
"MSFT\n",
"AMZN\n",
"GOOGL\n",
"FB\n",
"BRK.A\n",
"TSLA\n",
"BABA\n",
"TCEHY\n",
"JNJ\n",
"V\n",
"WMT\n",
"NVDA\n",
"SSNLF\n",
"PG\n"
]
}
],
"source": [
"for index, row in stocks_df.iterrows():\n",
" print(row['symbol'])"
]
}
],
"metadata": {
Expand All @@ -961,7 +1188,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
"version": "3.12.4"
}
},
"nbformat": 4,
Expand Down
8 changes: 2 additions & 6 deletions notebooks/kafka.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,9 @@
"output_type": "stream",
"text": [
"Topics in Kafka cluster:\n",
"docker-connect-status\n",
"_schemas\n",
"scene_parameters\n",
"__consumer_offsets\n",
"_confluent-ksql-ksqldb-server__command_topic\n",
"docker-connect-offsets\n",
"docker-connect-configs\n"
"__consumer_offsets\n"
]
}
],
Expand Down Expand Up @@ -256,7 +252,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
"version": "3.8.10"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit ec4992a

Please sign in to comment.