PatternFinder_v0_2a.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DNu7L-Kzgjqv"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "#@#main_proc\n",
        "if True:\n",
        "    import os\n",
        "    os.chdir(r\"c:\\Users\\content\")\n",
        "    from os.path import exists\n",
        "    from scipy.stats import pearsonr\n",
        "    from calc_error import multi_calc_total_error\n",
        "    import statsmodels.tools.eval_measures as metrics\n",
        "    import numpy as np\n",
        "    from datetime import datetime, timedelta\n",
        "    import matplotlib.cm as cm\n",
        "    from scipy.spatial import distance\n",
        "    import pandas as pd\n",
        "    from IPython.display import clear_output\n",
        "    import matplotlib.pyplot as plt\n",
        "    from numpy import array as ar\n",
        "    from sklearn.cluster import KMeans\n",
        "    from sklearn.metrics import mean_squared_error as mse\n",
        "    from execthatcell import *\n",
        "    from sklearn.metrics import *\n",
        "    from sklearn import preprocessing\n",
        "    from time import sleep\n",
        "    from tqdm import tqdm\n",
        "    import multiprocessing\n",
        "    from threading import Thread\n",
        "    import copy\n",
        "    import ftplib\n",
        "    import ta_py as ta\n",
        "    from random import randint as rnd\n",
        "    import requests\n",
        "    import json\n",
        "\n",
        "    botToken = '222222222:AAF2f-2bTMstIZFfkAMtOROiHV0qzVbm0xo'\n",
        "    chatId = 222222222222\n",
        "\n",
        "    pd.options.mode.chained_assignment = None\n",
        "    pd.options.display.precision = 10\n",
        "    pd.set_option('display.max_rows', None)\n",
        "    plt.style.use('fivethirtyeight')\n",
        "    plt.rcParams['lines.linewidth'] = 2.0\n",
        "    plt.rcParams['figure.figsize'] = [9, 5]\n",
        "    # pd.set_option(\"precision\", 13)\n",
        "\n",
        "\n",
        "# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
        "#                               MAIN_PROC\n",
        "# vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n",
        "\n",
        "def norm(data,do=True):\n",
        "    if do:\n",
        "        _min = np.min(data)\n",
        "        _max = np.max(data)\n",
        "        return (data - _min) / (_max - _min)\n",
        "    else: return np.array(data)\n",
        "def GetCurrentTime(DELTA_HOUR=0):\n",
        "    return (datetime.now()+timedelta(hours=DELTA_HOUR)).strftime(\"%Y-%m-%d %H:%M:%S\")\n",
        "def calc_candle_sizes(df):\n",
        "    df[\"body_size\"] = abs(df[\"open\"] - df[\"close\"])\n",
        "    df[\"range_size\"] = df[\"high\"] - df[\"low\"]\n",
        "    df[\"upper_shadow_size\"] = abs(df[\"high\"] - df[[\"close\",\"open\"]].max(axis=1))\n",
        "    df[\"lower_shadow_size\"] = abs(df[\"low\"] - df[[\"close\",\"open\"]].min(axis=1))\n",
        "    return df\n",
        "\n",
        "## ------------------------ load db --------------------------------------------\n",
        "hist_data = pd.read_csv('./root_coins/csv_files/BTCUSDT-15m-data-big.csvx')#('./CNN-TA/root_coins/csv_files/BNM_last_price_db_v5.csv')#\n",
        "# hist_data = hist_data[['date','open','high','low','close']]\n",
        "\n",
        "HCL = np.stack([hist_data['high'].to_numpy(),hist_data['close'].to_numpy(),hist_data['low'].to_numpy()],axis=1)\n",
        "# HL = np.stack([hist_data['high'].to_numpy(),hist_data['low'].to_numpy()],axis=1)\n",
        "\n",
        "HISTORY_PRICE = ((hist_data['low']+hist_data['high']+hist_data['close'])/3).to_numpy()\n",
        "hist_len = len(HISTORY_PRICE)\n",
        "\n",
        "# ---------------------------- add indicators ----------------------------------\n",
        "\n",
        "# x = ta.ichimoku(HCL)\n",
        "# align = abs(len(HCL) - len(x))\n",
        "# ichi = ([[0,0,0,0,0]] * align) + x\n",
        "# a, b, c, d, e = zip(*ichi)\n",
        "# ichi_a,ichi_b,ichi_c,ichi_d,ichi_e= np.array(a),\\\n",
        "#                                     np.array(b),\\\n",
        "#                                     np.array(c),\\\n",
        "#                                     np.array(d),\\\n",
        "#                                     np.array(e)\n",
        "\n",
        "\n",
        "# x = ta.don(HL)\n",
        "# align = abs(len(HL) - len(x))\n",
        "# donch = ([[0,0,0]] * align) + x\n",
        "# a, b, c = zip(*donch)\n",
        "# donch_a,donch_b,donch_c = np.array(a),\\\n",
        "#                           np.array(b),\\\n",
        "#                           np.array(c)\n",
        "\n",
        "x = ta.macd(HISTORY_PRICE)\n",
        "align = abs(hist_len-len(x))\n",
        "macd = ([0]*align)+x\n",
        "\n",
        "x = ta.rsi(HISTORY_PRICE)\n",
        "align = abs(hist_len-len(x))\n",
        "rsi = ([0]*align)+x\n",
        "\n",
        "# x = ta.atr(HCL)\n",
        "# align = abs(hist_len-len(x))\n",
        "# atr = ([0]*align)+x\n",
        "\n",
        "# x = ta.asi(HCL)\n",
        "# align = abs(hist_len-len(x))\n",
        "# asi = ([0]*align)+x\n",
        "\n",
        "# x = ta.hull(hist_data['volume'].to_numpy(),21)\n",
        "# align = abs(hist_len-len(x))\n",
        "# VOL = ([0]*align)+x\n",
        "\n",
        "avg_price = HISTORY_PRICE\n",
        "VOL = hist_data['volume'].to_numpy()\n",
        "trades = hist_data['trades'].to_numpy()\n",
        "\n",
        "x = ta.wrsi(HISTORY_PRICE)\n",
        "align = abs(hist_len-len(x))\n",
        "wrsi = ([0]*align)+x\n",
        "\n",
        "x = ta.bandwidth(HISTORY_PRICE)\n",
        "align = abs(hist_len-len(x))\n",
        "band = ([0]*align)+x\n",
        "\n",
        "# ichi_a,ichi_b,ichi_c,ichi_d,ichi_e = hist_data['ichi_a'].to_numpy(),\\\n",
        "#                                      hist_data['ichi_b'].to_numpy(),\\\n",
        "#                                      hist_data['ichi_c'].to_numpy(),\\\n",
        "#                                      hist_data['ichi_d'].to_numpy(),\\\n",
        "#                                      hist_data['ichi_e'].to_numpy()\n",
        "\n",
        "# donch_a,donch_b,donch_c = hist_data['donch_a'].to_numpy(),\\\n",
        "#                           hist_data['donch_b'].to_numpy(),\\\n",
        "#                           hist_data['donch_c'].to_numpy()\n",
        "\n",
        "hist_data = calc_candle_sizes(hist_data)\n",
        "bs,rs,uss,lss = hist_data['body_size'].to_numpy(),\\\n",
        "                hist_data['range_size'].to_numpy(),\\\n",
        "                hist_data['upper_shadow_size'].to_numpy(),\\\n",
        "                hist_data['lower_shadow_size'].to_numpy()\n",
        "\n",
        "# ---------------------------- end indicators ----------------------------------\n",
        "\n",
        "\n",
        "## ---------------------- for pattern processing -------------------------------\n",
        "def __processing(update=False):\n",
        "    if update: execthatcell(\"upd_binance_price\", cell_marker=\"#@#\", latest=True) #get_binomo_price\n",
        "    last_data = pd.read_csv('./root_coins/csv_files/BTCUSDT-15m-data.csv')#('./CNN-TA/root_coins/csv_files/BNM_last_price.csv')#\n",
        "    # last_data = last_data[['date','open','high','low','close']]\n",
        "    # last_data = pd.concat([history_data, last_data], axis=0).reset_index(drop=True)\n",
        "    return last_data\n",
        "\n",
        "\n",
        "\n",
        "def send_photo(pic):\n",
        "    image = pic\n",
        "    address = f'https://api.telegram.org/bot{botToken}/sendPhoto'\n",
        "    data = {\"chat_id\": chatId, \"caption\": GetCurrentTime()}\n",
        "    with open(image, \"rb\") as imageFile:\n",
        "        result = requests.post(address, files={\"photo\": imageFile}, data=data).json()\n",
        "        if result[\"ok\"]:\n",
        "            return result[\"result\"][\"message_id\"]\n",
        "        else:\n",
        "            raise Exception(result[\"description\"])\n",
        "\n",
        "path_to_file = 'untitled'\n",
        "try: os.remove(path_to_file)\n",
        "except: pass\n",
        "\n",
        "night_mode = 0\n",
        "result_table = []\n",
        "sz_list = [5,15,25,30]#302\n",
        "\n",
        "# sz_list = [x for x in range(5, 51, 5)]\n",
        "\n",
        "horizon = 4*12 # 24hour?? :D\n",
        "TRAIN = 0\n",
        "\n",
        "\n",
        "while True:\n",
        "    if exists(path_to_file): break # STOP execution\n",
        "    night_mode += 1\n",
        "    for sz in sz_list:\n",
        "\n",
        "        t_error = 0.0\n",
        "        n=800\n",
        "        for _ in range(1):\n",
        "\n",
        "            if TRAIN:\n",
        "                n = rnd(100,hist_len-sz-horizon)\n",
        "\n",
        "                # concat all_in_one\n",
        "                indexes = range(hist_len) # !!!!!!!!!!! recheck!\n",
        "                all_stack = np.stack([\n",
        "                                      # macd,\n",
        "                                      # rsi,\n",
        "                                      # atr,\n",
        "                                      # asi,\n",
        "                                      avg_price,\n",
        "                                      # VOL,\n",
        "                                      # trades,\n",
        "                                      # wrsi,\n",
        "                                      # band,\n",
        "                                      # ichi_a,ichi_b,ichi_c,ichi_d,ichi_e,\n",
        "                                      # donch_a,donch_b,donch_c,\n",
        "                                      # bs,rs,uss,lss,\n",
        "                                      indexes],axis=0)  # last row we save our indexes array\n",
        "\n",
        "                pattrn = []\n",
        "                # get pattern seq\n",
        "                for x in range(len(all_stack)-1):\n",
        "                    pattrn.append(all_stack[x][n:n+sz].copy())\n",
        "\n",
        "                print('n=',n)\n",
        "                _true_price_ = norm(HISTORY_PRICE[n+0:n+sz+horizon].copy())\n",
        "\n",
        "                # fill 0 ptrn from list\n",
        "                del_idxs = [i for i in range(n-1,n+sz)]#\n",
        "                for x in range(len(all_stack)-1):\n",
        "                    np.put(all_stack[x], del_idxs,[0xBAD])\n",
        "                    # recheck all sizes for same\n",
        "                    assert len(all_stack[x]) == (hist_len)\n",
        "\n",
        "                n += 10\n",
        "            else:    # ---- TEST ----\n",
        "                LAST = __processing(update=1)\n",
        "                PRICE = ((LAST['high']+LAST['low']+LAST['close'])/3).to_numpy() #LAST['close'].to_numpy() #\n",
        "                # HCL = np.stack([LAST['high'].to_numpy(),\n",
        "                #                 LAST['close'].to_numpy(),\n",
        "                #                 LAST['low'].to_numpy()],axis=1)\n",
        "                # HL  = np.stack([LAST['high'].to_numpy(),\n",
        "                #                 LAST['low'].to_numpy()],axis=1)\n",
        "                # last_len = len(PRICE)\n",
        "\n",
        "                # ---------------- get indicators from last seq ----------------\n",
        "\n",
        "                L_macd = ta.macd(PRICE)\n",
        "                L_rsi = ta.rsi(PRICE)\n",
        "                # L_atr = ta.atr(HCL)\n",
        "                # L_asi = ta.asi(HCL)\n",
        "                L_avg_price = PRICE\n",
        "                L_vol = LAST['volume'].to_numpy()\n",
        "                L_trades = LAST['trades'].to_numpy()\n",
        "                L_wrsi = ta.wrsi(PRICE)\n",
        "                L_band = ta.bandwidth(PRICE)\n",
        "\n",
        "\n",
        "                # a, b, c, d, e = zip(*ta.ichimoku(HCL))\n",
        "                # L_ichi_a,L_ichi_b,L_ichi_c,L_ichi_d,L_ichi_e = np.array(a),\\\n",
        "                #                                                np.array(b),\\\n",
        "                #                                                np.array(c),\\\n",
        "                #                                                np.array(d),\\\n",
        "                #                                                np.array(e)\n",
        "\n",
        "                # a, b, c = zip(*ta.don(HL))\n",
        "                # L_donch_a,L_donch_b,L_donch_c = np.array(a),\\\n",
        "                #                                 np.array(b),\\\n",
        "                #                                 np.array(c)\n",
        "\n",
        "\n",
        "                LAST = calc_candle_sizes(LAST)\n",
        "                L_bs = LAST['body_size'].to_numpy()\n",
        "                L_rs = LAST['range_size'].to_numpy()\n",
        "                L_uss = LAST['upper_shadow_size'].to_numpy()\n",
        "                L_lss = LAST['lower_shadow_size'].to_numpy()\n",
        "\n",
        "\n",
        "                pattrn = []\n",
        "                # get pattern seq\n",
        "                for x in [\n",
        "                          # L_macd,\n",
        "                          L_rsi,\n",
        "                          # L_atr,\n",
        "                          # L_asi,\n",
        "                          # L_avg_price,\n",
        "                          # L_vol,\n",
        "                          # L_trades,\n",
        "                          # L_wrsi,\n",
        "                          # L_band,\n",
        "                          # L_ichi_a,L_ichi_b,L_ichi_c,L_ichi_d,L_ichi_e,\n",
        "                          # L_donch_a,L_donch_b,L_donch_c,\n",
        "                          # L_bs,L_rs,L_uss,L_lss,\n",
        "                          ]:\n",
        "                    pattrn.append(x[-sz:].copy())\n",
        "\n",
        "                _true_price_ = PRICE[-sz:].copy()\n",
        "\n",
        "                # concat all_in_one\n",
        "                indexes = range(hist_len) # !!!!!!!!!!! recheck!\n",
        "                all_stack = np.stack([\n",
        "                                      # macd,\n",
        "                                      rsi,\n",
        "                                      # atr,\n",
        "                                      # asi,\n",
        "                                      # avg_price,\n",
        "                                      # VOL,\n",
        "                                      # trades,\n",
        "                                      # wrsi,\n",
        "                                      # band,\n",
        "                                      # ichi_a,ichi_b,ichi_c,ichi_d,ichi_e,\n",
        "                                      # donch_a,donch_b,donch_c,\n",
        "                                      # bs,rs,uss,lss,\n",
        "                                      indexes],axis=0)  # last row we save our indexes array\n",
        "\n",
        "                # --------------------------- end ------------------------------\n",
        "\n",
        "\n",
        "            # ---------- threading - find similarity -----------------------------\n",
        "            if True:\n",
        "                SPLIT = 6\n",
        "                tasks = range(SPLIT)\n",
        "\n",
        "                multi_stack = np.array_split(all_stack, SPLIT, axis=1) #hsplit -- split two parts! for threading! =)\n",
        "\n",
        "                queue = multiprocessing.SimpleQueue()\n",
        "\n",
        "                for x in tasks:\n",
        "                    multiprocessing.Process(target=multi_calc_total_error,args=(metrics.rmse,\n",
        "                                                                                pattrn,\n",
        "                                                                                multi_stack[x],\n",
        "                                                                                sz,\n",
        "                                                                                horizon,\n",
        "                                                                                1,\n",
        "                                                                                False,\n",
        "                                                                                True,\n",
        "                                                                                queue,)).start()\n",
        "                error_table = []\n",
        "                for _ in tasks: error_table += queue.get()\n",
        "            # ------------------------------------------------------------------\n",
        "\n",
        "            SLICE = 20\n",
        "            df = pd.DataFrame(error_table).sort_values('error',ascending=True)[:SLICE]\n",
        "            debug = df.copy()\n",
        "\n",
        "            if TRAIN:\n",
        "\n",
        "                preds_list = []\n",
        "                __List = []\n",
        "\n",
        "                for x in range(SLICE):\n",
        "                    # get top-n index\n",
        "                    index = int(df.iloc[x]['index'])\n",
        "                    # get predict from HISTORY_PRICE\n",
        "                    pred = norm(HISTORY_PRICE[index+0 : index+sz+horizon],True)\n",
        "                    if (x <= 0): preds_list.append(pred) # add only first 8 element!\n",
        "                    __List.append({'err':metrics.rmse(_true_price_[:-horizon],pred[:-horizon]),\n",
        "                                   'vector':pred})\n",
        "\n",
        "\n",
        "                # df = pd.DataFrame(__List)#.sort_values('err',ascending=True)[:]\n",
        "                # # print('df_train:',df)\n",
        "                # for x in range(SLICE):\n",
        "                #     preds_list.append(df.iloc[x]['vector'])\n",
        "\n",
        "                # get average from all predict's\n",
        "                pred_avg = np.mean(np.array(preds_list),axis=0)\n",
        "\n",
        "                # get error's\n",
        "                # t_error += distance.euclidean(pct0,pred_avg)\n",
        "\n",
        "                # Plotting\n",
        "                plt.title(f'[Train] - {sz}')\n",
        "                plt.plot(_true_price_,label='actual')\n",
        "                # for q,x in enumerate(preds_list): plt.scatter(y=x,x=range(len(x)),alpha=0.12)#,label=f'pred{q}')\n",
        "                plt.plot(pred_avg,label='pred_avg',ls='--',c='red',alpha=0.8)\n",
        "                # plt.plot([],[],' ',label=f\"err: {round(y_pred0['mse'])}\")\n",
        "                plt.axvline(x=len(_true_price_)-horizon-1,c='grey',ls='--')\n",
        "                plt.legend()\n",
        "                # plt.savefig('grid.svg', bbox_inches='tight')\n",
        "                # upload_on_ftp('grid.svg')\n",
        "                plt.show()\n",
        "                plt.close()\n",
        "            else: # ---TEST---it---\n",
        "\n",
        "                preds_list = []\n",
        "                tmp_list = []\n",
        "\n",
        "                for x in range(SLICE):\n",
        "                    # get top-n index\n",
        "                    index = int(df.iloc[x]['index'])\n",
        "                    # get predict from HISTORY_PRICE\n",
        "                    pred = HISTORY_PRICE[index+0 : index+sz+horizon]\n",
        "                    if (x <= 0): preds_list.append(norm(pred,False)) # add only first 8 element!\n",
        "                    tmp_list.append({'err':metrics.rmse(norm(_true_price_,False),norm(pred[:-horizon],False)),\n",
        "                                   'vector':norm(pred,False)})\n",
        "\n",
        "                df = pd.DataFrame(tmp_list)\n",
        "                pred = df.iloc[0]['vector']\n",
        "                err1 = df.iloc[0]['err']\n",
        "\n",
        "                df = df.sort_values('err',ascending=True)\n",
        "                pred_sort = df.iloc[0]['vector']\n",
        "                err2 = df.iloc[0]['err']\n",
        "\n",
        "\n",
        "                # get average from all predict's\n",
        "                pred_avg = np.mean(np.array(preds_list),axis=0)\n",
        "\n",
        "                # Make plott grid\n",
        "                rows = 1\n",
        "                columns = 2\n",
        "                fig = plt.figure(figsize=(21,6))\n",
        "\n",
        "                # Plott 1st pic ----------------------\n",
        "                fig.add_subplot(rows, columns, 1)\n",
        "                plt.title(f'[Live]- {sz}')\n",
        "                plt.plot(norm(_true_price_,False),label='actual',linewidth=2.2)\n",
        "                # plt.plot(norm(pred_avg[:-horizon],True),label='pred_avg',ls='--',c='red',linewidth=2.5,alpha=0.7)\n",
        "                # plt.axvline(x=len(pred)-horizon-1,c='grey',ls='--')\n",
        "                plt.legend()\n",
        "\n",
        "                # Plott 2nd pic ----------------------\n",
        "                fig.add_subplot(rows, columns, 2)\n",
        "                # plt.title('[Live]')\n",
        "                plt.plot(pred_avg[:],label='predict',ls='-',c='red',linewidth=2.2,alpha=0.6)\n",
        "                plt.plot([],[],' ',label=f\"err. {round(err1)}\")\n",
        "                plt.axvline(x=len(pred)-horizon-1,c='grey',ls='--')\n",
        "                plt.legend()\n",
        "\n",
        "                # Save plot\n",
        "                plt.savefig('grid.png', bbox_inches='tight')\n",
        "                # edit_tg_photo(246874,'grid.png')\n",
        "                send_photo('grid.png')\n",
        "                # plt.show()\n",
        "                plt.close()\n",
        "\n",
        "        result_table.append({'sz':sz,'t_error':debug.iloc[0]['error']})\n",
        "\n",
        "    # if TRAIN: break\n",
        "    # if night_mode>110: break\n",
        "    # break\n",
        "    sleep(7*60)"
      ]
    }
  ]
}