{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"},{"sourceId":8418820,"sourceType":"datasetVersion","datasetId":5011657},{"sourceId":8444464,"sourceType":"datasetVersion","datasetId":5031627},{"sourceId":8670784,"sourceType":"datasetVersion","datasetId":5196428}],"dockerImageVersionId":30154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0\"></a>\n# Competition \"[G-Research Crypto Forecasting](https://www.kaggle.com/c/g-research-crypto-forecasting)\"","metadata":{"papermill":{"duration":0.025818,"end_time":"2022-01-05T00:26:31.467962","exception":false,"start_time":"2022-01-05T00:26:31.442144","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Криптовалюта** - это набор двоичных данных, предназначенный для использования в качестве средства обмена. Записи о владельцах отдельных монет хранятся в реестре, который представляет собой компьютеризированную базу данных, использующую надежную криптографию для защиты записей о транзакциях, контроля создания дополнительных монет и проверки передачи права собственности на монеты.\n\nКриптовалюта, форма цифровых токенов или “монет”, которые существуют в распределенном и децентрализованном реестре, называемом блокчейном, в настоящее время является популярной инвестицией.\n\n**Оценка результатов**\n\nДля оценки результатов будем использовать корреляцию Писрона.\n\nКоэффициент корреляции Пирсона (или r-Пирсона) применяется для исследования взаимосвязи двух переменных, измеренных в метрических шкалах на одной и той же выборке. Он позволяет определить, насколько пропорциональная изменчивость двух переменных.\n\nЭтот коэффициент измеряет степень линейной зависимости между двумя переменными. Если коэффициент равен по модулю единице, это свидетельствует о функциональной связи, то есть, изменения двух величин можно описать математической функцией. Однако следует помнить, что коэффициенты корреляции устанавливают лишь статистические взаимосвязи и не обязательно имеют причинно-следственный характер.","metadata":{}},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0.1\"></a>\n\n## Table of Contents\n\n1. [Import libraries](#1)\n1. [Download datasets](#2)\n1. [EDA](#3)\n1. [Проверка Target-а](#4)\n1. [Feature Engineering](#5)\n1. [Model training](#6)\n    - [Скользящее среднее (MV)](#6.1)\n    - [LinearRegression](#6.2)\n    - [kNN](#6.3)\n    - [Prophet](#6.4)\n    - [ARIMA](#6.5)\n    - [AutoARIMA](#6.6)\n    - [LGBM](#6.7)\n    - [LSTM](#6.8)\n1. [Tests](#7)","metadata":{"papermill":{"duration":0.02337,"end_time":"2022-01-05T00:26:31.562419","exception":false,"start_time":"2022-01-05T00:26:31.539049","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip list","metadata":{"execution":{"iopub.status.busy":"2024-06-17T16:23:28.036794Z","iopub.execute_input":"2024-06-17T16:23:28.037164Z","iopub.status.idle":"2024-06-17T16:23:31.385099Z","shell.execute_reply.started":"2024-06-17T16:23:28.037122Z","shell.execute_reply":"2024-06-17T16:23:31.383850Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pmdarima==1.8.2","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:02:02.929920Z","iopub.execute_input":"2024-06-18T10:02:02.930633Z","iopub.status.idle":"2024-06-18T10:02:13.611399Z","shell.execute_reply.started":"2024-06-18T10:02:02.930536Z","shell.execute_reply":"2024-06-18T10:02:13.610616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Import libraries <a class=\"anchor\" id=\"1\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.02353,"end_time":"2022-01-05T00:26:31.609677","exception":false,"start_time":"2022-01-05T00:26:31.586147","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#standart\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\nimport time\nfrom datetime import datetime\n\nfrom tqdm import tqdm\n\nimport pickle\nimport gc\nimport zipfile\nimport os\nimport gresearch_crypto\n\nimport pyarrow.csv as pc\nimport plotly.express as px\n\n# lgb\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\n\n# linear model\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.multioutput import MultiOutputRegressor\n\nfrom sklearn.model_selection import GridSearchCV\n\n#arima\nimport statsmodels.api as sm\nfrom statsmodels.tsa.ar_model import AutoReg\nfrom statsmodels.tsa.arima_model import ARMA\nfrom statsmodels.tsa.stattools import adfuller\nfrom statsmodels.graphics.tsaplots import plot_acf\nfrom statsmodels.graphics.tsaplots import plot_pacf\nfrom statsmodels.tsa.arima_model import ARIMA\nfrom statsmodels.tsa.statespace.sarimax import SARIMAX\nfrom statsmodels.tsa.seasonal import seasonal_decompose\nfrom scipy import stats\nfrom itertools import product\n\n#autoarima\nimport pmdarima as pm\n\n#lstm\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom keras.models import Sequential\nfrom keras.layers import Dense, LSTM\nimport tensorflow as tf\nimport matplotlib.ticker as ticker\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn import neighbors\n\nuse_supple_for_train = True\ndata_folder = \"/kaggle/input/g-research-crypto-forecasting/\"\ndirectory_path = '/kaggle/working/'\nlgb_list = ['/kaggle/input/new-lgb']\nn_fold_list= [7]\nn_fold = 7\ntf.device('/GPU:0'),\n\n# and create standart functions\ndef fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n\nfix_all_seeds(42)\n\n\ndef reduce_memory_usage(df):\n    \"\"\" RAM optimizer\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(\n        100 * (start_mem - end_mem) / start_mem))\n\n    return df\n\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n# Log Return \ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\ndef mean_absolute_percentage_error(y_true, y_pred): \n    y_true, y_pred = np.array(y_true), np.array(y_pred)\n    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100\n\ndef clean_output(os_dir=directory_path):\n    file_names = os.listdir(os_dir)\n    if clean_output:\n        for file_name in file_names:\n            if file_name != 'submission.csv':\n                os.remove(file_name)\n\ndef save_model(model, asset_name):    \n    #save model \n    file = f'trained_model_name_{asset_name}.pkl'\n    pickle.dump(model, open(file, 'wb'))\n    print(f\"Trained model was saved to 'trained_model_fold_{asset_name}.pkl'\")\n    \n    \ndef save_zip_file(os_dir=directory_path, zip_file_name='new_zip_file.zip'):\n    # Получаем список файлов в директории\n    file_names = os.listdir(os_dir)\n    # Создаем объект ZipFile\n    with zipfile.ZipFile(zip_file_name, 'w') as zip_object:\n        # Добавляем каждый файл в zip-архив\n        for file_name in file_names:\n            if file_name != 'submission.csv':\n                file_path = os.path.join(directory_path, file_name)\n                zip_object.write(file_path, compress_type=zipfile.ZIP_DEFLATED)\n    print('Zip archive created')","metadata":{"papermill":{"duration":3.725834,"end_time":"2022-01-05T00:26:53.203077","exception":false,"start_time":"2022-01-05T00:26:49.477243","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-18T10:02:15.931273Z","iopub.execute_input":"2024-06-18T10:02:15.932010Z","iopub.status.idle":"2024-06-18T10:02:26.760556Z","shell.execute_reply.started":"2024-06-18T10:02:15.931970Z","shell.execute_reply":"2024-06-18T10:02:26.759834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### API for test","metadata":{}},{"cell_type":"code","source":"# Данный api я взыл отсюда https://www.kaggle.com/code/jagofc/local-api/\n\ndtypes = {'timestamp': np.int64, 'Asset_ID': np.int8,\n          'Count': np.int32,     'Open': np.float64,\n          'High': np.float64,    'Low': np.float64,\n          'Close': np.float64,   'Volume': np.float64,\n          'VWAP': np.float64,    'Target': np.float64}\n\nid_2_weight = {0:  4.30406509320417,\n               1:  6.779921907472252,\n               2:  2.3978952727983707,\n               3:  4.406719247264253,               \n               4:  3.555348061489413,\n               5:  1.3862943611198906,\n               6:  5.8944028342648505,\n               7:  2.079441541679836,\n               8:  1.0986122886681098,\n               9:  2.3978952727983707,\n               10: 1.0986122886681098,\n               11: 1.6094379124341005,\n               12: 2.079441541679836,\n               13: 1.791759469228055}\n\nname_2_code = {\n    'Bitcoin Cash': 'BCH',\n    'Binance Coin': 'BNB',\n    'Bitcoin': 'BTC',\n    'EOS.IO': 'EOS',\n    'Ethereum Classic': 'ETC',\n    'Ethereum': 'ETH',\n    'Litecoin': 'LTC',\n    'Monero': 'XMR',\n    'TRON': 'TRX',\n    'Stellar': 'XLM',\n    'Cardano': 'ADA',\n    'IOTA': 'MIOTA',\n    'Maker': 'MKR',\n    'Dogecoin': 'DOGE'\n}\n\n\ndef datestring_to_timestamp(ts):\n    return int(pd.Timestamp(ts).timestamp())\n\n\ndef read_csv_slice(file_path, dtypes=dtypes, use_window=None):\n    df = pd.read_csv(file_path, dtype=dtypes)\n    if use_window is not None:\n        df = df[(df.timestamp >= use_window[0]) &\n                (df.timestamp < use_window[1])]\n    return df\n\n\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n    return corr\n\n\nLB_WINDOW = (datestring_to_timestamp('2021-06-13T00:00'),\n             datestring_to_timestamp('2021-09-22T00:00'))\n\n\nclass API:\n    def __init__(self, df, use_window=None):\n        df = df.astype(dtypes)\n        if use_window is not None:\n            df = df[(df.timestamp >= use_window[0]) &\n                    (df.timestamp <= use_window[1])]\n        df['row_id'] = df.index\n        dfg = df.groupby('timestamp')\n\n        self.data_iter = dfg.__iter__()\n        self.init_num_times = len(dfg)\n        self.next_calls = 0\n        self.pred_calls = 0\n        self.predictions = []\n        self.targets = []\n\n        print(\"This version of the API is not optimized and should not be used to\",\n              \"estimate the runtime of your code on the hidden test set. ;)\")\n\n    def __iter__(self):\n        return self\n\n    def __len__(self):\n        return self.init_num_times - self.next_calls\n\n    def __next__(self):\n        assert self.pred_calls == self.next_calls, \\\n            \"You must call `predict()` successfully before you can get the next batch of data.\"\n        timestamp, df = next(self.data_iter)\n        self.next_calls += 1\n        data_df = df.drop(columns=['Target'])\n        true_df = df.drop(\n            columns=['timestamp', 'Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP'])\n        true_df = true_df[['row_id', 'Target', 'Asset_ID']]\n        self.targets.append(true_df)\n        pred_df = true_df.drop(columns=['Asset_ID'])\n        pred_df['Target'] = 0.\n        return data_df, pred_df\n\n    def predict(self, pred_df):\n        assert self.pred_calls == self.next_calls - 1, \\\n            \"You must get the next batch of data from the API before making a new prediction.\"\n        assert pred_df.columns.to_list() == ['row_id', 'Target'], \\\n            \"Prediction dataframe should have columns `row_id` and `Target`.\"\n        pred_df = pred_df.astype({'row_id': np.int64, 'Target': np.float64})\n        self.predictions.append(pred_df)\n        self.pred_calls += 1\n\n    def score(self, id_2_weight=id_2_weight):\n        pred_df = pd.concat(self.predictions).rename(\n            columns={'Target': 'Prediction'})\n        true_df = pd.concat(self.targets)\n        scoring_df = pd.merge(true_df, pred_df, on='row_id', how='left')\n        scoring_df['Weight'] = scoring_df.Asset_ID.map(id_2_weight)\n        scoring_df = scoring_df[scoring_df.Target.isna() == False]\n        if scoring_df.Prediction.var(ddof=0) < 1e-10:\n            score = -1\n        else:\n            score = weighted_correlation(\n                scoring_df.Prediction, scoring_df.Target, scoring_df.Weight)\n        return scoring_df, score\n","metadata":{"execution":{"iopub.status.busy":"2024-06-15T14:05:01.005037Z","iopub.execute_input":"2024-06-15T14:05:01.005419Z","iopub.status.idle":"2024-06-15T14:05:01.046092Z","shell.execute_reply.started":"2024-06-15T14:05:01.005380Z","shell.execute_reply":"2024-06-15T14:05:01.042653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Download datasets <a class=\"anchor\" id=\"2\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.041177,"end_time":"2022-01-05T00:26:53.286842","exception":false,"start_time":"2022-01-05T00:26:53.245665","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df = pc.read_csv(data_folder + \"train.csv\").to_pandas()\ntrain_df = reduce_memory_usage(train_df)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:40:46.971949Z","iopub.execute_input":"2024-06-18T10:40:46.972712Z","iopub.status.idle":"2024-06-18T10:41:00.869582Z","shell.execute_reply.started":"2024-06-18T10:40:46.972669Z","shell.execute_reply":"2024-06-18T10:41:00.868829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Немного о данных:\n- Asset_id - идентификатор актива\n- count - общее количество сделок за временной интервал\n- open - цена открытия временного интервала\n- high - самая высокая цена, достигнутая за временной интервал\n- low - самая низкая цена, достигнутая за временной интервал\n- close - цена закрытия временного интервала\n- volume - количество купленных или проданных активов\n- vwap - средняя цена актива за временной интервал, взвешенная по объему\n- target - остаточный журнал, доходность актива за 15-минутный период","metadata":{}},{"cell_type":"code","source":"asset_details = pd.read_csv(data_folder + 'asset_details.csv')\nasset_details","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:41:06.794451Z","iopub.execute_input":"2024-06-18T10:41:06.795101Z","iopub.status.idle":"2024-06-18T10:41:06.808511Z","shell.execute_reply.started":"2024-06-18T10:41:06.795063Z","shell.execute_reply":"2024-06-18T10:41:06.807749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"В asset_details содержатся данные о названиях активов и их вес","metadata":{}},{"cell_type":"code","source":"asset_details.sort_values(by=['Asset_ID'], inplace=True, ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:05:47.244834Z","iopub.execute_input":"2024-06-18T10:05:47.245454Z","iopub.status.idle":"2024-06-18T10:05:47.250115Z","shell.execute_reply.started":"2024-06-18T10:05:47.245412Z","shell.execute_reply":"2024-06-18T10:05:47.249339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(data_folder + \"example_test.csv\")\nmapping = dict(asset_details[['Asset_ID', 'Asset_Name']].values)\ntrain_df[\"Asset name\"] = train_df[\"Asset_ID\"].map(mapping)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:49:45.174495Z","iopub.execute_input":"2024-06-18T09:49:45.174854Z","iopub.status.idle":"2024-06-18T09:49:46.787027Z","shell.execute_reply.started":"2024-06-18T09:49:45.174805Z","shell.execute_reply":"2024-06-18T09:49:46.785873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. EDA <a class=\"anchor\" id=\"3\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"markdown","source":"### Посмотрим на распределение весов валют","metadata":{}},{"cell_type":"code","source":"asset_details.sort_values(by=['Weight'], inplace=True, ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:39:12.578846Z","iopub.execute_input":"2024-06-18T10:39:12.579521Z","iopub.status.idle":"2024-06-18T10:39:12.584267Z","shell.execute_reply.started":"2024-06-18T10:39:12.579477Z","shell.execute_reply":"2024-06-18T10:39:12.583482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20,10))\nsns.barplot(x='Asset_Name', y='Weight', data=asset_details)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:05:32.328733Z","iopub.execute_input":"2024-06-16T18:05:32.329101Z","iopub.status.idle":"2024-06-16T18:05:32.694490Z","shell.execute_reply.started":"2024-06-16T18:05:32.329060Z","shell.execute_reply":"2024-06-16T18:05:32.693635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details[\"Weight\"].sum()","metadata":{"execution":{"iopub.status.busy":"2024-06-10T08:20:44.773271Z","iopub.execute_input":"2024-06-10T08:20:44.773604Z","iopub.status.idle":"2024-06-10T08:20:44.780181Z","shell.execute_reply.started":"2024-06-10T08:20:44.773567Z","shell.execute_reply":"2024-06-10T08:20:44.779382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_det_percentage = asset_details.copy()\nasset_det_percentage[\"weight_percentage\"] = (asset_details[\"Weight\"] / asset_details[\"Weight\"].sum()) * 100\nasset_det_percentage.sort_values(\"Weight\", ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:05:41.670372Z","iopub.execute_input":"2024-06-16T18:05:41.671211Z","iopub.status.idle":"2024-06-16T18:05:41.687793Z","shell.execute_reply.started":"2024-06-16T18:05:41.671166Z","shell.execute_reply":"2024-06-16T18:05:41.686744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.pie(asset_det_percentage, values='weight_percentage', names='Asset_Name', title='Weights given to each cryptocurrency')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:05:45.951154Z","iopub.execute_input":"2024-06-16T18:05:45.951467Z","iopub.status.idle":"2024-06-16T18:05:47.106741Z","shell.execute_reply.started":"2024-06-16T18:05:45.951435Z","shell.execute_reply":"2024-06-16T18:05:47.105606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names_dict = {row[\"Asset_Name\"]:row[\"Asset_ID\"] for ind, row in asset_details.iterrows()}\n\nasset_names = [\n    'Bitcoin',\n    'Ethereum',\n    'Cardano',\n    'Binance Coin',\n    'Dogecoin',\n    'Bitcoin Cash',\n    'Litecoin',\n    'Ethereum Classic',\n    'Stellar',\n    'TRON',\n    'Monero',\n    'EOS.IO',\n    'IOTA',\n    'Maker'\n]\n\ntime_list = []\nfor coin in asset_names:\n    coin_df = train_df[train_df[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    start_time = coin_df.index[0].astype('datetime64[s]')\n    end_time = coin_df.index[-1].astype('datetime64[s]')\n    time_list.append([coin, start_time, end_time])\ntime_df = pd.DataFrame(time_list)\ntime_df.columns = [\"Asset_Name\", \"Start_Time\", \"End_Time\"]\ntime_df","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:40:25.008310Z","iopub.execute_input":"2024-06-18T10:40:25.008624Z","iopub.status.idle":"2024-06-18T10:40:25.042529Z","shell.execute_reply.started":"2024-06-18T10:40:25.008587Z","shell.execute_reply":"2024-06-18T10:40:25.041450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-06-16T17:51:51.056171Z","iopub.execute_input":"2024-06-16T17:51:51.056484Z","iopub.status.idle":"2024-06-16T17:51:51.063732Z","shell.execute_reply.started":"2024-06-16T17:51:51.056452Z","shell.execute_reply":"2024-06-16T17:51:51.062656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_list = []\nfor coin in asset_names:\n    coin_df = train_df[train_df[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    missing_list.append([coin, coin_df.shape[0]] + coin_df.isna().sum().tolist())\nmissing_df = pd.DataFrame(missing_list)\nmissing_df.columns = [\"Asset_Name\", \"TotalRows\", 'Missing_Asset_ID', 'Missing_Count', 'Missing_Open', \n                      'Missing_High', 'Missing_Low', 'Missing_Close', 'Missing_Volume', 'Missing_VWAP', 'Missing_Target']","metadata":{"execution":{"iopub.status.busy":"2024-06-16T17:52:30.122489Z","iopub.execute_input":"2024-06-16T17:52:30.122861Z","iopub.status.idle":"2024-06-16T17:52:33.029455Z","shell.execute_reply.started":"2024-06-16T17:52:30.122822Z","shell.execute_reply":"2024-06-16T17:52:33.028605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_df","metadata":{"execution":{"iopub.status.busy":"2024-06-16T17:52:36.602805Z","iopub.execute_input":"2024-06-16T17:52:36.603147Z","iopub.status.idle":"2024-06-16T17:52:36.619489Z","shell.execute_reply.started":"2024-06-16T17:52:36.603109Z","shell.execute_reply":"2024-06-16T17:52:36.618599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Мы знаем, что данные собираются каждые 60 секунд. Если у нас есть карта с интервалом более одной минуты между последовательными строками, то это означает, что данные отсутствуют между ними. Давайте проверим это для каждой из монет.","metadata":{}},{"cell_type":"code","source":"gap_list = []\nfor coin in asset_names:\n    coin_df = train_df[train_df[\"Asset_ID\"] ==\n                       asset_names_dict[coin]].set_index(\"timestamp\")\n    gap_df = (coin_df.index[1:]-coin_df.index[:-1]\n              ).value_counts().reset_index()\n    n_gaps = gap_df[gap_df[\"index\"] != 60][\"timestamp\"].sum()\n    max_gap = gap_df[\"index\"].max()\n    gap_list.append([coin, n_gaps, max_gap])\ngap_df = pd.DataFrame(gap_list)\ngap_df.columns = [\"Asset_Name\", \"number_of_gaps\", \"max_gap_in_seconds\"]\ngap_df","metadata":{"execution":{"iopub.status.busy":"2024-06-16T17:55:24.109017Z","iopub.execute_input":"2024-06-16T17:55:24.109415Z","iopub.status.idle":"2024-06-16T17:55:26.819729Z","shell.execute_reply.started":"2024-06-16T17:55:24.109376Z","shell.execute_reply":"2024-06-16T17:55:26.818696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вывод:\n\nВ наборе данных есть несколько пробелов, и нам нужно внимательно отнестись к ним\nМаксимальный пробел в Maker coin составляет 281 700 секунд, или около 78 часов\nКогда у нас есть пробелы в данных, мы не можем рассчитать доходность за период времени, который наступает сразу после этих пробелов. Я думаю, что именно по этой причине в наших данных отсутствуют целевые значения.","metadata":{}},{"cell_type":"code","source":"import matplotlib.colors as mcolors\n# cmap = mcolors.ListedColormap([\"w\", \"grey\", \"green\", \"red\"])\n\ncmap = plt.cm.get_cmap('tab10')\nf = plt.figure(figsize=(15, 30))\n\nfor ind, coin in enumerate(asset_names):\n    coin_df = train_df[train_df[\"Asset_ID\"] ==\n                       asset_names_dict[coin]].set_index(\"timestamp\")\n    # fill missing values\n    coin_df = coin_df.reindex(\n        range(coin_df.index[0], coin_df.index[-1]+60, 60), method='pad')\n    ax = f.add_subplot(7, 2, ind+1)\n    plt.plot(coin_df['Close'], label=coin, color=cmap.colors[ind % 10])\n    plt.legend()\n    plt.xlabel('Time')\n    plt.ylabel(coin)\n    plt.title(coin)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:06:50.900467Z","iopub.execute_input":"2024-06-16T18:06:50.900830Z","iopub.status.idle":"2024-06-16T18:07:25.409896Z","shell.execute_reply.started":"2024-06-16T18:06:50.900795Z","shell.execute_reply":"2024-06-16T18:07:25.408629Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вывод:\n\nПо-видимому, существует хорошая корреляция между различными криптоактивами, а также, по-видимому, они различаются на разных временных шкалах.\nУ нас есть данные о медвежьих показателях за 2018 год, массовые данные за 2021 год, побочные данные за 2019 и 2020 годы. Таким образом, этот набор данных отражает различные типы рынков, которые мы можем смоделировать.","metadata":{}},{"cell_type":"code","source":"# create intervals\nf = plt.figure(figsize=(15,30))\n\nfor ind, coin in enumerate(asset_names):\n    coin_df = train_df[train_df[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    # fill missing values \n    coin_df = coin_df.reindex(range(coin_df.index[0],coin_df.index[-1]+60,60),method='pad')\n    coin_df = coin_df.loc[totimestamp('01/07/2021'):totimestamp('21/09/2021')]\n    ax = f.add_subplot(7,2,ind+1)\n    plt.plot(coin_df['Close'], label=coin, color=cmap.colors[ind%10])\n    plt.legend()\n    plt.xlabel('Time')\n    plt.ylabel(coin)\n    plt.title(coin)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:07:45.365333Z","iopub.execute_input":"2024-06-16T18:07:45.365659Z","iopub.status.idle":"2024-06-16T18:07:58.054551Z","shell.execute_reply.started":"2024-06-16T18:07:45.365625Z","shell.execute_reply":"2024-06-16T18:07:58.053633Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_assets_df = pd.DataFrame([])\nfor ind, coin in enumerate(asset_names):\n    coin_df = train_df[train_df[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    coin_df = coin_df.reindex(range(coin_df.index[0],coin_df.index[-1]+60,60),method='pad')\n    # fill missing values\n    coin_df = coin_df.loc[totimestamp('01/07/2021'):totimestamp('21/09/2021')]\n    close_values = coin_df[\"Close\"].fillna(0)\n    close_values.name = coin\n    all_assets_df = all_assets_df.join(close_values, how=\"outer\")\n\ncorrmat = all_assets_df.corr()\nfig, ax = plt.subplots(figsize=(15, 9))\nsns.heatmap(corrmat, vmax=1., square=True, cmap=\"rocket_r\")\nplt.title(\"Cryptocurrency correlation map on actual price values\", fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:41:15.724500Z","iopub.execute_input":"2024-06-18T10:41:15.724757Z","iopub.status.idle":"2024-06-18T10:41:21.454127Z","shell.execute_reply.started":"2024-06-18T10:41:15.724731Z","shell.execute_reply":"2024-06-18T10:41:21.453386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вывод:\n\nМы могли четко видеть высокую корреляцию между различными активами\nСамая низкая корреляция находится в диапазоне 0,4x\nСуществует высокая корреляция между\nBitcoin и Ethereum\nCardano и Binance coin\nBitcoin Cash и Litecoin","metadata":{}},{"cell_type":"markdown","source":"\nКорреляция на основе логарифмических данных\nНо мы получили корреляцию на основе фактических значений. Разные монеты находятся в разных ценовых диапазонах. Поэтому давайте воспользуемся логарифмическими данными, как предложено в учебном пособии, и проверим корреляцию за последний период времени.\n\nИз учебного пособия,\n\nЛогарифмические данные\n\nЧтобы проанализировать изменения цен на актив, мы можем учитывать разницу в цене. Однако разные активы имеют разный масштаб цен, поэтому их доходность не всегда сопоставима. Мы можем решить эту проблему, рассчитав процентное изменение цены, также известное как доходность. Эта доходность совпадает с процентным изменением нашего инвестированного капитала.\n\nДоходность широко используется в финансах, однако логарифмическая доходность предпочтительнее для математического моделирования временных рядов, поскольку она аддитивна во времени. Кроме того, хотя обычная доходность не может быть ниже -100%, логарифмическая доходность не ограничена.\n\nЧтобы вычислить логарифмический результат, мы можем просто взять логарифм отношения между двумя последовательными ценами. Первая строка будет содержать пустой результат, поскольку предыдущее значение неизвестно, поэтому пустая точка данных для возврата будет удалена.","metadata":{}},{"cell_type":"code","source":"\nall_assets_df = pd.DataFrame([])\nfor ind, coin in enumerate(asset_names):\n    coin_df = train_df[train_df[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    coin_df = coin_df.reindex(range(coin_df.index[0],coin_df.index[-1]+60,60),method='pad')\n    # fill missing values\n    coin_df = coin_df.loc[totimestamp('01/07/2021'):totimestamp('21/09/2021')]\n    return_values = log_return(coin_df[\"Close\"].fillna(0))[1:]\n    return_values.name = coin\n    all_assets_df = all_assets_df.join(return_values, how=\"outer\")\n\ncorrmat = all_assets_df.corr()\nfig, ax = plt.subplots(figsize=(15, 7))\nsns.heatmap(corrmat, vmax=1., square=True, cmap=\"rocket_r\")\nplt.title(\"Correlation map on log returns of the coins\", fontsize=15)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:52:18.152542Z","iopub.execute_input":"2024-06-18T10:52:18.153281Z","iopub.status.idle":"2024-06-18T10:52:24.532058Z","shell.execute_reply.started":"2024-06-18T10:52:18.153242Z","shell.execute_reply":"2024-06-18T10:52:24.531355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вывод:\n\nВ последние месяцы, когда мы смотрим на доходность журналов, у IOTA очень низкая корреляция с другими монетами\nБиткоин и Эфириум по-прежнему демонстрируют высокую корреляцию.\nПохоже, что корреляция между активами меняется с течением времени, и поэтому это необходимо учитывать при построении моделей","metadata":{}},{"cell_type":"code","source":"all_assets_2021 = pd.DataFrame([])\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n  asset = train_df[train_df[\"Asset_ID\"] == asset_id].set_index(\"timestamp\")\n  asset = asset.loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n  asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n  lret = log_return(asset.Close.fillna(0))[1:]\n  all_assets_2021 = all_assets_2021.join(lret, rsuffix=asset_name, how=\"outer\")\n\nplt.imshow(all_assets_2021.corr());\nplt.yticks(range(14), asset_details.Asset_Name.values);\nplt.xticks(range(14), asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:41:33.778938Z","iopub.execute_input":"2024-06-18T10:41:33.779577Z","iopub.status.idle":"2024-06-18T10:41:36.587401Z","shell.execute_reply.started":"2024-06-18T10:41:33.779534Z","shell.execute_reply":"2024-06-18T10:41:36.586653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Далее основной датасет","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:29:13.199891Z","iopub.execute_input":"2024-06-16T18:29:13.200243Z","iopub.status.idle":"2024-06-16T18:29:13.809551Z","shell.execute_reply.started":"2024-06-16T18:29:13.200204Z","shell.execute_reply":"2024-06-16T18:29:13.808615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info(show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T18:29:18.471541Z","iopub.execute_input":"2024-06-16T18:29:18.471871Z","iopub.status.idle":"2024-06-16T18:29:19.153744Z","shell.execute_reply.started":"2024-06-16T18:29:18.471834Z","shell.execute_reply":"2024-06-16T18:29:19.152577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train_df.reset_index(drop = True) \ntemp['TIME'] = pd.to_datetime(temp['timestamp'], unit='s')","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:41:41.957245Z","iopub.execute_input":"2024-06-18T10:41:41.958032Z","iopub.status.idle":"2024-06-18T10:41:43.327119Z","shell.execute_reply.started":"2024-06-18T10:41:41.957993Z","shell.execute_reply":"2024-06-18T10:41:43.326308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp['Year'] = temp['TIME'].dt.year\ntemp.columns = [column.replace(\" \", \"_\") for column in temp.columns]\ntemp.dropna()[['Open', 'High', 'Low', 'Close', 'Volume','VWAP']]\nvolume_yr = temp.groupby(['Year','Asset_name'])['Volume'].max().reset_index()\nvolume_yr = volume_yr.query(\"Year == 2021\")","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:41:45.247602Z","iopub.execute_input":"2024-06-18T10:41:45.248258Z","iopub.status.idle":"2024-06-18T10:41:50.226189Z","shell.execute_reply.started":"2024-06-18T10:41:45.248218Z","shell.execute_reply":"2024-06-18T10:41:50.225214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.rcParams['figure.dpi'] = 300\n# fig = plt.figure(figsize=(3, 5), facecolor='#F7F7F7')\n# gs = fig.add_gridspec(2, 1)\n# gs.update(wspace=1.5, hspace=1.1)\n\n# background_color = \"#F7F7F7\"\n# color_map = [\"#E8F0F2\" for _ in range(20)]\n# color_map[4] = \"#C9CCD5\" #F2A154 79B4B7\n# color_map[1] = \"#F2A154\"\n# color_map[6] = \"#F2A154\"\n# sns.set_palette(sns.color_palette(color_map))\n\n# ax0 = fig.add_subplot(gs[0, 0])\n# ax0.set_facecolor(background_color)\n\n# for s in [\"right\", \"top\"]:\n#     ax0.spines[s].set_visible(False)\n\n# for s in ['bottom','left']:\n#   ax0.spines[s].set_linewidth(0.5)\n#   ax0.spines[s].set_color('grey')\n    \n# #graph\n# ax0 = sns.barplot(ax=ax0, y=volume_yr['Volume']/1000, x=volume_yr['Asset_name'], \n#                       zorder=2, linewidth=0.3, edgecolor=\"#7F7C82\", \n#                       orient='v', saturation=0.9, alpha=0.7)\n# ax0.grid(which='major', axis='y', zorder=0, color='#CDD0CB', linewidth=0.2, alpha=0.5)\n# ax0.set_yscale('symlog')\n\n# #format axis\n# ax0.set_ylabel(\"Volume\",fontsize=3, weight='semibold')\n# ax0.set_xlabel(\"Crypto Currency\",fontsize=3, weight='semibold')\n# ax0.tick_params(labelsize=3, width=0.2, length=1)\n# plt.setp( ax0.xaxis.get_majorticklabels(), rotation=45, ha=\"right\" )\n\n# #title\n# x0, x1 = ax0.get_xlim()\n# y0, y1 = ax0.get_ylim()\n# ax0.text(x0, y1+2000000, 'Volume by Year', fontsize=4, ha='left', va='top', weight='semibold')\n\n# y_format = ticker.FuncFormatter(lambda x, p: format(int(x), ','))\n# ax0.yaxis.set_major_formatter(y_format)\n\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:41:54.120485Z","iopub.execute_input":"2024-06-18T10:41:54.121132Z","iopub.status.idle":"2024-06-18T10:41:54.370350Z","shell.execute_reply.started":"2024-06-18T10:41:54.121093Z","shell.execute_reply":"2024-06-18T10:41:54.369370Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Для технического анализа будем использовать скользящую среднюю(SMA) и индекс относительной силы(RSI).","metadata":{}},{"cell_type":"code","source":"# import plotly.graph_objects as go\n# from plotly.offline import iplot, init_notebook_mode\n# init_notebook_mode()\n\n# for currency_id in asset_details.Asset_ID:\n#     currency_value = train_df[train_df[\"Asset_ID\"] == currency_id].iloc[-365:].set_index(\"timestamp\")\n#     fig = go.Figure(data=go.Candlestick(x=currency_value.index, open=currency_value['Open'], high=currency_value['High'], low=currency_value['Low'], close=currency_value['Close']))\n#     name = asset_details[asset_details.Asset_ID == currency_id]['Asset_Name']\n#     fig.update_layout(title=f\"{name}\")\n#     fig.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:42:01.987850Z","iopub.execute_input":"2024-06-18T10:42:01.988113Z","iopub.status.idle":"2024-06-18T10:42:02.213569Z","shell.execute_reply.started":"2024-06-18T10:42:01.988084Z","shell.execute_reply":"2024-06-18T10:42:02.212558Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total Null Target Rows = \" ,train_df[\"Target\"].isnull().sum())\nprint(\"Percentage of NUll rows in Training Data = {:.2f}%\".format(train_df[\"Target\"].isnull().sum()*100 / train_df.shape[0] ))","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:42:05.850762Z","iopub.execute_input":"2024-06-18T10:42:05.851045Z","iopub.status.idle":"2024-06-18T10:42:06.001912Z","shell.execute_reply.started":"2024-06-18T10:42:05.851011Z","shell.execute_reply":"2024-06-18T10:42:06.001157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Иногда отсутсвие данных выражается не ввиде NaN, а в виде отсутсвия этих строк.\n\nНе так уж много данных пропущено. Мы можем удалить эти строки, вычислить значения и т.д.\n\nБолее скрытая потеря данных - это когда у нас вообще нет никакой информации за определенную минуту. Это означает, что в нашем наборе данных могут отсутствовать целые строки.\n\nУ нас должна быть одна строка в минуту для каждого актива. Поскольку мы извлекли данные для одного актива, мы ожидаем, что в последовательных строках разница между значениями индекса составит 60 секунд.\n\nПервый шаг - посмотреть на временную задержку между последовательными записями в нашем наборе данных.","metadata":{}},{"cell_type":"code","source":"for currency_id in asset_details.Asset_ID:\n    currency_value = train_df[train_df[\"Asset_ID\"] == currency_id]\n    print(asset_details[asset_details['Asset_ID'] == currency_id]['Asset_Name'], (currency_value.index[1:] - currency_value.index[:-1]).value_counts().head())","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:42:11.395065Z","iopub.execute_input":"2024-06-18T10:42:11.395399Z","iopub.status.idle":"2024-06-18T10:42:13.399506Z","shell.execute_reply.started":"2024-06-18T10:42:11.395360Z","shell.execute_reply":"2024-06-18T10:42:13.398687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Поскольку пробелы в данных очень малы, мы можем использовать простой метод вычисления: заполнить недостающие данные значением за последнюю доступную минуту.\n\nИменно это делает параметр method = 'pad' функции напоминания, приведенной ниже.","metadata":{}},{"cell_type":"code","source":"currency_values = {}\nfor currency_id in asset_details.Asset_ID:\n    \n    currency_value = train_df[train_df[\"Asset_ID\"] == currency_id]\n    currency_value = currency_value.reindex(range(currency_value.index[0], currency_value.index[-1] + 60, 60), method='pad')\n    name = asset_details[asset_details.Asset_ID == currency_id]['Asset_Name'].values[0]\n    currency_values[str(name)] = currency_value\n    \n    print(name, (currency_value.index[1:] - currency_value.index[:-1]).value_counts().head())","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:42:18.292565Z","iopub.execute_input":"2024-06-18T10:42:18.292879Z","iopub.status.idle":"2024-06-18T10:42:21.922432Z","shell.execute_reply.started":"2024-06-18T10:42:18.292829Z","shell.execute_reply":"2024-06-18T10:42:21.921629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Все, в нашем датасете нет временных пробелов","metadata":{}},{"cell_type":"markdown","source":"### Data Visualisation","metadata":{}},{"cell_type":"code","source":"for key,value in currency_values.items():\n    f = plt.figure(figsize=(15, 4))\n    plt.plot(value['Close'], label=f'{key}')\n    plt.legend()\n    plt.xlabel('Time')\n    plt.ylabel(f'{key}')\n\n    plt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:42:24.984051Z","iopub.execute_input":"2024-06-18T10:42:24.984375Z","iopub.status.idle":"2024-06-18T10:42:39.595057Z","shell.execute_reply.started":"2024-06-18T10:42:24.984310Z","shell.execute_reply":"2024-06-18T10:42:39.594217Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Добавим датасет asset_details для анализа","metadata":{}},{"cell_type":"code","source":"df_analysis = pd.merge(train_df, asset_details, how='left', on=['Asset_ID'])\ndf_analysis.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:42:47.348254Z","iopub.execute_input":"2024-06-18T10:42:47.349011Z","iopub.status.idle":"2024-06-18T10:42:52.511388Z","shell.execute_reply.started":"2024-06-18T10:42:47.348963Z","shell.execute_reply":"2024-06-18T10:42:52.510611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20,10))\nsns.countplot(x='Asset_Name', data = df_analysis.dropna())","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:43:15.721995Z","iopub.execute_input":"2024-06-18T10:43:15.722875Z","iopub.status.idle":"2024-06-18T10:43:49.452928Z","shell.execute_reply.started":"2024-06-18T10:43:15.722827Z","shell.execute_reply":"2024-06-18T10:43:49.452233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_analysis.dropna()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:43:55.320242Z","iopub.execute_input":"2024-06-18T10:43:55.320528Z","iopub.status.idle":"2024-06-18T10:44:00.266637Z","shell.execute_reply.started":"2024-06-18T10:43:55.320497Z","shell.execute_reply":"2024-06-18T10:44:00.265878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20,10))\nsns.heatmap(df.corr())","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:44:03.310388Z","iopub.execute_input":"2024-06-18T10:44:03.310668Z","iopub.status.idle":"2024-06-18T10:44:13.250367Z","shell.execute_reply.started":"2024-06-18T10:44:03.310635Z","shell.execute_reply":"2024-06-18T10:44:13.249577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['fulldate'] = pd.to_datetime(df['timestamp'], unit='s')\ndf['date'] = df['fulldate'].apply(lambda d: d.date())\ndf['time'] = df['fulldate'].apply(lambda d: d.time())\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:44:17.997339Z","iopub.execute_input":"2024-06-18T10:44:17.998099Z","iopub.status.idle":"2024-06-18T10:49:01.267744Z","shell.execute_reply.started":"2024-06-18T10:44:17.998062Z","shell.execute_reply":"2024-06-18T10:49:01.266969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtrain_sample = df.sample(10000)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:49:33.731105Z","iopub.execute_input":"2024-06-18T10:49:33.731832Z","iopub.status.idle":"2024-06-18T10:49:36.916066Z","shell.execute_reply.started":"2024-06-18T10:49:33.731791Z","shell.execute_reply":"2024-06-18T10:49:36.915001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axes = plt.subplots(7, 2, figsize=(20, 50))\n# for i,asset in enumerate(asset_details['Asset_Name']):\n#     df_crypt = dtrain_sample[dtrain_sample['Asset_Name'] == asset]\n#     sns.lineplot(x=\"date\", y=\"VWAP\", data=df_crypt, ax=axes[int(i/2),i%2])\n#     axes[int(i/2),i%2].set_title(asset)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:51:37.502229Z","iopub.execute_input":"2024-06-18T10:51:37.502982Z","iopub.status.idle":"2024-06-18T10:51:37.506153Z","shell.execute_reply.started":"2024-06-18T10:51:37.502946Z","shell.execute_reply":"2024-06-18T10:51:37.505378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Available data period\")\nfor i,asset in enumerate(asset_details['Asset_Name']):\n    df_crypt = df[df['Asset_Name'] == asset]\n    print( \"{} to {} ------> {}\".format(df_crypt.sort_values(by=['timestamp'])['date'].iloc[0], df_crypt.sort_values(by=['timestamp'])['date'].iloc[-1], asset ))","metadata":{"execution":{"iopub.status.busy":"2024-06-10T08:42:10.661022Z","iopub.execute_input":"2024-06-10T08:42:10.661933Z","iopub.status.idle":"2024-06-10T08:42:50.664462Z","shell.execute_reply.started":"2024-06-10T08:42:10.661883Z","shell.execute_reply":"2024-06-10T08:42:50.663528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intervals_currency = {}\nfor currency_id in asset_details.Asset_ID:\n    currency_value = train_df[train_df[\"Asset_ID\"] == currency_id].set_index('timestamp')\n    name = asset_details[asset_details.Asset_ID == currency_id]['Asset_Name'].values[0]\n    intervals_currency[f\"{name}_min_2021\"] = currency_value.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:51:10.022877Z","iopub.execute_input":"2024-06-18T10:51:10.023405Z","iopub.status.idle":"2024-06-18T10:51:12.107124Z","shell.execute_reply.started":"2024-06-18T10:51:10.023364Z","shell.execute_reply":"2024-06-18T10:51:12.106482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key, value in intervals_currency.items():\n    f = plt.figure(figsize=(8, 4))\n    plt.plot(value['Close'], label=f'{key}')\n    plt.legend()\n    plt.xlabel('Time')\n    plt.ylabel(f'{key}')\n\n    plt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:51:25.852748Z","iopub.execute_input":"2024-06-18T10:51:25.853399Z","iopub.status.idle":"2024-06-18T10:51:25.856853Z","shell.execute_reply.started":"2024-06-18T10:51:25.853361Z","shell.execute_reply":"2024-06-18T10:51:25.856004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Проверка Target-а <a class=\"anchor\" id=\"4\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"check_df = train_df.copy()\ncheck_df['Time'] = pd.to_datetime(check_df['timestamp'], unit='s')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Затем рассчитаем Target по следующей формуле\n\n$$R^a(t) = log (P^a(t+16)\\ /\\ P^a(t+1))$$\n\nЭто делается для каждого актива отдельно. Мы не знаем, какую цену следует использовать. Существует пять различных цен: Открытие, Максимум, минимум, закрытие и VWAP. Вероятно, существует такая комбинация, как цена открытия за время + 1 минута и цена закрытия за время + 16 минут. В приведенных ниже расчетах мы используем цену закрытия.","metadata":{}},{"cell_type":"code","source":"# use mix price\nprice_column = 'Close'\n\nids = list(asset_details.Asset_ID)\nchunks = []\n\nfor id in ids:\n    asset = check_df[train_df.Asset_ID == id].copy()\n    asset.sort_values(by='Time', inplace=True)\n    asset.set_index(keys='Time', inplace=True)\n    asset['p1'] = asset[price_column].shift(freq='-1T')\n    asset['p16'] = asset[price_column].shift(freq='-16T')\n    asset['r'] = np.log(asset.p16/asset.p1)\n    asset.drop(['p1', 'p16'], axis=1, inplace=True)\n    asset.reset_index(inplace=True)\n    chunks.append(asset)\n    \ncheck_df = pd.concat(chunks)\ncheck_df.sort_values(by='Time', inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Затем назначим вес для каждой строки. И рассчитаем M(t). Обратите внимание, что M(t) одинаков для всех активов и зависит только от времени.\n\n$$M(t) = \\frac{\\sum_a w^a R^a(t)}{\\sum_a w^a}$$\n\nНе известно, ${\\sum_a w^a}$ должна быть рассчитана для всех активов или только для активов, имеющих данные на момент времени t.\n","metadata":{}},{"cell_type":"code","source":"check_df['w'] = check_df['Asset_ID'].map(asset_details.set_index(keys='Asset_ID')['Weight'])\nweight_sum = asset_details.Weight.sum()\n\ncheck_df['weighted_asset_r'] = check_df.w * check_df.r\ntime_group = check_df.groupby('Time')\n\nm = time_group['weighted_asset_r'].sum() / time_group['w'].sum()\n\ncheck_df.set_index(keys=['Time'], inplace=True)\ncheck_df['m'] = m\ncheck_df.reset_index(inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"После этого рассчитывается бета-версия. В скобках $\\langle .\\rangle$ представляет собой скользящее среднее значение по времени (3750-минутные интервалы). Если нет полного 3750-минутного интервала, $\\beta$ становится нулевым.\n$$\\beta^a = \\frac{\\langle M \\cdot R^a \\rangle}{\\langle M^2 \\rangle}$$","metadata":{}},{"cell_type":"code","source":"check_df['m2'] = check_df.m ** 2\ncheck_df['mr'] = check_df.m * check_df.r\n\nchunks = []\n\nfor id in ids:\n    asset = check_df[check_df.Asset_ID == id].copy()\n    asset.sort_values(by='Time', inplace=True)\n    asset.set_index(keys='Time', inplace=True)\n    asset['mr_rolling'] = asset['mr'].rolling(window='3750T', min_periods=3750).mean()\n    asset['m2_rolling'] = asset['m2'].rolling(window='3750T', min_periods=3750).mean()\n    asset.reset_index(inplace=True)\n    chunks.append(asset)\n    \n    debug = 1\n    \ncheck_df = pd.concat(chunks)\ncheck_df.sort_values(by='Time', inplace=True)\ncheck_df['beta'] = check_df['mr_rolling'] / check_df['m2_rolling']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Все, target вычислен\n$$\\text{Target}^a(t) = R^a(t) - \\beta^a M(t)$$","metadata":{}},{"cell_type":"code","source":"check_df['Target_recreated'] = check_df['r'] - check_df['beta'] * check_df['m']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Теперь сравним воссозданный Target","metadata":{}},{"cell_type":"code","source":"check_df['Target_diff'] = np.abs(check_df['Target'] - check_df['Target_recreated'])\n\nprint(f'Average absolute error {check_df.Target_diff.mean():8.6f}')\nprint(f'Max absolute error {check_df.Target_diff.max():8.6f}')\nprint(f'Standart absolute error {check_df.Target_diff.std():8.6f}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Нормальная цель изменяется в диапазоне [-0.5, 0.96], поэтому максимальная абсолютная ошибка 2.4 означает, что воссозданная цель полностью неверна для некоторых записей.","metadata":{}},{"cell_type":"code","source":"check_df['Target'].agg(['min', 'max'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Проверим сколько записей у нас с неправильными Targetами","metadata":{}},{"cell_type":"code","source":"(check_df.Target_recreated < -0.509351).sum(), (check_df.Target_recreated > 0.964170).sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Я пробовал разные цены, менял временные интервалы в формуле, заменял скользящее среднее с 3750 минут на 3750 последних записей и т.д., но результат тот же или хуже. Можно минимизировать максимальную ошибку, но среднее значение стало выше.\n\nПожалуйста, дайте мне знать, если вы обнаружите ошибку в расчетах или у вас будет какое-то представление о том, как на самом деле вычисляется исходный целевой показатель.\n\nДобавьте код, добавляющий Markdown\nОказывается, что этот код дает много значений NA в воссозданном целевом показателе, поэтому минимизация ошибок Abs/Max, возможно, вводила в заблуждение.","metadata":{}},{"cell_type":"markdown","source":"Проблема такая, что выше пречиленный код создает много значений NA, которых не было в оригинале. Для решения этой проблемы попробуем другой способ.","metadata":{}},{"cell_type":"code","source":"pd.isna(data.Target).sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.isna(data.Target_recreated).sum()pd.isna(data.Target_recreated).sum()pd.isna(data.Target_recreated).sum()pd.isna(data.Target_recreated).sum()pd.isna(data.Target_recreated).sum()pd.isna(data.Target_recreated).sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"В результате воссозданный целевой объект имеет более чем в 10 раз больше значений NA. Давайте проверим, когда были введены эти NA.","metadata":{}},{"cell_type":"code","source":"pd.isna(data.r).sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"При расчете R количество NA практически равно количеству целевых NA. Разница составляет 1086. Это может быть объяснено отсутствием первой минуты каждого месяца для каждого актива.\n\nИнтересно, что цель присутствует в наборе данных о поездах для таких дат, как ГГГГ-ММ-01 23:44:00 (t+16) и ГГГГ-ММ-01 23:59:00 (t+1). Для этих случаев алгоритм расчета требует наличия цены на момент ГГГГ-ММ-01 00:00:00, но это время отсутствует в тестовых данных. Это можно рассматривать как доказательство того, что цель была вычислена на основе других данных, а затем набор обучающих данных был создан с уже вычисленной целью.\n\nОшибка, из-за которой отсутствует первая минута месяца, дает приблизительно 44 месяца х 2 (каждая строка влияет на две цели для смены 1 и 16) х 14 (количество активов) => 1232. В некоторых активах отсутствуют данные за все месяцы.\n\nЕдинственное другое место, где можно получить NA, - это следующий код\n\nпереход(window='3750T', min_periods=3750)\n\nпропущенные значения в интервале 3750 минут приводят к значениям NA.\n\nТеперь давайте посмотрим на код, предоставленный хостом:\n\nчисло = df.умножение(mkt.значения, ось=0).переход (окно).среднее значение().значения\nзначение = mkt.умножение(mkt.значения, ось=0).переход(окно).среднее значение().значения\n\nСуществует два возможных типа окна, которые принимает rolling - это может быть либо shift/int (количество строк для сдвига), либо time offset (количество минут для сдвига), но параметра min_periods нет. Это означает, что для int shift он вообще не выдает NA, а для time offset он может выдавать NA, только если нет данных за последние 3750 минут.\n\nСледующая строка кода вычисляет beta и удаляет все результирующие значения, отличные от чисел, с нулем:\n\nбета = np.nan_to_num( число.T / denom, nan=0., posinf=0., neginf=0.)\n\nВывод здесь заключается в том, что добавление min_periods=3750, которое, казалось бы, улучшило наши показатели, на самом деле добавило множество значений NA, эффективно минимизируя данные, в которых эти показатели вычислялись. Основной код показывает, что скользящее среднее значение не приводит к дополнительным значениям NA.","metadata":{}},{"cell_type":"markdown","source":"Более оптимизированная версия¶\nЗдесь мы пытаемся сделать две вещи: решить проблему с NA и ускорить работу этого кода, а также повторно использовать код с хостинга. Код из хостинга показывает, что данные должны быть представлены немного по-другому, чтобы избежать дорогостоящих операций сдвига/переноса на основе минут и выполнить расчет для всех ресурсов сразу.\n\nдобавить код, добавить уценку\nМы начинаем с создания нового фрейма данных, в котором присутствуют все временные метки (минуты). В поезде для некоторых целей может не хватить нескольких минут.","metadata":{}},{"cell_type":"code","source":"ids = list(asset_details.Asset_ID)\nasset_names = list(asset_details.Asset_Name)\n\nall_timestamps = np.sort(check_df['timestamp'].unique())\ntargets = pd.DataFrame(index=all_timestamps)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"calcualte r, некоторые столбцы могут содержать NA, так как некоторые даты со свдигом могут отсутствовать\n\nДалее мы вычисляем R для каждого актива и добавляем его значения в виде столбца в новый целевой фрейм данных. Обратите внимание, что некоторые строки будут содержать NA, так как требуемые цены со сдвигом могут отсутствовать.","metadata":{}},{"cell_type":"code","source":"for i, id in enumerate(ids):\n    asset = check_df[check_df.Asset_ID == id].set_index(keys='timestamp')\n    price = pd.Series(index=all_timestamps, data=asset[price_column])\n    \n    targets[asset_names[i]] = (\n        price.shift(periods=-16)/price.shift(periods=-1)\n    ) - 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Затем вычисляется как средневзвешенное значение столбцов (в настоящее время содержащих значение R для каждого актива) для каждой строки. В этой реализации недоступные значения R считаются равными нулю. Пожалуйста, обратите внимание, что это не единственный способ обработки значений NA при расчете средневзвешенного значения.","metadata":{}},{"cell_type":"code","source":"weights = np.array(list(asset_details.Weight))\ntargets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Наконец, пришло время применить код, предоставленный хостером, для расчета бета-версии и целевого показателя.","metadata":{}},{"cell_type":"code","source":"m = targets['m']\n\nnum = targets.multiply(m.values, axis=0).rolling(3750).mean().values\ndenom = m.multiply(m.values, axis=0).rolling(3750).mean().values\nbeta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n\ntargets = targets - (beta * m.values).T","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Давайте проверим, насколько это отличается от заданной цели и сколько у нас значений NA.","metadata":{}},{"cell_type":"code","source":"diffs = []\n\nfor i, id in enumerate(ids):\n    print(asset_names[i])\n    # type: pd.DataFrame\n    asset = check_df[check_df.Asset_ID == id].set_index(keys='timestamp')\n    print(f'asset size {asset.shape[0]}')\n    recreated = pd.Series(index=asset.index, data=targets[asset_names[i]])\n    diff = np.abs(asset['Target'] - recreated)\n    diffs.append(diff[~pd.isna(diff)].values)\n    print(f'Average absolute error {diff.mean():8.6f}')\n    print(f'Max absolute error {diff.max():8.6f}')\n    print(f'Standard deviation {diff.std():8.6f}')\n    print(f'Target na {pd.isna(asset.Target).sum()}')\n    print(f'Target_calculated na {pd.isna(recreated).sum()}')\n    print()\n\ndiffs = np.concatenate(diffs, axis=0)\nprint('For all assets')\nprint(f'Average absolute error {diffs.mean():8.6f}')\nprint(f'Max absolute error {diffs.max():8.6f}')\nprint(f'Standard deviation {diffs.std():8.6f}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Это все еще не идеально, но, по крайней мере, наши показатели рассчитываются практически по всем доступным данным, а количество NA почти такое же, как в Target.\n\nдобавляем код, добавляем Markdown\nОбсуждается, что бета равна нулю. Это превращает расчет Target просто в расчет R. Давайте посмотрим, сколько у нас есть нечисловых значений и сколько строк имеют значение beta=0.","metadata":{}},{"cell_type":"code","source":"beta_ = num.T / denom\n\nfor i, id in enumerate(ids):\n    print(asset_names[i])\n    print(f'Infiinte beta rows {np.isinf(beta_[i]).sum()}')\n    nan_sum = np.isnan(beta_[i]).sum()\n    print(f'NAN beta rows {nan_sum} ({100 * nan_sum / beta_.shape[1]:5.2f}%)')\n\n    eps = 1e-6\n    zero_sum = ((beta[i] > -eps) & (beta[i] < eps)).sum()\n    print(f'Zero beta rows {zero_sum} ({100 * zero_sum / beta.shape[1]:5.2f}%)')\n    print()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ниже приведен тот же код, но в виде функции, которую вы можете скопировать и вставить. Обратите внимание, что для этого не требуется столбец времени, вместо него используется временная метка.","metadata":{}},{"cell_type":"code","source":"def calculate_target(data: pd.DataFrame, details: pd.DataFrame, price_column: str):\n    \"\"\"Данная функция перерасчитывает наш Target по формуле, которая дается в качестве описания проекта\"\"\"\n    ids = list(details.Asset_ID)\n    asset_names = list(details.Asset_Name)\n    weights = np.array(list(details.Weight))\n\n    all_timestamps = np.sort(data['timestamp'].unique())\n    targets = pd.DataFrame(index=all_timestamps)\n\n    for i, id in enumerate(ids):\n        asset = data[data.Asset_ID == id].set_index(keys='timestamp')\n        price = pd.Series(index=all_timestamps, data=asset[price_column])\n        targets[asset_names[i]] = (\n            price.shift(periods=-16) /\n            price.shift(periods=-1)\n        ) - 1\n    \n    targets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)\n    \n    m = targets['m']\n\n    num = targets.multiply(m.values, axis=0).rolling(3750).mean().values\n    denom = m.multiply(m.values, axis=0).rolling(3750).mean().values\n    beta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n\n    targets = targets - (beta * m.values).T\n    targets.drop('m', axis=1, inplace=True)\n    \n    return targets","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df.columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Feature engineering <a class=\"anchor\" id=\"5\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.047191,"end_time":"2022-01-05T00:26:54.355054","exception":false,"start_time":"2022-01-05T00:26:54.307863","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_features(df):\n    \"\"\"Feature engineering by row\n    \"\"\"\n    df['upper_shadow'] = df['High'] / df[['Close', 'Open']].max(axis=1)\n    df['lower_shadow'] = df[['Close', 'Open']].min(axis=1) / df['Low']\n    df['open2close'] = df['Close'] / df['Open']\n    df['high2low'] = df['High'] / df['Low']\n    mean_price = df[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    median_price = df[['Open', 'High', 'Low', 'Close']].median(axis=1)\n    df['high2mean'] = df['High'] / mean_price\n    df['low2mean'] = df['Low'] / mean_price\n    df['high2median'] = df['High'] / median_price\n    df['low2median'] = df['Low'] / median_price\n    df['volume2count'] = df['Volume'] / (df['Count'] + 1)\n    return df ","metadata":{"execution":{"iopub.status.busy":"2024-06-18T04:15:21.558562Z","iopub.execute_input":"2024-06-18T04:15:21.559399Z","iopub.status.idle":"2024-06-18T04:15:21.567772Z","shell.execute_reply.started":"2024-06-18T04:15:21.559357Z","shell.execute_reply":"2024-06-18T04:15:21.567033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data_for_asset(df_train, asset_id):\n    # Get X and y\n    \n    df = df_train[df_train[\"Asset_ID\"] == asset_id]    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    #df_proc = df_proc[~df_proc.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    X = X.drop('Target', axis=1)\n    y = df_proc[\"y\"]\n    \n    return X, y","metadata":{"execution":{"iopub.status.busy":"2024-06-18T04:15:24.306069Z","iopub.execute_input":"2024-06-18T04:15:24.306807Z","iopub.status.idle":"2024-06-18T04:15:24.312362Z","shell.execute_reply.started":"2024-06-18T04:15:24.306765Z","shell.execute_reply":"2024-06-18T04:15:24.311576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Model training and prediction <a class=\"anchor\" id=\"6\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 1. Скользящее среднее(MV) <a class=\"anchor\" id=\"6.1\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"markdown","source":"Скользящая средняя\nВступление\n‘Средний\" - это, несомненно, одна из самых распространенных вещей, которые мы используем в повседневной жизни. Например, вычисление средних оценок для определения общей производительности или определение средней температуры за последние несколько дней, чтобы получить представление о сегодняшней температуре, – все это рутинные задачи, которые мы выполняем на регулярной основе. Таким образом, это хорошая отправная точка для использования в нашем наборе данных для составления прогнозов.\n\nПрогнозируемая цена закрытия на каждый день будет представлять собой среднее значение из набора ранее наблюдавшихся значений. Вместо использования простого среднего мы будем использовать метод скользящей средней, который использует последний набор значений для каждого прогноза. Другими словами, на каждом последующем шаге учитываются прогнозируемые значения, при этом из набора удаляется самое старое наблюдаемое значение. Вот простой рисунок, который поможет вам понять это с большей ясностью.\n\n\n\nМы применим этот метод к нашему набору данных. Первым шагом является создание фрейма данных, содержащего только столбцы даты и цены закрытия, а затем разделение его на наборы данных для подготовки и проверки для проверки наших прогнозов.","metadata":{}},{"cell_type":"code","source":"def moving_avg_prediction(df, asset_id, train_window, valid_window):\n    shape=df.shape[0]\n#     x, y = get_data_for_asset(df, asset_id)\n    df = df.set_index('timestamp')\n    x, y = get_data_for_asset(df, asset_id)\n    df_new = y #x[['']]\n#     print(df_new.head())\n    train_set = df_new.loc[train_window[0]:train_window[1]]#df_new.iloc[:ceil(shape*0.75)]\n    valid_set = df_new.loc[valid_window[0]:valid_window[1]]#df_new.iloc[ceil(shape*0.75):]\n    print('-----------Prediction by MOVING AVERAGE--------')\n    print('Shape of Training Set',train_set.shape)\n    print('Shape of Validation Set',valid_set.shape)\n    preds = []\n    len_valid_set = valid_set.shape[0]\n    for i in tqdm(range(0, len_valid_set)):\n        a = train_set[len(train_set) - len_valid_set+i:].sum() + sum(preds)\n        b = a / (len_valid_set)\n        preds.append(b)\n    rms=np.sqrt(np.mean(np.power((np.array(valid_set)-preds),2)))\n    print('RMSE value on validation set:',rms)\n    valid_sets = pd.DataFrame({'y': valid_set, 'predictions': preds})\n#     plt.plot(train_set)\n#     plt.plot(valid_sets[['y', 'predictions']])\n#     plt.xlabel('Date',size=20)\n#     plt.ylabel('Stock Price',size=20)\n#     plt.title(f'Asset_id_{asset_id}',size=20)\n#     plt.legend(['Model Training Data','Actual Data','Predicted Data'])\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T04:16:19.274251Z","iopub.execute_input":"2024-06-18T04:16:19.274570Z","iopub.status.idle":"2024-06-18T04:16:19.284231Z","shell.execute_reply.started":"2024-06-18T04:16:19.274540Z","shell.execute_reply":"2024-06-18T04:16:19.283411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_window = [totimestamp(\"01/01/2020\"), totimestamp(\"31/05/2020\")]\n# valid_window = [totimestamp(\"01/06/2020\"), totimestamp(\"01/07/2020\")]\n\ntrain_window = [totimestamp(\"01/01/2020\"), totimestamp(\"21/10/2020\")]\nvalid_window = [totimestamp(\"22/10/2020\"), totimestamp(\"01/01/2021\")]\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    moving_avg_prediction(train_df, asset_id, train_window, valid_window)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T04:17:02.259101Z","iopub.execute_input":"2024-06-18T04:17:02.259685Z","iopub.status.idle":"2024-06-18T08:19:37.197818Z","shell.execute_reply.started":"2024-06-18T04:17:02.259648Z","shell.execute_reply":"2024-06-18T08:19:37.195829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Разделим наш датасет на обучающую и тестовую выборки.\n\n### 2. BaseLine: LinearRegression <a class=\"anchor\" id=\"6.2\"></a>\n\n[Back to Table of Contents](#0.1)\n\nМы попробуем использовать простую модель линейной регрессии для разработанных нами функций.","metadata":{}},{"cell_type":"code","source":"def linear_regression_prediction(df, asset_id, train_window, valid_window):\n    df = df.set_index('timestamp')\n    X, y = get_data_for_asset(df, asset_id)  \n    max_value = np.nanmax(X['VWAP'][X['VWAP'] != np.inf ])\n    #replace inf and -inf in column with max value of column \n    X['VWAP']. replace([np.inf , -np.inf ], max_value, inplace = True)\n\n    X_asset_train = X.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  # filling NaN's with zeros\n    y_asset_train = y.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \n\n    X_asset_test = X.loc[valid_window[0]:valid_window[1]].fillna(0).to_numpy() \n    y_asset_test = y.loc[valid_window[0]:valid_window[1]].fillna(0).to_numpy() \n    \n    # Теперь мы стандартизируем входные данные. \n    # Стандартизация - это процесс приведения различных переменных к одной и той же шкале. \n    # В регрессионном анализе часто важно стандартизировать независимые переменные, иначе вы рискуете получить недостоверные результаты.\n    \n    scaler = StandardScaler()\n    X_asset_train_scaled = scaler.fit_transform(X_asset_train)\n    X_asset_test_scaled = scaler.transform(X_asset_test)\n    \n    # BaseLine model    \n    lr_mod = LinearRegression()\n    lr_mod.fit(X_asset_train_scaled, y_asset_train)\n    separete_lin_model.append(lr_mod)\n    \n    y_pred_lr_asset = lr_mod.predict(X_asset_test_scaled)\n\n    rms=np.sqrt(np.mean(np.power((np.array(y_asset_test) - y_pred_lr_asset), 2)))\n    print('RMSE value on validation set:',rms)\n#     valid_sets = pd.DataFrame({'y': y_asset_test, 'predictions': y_pred_lr_asset})\n#     plt.plot(y_asset_train)\n#     plt.plot(valid_sets[['y', 'predictions']])\n#     plt.xlabel('Date',size=20)\n#     plt.ylabel('Stock Price',size=20)\n#     plt.title(f'Asset_id_{asset_id}',size=20)\n#     plt.legend(['Model Training Data','Actual Data','Predicted Data'])\n#     plt.show()\n    \n    return lr_mod\n#     print(f'Test score for LR baseline: {asset_name}', f\"{np.corrcoef(y_asset_test, y_pred_lr_asset[0, 1])}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-17T07:27:11.115300Z","iopub.execute_input":"2024-06-17T07:27:11.115689Z","iopub.status.idle":"2024-06-17T07:27:11.132668Z","shell.execute_reply.started":"2024-06-17T07:27:11.115650Z","shell.execute_reply":"2024-06-17T07:27:11.131514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nseparete_lin_model = []\n\n# train_window = [totimestamp(\"01/01/2020\"), totimestamp(\"31/05/2020\")]\n# valid_window = [totimestamp(\"01/06/2020\"), totimestamp(\"01/07/2020\")]\n\ntrain_window = [totimestamp(\"01/01/2020\"), totimestamp(\"21/10/2020\")]\nvalid_window = [totimestamp(\"22/10/2020\"), totimestamp(\"01/01/2021\")]\n\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    separete_lin_model.append(linear_regression_prediction(train_df, asset_id, train_window, valid_window))","metadata":{"execution":{"iopub.status.busy":"2024-06-17T07:27:13.991029Z","iopub.execute_input":"2024-06-17T07:27:13.991523Z","iopub.status.idle":"2024-06-17T07:28:11.701697Z","shell.execute_reply.started":"2024-06-17T07:27:13.991466Z","shell.execute_reply":"2024-06-17T07:28:11.700592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_output_output()\nfor i, name in enumerate(asset_details.Asset_Name):\n    save_model(separete_lin_model[i], name)\nsave_zip_file(zip_file_name='lin_model')","metadata":{"execution":{"iopub.status.busy":"2024-06-12T06:35:21.184215Z","iopub.execute_input":"2024-06-12T06:35:21.185352Z","iopub.status.idle":"2024-06-12T06:35:21.229407Z","shell.execute_reply.started":"2024-06-12T06:35:21.185297Z","shell.execute_reply":"2024-06-12T06:35:21.228329Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. kNN <a claas='acnhor' id='6.3'></a>\n\n[Back to Table of Contents](#0.1)\n\nK-Ближайшие соседи\nВступление\nДругим интересным алгоритмом ML, который можно использовать здесь, является kNN (k ближайших соседей). На основе независимых переменных kNN находит сходство между новыми и старыми точками данных.","metadata":{}},{"cell_type":"code","source":"def k_nearest_neighbours_predict(df, asset_id, train_window, valid_window):\n    shape=df.shape[0]\n    df = df.set_index('timestamp')\n    x, y = get_data_for_asset(df, asset_id)\n    df_new = pd.DataFrame({'y': y}) #x[['Close']]\n    train_set = df_new.loc[train_window[0]:train_window[1]]\n    valid_set = df_new.loc[valid_window[0]:valid_window[1]]\n    \n    y_asset_train = y.loc[train_window[0]:train_window[1]].fillna(0).to_numpy() \n    y_asset_test = y.loc[valid_window[0]:valid_window[1]].fillna(0).to_numpy() \n    print('----------- PREDICTION BY K-NEAREST NEIGHBORS-----------')\n    print('Shape of Training Set',train_set.shape)\n    print('Shape of Validation Set',valid_set.shape)\n    train = train_set.reset_index()\n    valid = valid_set.reset_index()\n    \n#     print(train)\n    x_train = pd.to_datetime(train['timestamp'], unit='s').dt.date.map(datetime.toordinal)#x.loc[train_window[0]:train_window[1]]#\n    y_train = y_asset_train\n    x_valid = pd.to_datetime(valid['timestamp'], unit='s').map(datetime.toordinal)#x.loc[valid_window[0]:valid_window[1]]\n    y_valid = y_asset_test\n    \n    scaler = MinMaxScaler(feature_range=(0, 1))    \n    x_train_scaled = scaler.fit_transform(np.array(x_train).reshape(-1, 1))\n    x_train = pd.DataFrame(x_train_scaled)\n    x_valid_scaled = scaler.fit_transform(np.array(x_valid).reshape(-1, 1))\n    x_valid = pd.DataFrame(x_valid_scaled)\n    \n    params = {'n_neighbors': [2,3,4,5,6,7,8,9]}\n    knn = neighbors.KNeighborsRegressor()\n    model = GridSearchCV(knn, params, cv=5)\n    model.fit(x_train,y_train)\n    preds = model.predict(x_valid)\n    rms=np.sqrt(np.mean(np.power((np.array(y_valid) - np.array(preds)), 2)))\n    print('RMSE value on validation set:',rms)\n    \n#     valid_set['Predictions'] = preds\n#     plt.plot(train_set['Close'])\n#     plt.plot(valid_set[['Close', 'Predictions']])\n#     plt.xlabel('Date',size=20)\n#     plt.ylabel('Stock Price',size=20)\n#     plt.title('Stock Price Prediction by K-Nearest Neighbors',size=20)\n#     plt.legend(['Model Training Data','Actual Data','Predicted Data'])","metadata":{"execution":{"iopub.status.busy":"2024-06-18T08:24:48.904863Z","iopub.execute_input":"2024-06-18T08:24:48.905152Z","iopub.status.idle":"2024-06-18T08:24:48.921120Z","shell.execute_reply.started":"2024-06-18T08:24:48.905122Z","shell.execute_reply":"2024-06-18T08:24:48.920092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nknn_model = []\n\ntrain_window = [totimestamp(\"01/01/2020\"), totimestamp(\"21/10/2020\")]\nvalid_window = [totimestamp(\"22/10/2020\"), totimestamp(\"01/01/2021\")]\n\n# train_window = [totimestamp(\"01/01/2018\"), totimestamp(\"21/09/2020\")]\n# valid_window = [totimestamp(\"22/09/2020\"), totimestamp(\"21/09/2021\")]\n\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    knn_model.append(k_nearest_neighbours_predict(train_df, asset_id, train_window, valid_window))","metadata":{"execution":{"iopub.status.busy":"2024-06-18T08:24:50.696286Z","iopub.execute_input":"2024-06-18T08:24:50.697055Z","iopub.status.idle":"2024-06-18T08:37:58.071995Z","shell.execute_reply.started":"2024-06-18T08:24:50.697012Z","shell.execute_reply":"2024-06-18T08:37:58.071109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_output_output()\nfor i, name in enumerate(asset_details.Asset_Name):\n    save_model(separete_lin_model[i], name)\nsave_zip_file(zip_file_name='knn_model')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Prophet <a class=\"anchor\" id=\"6.4\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"(datetime(2020, 9, 21) - datetime(2018, 1, 1)).days, (datetime(2021, 9, 21) - datetime(2020, 9, 22)).days","metadata":{"execution":{"iopub.status.busy":"2024-06-14T18:33:14.891292Z","iopub.execute_input":"2024-06-14T18:33:14.892312Z","iopub.status.idle":"2024-06-14T18:33:14.900840Z","shell.execute_reply.started":"2024-06-14T18:33:14.892261Z","shell.execute_reply":"2024-06-14T18:33:14.899827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom fbprophet import Prophet\n# separete_proph_model = []\n# train_df = train_df.set_index(\"timestamp\")\n\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    x, y = get_data_for_asset(train_df, asset_id)\n    X = pd.DataFrame({\n        'ds': pd.to_datetime(x['timestamp'], unit='s').dt.date,\n        'y': y\n    })\n    X.drop_duplicates(subset=['ds'], inplace=True)\n    model_prophet = Prophet()\n    model_prophet.fit(X[:-364])\n    \n    future = model_prophet.make_future_dataframe(periods=364, freq='D')\n    forecast = model_prophet.predict(future)\n    print('Model trained')\n    separete_proph_model.append(model_prophet)\n    print('Model added in list')\n#     model_prophet.plot_components(forecast)\n    print(f'Test score for Proph model: {asset_name}', f\"{np.corrcoef(y_pred_proph_asset, y_asset_test)[0,1]:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-15T16:10:32.362770Z","iopub.execute_input":"2024-06-15T16:10:32.363134Z","iopub.status.idle":"2024-06-15T16:10:45.700631Z","shell.execute_reply.started":"2024-06-15T16:10:32.363093Z","shell.execute_reply":"2024-06-15T16:10:45.699329Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_output()\nfor i, name in enumerate(asset_details.Asset_Name):\n    save_model(separete_proph_model[i], name)\nsave_zip_file(zip_file_name='proph_model')","metadata":{"execution":{"iopub.status.busy":"2024-06-15T14:37:56.622459Z","iopub.execute_input":"2024-06-15T14:37:56.622761Z","iopub.status.idle":"2024-06-15T14:38:24.206062Z","shell.execute_reply.started":"2024-06-15T14:37:56.622728Z","shell.execute_reply":"2024-06-15T14:38:24.204905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nfor model, asset_id, asset_name in zip(separete_proph_model,asset_details.Asset_ID ,asset_details.Asset_Name):\n    x, y = get_data_for_asset(train_df, asset_id)\n    X = pd.DataFrame({\n        'ds': pd.to_datetime(x['timestamp'], unit='s').dt.date,#pd.to_datetime(train_df['timestamp'], unit='s').dt.date,\n        'y': y\n    })\n    X.drop_duplicates(subset=['ds'], inplace=True)\n#     print(X[:-364].shape)\n#     future = model.make_future_dataframe(X['y'][-364:])\n    forecast = model.predict(X[-364:])\n#     X[-364:]['y']\n#     print(future.shape)\n#     print(f'Test score for Prophet: {asset_name}', f\"{np.c(forecast['yhat'], X['y'][-364:]):.2f}\")\n    print(f'Test score for Prophet: {asset_name}', f\"{np.sqrt(mean_squared_error(X['y'][-364:], forecast['yhat']))}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-15T16:04:03.929529Z","iopub.execute_input":"2024-06-15T16:04:03.929863Z","iopub.status.idle":"2024-06-15T16:05:53.097837Z","shell.execute_reply.started":"2024-06-15T16:04:03.929823Z","shell.execute_reply":"2024-06-15T16:05:53.096590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. ARIMA models <a class=\"anchor\" id=\"6.5\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"def c_time_sub(asset_id, data=train_df):\n    df = data[train_df[\"Asset_ID\"] == asset_id].set_index(\"timestamp\")\n    df = df.reindex(range(df.index[0], df.index[-1]+60,60), method=\"pad\")\n    return df\n\ndict_crypt = {}\nfor i in asset_details.Asset_ID:\n    dict_crypt[i] = c_time_sub(i)\n    \n    \ntotimestamp= lambda s: np.int32(time.mktime(datetime.strptime(s,\"%d/%m/%Y\").timetuple()))\n\n\n# Data Selection\ndef crypto_sub(asset_id ,data = train_df):\n    df = data[data[\"Asset_ID\"] == asset_id].reset_index(drop = True)\n    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s')\n    df = df.set_index('timestamp')\n    return df\n\n\ndict_crypt_2 = {}\nfor i in asset_details.Asset_ID:\n    dict_crypt_2[i] = crypto_sub(i)\n    \n    \n# # Create time interval for 2021\n# def dur(start,end,data):\n#     df=data.loc[totimestamp(start): totimestamp(end)]\n#     return df\n\n# dict_crypt_2021 = {}\n# for i, value in zip(asset_details.Asset_ID, dict_crypt_2.values()):\n#     dict_crypt_2021[i] = dur(start=\"01/01/2021\", end=\"21/09/2021\", data=value)\n\n\n# 4.1 Seasonal Decomposition\n# Function to Subset the Price variables\ndef mini_data(data):\n    df=data[[\"Close\",\"Low\",\"High\",\"Open\"]]\n    return df\n    \ndef season_df(data, label):\n    df=data.resample(\"M\").mean()\n    seasonal_decompose(df.Close).plot()\n    print(label)\n    return plt.show()\n\n\ndict_crypt_mini = {}\nfor i, value in zip(asset_details.Asset_ID, dict_crypt_2.values()):\n    dict_crypt_mini[i] = mini_data(value)\n    \n    \nplt.rcParams[\"figure.figsize\"]=(15,7)\n# нужно проверить, что id соответсвует нужной валюте\nfor i, name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    season_df(data=dict_crypt_mini[i], label=f\"{name} Seasonal Decomposition\")\n    \n    \n# 4.2 Box - Cox Transformation\ndict_crypt_month = {}\nlambda_dict = {}\nfor i, value in zip(asset_details.Asset_ID, dict_crypt_mini.values()):\n    dict_crypt_month[i] = value.resample('M').mean()\n    dict_crypt_month[i]['close_box'], lambda_dict[i] = stats.boxcox(dict_crypt_month[i].Close)\n    \n\nfrom numpy.linalg import LinAlgError\nqs = range(0, 5)\nps = range(0, 5)\nd = 1\nparameters = product(ps, qs)\nparameters_list = list(parameters)\nlen(parameters_list)\nbest_model_arr = []\n\nfor value in dict_crypt_month.values():\n    # Model Selection\n    results = []\n    best_aic = float(\"inf\")\n    warnings.filterwarnings('ignore')\n    \n    for param in parameters_list:\n        try:\n            model = SARIMAX(value.close_box, order=(param[0], d, param[1])).fit(disp=-1)\n        except (ValueError, LinAlgError):\n            print('bad parameter combination:', param)\n            continue\n        aic = model.aic\n        if aic < best_aic:\n            best_model = model\n            best_aic = aic\n            best_param = param\n        results.append([param, model.aic])\n        \n    result_table = pd.DataFrame(results)\n    result_table.columns = ['parameters', 'aic']\n    print(result_table.sort_values(by = 'aic', ascending=True).head())\n    print(best_model.summary())\n    best_model_arr.append(best_model)\n    \n    best_model.plot_diagnostics(figsize=(15, 12))\n    plt.show()\n    \n    \ndef invboxcox(y,lmbda):\n    if lmbda == 0:\n        return(np.exp(y))\n    else:\n        return(np.exp(np.log(lmbda*y+1)/lmbda))\n    \n    \n\n# Prediction\nfor value, name, best_model, lamda in zip(dict_crypt_month.values(), asset_details.Asset_Name, best_model_arr, lambda_dict):\n#     value =  dict_crypt_month[2]\n#     name = asset_details[asset_details.Asset_ID == 2]['Asset_Name']\n#     tmp = tmp_val.reset_index()['timestamp'].apply(lambda d: d.date())\n    pred = value[['Close']]\n    date_list = [datetime(2021, 6, 20), datetime(2021, 7, 20), datetime(2021, 8, 21), datetime(2021, 9, 21)]\n    future = pd.DataFrame(index=date_list, columns= value.columns)\n    pred = pd.concat([pred, future])\n    \n    pred['forecast'] = invboxcox(best_model.predict(start=datetime(2020, 1, 31), \n                                                    end=datetime(2021, 12, 31)), lamda)\n    plt.figure(figsize=(18,10))\n        \n    pred.Close.plot(color='#fede86')\n    pred.forecast.plot(color='#b03f03', ls='--', label='Predicted Close')\n    plt.legend()\n    plt.title(f'{name} monthly forecast')\n    plt.ylabel('USD')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6. AutoARIMA <a class='anchor' id='6.6'></a>\n\n[Back to Table of Contents](#0.1)\n\n\nВступление¶\nARIMA - очень популярный статистический метод прогнозирования временных рядов. Модели ARIMA учитывают прошлые значения для прогнозирования будущих значений. В ARIMA есть три важных параметра:\n\np (прошлые значения, используемые для прогнозирования следующего значения)\nq (ошибки прошлых прогнозов, используемые для прогнозирования будущих значений)\nd (порядок расхождений)\n\n\nНастройка параметров для ARIMA занимает много времени. Поэтому мы будем использовать auto ARIMA, который автоматически выбирает наилучшую комбинацию (p,q,d), которая обеспечивает наименьшую ошибку. Чтобы узнать больше о том, как работает auto ARIMA, обратитесь к этой статье: https://alkaline-ml.com/pmdarima/modules/generated/pmdarima.arima.auto_arima.html","metadata":{}},{"cell_type":"code","source":"def auto_arima_prediction(df, asset_id, train_window, valid_window):\n    shape=df.shape[0]\n    df = df.set_index('timestamp')\n    x, y = get_data_for_asset(df, asset_id)\n    df_new = df.sort_index(ascending=True, axis=0)\n#     df_new = pd.DataFrame({'y': y}) #x[['Close']]\n    train_set = df_new.loc[train_window[0]:train_window[1]]\n    valid_set = df_new.loc[valid_window[0]:valid_window[1]]\n    y_asset_train = y.loc[train_window[0]:train_window[1]].fillna(0).to_numpy() \n    y_asset_test = y.loc[valid_window[0]:valid_window[1]].fillna(0).to_numpy() \n    print('----------------------------------------------')\n    print('-----------PREDICTION BY AUTO ARIMA-----------')\n    print('----------------------------------------------')\n    print('Shape of Training Set',train_set.shape)\n    print('Shape of Validation Set',valid_set.shape)\n    training = y_asset_train#train_set['Close']\n    validation = y_asset_test#valid_set['Close']\n    model = pm.auto_arima(training, start_p=1, start_q=1,max_p=3, max_q=3, m=12,start_P=0, seasonal=True,d=1, D=1, trace=True,error_action='ignore',suppress_warnings=True)\n    model.fit(training)\n    forecast = model.predict(n_periods=ceil(floor(df.shape[0]*0.25)))\n    forecast = pd.DataFrame(forecast,index = valid_set.index,columns=['Prediction'])\n    rms=np.sqrt(np.mean(np.power((np.array(valid_set['Close'])-np.array(forecast['Prediction'])),2)))\n    print('RMSE value on validation set:',rms)\n    \n#     visualitsate result\n#     plt.plot(train_set['Close'])\n#     plt.plot(valid_set['Close'])\n#     plt.plot(forecast['Prediction'])\n#     plt.xlabel('Date',size=20)\n#     plt.ylabel('Stock Price',size=20)\n#     plt.title('Stock Price Prediction by Auto ARIMA',size=20)\n#     plt.legend(['Model Training Data','Actual Data','Predicted Data'])","metadata":{"execution":{"iopub.status.busy":"2024-06-17T16:29:03.780276Z","iopub.execute_input":"2024-06-17T16:29:03.781398Z","iopub.status.idle":"2024-06-17T16:29:03.796955Z","shell.execute_reply.started":"2024-06-17T16:29:03.781341Z","shell.execute_reply":"2024-06-17T16:29:03.795576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nauto_arima_model = []\n\ntrain_window = [totimestamp(\"01/01/2020\"), totimestamp(\"31/05/2020\")]\nvalid_window = [totimestamp(\"01/06/2020\"), totimestamp(\"01/07/2020\")]\n\n# train_window = [totimestamp(\"01/01/2018\"), totimestamp(\"21/09/2020\")]\n# valid_window = [totimestamp(\"22/09/2020\"), totimestamp(\"21/09/2021\")]\n\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    auto_arima_model.append(auto_arima_prediction(train_df, asset_id, train_window, valid_window))","metadata":{"execution":{"iopub.status.busy":"2024-06-17T16:29:06.244631Z","iopub.execute_input":"2024-06-17T16:29:06.245022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LGBM решение <a class=\"anchor\" id=\"6.7\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"%%time\n\ndf_train = pd.read_csv(data_folder + \"train.csv\", usecols=['timestamp','Asset_ID', 'Close', 'Target'])\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(data_folder + 'supplemental_train.csv', usecols=['timestamp','Asset_ID', 'Close', 'Target'])\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\n    \ndf_train = reduce_memory_usage(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:02:34.320446Z","iopub.execute_input":"2024-06-18T10:02:34.320708Z","iopub.status.idle":"2024-06-18T10:03:31.941355Z","shell.execute_reply.started":"2024-06-18T10:02:34.320680Z","shell.execute_reply":"2024-06-18T10:03:31.940589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm(range(14)):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:03:38.484560Z","iopub.execute_input":"2024-06-18T10:03:38.485174Z","iopub.status.idle":"2024-06-18T10:03:55.274602Z","shell.execute_reply.started":"2024-06-18T10:03:38.485138Z","shell.execute_reply":"2024-06-18T10:03:55.273747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for id in range(14):\n    train_merged[f'Close_{id}'] = train_merged[f'Close_{id}'].fillna(method='ffill', limit=100)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:04:03.061598Z","iopub.execute_input":"2024-06-18T10:04:03.062378Z","iopub.status.idle":"2024-06-18T10:04:03.156767Z","shell.execute_reply.started":"2024-06-18T10:04:03.062336Z","shell.execute_reply":"2024-06-18T10:04:03.155877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        if use_supple_for_train:\n            df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n\n   \n    for id in range(14):    \n        for lag in lags:\n            df[f'log_close/mean_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n            df[f'log_return_{lag}_id{id}']     = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n    for lag in lags:\n        df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n        df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n        for id in range(14):\n            df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n            df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    if train == True:\n        for id in range(14):\n            df = df.drop([f'Close_{id}'], axis=1)\n        oldest_use_window = [totimestamp(\"12/01/2019\")]\n        df = df[  df['timestamp'] >= oldest_use_window[0]   ]\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:04:06.757239Z","iopub.execute_input":"2024-06-18T10:04:06.757538Z","iopub.status.idle":"2024-06-18T10:04:06.772102Z","shell.execute_reply.started":"2024-06-18T10:04:06.757502Z","shell.execute_reply":"2024-06-18T10:04:06.771296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [60,300,900]\nn_fold = 4\nseed0 = 42\nnot_use_overlap_to_train = False\nparams = {\n    'early_stopping_rounds': 50,\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'min_data_in_leaf':50,\n    'learning_rate': 0.03,\n    'subsample': 0.7,\n    'subsample_freq': 1,\n    'feature_fraction': 1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2,\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_fraction_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'extra_trees': True,\n    'extra_seed': seed0,\n    'zero_as_missing': True,\n    \"first_metric_only\": True,\n    'device': 'gpu',\n    'max_bin': 255\n         }\n#     'device': 'gpu',","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:04:10.243603Z","iopub.execute_input":"2024-06-18T10:04:10.244424Z","iopub.status.idle":"2024-06-18T10:04:10.251194Z","shell.execute_reply.started":"2024-06-18T10:04:10.244374Z","shell.execute_reply":"2024-06-18T10:04:10.250430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:04:21.595277Z","iopub.execute_input":"2024-06-18T10:04:21.596098Z","iopub.status.idle":"2024-06-18T10:04:47.861351Z","shell.execute_reply.started":"2024-06-18T10:04:21.596060Z","shell.execute_reply":"2024-06-18T10:04:47.860716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_use_features_train = ['timestamp', 'train_flg']\nfor id in range(14):\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:04:57.863864Z","iopub.execute_input":"2024-06-18T10:04:57.864472Z","iopub.status.idle":"2024-06-18T10:04:57.871578Z","shell.execute_reply.started":"2024-06-18T10:04:57.864431Z","shell.execute_reply":"2024-06-18T10:04:57.870788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:05:02.906803Z","iopub.execute_input":"2024-06-18T10:05:02.907527Z","iopub.status.idle":"2024-06-18T10:05:03.123692Z","shell.execute_reply.started":"2024-06-18T10:05:02.907487Z","shell.execute_reply":"2024-06-18T10:05:03.122897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train model","metadata":{}},{"cell_type":"code","source":"def correlation(a, train_data):\n    \"\"\"Return correlation between a and train_data\"\"\"\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    \"\"\"Return correlation between prediction and validation data\"\"\"\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr\n\n# For CV score calculation\ndef wcorr_score(pred, valid, weight):\n    \"\"\"Return weighted correlation between prediction and validaion data, use weight\"\"\"\n    len_data = len(pred)\n    sum_w = np.sum(weight)\n    mean_pred = np.sum(pred * weight) / sum_w\n    mean_valid = np.sum(valid * weight) / sum_w\n    var_pred = np.sum(weight * np.square(pred - mean_pred)) / sum_w\n    var_valid = np.sum(weight * np.square(valid - mean_valid)) / sum_w\n\n    cov = np.sum((pred * valid * weight)) / sum_w - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:05:06.542414Z","iopub.execute_input":"2024-06-18T10:05:06.543146Z","iopub.status.idle":"2024-06-18T10:05:06.555619Z","shell.execute_reply.started":"2024-06-18T10:05:06.543104Z","shell.execute_reply":"2024-06-18T10:05:06.554793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# # (used in nyanp's Optiver solution)\n# def plot_importance(importances, features_names = features, PLOT_TOP_N = 20, figsize=(10, 10)):\n#     \"\"\"Plotting the importance of features\"\"\"\n#     importance_df = pd.DataFrame(data=importances, columns=features)\n#     sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n#     sorted_importance_df = importance_df.loc[:, sorted_indices]\n#     plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n#     _, ax = plt.subplots(figsize=figsize)\n#     ax.grid()\n#     ax.set_xscale('log')\n#     ax.set_ylabel('Feature')\n#     ax.set_xlabel('Importance')\n#     sns.boxplot(data=sorted_importance_df[plot_cols],\n#                 orient='h',\n#                 ax=ax)\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-17T14:02:23.954008Z","iopub.execute_input":"2024-06-17T14:02:23.954350Z","iopub.status.idle":"2024-06-17T14:02:23.963834Z","shell.execute_reply.started":"2024-06-17T14:02:23.954308Z","shell.execute_reply":"2024-06-17T14:02:23.962695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\ndef get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3750):\n    \"\"\"This function split data on test and train data\"\"\"\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem > 0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        test_split_max = int(np.max(test_split))\n        test_split_min = int(np.min(test_split))\n        # get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        embargo_sec = 60*embargo\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    # convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:05:10.338815Z","iopub.execute_input":"2024-06-18T10:05:10.339100Z","iopub.status.idle":"2024-06-18T10:05:10.349268Z","shell.execute_reply.started":"2024-06-18T10:05:10.339067Z","shell.execute_reply":"2024-06-18T10:05:10.348385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    \"\"\"Train model\"\"\"\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in tqdm(enumerate(train_test_zip)):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n    \n        train_dataset = lgb.Dataset(df_proc.loc[train_split_index, features],\n                                    df_proc.loc[train_split_index, f'Target_{asset_id}'].values, \n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(df_proc.loc[test_split_index, features], \n                                  df_proc.loc[test_split_index, f'Target_{asset_id}'].values, \n                                  feature_name = features, \n                                 )\n\n        print(f\"number of train data: {len(df_proc.loc[train_split_index])}\")\n        print(f\"number of val data:   {len(df_proc.loc[test_split_index])}\")\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 5000,\n                          verbose_eval = 100,     \n                          feval = correlation,\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n        \n        file = f'trained_model_id{asset_id}_fold{split}.pkl'\n        pickle.dump(model, open(file, 'wb'))\n        print(f\"Trained model was saved to 'trained_model_id{asset_id}_fold{split}.pkl'\")\n        print(\"\")\n            \n        oof_pred += list(model.predict(df_proc.loc[test_split_index, features]))\n        oof_valid += list(df_proc.loc[test_split_index, f'Target_{asset_id}'].values)\n    \n    \n#     plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5))\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:07:24.613251Z","iopub.execute_input":"2024-06-18T10:07:24.613544Z","iopub.status.idle":"2024-06-18T10:07:24.627206Z","shell.execute_reply.started":"2024-06-18T10:07:24.613510Z","shell.execute_reply":"2024-06-18T10:07:24.626467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = [[] for id in range(14)]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\n\nfor asset_id, asset_name in zip(asset_details['Asset_ID'], asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    weight_temp = float(asset_details.loc[asset_details['Asset_ID'] == asset_id  , 'Weight']  )\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = corr_score(np.array(oof_pred), np.array(oof_valid))\n    \n    print(f'Score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')","metadata":{"execution":{"iopub.status.busy":"2024-06-18T10:07:26.879847Z","iopub.execute_input":"2024-06-18T10:07:26.880135Z","iopub.status.idle":"2024-06-18T10:38:41.065731Z","shell.execute_reply.started":"2024-06-18T10:07:26.880100Z","shell.execute_reply":"2024-06-18T10:38:41.064921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference model","metadata":{}},{"cell_type":"code","source":"models = [[[] for split in range(sum(n_fold_list))] for asset_id in range(14)]\n\nfor asset_id in range(14):\n    for input_number in range(len(lgb_list)):\n        for split in range(n_fold_list[input_number]):\n            models[asset_id][sum(n_fold_list[:input_number])+split] = pickle.load(open(f'{lgb_list[input_number]}/trained_model_id{asset_id}_fold{split}.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:56:31.469735Z","iopub.execute_input":"2024-06-18T09:56:31.470095Z","iopub.status.idle":"2024-06-18T09:56:32.046635Z","shell.execute_reply.started":"2024-06-18T09:56:31.470053Z","shell.execute_reply":"2024-06-18T09:56:32.045846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model in models:\n    print(models[7])","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:57:55.570260Z","iopub.execute_input":"2024-06-18T09:57:55.570998Z","iopub.status.idle":"2024-06-18T09:57:55.579279Z","shell.execute_reply.started":"2024-06-18T09:57:55.570948Z","shell.execute_reply":"2024-06-18T09:57:55.578193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# generate feature column names\ndf_train = pd.read_csv(data_folder + 'train.csv', usecols=['timestamp','Asset_ID', 'Close', 'Target'], nrows=max(lags)*20)\nprint(len(df_train['Asset_ID'].unique()))\ndf_train = reduce_memory_usage(df_train)\n# df_train\n\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm( range(14) ):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ndisplay(train_merged.head())\n\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in range(14):\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = get_features(train_merged, train=True).columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:55:42.688154Z","iopub.execute_input":"2024-06-18T09:55:42.688544Z","iopub.status.idle":"2024-06-18T09:55:43.034825Z","shell.execute_reply.started":"2024-06-18T09:55:42.688502Z","shell.execute_reply":"2024-06-18T09:55:43.033734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:55:52.904963Z","iopub.execute_input":"2024-06-18T09:55:52.905326Z","iopub.status.idle":"2024-06-18T09:55:53.152931Z","shell.execute_reply.started":"2024-06-18T09:55:52.905283Z","shell.execute_reply":"2024-06-18T09:55:53.151862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define max_lookback - an integer > (greater than) the furthest look back in your lagged features\nkeep_hist = max(lags)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:55:55.454225Z","iopub.execute_input":"2024-06-18T09:55:55.454560Z","iopub.status.idle":"2024-06-18T09:55:55.459309Z","shell.execute_reply.started":"2024-06-18T09:55:55.454520Z","shell.execute_reply":"2024-06-18T09:55:55.458229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_for_infer(df):\n    df_merged = pd.DataFrame()\n    df_merged[['timestamp', 'Asset_ID', 'Close']] = 0\n    for id in range(14):\n        df_merged = df_merged.merge(df.loc[df[\"Asset_ID\"] == id, ['timestamp', 'Close']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n \n    df_merged = df_merged.drop(['Asset_ID', 'Close'], axis=1)\n    return df_merged","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:55:57.920984Z","iopub.execute_input":"2024-06-18T09:55:57.921346Z","iopub.status.idle":"2024-06-18T09:55:57.930463Z","shell.execute_reply.started":"2024-06-18T09:55:57.921308Z","shell.execute_reply":"2024-06-18T09:55:57.929244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataframe to store data from the api to create lagged features\nhistory = pd.read_csv(data_folder + 'train.csv', usecols=['timestamp','Asset_ID', 'Close'])\nhistory = history.tail(keep_hist*14)\n\nhistory = reduce_memory_usage(history)\nhistory_merged = merge_for_infer(history)\nhistory_merged = history_merged.tail(keep_hist)\nhistory_merged","metadata":{"execution":{"iopub.status.busy":"2024-06-18T09:56:00.246691Z","iopub.execute_input":"2024-06-18T09:56:00.247069Z","iopub.status.idle":"2024-06-18T09:56:22.029419Z","shell.execute_reply.started":"2024-06-18T09:56:00.247029Z","shell.execute_reply":"2024-06-18T09:56:22.027910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_line = history_merged.tail(1).copy()\n\ndef merge_infer_2(df, df_one_line):\n    for asset_id, close in zip(   df['Asset_ID'].values,  df['Close'].values   ): \n        df_one_line[f'Close_{asset_id}'] = close\n    return df_one_line","metadata":{"execution":{"iopub.status.busy":"2024-05-18T04:49:47.099898Z","iopub.execute_input":"2024-05-18T04:49:47.100543Z","iopub.status.idle":"2024-05-18T04:49:47.107814Z","shell.execute_reply.started":"2024-05-18T04:49:47.100466Z","shell.execute_reply":"2024-05-18T04:49:47.106597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" ### 8. LSTM<a class=\"anchor\" id=\"6.8\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"df_train = train_df\ndf_asset_details = asset_details","metadata":{"execution":{"iopub.status.busy":"2024-06-17T16:55:48.314022Z","iopub.execute_input":"2024-06-17T16:55:48.314731Z","iopub.status.idle":"2024-06-17T16:55:48.318490Z","shell.execute_reply.started":"2024-06-17T16:55:48.314697Z","shell.execute_reply":"2024-06-17T16:55:48.317593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train_df.reset_index(drop = True) \ntemp['TIME'] = pd.to_datetime(temp['timestamp'], unit='s')","metadata":{"execution":{"iopub.status.busy":"2024-06-17T14:15:14.253719Z","iopub.execute_input":"2024-06-17T14:15:14.254034Z","iopub.status.idle":"2024-06-17T14:15:17.829819Z","shell.execute_reply.started":"2024-06-17T14:15:14.253998Z","shell.execute_reply":"2024-06-17T14:15:17.828836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp['Year'] = temp['TIME'].dt.year\ntemp.columns =[column.replace(\" \", \"_\") for column in temp.columns]\ntemp.dropna()[['Open', 'High', 'Low', 'Close', 'Volume','VWAP']]\nvolume_yr = temp.groupby(['Year','Asset_name'])['Volume'].max().reset_index()\nvolume_yr = volume_yr.query(\"Year == 2021\")","metadata":{"execution":{"iopub.status.busy":"2024-06-17T14:15:56.551827Z","iopub.execute_input":"2024-06-17T14:15:56.552076Z","iopub.status.idle":"2024-06-17T14:16:02.921019Z","shell.execute_reply.started":"2024-06-17T14:15:56.552045Z","shell.execute_reply":"2024-06-17T14:16:02.919294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.dpi'] = 300\nfig = plt.figure(figsize=(3, 5), facecolor='#F7F7F7')\ngs = fig.add_gridspec(2, 1)\ngs.update(wspace=1.5, hspace=1.1)\n\nbackground_color = \"#F7F7F7\"\ncolor_map = [\"#E8F0F2\" for _ in range(20)]\ncolor_map[4] = \"#C9CCD5\" #F2A154 79B4B7\ncolor_map[1] = \"#F2A154\"\ncolor_map[6] = \"#F2A154\"\nsns.set_palette(sns.color_palette(color_map))\n\nax0 = fig.add_subplot(gs[0, 0])\nax0.set_facecolor(background_color)\n\nfor s in [\"right\", \"top\"]:\n    ax0.spines[s].set_visible(False)\n\nfor s in ['bottom','left']:\n  ax0.spines[s].set_linewidth(0.5)\n  ax0.spines[s].set_color('grey')\n    \n#graph\nax0 = sns.barplot(ax=ax0, y=volume_yr['Volume']/1000, x=volume_yr['Asset_name'], \n                      zorder=2, linewidth=0.3, edgecolor=\"#7F7C82\", \n                      orient='v', saturation=0.9, alpha=0.7)\nax0.grid(which='major', axis='y', zorder=0, color='#CDD0CB', linewidth=0.2, alpha=0.5)\nax0.set_yscale('symlog')\n\n#format axis\nax0.set_ylabel(\"Volume\",fontsize=3, weight='semibold')\nax0.set_xlabel(\"Crypto Currency\",fontsize=3, weight='semibold')\nax0.tick_params(labelsize=3, width=0.2, length=1)\nplt.setp( ax0.xaxis.get_majorticklabels(), rotation=45, ha=\"right\" )\n\n#title\nx0, x1 = ax0.get_xlim()\ny0, y1 = ax0.get_ylim()\nax0.text(x0, y1+2000000, 'Volume by Year', fontsize=4, ha='left', va='top', weight='semibold')\n\ny_format = ticker.FuncFormatter(lambda x, p: format(int(x), ','))\nax0.yaxis.set_major_formatter(y_format)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-17T14:16:02.921896Z","iopub.status.idle":"2024-06-17T14:16:02.922285Z","shell.execute_reply.started":"2024-06-17T14:16:02.922066Z","shell.execute_reply":"2024-06-17T14:16:02.922095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### LSTM arhitecture","metadata":{}},{"cell_type":"markdown","source":"Долговременная кратковременная память (LSTM)\nВступление\nLSTM широко используются для решения задач предсказания последовательности и доказали свою исключительную эффективность. Причина, по которой они так хорошо работают, заключается в том, что LSTM способен сохранять прошлую информацию, которая важна, и забывать информацию, которая не важна. LSTM имеет три входа:\n\n\n\nЭлемент ввода: Входные данные добавляют информацию к состоянию ячейки Элемент забывания: удаляет информацию, которая больше не требуется модели Элемент вывода: Элемент вывода в LSTM выбирает информацию, которая будет отображаться в качестве выходных данных Для более детального понимания LSTM и его архитектуры, вы можете ознакомиться с приведенная ниже статья: https://colah.github.io/posts/2015-08-Understanding-LSTMs/\n\nВведение в долговременную кратковременную память\nА пока давайте реализуем LSTM в виде черного ящика и проверим его работоспособность на наших конкретных данных.","metadata":{}},{"cell_type":"code","source":"def lstm_prediction(df, asset_id, train_window, valid_window):\n    start = time.time()\n    shape=df.shape[0]\n    \n    df = df.set_index('timestamp')\n    x, y = get_data_for_asset(df, asset_id)\n    df_new = x[['Close']]\n    dataset = df_new.values\n    train = df_new.loc[train_window[0]:train_window[1]]#df_new.iloc[:ceil(shape*0.75)]\n    valid = df_new.loc[valid_window[0]:valid_window[1]]#df_new.iloc[ceil(shape*0.75):]\n\n\n    print('-----------LSTM prediction-----------')\n    print('Shape of Training Set',train.shape)\n    print('Shape of Validation Set',valid.shape)\n    \n    scaler = MinMaxScaler(feature_range=(0, 1))\n    scaled_data = scaler.fit_transform(dataset)\n    \n    x_train, y_train = [], []\n    for i in tqdm(range(40,len(train))):\n        x_train.append(scaled_data[i-40:i,0])\n        y_train.append(scaled_data[i,0])\n        \n    x_train, y_train = np.array(x_train), np.array(y_train)\n    x_train = np.reshape(x_train, (x_train.shape[0],x_train.shape[1],1))\n    \n    #keras nn\n    model = Sequential()\n    model.add(LSTM(units=50, return_sequences=True, input_shape=(x_train.shape[1],1)))\n    model.add(LSTM(units=50))\n#     model.add(Dense(25)) #optional\n    model.add(Dense(1))\n    model.compile(loss='mean_squared_error', optimizer='adam')\n    model.fit(x_train, y_train, epochs=1, batch_size=1, verbose=2)\n    \n    end = time.time()\n        \n    inputs = df_new[len(df_new) - len(valid) - 40:].values\n    inputs = inputs.reshape(-1,1)\n    inputs  = scaler.transform(inputs)\n    X_test = []\n    for i in range(40,inputs.shape[0]):\n        X_test.append(inputs[i-40:i,0])\n        \n    X_test = np.array(X_test)\n    X_test = np.reshape(X_test, (X_test.shape[0],X_test.shape[1],1))\n    closing_price = model.predict(X_test)\n    closing_price = scaler.inverse_transform(closing_price)\n    rms=np.sqrt(np.mean(np.power((valid-closing_price),2)))\n    print('RMSE value on validation set:',rms)\n    print(f'Times work {end - start}')\n#     valid['Predictions'] = closing_price\n#     plt.plot(train['Close'])\n#     plt.plot(valid[['Close','Predictions']])\n#     plt.xlabel('Date',size=20)\n#     plt.ylabel('Stock Price',size=20)\n#     plt.title('Stock Price Prediction by Long Short Term Memory (LSTM)',size=20)\n#     plt.legend(['Model Training Data','Actual Data','Predicted Data'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-06-17T16:55:54.426388Z","iopub.execute_input":"2024-06-17T16:55:54.427162Z","iopub.status.idle":"2024-06-17T16:55:54.446001Z","shell.execute_reply.started":"2024-06-17T16:55:54.427125Z","shell.execute_reply":"2024-06-17T16:55:54.445065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_window = [totimestamp(\"01/01/2020\"), totimestamp(\"25/01/2020\")]\n# valid_window = [totimestamp(\"26/01/2020\"), totimestamp(\"01/02/2020\")]\n\ntrain_window = [totimestamp(\"01/01/2020\"), totimestamp(\"21/10/2020\")]\nvalid_window = [totimestamp(\"22/10/2020\"), totimestamp(\"01/01/2021\")]\n\nlstm_models = []\n\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    print(f'Train_model_for_{asset_name}')\n    lstm_models.append(lstm_prediction(train_df, asset_id, train_window, valid_window))","metadata":{"execution":{"iopub.status.busy":"2024-06-17T17:31:39.532653Z","iopub.execute_input":"2024-06-17T17:31:39.533384Z","iopub.status.idle":"2024-06-17T23:56:27.448122Z","shell.execute_reply.started":"2024-06-17T17:31:39.533344Z","shell.execute_reply":"2024-06-17T23:56:27.447391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_output()\nfor i, name in enumerate(asset_details.Asset_Name):\n    save_model(lstm_models[i], name)\nsave_zip_file()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T03:52:35.458446Z","iopub.execute_input":"2024-06-18T03:52:35.459282Z","iopub.status.idle":"2024-06-18T03:52:35.551283Z","shell.execute_reply.started":"2024-06-18T03:52:35.459172Z","shell.execute_reply":"2024-06-18T03:52:35.550312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tests <a class=\"anchor\" id=\"7\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"markdown","source":"Еще один вариант предсказания","metadata":{}},{"cell_type":"code","source":"api = API(train_df, use_window=LB_WINDOW)\n\nstart_time = time.time()\n\nfor i, (df_test, df_pred) in enumerate(tqdm(api)):\n    df_test_merged = merge_infer_2(df_test, one_line)    \n    history_merged = pd.concat([history_merged, df_test_merged])    \n    x_test = get_features(history_merged, False)\n    x_calc = x_test.iloc[-1]\n    \n    for j , (asset_id,row_id) in enumerate(zip(df_test['Asset_ID'].values,  df_test['row_id'].values)): \n        y_pred = 0       \n        y_pred_list = []\n        for split in range(n_fold):\n            y_pred_list.append(models[asset_id][split].predict(x_calc[features]))\n        y_pred = np.median(y_pred_list)     \n        df_pred.loc[df_pred['row_id'] == row_id,'Target'] = y_pred\n\n    history_merged = history_merged.tail(keep_hist)\n    \n    # Send submissions\n    api.predict(df_pred)    \nstop = time.time()\nprint(stop-start)\n\nfinish_time = time.time()\n\ntotal_time = finish_time - start_time\niter_speed = api.init_num_times/total_time\n\nprint(f\"    num_fold = {n_fold}\")\nprint(f\"Iterations/s = {round(iter_speed, 3)}\")\nprint(f\"s/Iterations = {round(1/iter_speed, 3)}\")\ntest_iters = 60 * 24 * 100\nprint(f\"Expected number of iterations in test set is approx. {test_iters}\",\n      f\"which will take {round(test_iters / (iter_speed * 3600), 2)} hours\",\n      \"using this API emulator while making dummy predictions.\")\n\ndf, score = api.score()\nprint(f\"Your LB score is {round(score, 5)}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction and submission\n# not lgbm test\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j, row in df_test.iterrows():\n        \n        try:\n            model = models[row['Asset_ID']]\n            x_test = get_features(row)\n            y_pred = model.predict([x_test])[0]\n\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        except:\n            print(f'{i}-th iteration of the test dataset, {j}-th row - there was the exception, then set Target = 0')\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n            \n        # Print just one sample row to get a feeling of what it looks like        \n        if i == 0 and j == 0:\n            print('Example of the x_test data')\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        print('Example of the prediction for test data')\n        display(df_pred)\n    df_pred['Target'] = df_pred['Target'].fillna(0)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\nfor (test_df, sample_prediction_df) in iter_test:\n    # feature engineering\n    test_df = get_row_feats(test_df)\n    \n    # inference\n    sample_prediction_df['Target'] = model.predict(test_df[features])  # make your predictions here\n    \n    # register your predictions\n    env.predict(sample_prediction_df)  ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"api = API(train_df, use_window=LB_WINDOW)\n# Пока тут только линейная модель\nmodels = separete_lin_model\nprediction = []\n\nstart_time = time.time()\n\nfor (df_test, df_pred) in tqdm(api):\n    test_df = get_features(df_test)\n    test_df.drop(['Asset name', 'timestamp'], axis=1, inplace=True)\n    \n    tmp = []\n    for i in range(len(models)):\n        tmp.append(models[i].predict(test_df))  # make your predictions here\n    df_pred['Target'] = np.mean(tmp)\n    # register your predictions\n    api.predict(df_pred) \n    \nstop = time.time()\nprint(stop - start)\n\nfinish_time = time.time()\n\ntotal_time = finish_time - start_time\niter_speed = api.init_num_times/total_time\n\nprint(f\"Iterations/s = {round(iter_speed, 3)}\")\nprint(f\"s/Iterations = {round(1/iter_speed, 3)}\")\ntest_iters = 60 * 24 * 100\nprint(f\"Expected number of iterations in test set is approx. {test_iters}\",\n      f\"which will take {round(test_iters / (iter_speed * 3600), 2)} hours\",\n      \"using this API emulator while making dummy predictions.\")\n\ndf, score = api.score()\nprint(f\"Your LB score is {round(score, 5)}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[Go to Top](#0)","metadata":{}}]}