{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport mxnet as mx\nfrom mxnet import gluon\n\n# !pip install gluonts\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-08T21:36:08.420058Z","iopub.execute_input":"2021-12-08T21:36:08.420685Z","iopub.status.idle":"2021-12-08T21:36:11.490678Z","shell.execute_reply.started":"2021-12-08T21:36:08.420564Z","shell.execute_reply":"2021-12-08T21:36:11.489859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**gluon in kaggle:**  \nhttps://www.kaggle.com/steverab/m5-forecasting-competition-gluonts-template  \nhttps://www.kaggle.com/aldente0630/demand-forecasting-with-deepar-and-deepvar\n\n**gluon multiforecasting::**  \nhttps://github.com/awslabs/gluon-ts/issues/1095  \nhttps://githubmemory.com/repo/awslabs/gluon-ts/issues/1095   \n\n**crypto forecasting:**   \nhttps://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition/notebook  \nhttps://github.com/pedrolarben/TimeSeriesForecasting-DeepLearning  \nhttps://www.kaggle.com/negstek/to-the-moon-g-research-crypto-forecasting-eda/edit  \nhttps://www.kaggle.com/yamqwe/crypto-prediction-catboost-regressor\n\n**lint et méthodo:**  \nhttps://blog.reviewnb.com/development-with-jupyter/   \nhttps://towardsdatascience.com/4-tools-for-reproducible-jupyter-notebooks-d7423721bd04  \n\nThis first test is based on [this notebook](https://www.kaggle.com/yamqwe/crypto-prediction-catboost-regressor/notebook)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gresearch_crypto \nfrom catboost import Pool, CatBoostClassifier, cv, CatBoostRegressor","metadata":{"execution":{"iopub.status.busy":"2021-12-08T21:36:11.492003Z","iopub.execute_input":"2021-12-08T21:36:11.492551Z","iopub.status.idle":"2021-12-08T21:36:11.764101Z","shell.execute_reply.started":"2021-12-08T21:36:11.492518Z","shell.execute_reply":"2021-12-08T21:36:11.763233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-12-08T21:36:11.765459Z","iopub.execute_input":"2021-12-08T21:36:11.766182Z","iopub.status.idle":"2021-12-08T21:36:11.769904Z","shell.execute_reply.started":"2021-12-08T21:36:11.766138Z","shell.execute_reply":"2021-12-08T21:36:11.769239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-08T21:36:11.771254Z","iopub.execute_input":"2021-12-08T21:36:11.771954Z","iopub.status.idle":"2021-12-08T21:37:04.747774Z","shell.execute_reply.started":"2021-12-08T21:36:11.771918Z","shell.execute_reply":"2021-12-08T21:37:04.746797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-08T21:37:04.749095Z","iopub.execute_input":"2021-12-08T21:37:04.749628Z","iopub.status.idle":"2021-12-08T21:37:04.774491Z","shell.execute_reply.started":"2021-12-08T21:37:04.749582Z","shell.execute_reply":"2021-12-08T21:37:04.77375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.replace([np.inf, -np.inf], np.nan, inplace=True)\ndf_train.dropna(inplace=True)\ndf_train.drop_duplicates(inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# Credit: https://www.kaggle.com/swaralipibose/64-new-features-with-autoencoders/code\ndef get_features(df, row = False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    df_feat['trade'] = df_feat['Close'] - df_feat['Open']\n    df_feat['gtrade'] = df_feat['trade'] / df_feat['Count']\n    df_feat['shadow1'] = df_feat['trade'] / df_feat['Volume']\n    df_feat['shadow3'] = df_feat['upper_Shadow'] / df_feat['Volume']\n    df_feat['shadow5'] = df_feat['lower_Shadow'] / df_feat['Volume']\n    df_feat['diff1'] = df_feat['Volume'] - df_feat['Count']\n    df_feat['mean1'] = (df_feat['shadow5'] + df_feat['shadow3']) / 2\n    df_feat['mean2'] = (df_feat['shadow1'] + df_feat['Volume']) / 2\n    df_feat['mean3'] = (df_feat['trade'] + df_feat['gtrade']) / 2\n    df_feat['mean4'] = (df_feat['diff1'] + df_feat['upper_Shadow']) / 2\n    df_feat['mean5'] = (df_feat['diff1'] + df_feat['lower_Shadow']) / 2\n    return df_feat\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    model = CatBoostRegressor(iterations=1000, learning_rate=0.05, depth=10, random_seed=42, verbose = 0)\n    model.fit(X, y)\n\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-12-08T21:37:04.775806Z","iopub.execute_input":"2021-12-08T21:37:04.776042Z","iopub.status.idle":"2021-12-08T21:37:04.787888Z","shell.execute_reply.started":"2021-12-08T21:37:04.776015Z","shell.execute_reply":"2021-12-08T21:37:04.787014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-12-08T21:37:04.789275Z","iopub.execute_input":"2021-12-08T21:37:04.789508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model interface\nx = get_features(df_train.iloc[1],row=True)\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pdb; pdb.set_trace()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = get_features(row,row=True)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}