{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Creation\n\nI have been trying to build models on this dataset, but due to the size I felt it would be more comfortable to break it up into two parts. In this notebook I will create datasets with additional variables, and I will later use them in a separate notebook to create predictions and submit. \n\nThe second part can be found here - [https://www.kaggle.com/craniket/g-research-lgbm-basic-model](http://)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-06T11:25:16.503764Z","iopub.execute_input":"2022-01-06T11:25:16.504079Z","iopub.status.idle":"2022-01-06T11:25:16.558573Z","shell.execute_reply.started":"2022-01-06T11:25:16.503994Z","shell.execute_reply":"2022-01-06T11:25:16.557878Z"}}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gresearch_crypto\nimport matplotlib.pyplot as plt\nenv = gresearch_crypto.make_env()\nfrom datetime import datetime\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom sklearn.model_selection import StratifiedKFold,KFold\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:25:25.027064Z","iopub.execute_input":"2022-01-13T14:25:25.02817Z","iopub.status.idle":"2022-01-13T14:25:27.809974Z","shell.execute_reply.started":"2022-01-13T14:25:25.028063Z","shell.execute_reply":"2022-01-13T14:25:27.8092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we define a function to reduce the size of the data.","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(train_data):\n    start_mem = train_data.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    for col in train_data.columns:\n        col_type = train_data[col].dtype\n        if col_type != object:\n            c_min = train_data[col].min()\n            c_max = train_data[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    train_data[col] = train_data[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    train_data[col] = train_data[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    train_data[col] = train_data[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    train_data[col] = train_data[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    train_data[col] = train_data[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    train_data[col] = train_data[col].astype(np.float32)\n                else:\n                    train_data[col] = train_data[col].astype(np.float64)\n        else:\n            train_data[col] = train_data[col].astype('category')\n\n    end_mem = train_data.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return train_data","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:25:26.488618Z","iopub.execute_input":"2022-01-06T11:25:26.489362Z","iopub.status.idle":"2022-01-06T11:25:26.502178Z","shell.execute_reply.started":"2022-01-06T11:25:26.489323Z","shell.execute_reply":"2022-01-06T11:25:26.501293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = \"/kaggle/input/g-research-crypto-forecasting/\"\ncrypto_df = pd.read_csv(data_dir+\"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:25:28.10572Z","iopub.execute_input":"2022-01-06T11:25:28.106026Z","iopub.status.idle":"2022-01-06T11:26:25.250911Z","shell.execute_reply.started":"2022-01-06T11:25:28.10599Z","shell.execute_reply":"2022-01-06T11:26:25.250185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = reduce_mem_usage(crypto_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:25.252423Z","iopub.execute_input":"2022-01-06T11:26:25.252949Z","iopub.status.idle":"2022-01-06T11:26:29.354635Z","shell.execute_reply.started":"2022-01-06T11:26:25.252916Z","shell.execute_reply":"2022-01-06T11:26:29.353834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:29.355799Z","iopub.execute_input":"2022-01-06T11:26:29.35659Z","iopub.status.idle":"2022-01-06T11:26:29.377083Z","shell.execute_reply.started":"2022-01-06T11:26:29.356543Z","shell.execute_reply":"2022-01-06T11:26:29.376422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_name = pd.read_csv(data_dir+\"asset_details.csv\")\ncrypto_name","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:29.378934Z","iopub.execute_input":"2022-01-06T11:26:29.380436Z","iopub.status.idle":"2022-01-06T11:26:29.399263Z","shell.execute_reply.started":"2022-01-06T11:26:29.380397Z","shell.execute_reply":"2022-01-06T11:26:29.398606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#crypto_df.loc[crypto_df.Asset_ID == 2].plot(x = \"timestamp\",y = \"Target\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df = crypto_df.merge(crypto_name,on = \"Asset_ID\")\nplot_df = plot_df.sort_values(by = ['Asset_Name','timestamp'])\nplot_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:29.400467Z","iopub.execute_input":"2022-01-06T11:26:29.400686Z","iopub.status.idle":"2022-01-06T11:26:44.203607Z","shell.execute_reply.started":"2022-01-06T11:26:29.400656Z","shell.execute_reply":"2022-01-06T11:26:44.202705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets = plot_df.Asset_Name.unique()\nassets","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:44.205223Z","iopub.execute_input":"2022-01-06T11:26:44.205502Z","iopub.status.idle":"2022-01-06T11:26:46.599047Z","shell.execute_reply.started":"2022-01-06T11:26:44.205463Z","shell.execute_reply":"2022-01-06T11:26:46.598411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Adding features","metadata":{}},{"cell_type":"code","source":"plot_df['change'] = plot_df['Close']/plot_df['Open']\nplot_df['variation'] = plot_df['High']/plot_df['Low']\nplot_df['change_diff'] = plot_df['Close'] - plot_df['Open']\nplot_df['variation_diff'] = plot_df['High'] - plot_df['Low']\ndef hlco_ratio(df): return (df['High'] - df['Low'])/(df['Close']-df['Open'])\ndef upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\nplot_df['Upper_Shadow'] = upper_shadow(plot_df)\nplot_df['Lower_Shadow'] = lower_shadow(plot_df)\nplot_df['hlco_ration'] = hlco_ratio(plot_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:46.603639Z","iopub.execute_input":"2022-01-06T11:26:46.606285Z","iopub.status.idle":"2022-01-06T11:26:49.85107Z","shell.execute_reply.started":"2022-01-06T11:26:46.606248Z","shell.execute_reply":"2022-01-06T11:26:49.850302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del crypto_df","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:49.852685Z","iopub.execute_input":"2022-01-06T11:26:49.852936Z","iopub.status.idle":"2022-01-06T11:26:49.85744Z","shell.execute_reply.started":"2022-01-06T11:26:49.852902Z","shell.execute_reply":"2022-01-06T11:26:49.856716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def reindex(df):\n#    df = df.reindex(range(df.index[0],df.index[-1]+60,60),method='nearest')\n#    return df\nplot_df = plot_df.sort_values(by = [\"Asset_ID\",'timestamp'])\n#plot_df = plot_df.set_index(['timestamp',\"Asset_ID\"])\n#plot_df = reindex(plot_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:49.859129Z","iopub.execute_input":"2022-01-06T11:26:49.859766Z","iopub.status.idle":"2022-01-06T11:26:58.967851Z","shell.execute_reply.started":"2022-01-06T11:26:49.85973Z","shell.execute_reply":"2022-01-06T11:26:58.967118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Re-indexing\n\nThe timestamps are not continuous for each Crypto. We need to reindex each individually.","metadata":{}},{"cell_type":"code","source":"plot_df2 = pd.DataFrame()\ndef reindex(df):\n    df = df.reindex(range(df.index[0],df.index[-1]+60,60),method='nearest')\n    df = df.assign(filt = (df.index - df.index[0])/(df.index[-1]+60 - df.index[0]))\n    return df\nfor asset_id,asset_name in enumerate(assets):\n    df = plot_df.loc[plot_df[\"Asset_ID\"] == asset_id]\n    df = df.set_index(['timestamp'])\n    df = reindex(df)\n    plot_df2 = plot_df2.append(df)\nplot_df2.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:26:58.970198Z","iopub.execute_input":"2022-01-06T11:26:58.97039Z","iopub.status.idle":"2022-01-06T11:27:14.35964Z","shell.execute_reply.started":"2022-01-06T11:26:58.970366Z","shell.execute_reply":"2022-01-06T11:27:14.358998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df2.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:27:14.361102Z","iopub.execute_input":"2022-01-06T11:27:14.361601Z","iopub.status.idle":"2022-01-06T11:27:14.366915Z","shell.execute_reply.started":"2022-01-06T11:27:14.361563Z","shell.execute_reply":"2022-01-06T11:27:14.366277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df2_train = plot_df2.loc[plot_df2[\"filt\"]<=0.7]\nplot_df2_test = plot_df2.loc[plot_df2[\"filt\"]>0.7]\n[plot_df2_train.shape,plot_df2_test.shape]","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:27:14.368438Z","iopub.execute_input":"2022-01-06T11:27:14.368982Z","iopub.status.idle":"2022-01-06T11:27:18.659769Z","shell.execute_reply.started":"2022-01-06T11:27:14.368942Z","shell.execute_reply":"2022-01-06T11:27:18.657348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del plot_df2\n[x_train,y_train,x_test,y_test] = [plot_df2_train.drop([\"Target\",\"Asset_Name\",\"Weight\"],axis=1),\n                                  plot_df2_train.Target,\n                                  plot_df2_test.drop([\"Target\",\"Asset_Name\",\"Weight\"],axis=1),\n                                  plot_df2_test.Target]\n\n\n[x_train.shape,y_train.shape,x_test.shape,y_test.shape] \n","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:27:18.661206Z","iopub.execute_input":"2022-01-06T11:27:18.662385Z","iopub.status.idle":"2022-01-06T11:27:20.843007Z","shell.execute_reply.started":"2022-01-06T11:27:18.662347Z","shell.execute_reply":"2022-01-06T11:27:20.842185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_train,weight_test = plot_df2_train.Weight,plot_df2_test.Weight\n","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:27:20.844409Z","iopub.execute_input":"2022-01-06T11:27:20.844749Z","iopub.status.idle":"2022-01-06T11:27:20.849779Z","shell.execute_reply.started":"2022-01-06T11:27:20.844712Z","shell.execute_reply":"2022-01-06T11:27:20.849098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.to_csv(\"x_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:27:21.068845Z","iopub.execute_input":"2022-01-06T11:27:21.069194Z","iopub.status.idle":"2022-01-06T11:32:05.170382Z","shell.execute_reply.started":"2022-01-06T11:27:21.069134Z","shell.execute_reply":"2022-01-06T11:32:05.169576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test.to_csv(\"x_test.csv\")\ny_train.to_csv(\"y_train.csv\")\ny_test.to_csv(\"y_test.csv\")\nweight_train.to_csv(\"weight_train.csv\")\nweight_test.to_csv(\"weight_test.csv\")\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-06T11:34:13.222592Z","iopub.execute_input":"2022-01-06T11:34:13.223272Z","iopub.status.idle":"2022-01-06T11:34:36.219103Z","shell.execute_reply.started":"2022-01-06T11:34:13.223233Z","shell.execute_reply":"2022-01-06T11:34:36.218339Z"},"trusted":true},"execution_count":null,"outputs":[]}]}