{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-03T17:44:20.345562Z","iopub.execute_input":"2021-11-03T17:44:20.345864Z","iopub.status.idle":"2021-11-03T17:44:20.364434Z","shell.execute_reply.started":"2021-11-03T17:44:20.345780Z","shell.execute_reply":"2021-11-03T17:44:20.363777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\nsuptrain=pd.read_csv(\"../input/g-research-crypto-forecasting/supplemental_train.csv\")\nextest=pd.read_csv(\"../input/g-research-crypto-forecasting/example_test.csv\")\nexsub=pd.read_csv(\"../input/g-research-crypto-forecasting/example_sample_submission.csv\")\nassdet=pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\")\ntest=pd.read_csv(\"../input/g-research-crypto-forecasting/example_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:44:20.367332Z","iopub.execute_input":"2021-11-03T17:44:20.367536Z","iopub.status.idle":"2021-11-03T17:45:17.063776Z","shell.execute_reply.started":"2021-11-03T17:44:20.367512Z","shell.execute_reply":"2021-11-03T17:45:17.063045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:17.065120Z","iopub.execute_input":"2021-11-03T17:45:17.065480Z","iopub.status.idle":"2021-11-03T17:45:17.089897Z","shell.execute_reply.started":"2021-11-03T17:45:17.065439Z","shell.execute_reply":"2021-11-03T17:45:17.089041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"suptrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:17.091441Z","iopub.execute_input":"2021-11-03T17:45:17.091819Z","iopub.status.idle":"2021-11-03T17:45:17.105556Z","shell.execute_reply.started":"2021-11-03T17:45:17.091779Z","shell.execute_reply":"2021-11-03T17:45:17.104645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assdet.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:17.107007Z","iopub.execute_input":"2021-11-03T17:45:17.107456Z","iopub.status.idle":"2021-11-03T17:45:17.120811Z","shell.execute_reply.started":"2021-11-03T17:45:17.107420Z","shell.execute_reply":"2021-11-03T17:45:17.120046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assdet[\"weight-Percent\"] = (assdet[\"Weight\"] / assdet[\"Weight\"].sum()) * 100\nassdet.sort_values(\"Weight\", ascending=False)\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport seaborn as sns\ncmap = sns.color_palette()\nfig = px.bar(assdet, y='weight-Percent', x='Asset_Name', title='Weight_Percent/Cryptography')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:49:31.954800Z","iopub.execute_input":"2021-11-03T17:49:31.955078Z","iopub.status.idle":"2021-11-03T17:49:32.596078Z","shell.execute_reply.started":"2021-11-03T17:49:31.955048Z","shell.execute_reply":"2021-11-03T17:49:32.594342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_ID=1\ntgt_df = train[train[\"Asset_ID\"] == asset_ID]\ntgt_df = tgt_df[['Open', 'High', 'Low', 'Close', 'Volume','timestamp']]  ## ohlcvと時間の情報のみ必要。\ntgt_df[\"timestamp\"] = pd.to_datetime(tgt_df['timestamp'].astype(int), unit='s')\ntgt_df.set_index(\"timestamp\",inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T18:23:22.396765Z","iopub.execute_input":"2021-11-03T18:23:22.397029Z","iopub.status.idle":"2021-11-03T18:23:22.733358Z","shell.execute_reply.started":"2021-11-03T18:23:22.397000Z","shell.execute_reply":"2021-11-03T18:23:22.732600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T18:23:56.765673Z","iopub.execute_input":"2021-11-03T18:23:56.766308Z","iopub.status.idle":"2021-11-03T18:23:56.778866Z","shell.execute_reply.started":"2021-11-03T18:23:56.766264Z","shell.execute_reply":"2021-11-03T18:23:56.778094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:17.121850Z","iopub.execute_input":"2021-11-03T17:45:17.122113Z","iopub.status.idle":"2021-11-03T17:45:17.644174Z","shell.execute_reply.started":"2021-11-03T17:45:17.122078Z","shell.execute_reply":"2021-11-03T17:45:17.643364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.dropna()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:17.648698Z","iopub.execute_input":"2021-11-03T17:45:17.650796Z","iopub.status.idle":"2021-11-03T17:45:19.562554Z","shell.execute_reply.started":"2021-11-03T17:45:17.650755Z","shell.execute_reply":"2021-11-03T17:45:19.561808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:19.565915Z","iopub.execute_input":"2021-11-03T17:45:19.566543Z","iopub.status.idle":"2021-11-03T17:45:26.655697Z","shell.execute_reply.started":"2021-11-03T17:45:19.566512Z","shell.execute_reply":"2021-11-03T17:45:26.654966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.corr()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T17:45:26.657022Z","iopub.execute_input":"2021-11-03T17:45:26.657481Z","iopub.status.idle":"2021-11-03T17:45:33.860811Z","shell.execute_reply.started":"2021-11-03T17:45:26.657443Z","shell.execute_reply":"2021-11-03T17:45:33.860064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom lightgbm import LGBMRegressor\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    return df_feat\n\ndef get_Xy_and_model_for_asset(train, asset_id):\n    df = train[train[\"Asset_ID\"] == asset_id]\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    model = LGBMRegressor(n_estimators=1000)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-11-03T18:06:05.387486Z","iopub.execute_input":"2021-11-03T18:06:05.387757Z","iopub.status.idle":"2021-11-03T18:06:05.607692Z","shell.execute_reply.started":"2021-11-03T18:06:05.387726Z","shell.execute_reply":"2021-11-03T18:06:05.606974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(assdet['Asset_ID'], assdet['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-11-03T18:07:46.785803Z","iopub.execute_input":"2021-11-03T18:07:46.786057Z","iopub.status.idle":"2021-11-03T18:16:59.155663Z","shell.execute_reply.started":"2021-11-03T18:07:46.786029Z","shell.execute_reply":"2021-11-03T18:16:59.155045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = get_features(train.iloc[1])\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-03T18:16:59.157225Z","iopub.execute_input":"2021-11-03T18:16:59.157738Z","iopub.status.idle":"2021-11-03T18:16:59.169811Z","shell.execute_reply.started":"2021-11-03T18:16:59.157700Z","shell.execute_reply":"2021-11-03T18:16:59.169072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (test, df_pred) in enumerate(iter_test):\n    for j , row in test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = get_features(row)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T18:19:44.177896Z","iopub.execute_input":"2021-11-03T18:19:44.178275Z","iopub.status.idle":"2021-11-03T18:19:44.399433Z","shell.execute_reply.started":"2021-11-03T18:19:44.178237Z","shell.execute_reply":"2021-11-03T18:19:44.398800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Work in progress, please upvote","metadata":{}}]}