{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import and load dfs\n\nReferences: [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\nimport gresearch_crypto\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\ndef read_csv_strict(file_name='/kaggle/input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    train_df = df[df['datetime'] < '2021-06-13 00:00:00']\n    test_df = df[df['datetime'] >= '2021-06-13 00:00:00']\n    return train_df, test_df\n\n\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:38:35.056676Z","iopub.execute_input":"2021-12-23T01:38:35.057394Z","iopub.status.idle":"2021-12-23T01:38:35.069193Z","shell.execute_reply.started":"2021-12-23T01:38:35.057348Z","shell.execute_reply":"2021-12-23T01:38:35.068192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_test = read_csv_strict()","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:39:07.599911Z","iopub.execute_input":"2021-12-23T01:39:07.600231Z","iopub.status.idle":"2021-12-23T01:40:12.676635Z","shell.execute_reply.started":"2021-12-23T01:39:07.600197Z","shell.execute_reply":"2021-12-23T01:40:12.675565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:47:52.858976Z","iopub.execute_input":"2021-12-23T01:47:52.859313Z","iopub.status.idle":"2021-12-23T01:47:52.876468Z","shell.execute_reply.started":"2021-12-23T01:47:52.859280Z","shell.execute_reply":"2021-12-23T01:47:52.875854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df, row=False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    \n    \n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High/Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low/Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume/Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n\n    return df_feat\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=10)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:40:34.314332Z","iopub.execute_input":"2021-12-23T01:40:34.314662Z","iopub.status.idle":"2021-12-23T01:40:34.331672Z","shell.execute_reply.started":"2021-12-23T01:40:34.314630Z","shell.execute_reply":"2021-12-23T01:40:34.330672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loop over all assets","metadata":{}},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:40:35.181741Z","iopub.execute_input":"2021-12-23T01:40:35.182047Z","iopub.status.idle":"2021-12-23T01:41:28.696377Z","shell.execute_reply.started":"2021-12-23T01:40:35.182014Z","shell.execute_reply":"2021-12-23T01:41:28.695634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model interface\nx = get_features(df_train.iloc[1], row=True)\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:27:50.651002Z","iopub.execute_input":"2021-12-23T01:27:50.652038Z","iopub.status.idle":"2021-12-23T01:27:50.676030Z","shell.execute_reply.started":"2021-12-23T01:27:50.651997Z","shell.execute_reply":"2021-12-23T01:27:50.674973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = df_test[\"Target\"]\ny_pred = pd.Series(np.nan, index=y_true.index)\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    df_asset = df_test[df_test.Asset_ID == asset_id]\n    x_asset = get_features(df_asset)\n    y = models[asset_id].predict(x_asset)\n    y_pred[df_test.Asset_ID == asset_id] = y","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:43:29.008986Z","iopub.execute_input":"2021-12-23T01:43:29.009844Z","iopub.status.idle":"2021-12-23T01:43:31.302670Z","shell.execute_reply.started":"2021-12-23T01:43:29.009799Z","shell.execute_reply":"2021-12-23T01:43:31.301793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details_asset = df_asset_details.set_index(\"Asset_ID\")\nweight = df_asset_details_asset[\"Weight\"].to_dict()\nasset_ids = df_test.Asset_ID\nweight = asset_ids.replace(weight)","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:49:46.895020Z","iopub.execute_input":"2021-12-23T01:49:46.895375Z","iopub.status.idle":"2021-12-23T01:49:46.902432Z","shell.execute_reply.started":"2021-12-23T01:49:46.895339Z","shell.execute_reply":"2021-12-23T01:49:46.901273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_nan = np.isnan(y_true)\ny_true = y_true[~is_nan]\ny_pred = y_pred[~is_nan]\nweight = weight[~is_nan]\nprint(weighted_correlation(y_true, y_pred, weight))","metadata":{"execution":{"iopub.status.busy":"2021-12-23T01:53:46.284791Z","iopub.execute_input":"2021-12-23T01:53:46.285204Z","iopub.status.idle":"2021-12-23T01:53:46.471909Z","shell.execute_reply.started":"2021-12-23T01:53:46.285168Z","shell.execute_reply":"2021-12-23T01:53:46.470988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = get_features(row, row=True)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T00:45:51.361771Z","iopub.execute_input":"2021-11-06T00:45:51.362127Z","iopub.status.idle":"2021-11-06T00:45:51.981721Z","shell.execute_reply.started":"2021-11-06T00:45:51.362087Z","shell.execute_reply":"2021-11-06T00:45:51.981002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}