{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Description\nIn many cases to make a baseline prediction it's good to follow conservative strategy, or \"it will be the same\" rule. But what exactly will be the same: some value, derivative of some value or something else depends on the task. \n\nWe are trying to \"forecast short term returns\" (to be more specific returns of crypto assets in next 15 minutes), so let's calculate what \"short-term return\" we would have received if we bought this currency some \"short term\" ago. Say, from 10 to 20 minutes ago (averaging by time seem to be good idea due to high volatility). So let's calculate `Z = log(Close / Mean(Close_lag_10_to_20_minutes) )` and correlation of Z with target value.\n\nThere are three options:\n1. Positive correlation between Z and target. It may be interpreted as \"tendency to keep grow or fall\".\n2. Negative correlation between Z and target. It may be interpreted as \"tendency to smooth grow or fall by opposite movement in immediate future\".\n3. No correlation between Z and Target. \n\nIt's a good place to check you intuition: what option seems more probable?","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport gc\nimport math\nimport gresearch_crypto\n\nPRICE_COLUMN = \"Close\"\nINPUT_PATH = \"../input/g-research-crypto-forecasting\"\nFOLDS = 15 #apporx. 3 months\n\nWINDOW = 10 \nSHIFT = 10\n\ndef get_preprocess_data(df_data, df_assets, folds=10):\n    df_data['Time'] = pd.to_datetime(df_data['timestamp'], unit='s')\n    df_data['w'] = df_data['Asset_ID'].map(\n        df_assets.set_index(keys='Asset_ID')['Weight'])\n    ids = list(df_assets.Asset_ID)\n    chunks = []\n    for id in ids:\n        df_asset = df_data[df_data.Asset_ID == id].copy()\n        df_asset.sort_values(by='Time', inplace=True)\n        df_asset.set_index(keys='Time', inplace=True)\n        df_asset['p1'] = df_asset[PRICE_COLUMN].shift(freq='-1T')\n        df_asset['p16'] = df_asset[PRICE_COLUMN].shift(freq='-16T')\n        df_asset['r'] = np.log(df_asset.p16/df_asset.p1)\n        df_asset.reset_index(inplace=True)\n        chunks.append(df_asset)\n    df_data = pd.concat(chunks)\n    df_data.sort_values(by='Time', inplace=True)\n    \n    df_data.sort_values(by='Time', inplace=True)\n    df_data.set_index(keys='Time', inplace=True)\n    \n    min_value = df_data[\"timestamp\"].min()\n    step = (df_data[\"timestamp\"].max() - min_value) / folds\n    df_data[\"fold_id\"] = ((df_data[\"timestamp\"] - min_value - 1) / step).astype(np.int32)\n    \n    return df_data\n\ndef get_df_rolling_simple(df, df_assets, field=\"Close\", new_field=\"Close_RMEAN\", \n                          window=10, shift=10):\n    \"\"\"Apply rolling funcion to some column and calculate mean value of this column \n    on [-shift - window, -shift] records interval (if 0 is current record).\n    Shift is usefull to work with target-based features: \n    one should not look at values of target in 15 or less steps back\n    \"\"\"\n    chunks = []\n    for asset_id in df_assets[\"Asset_ID\"]:\n        df_asset = df[(df[\"Asset_ID\"] == asset_id)].copy()\n        df_asset[new_field] = df_asset[field].rolling(\n            window=window, min_periods=window, center=False).mean().values\n        df_asset[new_field] = df_asset[new_field].shift(freq=f\"{shift}T\")\n        df_asset.reset_index(inplace=True)\n        chunks.append(df_asset)\n    df_result = pd.concat(chunks)\n    df_result.sort_values(by='Time', inplace=True)\n    df_result.set_index(keys='Time', inplace=True)\n    return df_result\n    \ndef corr(a, b, w):\n    \"\"\"weighed correlation \n    \"\"\"\n    def cov(x, y):\n        return np.sum(\n            w * (x - np.average(x, weights=w)) *\n            (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n\ngc.collect()\ndf_assets: pd.DataFrame = pd.read_csv(os.path.join(INPUT_PATH, \n                                                   \"asset_details.csv\")).sort_values(\"Asset_ID\")\ndf_train = pd.read_csv(\n        os.path.join(INPUT_PATH, \"train.csv\"),\n        dtype={'timestamp': 'int64', 'Asset_ID': 'int8', 'Count': 'int32',\n               'row_id': 'int32', 'Count': 'int32', 'Open': 'float64',\n               'High': 'float64', 'Low': 'float64', 'Close': 'float64',\n               'Volume': 'float64', 'VWAP': 'float64', 'Target': 'float64'})\ndf_train: pd.DataFrame = get_preprocess_data(df_train, df_assets, folds=FOLDS)\ndf_train: pd.DataFrame = get_df_rolling_simple(df_train, df_assets, window=WINDOW, shift=SHIFT)\ndf_train[\"Z\"] = np.log(df_train[\"Close\"] / df_train[\"Close_RMEAN\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T22:38:27.024292Z","iopub.execute_input":"2022-01-18T22:38:27.024632Z","iopub.status.idle":"2022-01-18T22:40:02.464163Z","shell.execute_reply.started":"2022-01-18T22:38:27.024601Z","shell.execute_reply":"2022-01-18T22:40:02.463066Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrs = []\nfold_id = None\nasset_id = None\nfor field in [\"Z\",]:\n    for fold_id in list(df_train[\"fold_id\"].unique()) + [None,]:\n        for asset_id in list(df_assets[\"Asset_ID\"].unique()) + [None,]:\n            filter = ((df_train[\"Target\"].notna()) &\n                      (df_train[field].notna()) &\n                      ((df_train[\"fold_id\"] == fold_id) | (fold_id == None)) &\n                      ((df_train[\"Asset_ID\"] == asset_id) | (asset_id == None)))\n            records = len(df_train[filter])\n            if records > 10000:\n                score = corr(df_train.loc[filter, \"Target\"],\n                             df_train.loc[filter, field],\n                             df_train.loc[filter, \"w\"])\n                corrs.append({\"field\": field, \"fold_id\": fold_id, \"Asset_ID\": asset_id, \n                              \"score\": score, \"records\": records})\ndf_corr = pd.DataFrame(corrs)\n#print(df_corr[ (df_corr[\"Asset_ID\"].isna()) ])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T22:40:02.466151Z","iopub.execute_input":"2022-01-18T22:40:02.466512Z","iopub.status.idle":"2022-01-18T22:41:18.611713Z","shell.execute_reply.started":"2022-01-18T22:40:02.466478Z","shell.execute_reply":"2022-01-18T22:41:18.610944Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Score for each asset (correlation of target and Z calculated for each asset on entire time interval)","metadata":{}},{"cell_type":"code","source":"filter = (df_corr[\"fold_id\"].isna()) & (df_corr[\"Asset_ID\"].notna()) \nprint(df_corr[filter].sort_values(\"score\").join(\n    df_assets.set_index(\"Asset_ID\"), on=\"Asset_ID\")[[\"Asset_ID\", \"Asset_Name\", \"score\", \"records\"]])\n#df_corr[filter][\"score\"].hist()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-18T22:41:18.619374Z","iopub.execute_input":"2022-01-18T22:41:18.619614Z","iopub.status.idle":"2022-01-18T22:41:18.642607Z","shell.execute_reply.started":"2022-01-18T22:41:18.619584Z","shell.execute_reply":"2022-01-18T22:41:18.641746Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only one asset shows notable positive correlation, and it's a Maker coin. One can speculate whether it differs from other coin in some fundamental way (Maker coin have a tricky description), but for such small correlation it may be coincidence.","metadata":{}},{"cell_type":"markdown","source":"# Score by folds (weighted correlation of target and Z calculated for each folds)","metadata":{}},{"cell_type":"code","source":"filter = (df_corr[\"fold_id\"].notna()) & (df_corr[\"Asset_ID\"].isna()) \nprint(df_corr[filter].sort_values(\"fold_id\")[[\"fold_id\", \"score\", \"records\"]])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T22:41:18.643931Z","iopub.execute_input":"2022-01-18T22:41:18.644751Z","iopub.status.idle":"2022-01-18T22:41:18.664176Z","shell.execute_reply.started":"2022-01-18T22:41:18.644704Z","shell.execute_reply":"2022-01-18T22:41:18.663326Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Score","metadata":{}},{"cell_type":"code","source":"df_train.fillna({\"Z\":df_train[\"Z\"].mean()}, inplace=True)\nfilter = (df_train[\"Target\"].notna())\nscore = corr(df_train[filter][\"Target\"], -1 * df_train[filter][\"Z\"], df_train[filter][\"w\"])\nprint(f'Total score: {score:.5f}')","metadata":{"execution":{"iopub.status.busy":"2022-01-18T22:41:18.665618Z","iopub.execute_input":"2022-01-18T22:41:18.665871Z","iopub.status.idle":"2022-01-18T22:41:25.436583Z","shell.execute_reply.started":"2022-01-18T22:41:18.665841Z","shell.execute_reply":"2022-01-18T22:41:25.435347Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit simplified solution\nDo not average price in past - simply get value 15 minutes back and calculate `-1 * log(Close / Close_lag_15_minutes) ","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nreverse = True\nprices = {}\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j, row in df_test.iterrows():\n        key = f'{row[\"timestamp\"]}-{row[\"Asset_ID\"]}'\n        key_prev = f'{row[\"timestamp\"] - 15 * 60}-{row[\"Asset_ID\"]}'\n        prices[key] = row[\"Close\"]\n        df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"] = 0\n        if key_prev in prices.keys():\n            df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"] = \\\n                (-1 if reverse else 1) * \\\n                math.log(row[\"Close\"]/prices[key_prev])\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T22:41:25.437959Z","iopub.execute_input":"2022-01-18T22:41:25.438253Z","iopub.status.idle":"2022-01-18T22:41:25.526261Z","shell.execute_reply.started":"2022-01-18T22:41:25.43822Z","shell.execute_reply":"2022-01-18T22:41:25.525458Z"},"trusted":true},"execution_count":null,"outputs":[]}]}