{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.preprocessing import StandardScaler\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-22T14:27:55.635573Z","iopub.execute_input":"2021-11-22T14:27:55.636693Z","iopub.status.idle":"2021-11-22T14:27:55.642613Z","shell.execute_reply.started":"2021-11-22T14:27:55.636609Z","shell.execute_reply":"2021-11-22T14:27:55.641275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplemental_train = pd.read_csv(\"../input/g-research-crypto-forecasting/supplemental_train.csv\")\nasset_details = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\")\nss = pd.read_csv(\"../input/g-research-crypto-forecasting/example_sample_submission.csv\")\nexample_test=  pd.read_csv(\"../input/g-research-crypto-forecasting/example_test.csv\")\ntrain = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:55.644415Z","iopub.execute_input":"2021-11-22T14:27:55.644760Z","iopub.status.idle":"2021-11-22T14:28:39.558982Z","shell.execute_reply.started":"2021-11-22T14:27:55.644716Z","shell.execute_reply":"2021-11-22T14:28:39.557809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details_dict = dict(zip(asset_details[\"Asset_ID\"] , asset_details[\"Asset_Name\"]))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:28:39.560517Z","iopub.execute_input":"2021-11-22T14:28:39.560768Z","iopub.status.idle":"2021-11-22T14:28:39.566366Z","shell.execute_reply.started":"2021-11-22T14:28:39.560740Z","shell.execute_reply":"2021-11-22T14:28:39.565403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process(df: pd.DataFrame, DEBUG = False, scaler = None):\n    def nans(df_group: pd.DataFrame):\n        df_group.set_index(\"timestamp\")\n        asset_name = asset_details_dict[df_group[\"Asset_ID\"].iloc[0]]\n        if(DEBUG):\n            print(asset_name)\n        df_group.replace([np.inf, -np.inf], np.nan, inplace=True)\n        df_group.fillna(0, inplace= True)\n        if(DEBUG):\n            print(\"missing rows  gaps before [above 60 - missing rows]:\\n {}\".format((df_group.index[1:]-df_group.index[:-1]).value_counts().head()))\n        df_group=df_group.reindex(range(df_group.index[0],df_group.index[-1]+60,60),method='pad')\n        if(DEBUG):\n            print(\"missing rows  gaps after [above 60 - missing rows]:\\n {}\".format((df_group.index[1:]-df_group.index[:-1]).value_counts().head()))\n        return df_group\n    \n    if(DEBUG):\n        print(\"nan count before: {}\".format(df.isna().sum().sum()))\n    df = df.groupby(\"Asset_ID\").apply(nans).reset_index(drop = True)\n    if(DEBUG):\n        print(\"nan count after: {}\".format(df.isna().sum().sum()))\n        \n    return_target = False\n    if \"Target\" in df:\n        return_target = True\n        target = df[\"Target\"]\n        df = df.drop(\"Target\", axis = 1)\n        \n    if (scaler == None):\n        scaler = StandardScaler()\n        keep_Asset_ID = df[\"Asset_ID\"].copy()\n        keep_timestamp = df[\"timestamp\"].copy()\n\n        df = df.drop([\"Asset_ID\", \"timestamp\"], axis = 1)\n        df = pd.DataFrame(scaler.fit_transform(df), columns = df.columns)\n        \n        df[\"Asset_ID\"] = keep_Asset_ID\n        df[\"timestamp\"] = keep_timestamp\n\n    \n    else:\n        keep_Asset_ID = df[\"Asset_ID\"].copy()\n        keep_timestamp = df[\"timestamp\"].copy()\n        df = df.drop([\"Asset_ID\", \"timestamp\"], axis = 1)\n        df = pd.DataFrame(scaler.transform(df),  columns = df.columns)\n        \n        df[\"Asset_ID\"] = keep_Asset_ID\n        df[\"timestamp\"] = keep_timestamp\n\n\n\n    if return_target:\n        df[\"Target\"] = target\n        \n    return df, scaler\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:28:39.568110Z","iopub.execute_input":"2021-11-22T14:28:39.568344Z","iopub.status.idle":"2021-11-22T14:28:39.587043Z","shell.execute_reply.started":"2021-11-22T14:28:39.568317Z","shell.execute_reply":"2021-11-22T14:28:39.585927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def FE(df):\n    def get_prev(df, cols, depth, reps):\n        for col in cols:\n            for gap in range(depth, reps * depth):\n                gap_name = \"{}_{}\".format(col, gap)\n                df[gap_name] = df[col].shift(periods=gap, fill_value=0)\n        gc.collect()\n        return df\n\n    asset_ids = list(df.Asset_ID.unique())\n\n    cols = [\"Count\", \"Open\" , \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n    df = df.groupby(\"Asset_ID\").apply(get_prev, cols, 1, 2)\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:28:39.589442Z","iopub.execute_input":"2021-11-22T14:28:39.589733Z","iopub.status.idle":"2021-11-22T14:28:39.604435Z","shell.execute_reply.started":"2021-11-22T14:28:39.589701Z","shell.execute_reply":"2021-11-22T14:28:39.603300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_vars = %who_ls\nif \"train\" in  all_vars:\n    if train.isna().sum().sum() == 0: # pre_process already ran, restart\n        print(\"restart\")\n        train = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\n        \nelse: # train not loaded\n    print(\"loading train\")\n    train = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\ntrain, scaler = pre_process(train, DEBUG=False)\ntrain = FE(train)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:28:39.606127Z","iopub.execute_input":"2021-11-22T14:28:39.606444Z","iopub.status.idle":"2021-11-22T14:28:57.905896Z","shell.execute_reply.started":"2021-11-22T14:28:39.606389Z","shell.execute_reply":"2021-11-22T14:28:57.904918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom tqdm import tqdm\n\ntrain_columns = train.drop(\"Target\", axis = 1).columns\n\nmodels_dict = {}\nfor group_id in tqdm(train[\"Asset_ID\"].unique()):\n    temp_train = train.loc[train[\"Asset_ID\"] == int(group_id)]\n    models_dict[str(group_id)] = LinearRegression().fit(temp_train[train_columns], temp_train[\"Target\"])","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:28:57.907681Z","iopub.execute_input":"2021-11-22T14:28:57.907948Z","iopub.status.idle":"2021-11-22T14:29:02.064689Z","shell.execute_reply.started":"2021-11-22T14:28:57.907916Z","shell.execute_reply":"2021-11-22T14:29:02.063762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\n\n# gresearch_crypto.competition.make_env.__called__ = False\n\nenv = gresearch_crypto.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df, _ = pre_process(test_df.drop(\"row_id\", axis = 1),False, scaler)\n    test_df = FE(test_df)\n\n    group_id = train[\"Asset_ID\"].at[0]\n    clf = models_dict[str(group_id)]\n    sample_prediction_df['Target'] = clf.predict(test_df[train_columns])  # make your predictions here\n    sample_prediction_df.fillna(0, inplace=True)\n    env.predict(sample_prediction_df)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:29:02.066374Z","iopub.execute_input":"2021-11-22T14:29:02.066987Z","iopub.status.idle":"2021-11-22T14:29:08.801109Z","shell.execute_reply.started":"2021-11-22T14:29:02.066932Z","shell.execute_reply":"2021-11-22T14:29:08.800028Z"},"trusted":true},"execution_count":null,"outputs":[]}]}