{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Wed Dec  8 17:10:17 2021\n\n@source1: https://www.kaggle.com/tarlannazarov/g-research-crypto-starter-xgb-pipeline\n\n@source2: Memory reduction function from https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285721\n\n@source3: class MemReducer(...) from https://www.kaggle.com/jpmiller/skmem\n\n@source4: https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285289\n\n@source5: https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition\n\"\"\"\n\n#!pip install skmem #Cannot install on Kaggle. Class MemReducer(...) copied from Source3 mentioned above.\n\n##Import and load dfs\n#References: Tutorial to the G-Research Crypto Competition\nimport pandas as pd\nimport numpy as np\nimport gresearch_crypto\nimport xgboost as xgb\nimport traceback\nimport datatable as dt\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils import validation\nfrom datetime import datetime\nimport time\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\ncutoff_timestamp = totimestamp('13/06/2021')\n\nclass MemReducer(BaseEstimator, TransformerMixin):\n    def __init__(self, max_unique_pct=0.2, nullables=True):\n        self.max_unique_pct = max_unique_pct\n        self.nullables = nullables\n        \n    def fit(self, df, float_cols=None):\n        if not isinstance(df, pd.DataFrame):\n            raise TypeError(f\"'{type(df).__name__}' object is not a pandas \\\n                    dataframe.\")\n        \n        self.float_candidates = float_cols\n        return self\n\n    \n    # Helper functions for .transform()\n    def reduce_ints(self, df):\n        int_cols = df.select_dtypes('integer').columns\n        if not int_cols.empty:\n            print(\"Starting integers.\", flush=True)\n            start_types = df[int_cols].dtypes\n\n            mins = df[int_cols].min()\n            unsigneds = mins.index[mins >= 0]\n            df[unsigneds] = df[unsigneds].apply(pd.to_numeric,\n                                                downcast='unsigned')\n            signeds = mins.index[mins < 0]\n            df[signeds] = df[signeds].apply(pd.to_numeric,\n                                            downcast='signed')\n            end_types = df[int_cols].dtypes\n            changed = end_types[~end_types.eq(start_types)]\n            print(f\"Downcast {len(changed)} standard integer columns.\")\n        return df\n\n    \n    def reduce_floats(self, df, float_cols):\n        print(\"Starting floats.\", flush=True)\n        \n        if not isinstance(float_cols, list):\n            print(f\"'{type(float_cols).__name__}' object is not a list,\\\n                    skipping floats.\")\n        else:\n            true_float_cols = df.select_dtypes(np.float64).columns.tolist()\n            non_float64s = [f for f in float_cols if f not in true_float_cols]\n            if non_float64s:\n                print(\"Skipping columns that are not np.float64\")\n\n            convertibles = [f for f in float_cols if f in true_float_cols]\n            if convertibles:\n                df[convertibles] = df[convertibles].astype(np.float32)\n                print(f\"Downcast {len(convertibles)} float columns.\")\n        return df\n    \n    \n    def reduce_objs(self, df, max_pct):   \n        if not 0<=max_pct<=1:\n            raise ValueError(\"max_unique_pct must be between 0 and 1\")\n\n        obj_cols = df.select_dtypes('object').columns\n        if not obj_cols.empty:\n            print(\"Starting objects.\", flush=True)\n            for oc in obj_cols:\n                try:\n                    df[oc] = pd.to_numeric(df[oc], downcast='integer')\n                except:\n                    pass\n                else: \n                    print(f\"Converted {len(oc)} columns to numbers.\")\n                    \n        new_obj_cols = df.select_dtypes('object').columns    \n        if not new_obj_cols.empty:\n            category_mask = df[new_obj_cols].nunique().to_numpy()/len(df) <= max_pct\n            cat_cols = new_obj_cols[category_mask]\n            if not cat_cols.empty:\n                df[cat_cols] = df[cat_cols].astype('category')\n                print(f\"Converted {len(cat_cols)} columns to categories.\")\n        return df\n    \n    \n    def reduce_nullables(self, df):\n        print(\"Starting nullables.\", flush=True)\n        \n        true_float_cols = df.select_dtypes('float').columns\n        remainders = df[true_float_cols].mod(1).max(axis=0)\n        nulls = df[true_float_cols].isnull().sum()\n        \n        convertibles = remainders[remainders==0].index \\\n                        .intersection(nulls[nulls!=0].index) \\\n                        .tolist()\n        if convertibles:\n            start_types = df[convertibles].dtypes\n            df[convertibles] = df[convertibles].convert_dtypes()\n            end_types = df[convertibles].dtypes\n\n            changed = end_types[~end_types.eq(start_types)]                     \n            changed_nums = changed[end_types!='string'].index\n                        \n            #TODO: change ifs and loops to np.arrays or similar\n            #       and add in unsigned ints\n            \n            for cc in changed_nums:  \n                max_int = df[cc].abs().max()\n                if 32767 < max_int <=  2147483647:\n                    df[cc] = df[cc].astype('Int32')\n                elif 127 < max_int <= 32767:\n                    df[cc] = df[cc].astype('Int16')\n                elif max_int <= 127:\n                    df[cc] = df[cc].astype('Int8')\n                    \n            print(f\"Converted {len(changed)} columns to nullable types.\")\n        \n        else:\n            print(\"No candidates for nullable integers.\")\n\n        return df\n\n    \n    def transform(self, df):\n        \"\"\" Convert dataframe columns to dtypes requiring lower memory.\n\n        Parameters\n        ----------\n        df : pandas DataFrame\n            The dataframe to be converted.\n        \"\"\"\n\n        validation.check_is_fitted(self, 'float_candidates')\n\n        print(\"Getting memory usage.\")\n        memory_MB_in = df.memory_usage(deep=True).sum()/(1024**2)\n        print(f\"Memory in: {memory_MB_in:.2f} MB\")\n\n        df = self.reduce_ints(df)\n        if self.float_candidates:\n            df = self.reduce_floats(df, self.float_candidates)\n        df = self.reduce_objs(df, self.max_unique_pct)\n        if self.nullables:\n            df = self.reduce_nullables(df)\n\n        memory_MB_out = df.memory_usage(deep=True).sum()/(1024**2)\n        print(f\"Memory out: {memory_MB_out:.2f} MB\",\n              f\"Reduction: {1 - memory_MB_out/memory_MB_in:.1%}\\n\")\n\n        return df\n    \n    \ndef drop_memory(path):\n    df = dt.fread(path).to_pandas()\n    df = df[df['timestamp'] < cutoff_timestamp]\n    mr = MemReducer()\n    floats = df.select_dtypes('float').columns.tolist()\n    df = mr.fit_transform(df, float_cols=floats)\n    return df\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\ndf_train = drop_memory(TRAIN_CSV)\ndf_train.head()\n\ndf_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details\n\ndf_train.replace([np.inf, -np.inf], np.nan)\ndf_train = df_train.dropna(how=\"any\")\n\n##Training\n#Utility functions to train a model for one asset\n\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    return df_feat\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    #df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    # TODO: Try different models here!\n    #model = LGBMRegressor(random_state=1111, n_estimators=1200)\n    #model.fit(X, y)\n    #return X, y, model\n    \n    model = xgb.XGBRegressor(\n    n_estimators=1787,\n    learning_rate=0.05,\n    max_depth=12,\n    subsample=0.9,\n    colsample_bytree=0.7,\n    #colsample_bylevel=0.75,\n    missing=-999,\n    random_state=1111,\n    tree_method='gpu_hist'  \n    )\n    \n    model.fit(X, y)\n    return X, y, model\n\n##Loop over all assets\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    try:\n        X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n        Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model\n    except: \n        traceback.print_exc()\n        Xs[asset_id], ys[asset_id], models[asset_id] = None, None, None    \n\n#sh?? Deal with the inf and nan\n\n# Check the model interface\n# x = get_features(df_train.iloc[1])\n#y_pred = models[0].predict([x])\n#y_pred[0]\n# y_pred = models[0].predict(pd.DataFrame([x]))\n# y_pred[0]\n\n##Predict & submit\n#References: Detailed API Introduction\n\n#Something that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n\n# import pdb; pdb.set_trace()\n\n#See Python Debugging With Pdb if you want to use it and you don't know how to.\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                x_test = get_features(row)\n                y_pred = model.predict(pd.DataFrame([x_test]))[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n        \n    env.predict(df_pred)\n\n#This version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]}]}