{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n \n\n Author: Vincent Weimer
\n Date: 2022-01-22\n \n \n
","metadata":{}},{"cell_type":"code","source":"%%capture\n\n## Import Libraries\nimport numpy as np # linear algebra\nfrom numpy.random import seed \nimport math \nfrom math import sqrt, log\n\nfrom scipy.stats import normaltest\n\nimport pandas as pd # data processing \npd.options.display.max_rows = 100\npd.options.display.max_seq_items = 2000\npd.set_option('display.max_columns', None)\npd.set_option('display.expand_frame_repr', False)\npd.set_option('max_colwidth', -1)\n\nimport datetime as dt\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nimport matplotlib.dates as mpl_dates\nplt.rcParams.update({'font.size': 14})\nimport seaborn as sns\nplt.style.use('seaborn')\nsns.set_style('whitegrid')\n\n# !pip install talib-binary # install talib for feature engineering \n# import talib\n# from talib import RSI, BBANDS, MACD, ATR\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso\nfrom sklearn.pipeline import Pipeline\n\nimport warnings # Supress warnings \nwarnings.filterwarnings('ignore')\n\n# import statsmodels as sm\n\n# import joblib\n\n# Fix seed for reproducible results\nSEED = 42\nnp.random.seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:14:45.131020Z","iopub.execute_input":"2022-01-23T23:14:45.131392Z","iopub.status.idle":"2022-01-23T23:15:04.518897Z","shell.execute_reply.started":"2022-01-23T23:14:45.131292Z","shell.execute_reply":"2022-01-23T23:15:04.517838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Useful Functions\n\ndef read_dataset(folder, name, types=None):\n return pd.read_csv(folder + name + \".csv\", dtype=types)\n\ndef resample_timeseries(df, offset='H'):\n \"\"\"Resampes a timeseries dataframe given the offset forwarded in the function\"\"\"\n \n df = df.reset_index().groupby(['Asset_ID', 'Ticker']).resample(offset, on='timestamp', origin='start').agg(\n {\"Open\": \"first\", \n \"Close\": \"last\", \n \"Low\": \"min\", \n \"High\": \"max\",\n \"Volume\": \"sum\",\n \"VWAP\": \"max\"\n }\n ).dropna()[['Open', 'High', 'Low', 'Close', 'Volume','VWAP']]\n\n return df.reset_index().set_index('timestamp')\n\ndef generate_crypto_dfs(df):\n \"\"\"Returns a dictionary of dataframes with key(Ticker):value(Dataframe that belongs to that ticker)\"\"\"\n \n return {ticker:df.query(\"Ticker == @ticker\") for ticker in tickers}\n\ndef find_missing_rows(df, start, end, asset):\n \"\"\"This function calculates how many missing rows each crypto asset has.\"\"\"\n \n num_rows = len(df.index)\n num_missing_rows = len(pd.date_range(start=start, end=end, freq='min').difference(df.index))\n print(f\"{asset} total number of records: \" + BOLD + f\"{num_rows}\" + END + \\\n \", Total number of missing minute records: \" + BOLD + f\"{num_missing_rows}\" + END)\n \ndef populate_missing_rows(df):\n \"\"\"This function populates missing minute data rows with Null values.\"\"\"\n \n asset, start_date, end_date = df.iloc[0]['Asset_Name'], df.index[0], df.index[-1]\n print(f\"{asset} start date: {start_date}, end date: {end_date}\")\n find_missing_rows(df, start_date, end_date, asset)\n \n df = df.asfreq(freq='T', method='ffill') # forward fill missing time series data\n print(f\"Populating missing {asset} data...\")\n find_missing_rows(df, start_date, end_date, asset)\n print('\\n')\n \n return df\n\ndef close_close_volatility(df,window_size, N=365): # N=365 because crypto markets don't close \n # Compute log returns using close prices \n df['Log_Returns'] = np.log(df['Close'] / df['Close'].shift(1))\n \n # Compute historical volatility\n return df.Log_Returns.rolling(window_size).std() * np.sqrt(N)\n\ndef calc_daily_features(df):\n \n df['Date'] = pd.to_datetime(df.index.date)\n df['Date'] = df['Date'].dt.strftime('%Y%m%d').astype(int)\n \n # Momentum \n df['Rsi_14d'] = RSI(df.Close, timeperiod=14)\n df['Bbands_upper'], df['Bbands_middle'], df['Bbands_lower'] = BBANDS(df.Close, timeperiod=10, nbdevup=2, nbdevdn=2)\n \n # SMA\n df['Sma_10d'] = df.Close.rolling(window=10).mean()\n df['Sma_20d'] = df.Close.rolling(window=20).mean()\n df['Sma_30d'] = df.Close.rolling(window=30).mean()\n \n # Volume moving average\n df['Avg_volume_10d'] = df['Volume'].rolling(window=10).mean()\n df['Avg_volume_20d'] = df['Volume'].rolling(window=20).mean()\n df['Avg_volume_30d'] = df['Volume'].rolling(window=30).mean()\n\n # Volatility\n df['Close_close_vol_10d'] = close_close_volatility(df,10)\n df['Close_close_vol_20d'] = close_close_volatility(df,20)\n df['Close_close_vol_30d'] = close_close_volatility(df,30)\n df['Atr_14d'] = ATR(df.High, df.Low, df.Close, timeperiod=14)\n \n return df.iloc[:, 8:] \n\ndef compute_feature_importance(model):\n fi_df = pd.DataFrame()\n fi_df['features'] = features\n f_df = fi_df[:-1]\n fi_df['importance'] = model.booster_.feature_importance(importance_type=\"gain\")\n \n # plot feature importance\n fig, ax = plt.subplots(1, 1, figsize=(7, 15))\n sns.barplot(\n x='importance', \n y='features',\n data=fi_df.sort_values(by=['importance'], ascending=False),\n ax=ax)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:15:04.521908Z","iopub.execute_input":"2022-01-23T23:15:04.522262Z","iopub.status.idle":"2022-01-23T23:15:04.547454Z","shell.execute_reply.started":"2022-01-23T23:15:04.522220Z","shell.execute_reply":"2022-01-23T23:15:04.546673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\ndef reduce_mem_usage(df): ## Copied directly from other Kagglers! ##\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n \n for col in df.columns:\n col_type = df[col].dtype\n \n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n# else:\n# df[col] = df[col].astype('category')\n\n end_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n \n return df","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:15:04.549245Z","iopub.execute_input":"2022-01-23T23:15:04.549827Z","iopub.status.idle":"2022-01-23T23:15:04.570109Z","shell.execute_reply.started":"2022-01-23T23:15:04.549782Z","shell.execute_reply":"2022-01-23T23:15:04.569217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# text formatting\ntext_formats = {\n 'PURPLE': '\\033[95m',\n 'CYAN': '\\033[96m',\n 'DARKCYAN': '\\033[36m',\n 'BLUE': '\\033[94m',\n 'GREEN': '\\033[92m',\n 'YELLOW': '\\033[93m',\n 'RED': '\\033[91m',\n 'BOLD': '\\033[1m',\n 'UNDERLINE': '\\033[4m',\n 'END': '\\033[0m'\n}\n\nRED = text_formats['RED']\nBOLD = text_formats['BOLD']\nEND = text_formats['END']","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:15:04.571413Z","iopub.execute_input":"2022-01-23T23:15:04.572020Z","iopub.status.idle":"2022-01-23T23:15:04.583355Z","shell.execute_reply.started":"2022-01-23T23:15:04.571977Z","shell.execute_reply":"2022-01-23T23:15:04.582646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n## Import data sets\n\n# folder\ndata_folder = \"../input/g-research-crypto-forecasting/\"\n!ls $data_folder\n\n# tickers \ntickers = ['BNB', 'BTC', 'BCH', 'ADA', 'DOGE', 'EOS', 'ETH', 'ETC', 'IOTA', 'LTC', 'MKR', 'XMR', 'XLM', 'TRX']\n\n# data types\ndtypes = {\n 'timestamp': str,\n 'Asset_ID': np.int8,\n 'Count': np.int32,\n 'Open': np.float64,\n 'High': np.float64,\n 'Low': np.float64,\n 'Close': np.float64,\n 'Volume': np.float64,\n 'VWAP': np.float64,\n 'Target': np.float64,\n}\n\n# data sets \n# example submission\nexample_submission_df = read_dataset(data_folder, \"example_sample_submission\")\n\n# test df\ntest_df = read_dataset(data_folder, \"example_test\")\ntest_df['timestamp'] = pd.to_datetime(test_df['timestamp'], unit='s') \n\n# asset details \nassets_df = read_dataset(data_folder, \"asset_details\")\nassets_df = assets_df.sort_values(by='Asset_ID')\n\nmapping_name = dict(assets_df[['Asset_ID', 'Asset_Name']].values)\nmapping_weight = dict(assets_df[['Asset_ID', 'Weight']].values)\nmapping_tickers = dict(enumerate(tickers))\nassets_df['Ticker'] = assets_df[\"Asset_ID\"].map(mapping_tickers)\n\n# train df\ntrain_df_raw = read_dataset(data_folder, \"train\", dtypes)\n\nprint(f\"Shape example submission file: {example_submission_df.shape}\")\nprint(f\"Shape test data file: {test_df.shape}\")\nprint(f\"Shape asset details file: {assets_df.shape}\")\nprint(f\"Shape train data file: {train_df_raw.shape}\")\n\ntrain_df_raw.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:15:04.585772Z","iopub.execute_input":"2022-01-23T23:15:04.585991Z","iopub.status.idle":"2022-01-23T23:16:17.146132Z","shell.execute_reply.started":"2022-01-23T23:15:04.585966Z","shell.execute_reply":"2022-01-23T23:16:17.144935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assets_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.147798Z","iopub.execute_input":"2022-01-23T23:16:17.148059Z","iopub.status.idle":"2022-01-23T23:16:17.152959Z","shell.execute_reply.started":"2022-01-23T23:16:17.148024Z","shell.execute_reply":"2022-01-23T23:16:17.152005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.154226Z","iopub.execute_input":"2022-01-23T23:16:17.154453Z","iopub.status.idle":"2022-01-23T23:16:17.166126Z","shell.execute_reply.started":"2022-01-23T23:16:17.154423Z","shell.execute_reply":"2022-01-23T23:16:17.165333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Important!! Test data starts at 2021-06-13 and training data ends at 2021-09-21. To prevent data leakage, we need to filter train_df before the test data starts.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Prepare first train data\nTaking in the raw train data, we wrangle the data in a first step perform the following steps:\n* Transform unix timestamp to a regular timestamp\n* Extract simple date values like date, year, and month\n* Add colomns from asset details including the ticket, which makes the Data Frames more easy to filter\n* Rename count to interval_trade_count","metadata":{}},{"cell_type":"code","source":"# %%time\n\n# def prepare_train_data(df):\n# \"\"\"This function takes in the raw training data and wrangles the data in the shape and types \n# required for further research\"\"\"\n \n# CUTOFF_DATE = '2021-06-13 00:00:00'\n \n# df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s') # convert from unix to datetime timestamp\n# df = df.set_index('timestamp') # set date as index column\n \n# dates = df.index.get_level_values('timestamp')\n# df['Date'] = pd.to_datetime(dates.date)\n# df['Date'] = df['Date'].dt.strftime('%Y%m%d').astype(int) # SLOWS IT DOWN BY 3 MINS! NEED STH BETTER\n# df['Year'] = dates.year\n# df['Month'] = dates.month\n \n# df[\"Asset_Name\"] = df[\"Asset_ID\"].map(mapping_name)\n# df[\"Asset_Weight\"] = df[\"Asset_ID\"].map(mapping_weight)\n# df[\"Ticker\"] = df[\"Asset_ID\"].map(mapping_tickers)\n \n# df.sort_index()\n# df = df[(df.index < CUTOFF_DATE)]\n \n# total_num_records = df.shape[0]\n# print(f\"Total number of records in train dataset: {total_num_records}\")\n \n# return df\n\n# df_train = prepare_train_data(train_df_raw)\n# print(df_train.dtypes)\n# df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.167575Z","iopub.execute_input":"2022-01-23T23:16:17.168098Z","iopub.status.idle":"2022-01-23T23:16:17.178487Z","shell.execute_reply.started":"2022-01-23T23:16:17.168054Z","shell.execute_reply":"2022-01-23T23:16:17.177677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Populate missing time series rows ","metadata":{}},{"cell_type":"code","source":"# %%time\n# crypto_dfs = list()\n \n# for i in list(range(14)):\n# symbol_df = df_train[df_train['Asset_ID'] == i]\n# crypto_dfs.append(populate_missing_rows(symbol_df))\n\n# df_train = pd.concat(crypto_dfs).sort_index() \n# df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.181486Z","iopub.execute_input":"2022-01-23T23:16:17.182163Z","iopub.status.idle":"2022-01-23T23:16:17.193596Z","shell.execute_reply.started":"2022-01-23T23:16:17.182121Z","shell.execute_reply":"2022-01-23T23:16:17.192634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# # all_cryptos = generate_crypto_dfs(df_train)\n# btc = df_train.query(\"Ticker == 'BTC'\")\n# btc.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.195455Z","iopub.execute_input":"2022-01-23T23:16:17.196334Z","iopub.status.idle":"2022-01-23T23:16:17.203007Z","shell.execute_reply.started":"2022-01-23T23:16:17.196295Z","shell.execute_reply":"2022-01-23T23:16:17.202415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(df_train.index))","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.205082Z","iopub.execute_input":"2022-01-23T23:16:17.205399Z","iopub.status.idle":"2022-01-23T23:16:17.213962Z","shell.execute_reply.started":"2022-01-23T23:16:17.205356Z","shell.execute_reply":"2022-01-23T23:16:17.213111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n### Import preprepped train data\nfolder_train_prepped = \"../input/g-research/\"\n!ls $data_folder_prepped\ndf_train_prepped = pd.read_csv(folder_train_prepped + \"df_train_prep.csv\").set_index('timestamp') \ndf_train_prepped.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:16:17.215749Z","iopub.execute_input":"2022-01-23T23:16:17.216207Z","iopub.status.idle":"2022-01-23T23:17:58.575589Z","shell.execute_reply.started":"2022-01-23T23:16:17.216161Z","shell.execute_reply":"2022-01-23T23:17:58.574524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_prepped.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:58.577711Z","iopub.execute_input":"2022-01-23T23:17:58.578812Z","iopub.status.idle":"2022-01-23T23:17:58.585539Z","shell.execute_reply.started":"2022-01-23T23:17:58.578759Z","shell.execute_reply":"2022-01-23T23:17:58.584680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for ticker in tickers:\n# ticker_df = df_train_prepped.query(\"Ticker == @ticker\")\n# print(f\"{ticker} contains how many inf VWAP values: {np.isinf(ticker_df.VWAP).values.sum()}\")\n# ticker_df['VWAP']= ticker_df['VWAP'].replace(np.inf, ticker_df.VWAP.mean())\n# print(f\"{ticker} contains how many inf VWAP values: {np.isinf(ticker_df.VWAP).values.sum()}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:58.589242Z","iopub.execute_input":"2022-01-23T23:17:58.590038Z","iopub.status.idle":"2022-01-23T23:17:58.598973Z","shell.execute_reply.started":"2022-01-23T23:17:58.590002Z","shell.execute_reply":"2022-01-23T23:17:58.598111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\n\nFeatures considered:\n\n* Price transformation features\n* RSI\n* Historical Volatility\n* Momentum features\n* Rolling Average Volume","metadata":{}},{"cell_type":"code","source":"features_submission = ['VWAP', 'Avg_price', 'Weighted_close', 'Typical_Price', 'Median_price', \\\n 'Dollar_volume', 'Volume_per_trade', 'Dollar_volume_per_trade', 'Upper_shadow', 'Lower_shadow']\n\ndef calc_features_submission(df):\n \n # Price transformation features\n df['Avg_price'] = (df['Close'] + df['Open'] + df['Low'] + df['High']) / 4\n df['Weighted_close'] = ((df['Close'] * 2) + df['High'] + df['Low']) / 4 # extra weight on the close pri\n df['Typical_price'] = (df['High'] + df['Low'] + df['Close']) / 3 \n df['Median_price'] = (df['High'] + df['Low']) / 2\n \n df['Dollar_volume'] = df['Close'] * df['Volume']\n df['Volume_per_trade'] = df['Volume'] / df['Count']\n df['Dollar_volume_per_trade'] = df['Dollar_volume'] / df['Count']\n \n df['Upper_shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n df['Lower_shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n \n# df_feat = pd.DataFrame(df, columns=features_submission)\n \n return df","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:58.600104Z","iopub.execute_input":"2022-01-23T23:17:58.600328Z","iopub.status.idle":"2022-01-23T23:17:58.616068Z","shell.execute_reply.started":"2022-01-23T23:17:58.600300Z","shell.execute_reply":"2022-01-23T23:17:58.615143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_features(df, daily=True):\n \n# df = reduce_mem_usage(df)\n \n CLOSE = df.Close\n DF_DAILY = resample_timeseries(df, 'D')\n \n # Price transformation features\n df['Avg_price'] = (df['Close'] + df['Open'] + df['Low'] + df['High']) / 4\n df['Weighted_close'] = ((df['Close'] * 2) + df['High'] + df['Low']) / 4 # extra weight on the close pri\n df['Typical_price'] = (df['High'] + df['Low'] + df['Close']) / 3 \n df['Median_price'] = (df['High'] + df['Low']) / 2\n \n df['Dollar_volume'] = df['Close'].mul(df['Volume'])\n df['Volume_per_trade'] = df['Volume'].div(df['Count'])\n df['Dollar_volume_per_trade'] = df['Dollar_volume'].div(df['Count'])\n \n df_features_daily = calc_daily_features(DF_DAILY) \n\n # Join daily features back to original df which contains minute data\n if daily:\n return df_features_daily\n else:\n df['timestamp'] = df.index\n df = df.merge(df_features_daily, on='Date', how='left').set_index(df['timestamp']).drop(columns='timestamp')\n df = df.dropna(subset=['Close_close_vol_30d'])\n return df ","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:58.617448Z","iopub.execute_input":"2022-01-23T23:17:58.617712Z","iopub.status.idle":"2022-01-23T23:17:58.631820Z","shell.execute_reply.started":"2022-01-23T23:17:58.617681Z","shell.execute_reply":"2022-01-23T23:17:58.630824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# btc_feat = calc_features(btc, False) \n# btc_feat.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:58.633674Z","iopub.execute_input":"2022-01-23T23:17:58.634266Z","iopub.status.idle":"2022-01-23T23:17:58.647081Z","shell.execute_reply.started":"2022-01-23T23:17:58.634221Z","shell.execute_reply":"2022-01-23T23:17:58.646298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"%%time\nfrom xgboost import XGBRegressor\nimport lightgbm as lgb\nfrom sklearn.preprocessing import StandardScaler\n\nfeatures = ['vwap', 'weighted_close', 'avg_price', 'median_price', 'typical_price', 'sma_10d', 'sma_20d', 'rsi_14d', 'atr_14d', 'close_close_vol_10d', \\\n 'close_close_vol_20d'] \n\n# parameters\nparams = {\n 'objective': 'regression',\n 'metric': 'rmse',\n 'boosting_type': 'gbdt',\n 'max_depth': -1,\n 'learning_rate': 0.01\n }\n\ndef calc_Xy_and_model_for_ticker(df):\n \n name = df.iloc[0]['Asset_Name']\n df.replace([np.inf, -np.inf], np.nan,inplace=True) # Replace inf values (Maker has a few inf VWAP values)\n \n # TODO: Try different features here!\n df_feat = calc_features_submission(df)\n# df_feat = reduce_mem_usage(df_feat) # reduce mem usage\n df_feat['y'] = df['Target']\n df_feat = df_feat.dropna(how=\"any\") # drop rows with null target rows \n \n X = pd.DataFrame(df_feat, columns=features_submission)\n scaler = StandardScaler()\n X_scaled = scaler.fit_transform(X)\n \n y = df_feat[\"y\"]\n \n print(f\"Shape {name} X_scaled: {X_scaled.shape}\")\n print(f\"Shape {name} y: {y.shape}\")\n \n # TODO: Try different models here!\n model = lgb.LGBMRegressor(**params, n_estimators=10)\n print(f\"Building model for {name}..\")\n model.fit(X_scaled, y)\n \n# compute_feature_importance(model)\n \n return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:58.648450Z","iopub.execute_input":"2022-01-23T23:17:58.649377Z","iopub.status.idle":"2022-01-23T23:17:59.948478Z","shell.execute_reply.started":"2022-01-23T23:17:58.649330Z","shell.execute_reply":"2022-01-23T23:17:59.947377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs, ys, models = {}, {}, {}\n\nfor asset_id, ticker in zip(assets_df['Asset_ID'], assets_df['Ticker']):\n \n asset_df = df_train_prepped.query(\"Ticker == @ticker\")\n X, y, model = calc_Xy_and_model_for_ticker(asset_df) \n Xs[ticker], ys[ticker], models[ticker] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:17:59.950209Z","iopub.execute_input":"2022-01-23T23:17:59.950440Z","iopub.status.idle":"2022-01-23T23:19:20.042874Z","shell.execute_reply.started":"2022-01-23T23:17:59.950411Z","shell.execute_reply":"2022-01-23T23:19:20.042138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# btc = df_train.query(\"Ticker == 'BTC'\")\n# btc_ticker = btc.iloc[0]['Ticker']\n\nprint(models.keys())","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:19:20.044339Z","iopub.execute_input":"2022-01-23T23:19:20.044783Z","iopub.status.idle":"2022-01-23T23:19:20.050006Z","shell.execute_reply.started":"2022-01-23T23:19:20.044750Z","shell.execute_reply":"2022-01-23T23:19:20.049061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:19:20.051315Z","iopub.execute_input":"2022-01-23T23:19:20.051770Z","iopub.status.idle":"2022-01-23T23:19:20.082861Z","shell.execute_reply.started":"2022-01-23T23:19:20.051739Z","shell.execute_reply":"2022-01-23T23:19:20.081790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"\nfor i, (df_test, df_pred) in enumerate(iter_test):\n \n df_test['Ticker'] = df_test[\"Asset_ID\"].map(mapping_tickers)\n \n for j , row in df_test.iterrows():\n \n if models[row['Ticker']] is not None:\n try:\n model = models[row['Ticker']]\n x_test = calc_features_submission(row)\n x_test = pd.DataFrame([x_test], columns=features_submission)\n print(x_test)\n y_pred = model.predict(x_test)[0]\n df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n except:\n df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n traceback.print_exc()\n else: \n df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n \n env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T23:19:20.084543Z","iopub.execute_input":"2022-01-23T23:19:20.084866Z","iopub.status.idle":"2022-01-23T23:19:20.952184Z","shell.execute_reply.started":"2022-01-23T23:19:20.084824Z","shell.execute_reply":"2022-01-23T23:19:20.951432Z"},"trusted":true},"execution_count":null,"outputs":[]}]}