{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Machine Learning\n## Final Project LGBM","metadata":{"id":"IELL8z8wuhb5"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.linear_model import LassoCV\nimport matplotlib.pyplot as plt","metadata":{"id":"WkyKv1DfuhcA","execution":{"iopub.status.busy":"2022-01-16T14:52:05.706919Z","iopub.execute_input":"2022-01-16T14:52:05.711902Z","iopub.status.idle":"2022-01-16T14:52:05.722488Z","shell.execute_reply.started":"2022-01-16T14:52:05.711649Z","shell.execute_reply":"2022-01-16T14:52:05.721013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {\n    \"objective\":\"regression\",\n    \"n_estimators\":8000,     \n    \"num_leaves\":1000,      \n    \"learning_rate\":0.1,   \n    \"seed\":1231,\n    \"tree_learner\":\"feature\",\n    \"feature_fraction\":0.88,\n    \"verbose\":1,\n    \"device_type\":\"gpu\"}","metadata":{"id":"0g__lDqPuhcB","execution":{"iopub.status.busy":"2022-01-16T14:37:47.697328Z","iopub.execute_input":"2022-01-16T14:37:47.702879Z","iopub.status.idle":"2022-01-16T14:37:47.719035Z","shell.execute_reply.started":"2022-01-16T14:37:47.70283Z","shell.execute_reply":"2022-01-16T14:37:47.718226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\ndef get_features(df_feat):\n\n    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    df_feat[\"open_sub_close\"] = df_feat[\"Open\"] - df_feat[\"Close\"]\n\n    df_feat['trade']=df_feat['Close']-df_feat['Open']\n    df_feat['gtrade']=df_feat['trade']/df_feat['Count']\n\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_Shadow'] = lower_shadow(df_feat)   \n    df_feat['shadow1']=df_feat['trade']/df_feat['Volume']\n    df_feat['shadow2']=df_feat['upper_Shadow']/df_feat['Low']\n    df_feat['shadow3']=df_feat['upper_Shadow']/df_feat['Volume']\n    df_feat['shadow4']=df_feat['lower_Shadow']/df_feat['High']\n    df_feat['shadow5']=df_feat['lower_Shadow']/df_feat['Volume']    \n\n    df_feat['spread'] = df_feat['High'] - df_feat['Low']\n    df_feat['mean_trade'] = df_feat['Volume']/df_feat['Count']\n    df_feat['diff1'] = df_feat['Volume'] - df_feat['Count']\n    df_feat['mean1'] = (df_feat['shadow5'] + df_feat['shadow3']) / 2\n    df_feat['mean2'] = (df_feat['shadow1'] + df_feat['Volume']) / 2\n    df_feat['mean3'] = (df_feat['trade'] + df_feat['gtrade']) / 2\n    df_feat['mean4'] = (df_feat['diff1'] + df_feat['upper_Shadow']) / 2\n    df_feat['mean5'] = (df_feat['diff1'] + df_feat['lower_Shadow']) / 2\n    df_feat['UPS'] = (df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open']))\n    df_feat['UPS'] = df_feat['UPS']\n    df_feat['LOS'] = (np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low'])\n    df_feat['LOS'] = df_feat['LOS']\n    df_feat['RNG'] = ((df_feat['High'] - df_feat['Low']) / df_feat['VWAP'])\n    df_feat['RNG'] = df_feat['RNG']\n    df_feat['MOV'] = ((df_feat['Close'] - df_feat['Open']) / df_feat['VWAP'])\n    df_feat['MOV'] = df_feat['MOV']\n    df_feat['CLS'] = ((df_feat['Close'] - df_feat['VWAP']) / df_feat['VWAP'])\n    df_feat['CLS'] = df_feat['CLS']\n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"]\n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"]\n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"]\n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis = 1)\n    df_feat[\"High/Mean\"] = df_feat[\"High\"] / df_feat[\"Mean\"]\n    df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n    df_feat[\"Volume/Count\"] = df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1)\n    mean_price = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    median_price = df_feat[['Open', 'High', 'Low', 'Close']].median(axis=1)\n    df_feat['high2mean'] = df_feat['High'] / mean_price\n    df_feat['low2mean'] = df_feat['Low'] / mean_price\n    df_feat['high2median'] = df_feat['High'] / median_price\n    df_feat['low2median'] = df_feat['Low'] / median_price\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:44:33.222014Z","iopub.execute_input":"2022-01-16T14:44:33.222383Z","iopub.status.idle":"2022-01-16T14:44:33.249306Z","shell.execute_reply.started":"2022-01-16T14:44:33.222334Z","shell.execute_reply":"2022-01-16T14:44:33.245379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"id":"cC5Hl5sduhcC","execution":{"iopub.status.busy":"2022-01-16T14:41:35.571597Z","iopub.execute_input":"2022-01-16T14:41:35.572362Z","iopub.status.idle":"2022-01-16T14:41:35.59277Z","shell.execute_reply.started":"2022-01-16T14:41:35.572312Z","shell.execute_reply":"2022-01-16T14:41:35.591828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import gresearch_crypto\n# env = gresearch_crypto.make_env()\n# iter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ndf_train_sup = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')\ndf_asset = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')","metadata":{"id":"vY-hEvgtuhcH","execution":{"iopub.status.busy":"2022-01-16T14:44:33.251415Z","iopub.execute_input":"2022-01-16T14:44:33.251785Z","iopub.status.idle":"2022-01-16T14:45:28.362741Z","shell.execute_reply.started":"2022-01-16T14:44:33.251735Z","shell.execute_reply":"2022-01-16T14:45:28.361638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def features_selection(df) :\n#     features = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\", \"timestamp\"]\n#     x = df[features].copy()\n#     return x","metadata":{"id":"6_peXrMOuhcJ","execution":{"iopub.status.busy":"2022-01-16T14:45:33.800654Z","iopub.execute_input":"2022-01-16T14:45:33.801478Z","iopub.status.idle":"2022-01-16T14:45:33.808556Z","shell.execute_reply.started":"2022-01-16T14:45:33.80144Z","shell.execute_reply":"2022-01-16T14:45:33.807472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[~df_train.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\ndf_train_sup = df_train_sup[~df_train_sup.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\ndf_combine = pd.concat(\n    [df_train, df_train_sup], axis = 0\n)\n\ndel df_train, df_train_sup","metadata":{"id":"6j6y5o4quhcK","execution":{"iopub.status.busy":"2022-01-16T14:45:33.810415Z","iopub.execute_input":"2022-01-16T14:45:33.811007Z","iopub.status.idle":"2022-01-16T14:45:39.604598Z","shell.execute_reply.started":"2022-01-16T14:45:33.810949Z","shell.execute_reply":"2022-01-16T14:45:39.60347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.mode.chained_assignment = None\nSEED = 1126\nfor asset_id in range(14):\n    print(f\"Asset ID = {asset_id}\")\n    df_sub = df_combine[df_combine[\"Asset_ID\"] == asset_id]\n    df_sub = get_features(df_sub)\n    \n    #preprocess\n    df_sub = df_sub.drop([\"timestamp\", \"Asset_ID\"], axis=1)\n    X_sub = df_sub.loc[:, df_sub.columns != \"Target\"]\n    y_sub = df_sub[\"Target\"]\n    \n    del df_sub\n    \n    #use L1 reg.\n    X_sub = X_sub.fillna(0)\n    clf = LassoCV(\n        random_state=SEED, max_iter = 50000, cv = 5\n        ).fit(X_sub, y_sub)\n    \n    final_features_l1Reg = X_sub.columns[\n        (abs(clf.coef_) > 0).flatten()\n    ]\n    print(f\"There are {len(final_features_l1Reg)} features selected by L1 regression\")\n    print(final_features_l1Reg.tolist())\n    \n    \n    THRESHOLD = 0.01\n    corr_value = X_sub.corrwith(y_sub)\n    corr_abs = abs(corr_value).sort_values(ascending=False)\n    final_features_corr = corr_abs[corr_abs > THRESHOLD].index\n    print(f\"There are {len(final_features_corr)} features selected by correlation coefficient\")\n    print(final_features_corr.tolist())\n    \n    final_features = list(\n    set(final_features_l1Reg).union(\n            set(final_features_corr)\n        )\n    )\n    print(f\"There are {len(final_features)} final features selected, union by L1 and corr coef. features:\")\n    print(final_features)\n    \n    plt.figure(figsize=(10,10))\n    plt.plot(corr_abs)\n    plt.xticks(rotation=45) \n    plt.show()\n    del X_sub, y_sub","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:59:32.609724Z","iopub.execute_input":"2022-01-16T14:59:32.610074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def data_preprocess(df_data ,asset_id):\n#     df = df_data[df_data[\"Asset_ID\"] == asset_id]\n#     df = get_features(df)\n#     return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def training(asset_id):\n#     df = data_preprocess(df_train, asset_id)\n#     df = df.append(data_preprocess(df_train_sup, asset_id), ignore_index = True)\n#     x = features_selection(df)\n#     y = df['Target'].copy()\n#     model = lgb.LGBMRegressor(**lgb_params)\n#     model.fit(x, y)\n#     return model","metadata":{"id":"oLUiD2PfuhcL","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models = {}\n# for asset_id in range(14):\n#     print(\"Training model for\", df_asset[df_asset[\"Asset_ID\"] == asset_id].iloc[0][\"Asset_Name\"])\n#     model = training(asset_id)\n#     models[asset_id] = model","metadata":{"id":"yNCOc5juuhcN","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i, (df_test, df_pred) in enumerate(iter_test):\n#     for j, df_row in df_test.iterrows():              \n#         model = models[df_row['Asset_ID']]\n#         x_test = features_selection(df_row)\n#         y_pred = model.predict(pd.DataFrame([x_test]))[0]\n#         df_pred.loc[df_pred['row_id'] == df_row['row_id'], 'Target'] = y_pred    \n#     env.predict(df_pred)","metadata":{"id":"P0HfNhq8uhcN","trusted":true},"execution_count":null,"outputs":[]}]}