{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84315,"databundleVersionId":9437184,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Required Libraries","metadata":{}},{"cell_type":"code","source":"!pip install -q fastbook\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:07:49.673073Z","iopub.execute_input":"2024-10-07T17:07:49.673793Z","iopub.status.idle":"2024-10-07T17:08:05.219456Z","shell.execute_reply.started":"2024-10-07T17:07:49.673753Z","shell.execute_reply":"2024-10-07T17:08:05.218178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q ta","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:08:05.221542Z","iopub.execute_input":"2024-10-07T17:08:05.221875Z","iopub.status.idle":"2024-10-07T17:08:19.136661Z","shell.execute_reply.started":"2024-10-07T17:08:05.221840Z","shell.execute_reply":"2024-10-07T17:08:19.135228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom sklearn import datasets, linear_model\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport requests\nimport joblib\n\nnp.random.seed(7)\nimport math\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import LSTM\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom fastai.tabular.all import *\nfrom fastbook import *\nimport ta\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-07T17:08:19.138799Z","iopub.execute_input":"2024-10-07T17:08:19.139225Z","iopub.status.idle":"2024-10-07T17:08:41.783884Z","shell.execute_reply.started":"2024-10-07T17:08:19.139177Z","shell.execute_reply":"2024-10-07T17:08:41.783125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.mode.chained_assignment = None","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:08:41.786443Z","iopub.execute_input":"2024-10-07T17:08:41.787279Z","iopub.status.idle":"2024-10-07T17:08:41.791170Z","shell.execute_reply.started":"2024-10-07T17:08:41.787232Z","shell.execute_reply":"2024-10-07T17:08:41.790273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/cryptocurrency-price-direction-predictor/train.csv\")\ndf_tst = pd.read_csv(\"/kaggle/input/cryptocurrency-price-direction-predictor/test.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:08:41.792250Z","iopub.execute_input":"2024-10-07T17:08:41.792539Z","iopub.status.idle":"2024-10-07T17:08:48.832476Z","shell.execute_reply.started":"2024-10-07T17:08:41.792507Z","shell.execute_reply":"2024-10-07T17:08:48.831679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:08:48.835843Z","iopub.execute_input":"2024-10-07T17:08:48.836235Z","iopub.status.idle":"2024-10-07T17:08:48.866585Z","shell.execute_reply.started":"2024-10-07T17:08:48.836200Z","shell.execute_reply":"2024-10-07T17:08:48.865721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:08:48.867615Z","iopub.execute_input":"2024-10-07T17:08:48.867907Z","iopub.status.idle":"2024-10-07T17:08:48.893878Z","shell.execute_reply.started":"2024-10-07T17:08:48.867875Z","shell.execute_reply":"2024-10-07T17:08:48.893063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:08:48.894817Z","iopub.execute_input":"2024-10-07T17:08:48.895147Z","iopub.status.idle":"2024-10-07T17:08:49.620554Z","shell.execute_reply.started":"2024-10-07T17:08:48.895113Z","shell.execute_reply":"2024-10-07T17:08:49.619486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualization ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.plot(df.index, df['close'])\nplt.title('Closing Price Over Time')\nplt.xlabel('Date')\nplt.ylabel('Price')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T16:55:26.283508Z","iopub.execute_input":"2024-10-07T16:55:26.283854Z","iopub.status.idle":"2024-10-07T16:55:26.831474Z","shell.execute_reply.started":"2024-10-07T16:55:26.283819Z","shell.execute_reply":"2024-10-07T16:55:26.830575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 16))\nsns.heatmap(df.corr(), annot=True, cmap='coolwarm', linewidths=0.5)\nplt.title('Correlation Heatmap of Features')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T16:56:48.362904Z","iopub.execute_input":"2024-10-07T16:56:48.363323Z","iopub.status.idle":"2024-10-07T16:56:49.836444Z","shell.execute_reply.started":"2024-10-07T16:56:48.363284Z","shell.execute_reply":"2024-10-07T16:56:49.835455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for column in df.columns:\n#    plt.figure(figsize=(10, 6))\n#    sns.histplot(df[column], kde=True)\n#    plt.title(f'Distribution of {column}')\n#    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for column in df.columns:\n#    if column != 'target':\n#        plt.figure(figsize=(10, 6))\n#        sns.boxplot(x='target', y=column, data=df)\n#        plt.title(f'Distribution of {column} by Target')\n#        plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sns.pairplot(df[['open', 'high', 'low', 'close', 'volume', 'target']], \n#             hue='target', height=2.5)\n#plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection","metadata":{}},{"cell_type":"code","source":"def add_technical_indicators(df):\n    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s')\n    df.set_index('timestamp', inplace=True)\n    # RSI\n    df['rsi'] = ta.momentum.RSIIndicator(df['close']).rsi()\n    df['returns'] = df['close'].pct_change()\n    df['price_range'] = df['high'] - df['low']\n    #df['volume_ma'] = df['volume'].rolling(window=20).mean()\n    # MACD\n    macd = ta.trend.MACD(df['close'])\n    df['macd'] = macd.macd()\n    df['macd_signal'] = macd.macd_signal()\n    #df['macd_diff'] = macd.macd_diff()\n\n    # Bollinger Bands\n    bollinger = ta.volatility.BollingerBands(df['close'])\n    df['bollinger_hband'] = bollinger.bollinger_hband()\n    #df['bollinger_lband'] = bollinger.bollinger_lband()\n    #df['bollinger_mavg'] = bollinger.bollinger_mavg()\n\n    # Volume indicators\n    df['volume_adi'] = ta.volume.AccDistIndexIndicator(df['high'], df['low'], df['close'], df['volume']).acc_dist_index()\n    #df['volume_obv'] = ta.volume.OnBalanceVolumeIndicator(df['close'], df['volume']).on_balance_volume()\n\n    # Volatility\n    df['volatility_atr'] = ta.volatility.AverageTrueRange(df['high'], df['low'], df['close']).average_true_range()\n    #Get scaled differences between open and close\n    df['delta'] = (df.close - df.open) / df.open\n    # Adding a previous close column\n    #df['prev_close'] = df['close'].shift(1)\n    return df\n\n#df = add_technical_indicators(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:08.727378Z","iopub.execute_input":"2024-10-07T17:19:08.727792Z","iopub.status.idle":"2024-10-07T17:19:08.737275Z","shell.execute_reply.started":"2024-10-07T17:19:08.727751Z","shell.execute_reply":"2024-10-07T17:19:08.736088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_time_features(df):\n    df['hour'] = df.index.hour\n    return df\n\n#df = add_time_features(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:09.043196Z","iopub.execute_input":"2024-10-07T17:19:09.044174Z","iopub.status.idle":"2024-10-07T17:19:09.049224Z","shell.execute_reply.started":"2024-10-07T17:19:09.044121Z","shell.execute_reply":"2024-10-07T17:19:09.048147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_roc_features(df):\n    df['roc_short'] = ta.momentum.ROCIndicator(df['close'], window=12).roc()\n    df['roc_medium'] = ta.momentum.ROCIndicator(df['close'], window=50).roc()\n    df['roc_long'] = ta.momentum.ROCIndicator(df['close'], window=120).roc()\n    return df\n\n#df = add_roc_features(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:09.774199Z","iopub.execute_input":"2024-10-07T17:19:09.775034Z","iopub.status.idle":"2024-10-07T17:19:09.781733Z","shell.execute_reply.started":"2024-10-07T17:19:09.774968Z","shell.execute_reply":"2024-10-07T17:19:09.780748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_ema_features(df):\n    df['ema_short'] = ta.trend.EMAIndicator(df['close'], window=10).ema_indicator()\n    #df['ema_medium'] = ta.trend.EMAIndicator(df['close'], window=50).ema_indicator()\n    #df['ema_long'] = ta.trend.EMAIndicator(df['close'], window=120).ema_indicator()\n    return df\n\ndf = add_ema_features(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:10.086753Z","iopub.execute_input":"2024-10-07T17:19:10.087602Z","iopub.status.idle":"2024-10-07T17:19:10.137242Z","shell.execute_reply.started":"2024-10-07T17:19:10.087550Z","shell.execute_reply":"2024-10-07T17:19:10.136234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_lag_features(df, lags=[1,30,60]):\n    for lag in lags:\n        #df[f'close_lag_{lag}'] = df['close'].shift(lag)\n        df[f'return_lag_{lag}'] = df['returns'].shift(lag)\n    return df\n\n#df = add_lag_features(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:10.419895Z","iopub.execute_input":"2024-10-07T17:19:10.420307Z","iopub.status.idle":"2024-10-07T17:19:10.425854Z","shell.execute_reply.started":"2024-10-07T17:19:10.420269Z","shell.execute_reply":"2024-10-07T17:19:10.424797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_features = df_tst.drop(columns=['row_id']).columns \ntest_id = df_tst['row_id']","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:11.698600Z","iopub.execute_input":"2024-10-07T17:19:11.698984Z","iopub.status.idle":"2024-10-07T17:19:11.727337Z","shell.execute_reply.started":"2024-10-07T17:19:11.698947Z","shell.execute_reply":"2024-10-07T17:19:11.726266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([df[base_features], df['target']], axis=1)\ntest_df = df_tst[base_features]","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:14.619624Z","iopub.execute_input":"2024-10-07T17:19:14.620028Z","iopub.status.idle":"2024-10-07T17:19:14.756798Z","shell.execute_reply.started":"2024-10-07T17:19:14.619969Z","shell.execute_reply":"2024-10-07T17:19:14.755884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = add_technical_indicators(train_df)\ntrain_df = add_time_features(train_df)\ntrain_df = add_roc_features(train_df)\ntrain_df = add_ema_features(train_df)\ntrain_df = add_lag_features(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:15.309681Z","iopub.execute_input":"2024-10-07T17:19:15.310100Z","iopub.status.idle":"2024-10-07T17:19:39.926289Z","shell.execute_reply.started":"2024-10-07T17:19:15.310059Z","shell.execute_reply":"2024-10-07T17:19:39.925326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = add_technical_indicators(test_df)\ntest_df = add_time_features(test_df)\ntest_df = add_roc_features(test_df)\ntest_df = add_ema_features(test_df)\ntest_df = add_lag_features(test_df)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:19:39.927647Z","iopub.execute_input":"2024-10-07T17:19:39.927944Z","iopub.status.idle":"2024-10-07T17:19:50.509654Z","shell.execute_reply.started":"2024-10-07T17:19:39.927910Z","shell.execute_reply":"2024-10-07T17:19:50.508780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(40, 32))\nsns.heatmap(train_df.corr(), annot=True, cmap='coolwarm', linewidths=0.5)\nplt.title('Correlation Heatmap of Features')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from pandas.plotting import autocorrelation_plot\n\n#plt.figure(figsize=(15, 10))\n#autocorrelation_plot(train_df['returns'])\n#plt.title('Autocorrelation of Returns')\n#plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Processing","metadata":{}},{"cell_type":"code","source":"procs = [Categorify, FillMissing]\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:12.029592Z","iopub.execute_input":"2024-10-07T17:28:12.029988Z","iopub.status.idle":"2024-10-07T17:28:12.034416Z","shell.execute_reply.started":"2024-10-07T17:28:12.029950Z","shell.execute_reply":"2024-10-07T17:28:12.033393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cond = (train_df.index.year<2020) | (train_df.index.month<9)\ntrain_idx = np.where( cond)[0]\nvalid_idx = np.where(~cond)[0]\nsplits = (list(train_idx),list(valid_idx))","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:12.716752Z","iopub.execute_input":"2024-10-07T17:28:12.717181Z","iopub.status.idle":"2024-10-07T17:28:13.018720Z","shell.execute_reply.started":"2024-10-07T17:28:12.717139Z","shell.execute_reply":"2024-10-07T17:28:13.017823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dep_var = 'target'\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:13.306655Z","iopub.execute_input":"2024-10-07T17:28:13.307071Z","iopub.status.idle":"2024-10-07T17:28:13.311982Z","shell.execute_reply.started":"2024-10-07T17:28:13.307024Z","shell.execute_reply":"2024-10-07T17:28:13.311042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cont,cat = cont_cat_split(train_df, 1, dep_var=dep_var)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:13.953327Z","iopub.execute_input":"2024-10-07T17:28:13.953700Z","iopub.status.idle":"2024-10-07T17:28:13.975243Z","shell.execute_reply.started":"2024-10-07T17:28:13.953664Z","shell.execute_reply":"2024-10-07T17:28:13.974243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to = TabularPandas(train_df, procs, cat, cont, y_names=dep_var, splits=splits)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:14.582436Z","iopub.execute_input":"2024-10-07T17:28:14.583452Z","iopub.status.idle":"2024-10-07T17:28:17.197340Z","shell.execute_reply.started":"2024-10-07T17:28:14.583404Z","shell.execute_reply":"2024-10-07T17:28:17.196332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to1 = TabularPandas(test_df, procs, cat, cont)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:20.835409Z","iopub.execute_input":"2024-10-07T17:28:20.835798Z","iopub.status.idle":"2024-10-07T17:28:21.696250Z","shell.execute_reply.started":"2024-10-07T17:28:20.835759Z","shell.execute_reply":"2024-10-07T17:28:21.695322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = to1.train.xs","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:21.805606Z","iopub.execute_input":"2024-10-07T17:28:21.805987Z","iopub.status.idle":"2024-10-07T17:28:21.917884Z","shell.execute_reply.started":"2024-10-07T17:28:21.805950Z","shell.execute_reply":"2024-10-07T17:28:21.916970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(to.train),len(to.valid)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:22.770916Z","iopub.execute_input":"2024-10-07T17:28:22.771866Z","iopub.status.idle":"2024-10-07T17:28:23.026265Z","shell.execute_reply.started":"2024-10-07T17:28:22.771824Z","shell.execute_reply":"2024-10-07T17:28:23.025077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to.show(3)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T17:28:24.327838Z","iopub.execute_input":"2024-10-07T17:28:24.328591Z","iopub.status.idle":"2024-10-07T17:28:24.646197Z","shell.execute_reply.started":"2024-10-07T17:28:24.328547Z","shell.execute_reply":"2024-10-07T17:28:24.645074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xs,y = to.train.xs,to.train.y\nvalid_xs,valid_y = to.valid.xs,to.valid.y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random forest","metadata":{}},{"cell_type":"code","source":"def rf(xs, y, n_estimators=40, max_samples=1500000,\n       max_features=0.5, min_samples_leaf=5, **kwargs):\n    return RandomForestClassifier(n_jobs=-1, n_estimators=n_estimators,\n        max_samples=max_samples, max_features=max_features,\n        min_samples_leaf=min_samples_leaf, oob_score=True).fit(xs, y)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:32:30.111376Z","iopub.execute_input":"2024-10-06T16:32:30.111736Z","iopub.status.idle":"2024-10-06T16:32:30.117029Z","shell.execute_reply.started":"2024-10-06T16:32:30.111698Z","shell.execute_reply":"2024-10-06T16:32:30.116046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = rf(xs, y,n_estimators=40);\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:32:30.118221Z","iopub.execute_input":"2024-10-06T16:32:30.118551Z","iopub.status.idle":"2024-10-06T16:39:36.504123Z","shell.execute_reply.started":"2024-10-06T16:32:30.118509Z","shell.execute_reply":"2024-10-06T16:39:36.503252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def acc(pred, y):\n    return  accuracy_score(y, pred)\ndef m_acc(m, xs, y): return acc(m.predict(xs), y)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:39:36.505488Z","iopub.execute_input":"2024-10-06T16:39:36.505889Z","iopub.status.idle":"2024-10-06T16:39:36.512185Z","shell.execute_reply.started":"2024-10-06T16:39:36.505846Z","shell.execute_reply":"2024-10-06T16:39:36.511152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_acc(m, xs, y), m_acc(m, valid_xs, valid_y)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:39:36.513745Z","iopub.execute_input":"2024-10-06T16:39:36.514216Z","iopub.status.idle":"2024-10-06T16:39:51.824959Z","shell.execute_reply.started":"2024-10-06T16:39:36.514171Z","shell.execute_reply":"2024-10-06T16:39:51.823916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rf_feat_importance(m, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}\n                       ).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:39:51.826096Z","iopub.execute_input":"2024-10-06T16:39:51.826412Z","iopub.status.idle":"2024-10-06T16:39:51.831789Z","shell.execute_reply.started":"2024-10-06T16:39:51.826379Z","shell.execute_reply":"2024-10-06T16:39:51.830717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature importance","metadata":{}},{"cell_type":"code","source":"fi = rf_feat_importance(m, xs)\nfi[:30]","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:40:48.780439Z","iopub.execute_input":"2024-10-06T16:40:48.781193Z","iopub.status.idle":"2024-10-06T16:40:48.941397Z","shell.execute_reply.started":"2024-10-06T16:40:48.781153Z","shell.execute_reply":"2024-10-06T16:40:48.940445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_keep = fi[fi.imp>0.035].cols\nlen(to_keep)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:41:01.784974Z","iopub.execute_input":"2024-10-06T16:41:01.785454Z","iopub.status.idle":"2024-10-06T16:41:01.794332Z","shell.execute_reply.started":"2024-10-06T16:41:01.785409Z","shell.execute_reply":"2024-10-06T16:41:01.793052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xs_imp = xs[to_keep]\nvalid_xs_imp = valid_xs[to_keep]","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:41:04.811531Z","iopub.execute_input":"2024-10-06T16:41:04.811914Z","iopub.status.idle":"2024-10-06T16:41:04.894327Z","shell.execute_reply.started":"2024-10-06T16:41:04.811875Z","shell.execute_reply":"2024-10-06T16:41:04.893464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = rf(xs_imp, y,n_estimators=100)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:41:06.849897Z","iopub.execute_input":"2024-10-06T16:41:06.850312Z","iopub.status.idle":"2024-10-06T16:55:40.277255Z","shell.execute_reply.started":"2024-10-06T16:41:06.850268Z","shell.execute_reply":"2024-10-06T16:55:40.276220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_acc(m, xs_imp, y), m_acc(m, valid_xs_imp, valid_y)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:55:40.279166Z","iopub.execute_input":"2024-10-06T16:55:40.279525Z","iopub.status.idle":"2024-10-06T16:56:09.828075Z","shell.execute_reply.started":"2024-10-06T16:55:40.279487Z","shell.execute_reply":"2024-10-06T16:56:09.827050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cluster_columns(xs)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:56:09.829721Z","iopub.execute_input":"2024-10-06T16:56:09.830536Z","iopub.status.idle":"2024-10-06T16:56:19.656009Z","shell.execute_reply.started":"2024-10-06T16:56:09.830476Z","shell.execute_reply":"2024-10-06T16:56:19.655066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### final_features","metadata":{}},{"cell_type":"code","source":"cluster_columns(xs_imp)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:56:19.658613Z","iopub.execute_input":"2024-10-06T16:56:19.659442Z","iopub.status.idle":"2024-10-06T16:56:24.505826Z","shell.execute_reply.started":"2024-10-06T16:56:19.659385Z","shell.execute_reply":"2024-10-06T16:56:24.504865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = ts[to_keep]","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:56:24.507250Z","iopub.execute_input":"2024-10-06T16:56:24.507941Z","iopub.status.idle":"2024-10-06T16:56:24.534901Z","shell.execute_reply.started":"2024-10-06T16:56:24.507890Z","shell.execute_reply":"2024-10-06T16:56:24.534220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#rf_preds = m.predict(ts)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:56:24.536069Z","iopub.execute_input":"2024-10-06T16:56:24.536436Z","iopub.status.idle":"2024-10-06T16:56:33.296866Z","shell.execute_reply.started":"2024-10-06T16:56:24.536400Z","shell.execute_reply":"2024-10-06T16:56:33.296045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_id = pd.DataFrame(test_id, columns = ['row_id'])\n#test_id['target'] = rf_preds\n#test_id.to_csv('rf.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T16:56:33.298318Z","iopub.execute_input":"2024-10-06T16:56:33.299052Z","iopub.status.idle":"2024-10-06T16:56:33.367158Z","shell.execute_reply.started":"2024-10-06T16:56:33.298982Z","shell.execute_reply":"2024-10-06T16:56:33.366160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LGBM","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom lightgbm import LGBMClassifier\nimport lightgbm as lgb\n\n# Assuming df is your DataFrame with all features and target\n\n# 1. Prepare the data\ndef prepare_data(df,tf):\n    df = df.replace([np.inf, -np.inf], np.nan).fillna(method='ffill')\n    \n    # Define features and target\n    features = [col for col in df.columns if col != 'target']\n    X = df[features]\n    y = tf['target']\n    \n    return X, y\n\nX, y = prepare_data(to[to_keep],train_df)\n\n# 2. Time Series Split\ntscv = TimeSeriesSplit(n_splits=5)\n\n# 3. LightGBM parameters\nparams = {\n    'objective': 'binary',\n    'metric': 'binary_logloss',\n    'boosting_type': 'gbdt',\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    #'feature_fraction': 0.9,\n    #'bagging_fraction': 0.8,\n    #'bagging_freq': 5,\n    #'verbose': 0\n}\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T17:32:34.094265Z","iopub.execute_input":"2024-10-06T17:32:34.094699Z","iopub.status.idle":"2024-10-06T17:32:35.353262Z","shell.execute_reply.started":"2024-10-06T17:32:34.094659Z","shell.execute_reply":"2024-10-06T17:32:35.352242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfl = to[to_keep]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.model_selection import GridSearchCV\n\n# Define the parameter grid for LGBMClassifier\n#param_grid_lgbm = {\n#    'n_estimators': [100, 200],\n#    'learning_rate': [ 0.05, 0.1]\n    \n#}\n\n# Initialize the model\n#lgbm = LGBMClassifier()\n\n# Grid search with cross-validation\n#grid_search_lgbm = GridSearchCV(estimator=lgbm, param_grid=param_grid_lgbm, cv=5, scoring='accuracy', n_jobs=-1)\n#grid_search_lgbm.fit(X_train, y_train)\n\n# Get the best parameters and accuracy score\n#print(f\"Best parameters for LGBM: {grid_search_lgbm.best_params_}\")\n#print(f\"Best accuracy from LGBM: {grid_search_lgbm.best_score_}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T19:46:15.487795Z","iopub.execute_input":"2024-10-06T19:46:15.488606Z","iopub.status.idle":"2024-10-06T19:57:46.144730Z","shell.execute_reply.started":"2024-10-06T19:46:15.488564Z","shell.execute_reply":"2024-10-06T19:57:46.143441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(dfl, train_df['target'], test_size=0.2)\n\n# Initialize the model\nmodel = LGBMClassifier(n_estimators = 200 ,learning_rate = 0.1 )\n\nmodel.fit(X_train, y_train)\n\ny_pred = model.predict(X_test)\n\naccuracy = accuracy_score(y_test, y_pred)\n\nprint(f\"Accuracy on test set: {accuracy * 100:.2f}%\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T19:59:08.814183Z","iopub.execute_input":"2024-10-06T19:59:08.814664Z","iopub.status.idle":"2024-10-06T19:59:53.518080Z","shell.execute_reply.started":"2024-10-06T19:59:08.814618Z","shell.execute_reply":"2024-10-06T19:59:53.516997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(ts)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T20:01:04.473419Z","iopub.execute_input":"2024-10-06T20:01:04.474284Z","iopub.status.idle":"2024-10-06T20:01:07.905449Z","shell.execute_reply.started":"2024-10-06T20:01:04.474239Z","shell.execute_reply":"2024-10-06T20:01:07.904392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_id = pd.DataFrame(test_id, columns = ['row_id'])\ntest_id['target'] = y_pred\ntest_id.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_scores = []\nfeature_importance_df = pd.DataFrame()\n\nfor fold, (train_index, val_index) in enumerate(tscv.split(X), 1):\n    print(f\"Training fold {fold}\")\n\n    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n\n    # Initialize and train the model\n    model = LGBMClassifier(n_estimators = 200 ,learning_rate = 0.1)\n    model.fit(X_train, y_train)\n              \n\n    # Make predictions\n    y_pred = model.predict(X_val)\n\n    # Evaluate\n    accuracy = accuracy_score(y_val, y_pred)\n    fold_scores.append(accuracy)\n\n    print(f\"Fold {fold} accuracy: {accuracy}\")\n    print(classification_report(y_val, y_pred))\n\n    # Feature importance\n    fold_importance = pd.DataFrame({\n        'feature': X.columns,\n        'importance': model.feature_importances_,\n        'fold': fold\n    })\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance], axis=0)\n\nprint(f\"Mean accuracy: {np.mean(fold_scores):.4f}\")\n\nfeature_importance_df = feature_importance_df.groupby('feature').agg({'importance': ['mean', 'std']})\nfeature_importance_df.columns = ['importance_mean', 'importance_std']\nfeature_importance_df = feature_importance_df.sort_values('importance_mean', ascending=False).reset_index()\n\nprint(\"Top 20 most important features:\")\nprint(feature_importance_df.head(20))\n\n# 6. Visualize Feature Importance\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(12, 8))\nsns.barplot(x='importance_mean', y='feature', data=feature_importance_df.head(20))\nplt.title('Top 20 Feature Importances')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T17:09:27.611144Z","iopub.execute_input":"2024-10-06T17:09:27.611532Z","iopub.status.idle":"2024-10-06T17:09:27.709177Z","shell.execute_reply.started":"2024-10-06T17:09:27.611500Z","shell.execute_reply":"2024-10-06T17:09:27.707793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LSTM","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MinMaxScaler\n\n\n\ndf = pd.read_csv('/kaggle/input/cryptocurrency-price-direction-predictor/train.csv') \n\nfeatures = df[['open', 'high', 'low', 'close', 'volume','quote_asset_volume','number_of_trades','taker_buy_base_volume','taker_buy_quote_volume']].values\nlabels = (df['target']).astype(int).values  \n\nscaler = MinMaxScaler(feature_range=(0, 1))\nfeatures_scaled = scaler.fit_transform(features)\n\nX_train, X_test, y_train, y_test = train_test_split(features_scaled, labels, test_size=0.2, shuffle=False)\n\nX_train_t = torch.tensor(X_train, dtype=torch.float32)\nX_test_t = torch.tensor(X_test, dtype=torch.float32)\ny_train_t = torch.tensor(y_train, dtype=torch.float32).unsqueeze(1)\ny_test_t = torch.tensor(y_test, dtype=torch.float32).unsqueeze(1)\n\n# Hyperparameters\ninput_size = X_train.shape[1]\nhidden_size = 50\nnum_layers = 2\noutput_size = 1\nnum_epochs = 10\nbatch_size = 64\nlearning_rate = 0.0001\n\ntrain_dataset = torch.utils.data.TensorDataset(X_train_t, y_train_t)\ntest_dataset = torch.utils.data.TensorDataset(X_test_t, y_test_t)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n\n# Define the LSTM model\nclass LSTMModel(nn.Module):\n    def __init__(self, input_size, hidden_size, num_layers, output_size):\n        super(LSTMModel, self).__init__()\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n        self.fc = nn.Linear(hidden_size, output_size)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        # Set initial hidden and cell states\n        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)  # hidden state\n        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)  # cell state\n        # Forward propagate LSTM\n        out, _ = self.lstm(x, (h0, c0)) \n        out = self.fc(out[:, -1, :])\n        out = self.sigmoid(out)\n        return out\n\n# Initialize the model, loss function, and optimizer\nmodel = LSTMModel(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, output_size=output_size)\ncriterion = nn.BCELoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\n# Training loop\nmodel.train()\nfor epoch in range(num_epochs):\n    for i, (inputs, targets) in enumerate(train_loader):\n        inputs = inputs.unsqueeze(1)  \n        targets = targets\n        \n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}')\n\n# Testing loop\nmodel.eval()\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    for inputs, targets in test_loader:\n        inputs = inputs.unsqueeze(1)\n        outputs = model(inputs)\n        predicted = (outputs > 0.5).float()\n        total += targets.size(0)\n        correct += (predicted == targets).sum().item()\n\n    accuracy = 100 * correct / total\n    print(f'Accuracy on test data: {accuracy:.2f}%')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T05:45:08.482534Z","iopub.execute_input":"2024-10-08T05:45:08.483270Z","iopub.status.idle":"2024-10-08T06:01:36.532203Z","shell.execute_reply.started":"2024-10-08T05:45:08.483215Z","shell.execute_reply":"2024-10-08T06:01:36.531169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    for inputs, targets in test_loader:\n        inputs = inputs.unsqueeze(1)\n        outputs = model(inputs)\n        predicted = (outputs > 0.4).float()\n        total += targets.size(0)\n        correct += (predicted == targets).sum().item()\n\n    accuracy = 100 * correct / total\n    print(f'Accuracy on test data: {accuracy:.2f}%')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T06:01:36.533496Z","iopub.execute_input":"2024-10-08T06:01:36.534032Z","iopub.status.idle":"2024-10-08T06:01:45.895511Z","shell.execute_reply.started":"2024-10-08T06:01:36.533997Z","shell.execute_reply":"2024-10-08T06:01:45.894526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/cryptocurrency-price-direction-predictor/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T06:01:45.896842Z","iopub.execute_input":"2024-10-08T06:01:45.897139Z","iopub.status.idle":"2024-10-08T06:01:48.106325Z","shell.execute_reply.started":"2024-10-08T06:01:45.897106Z","shell.execute_reply":"2024-10-08T06:01:48.105540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_test = test_df[['open', 'high', 'low', 'close', 'volume','quote_asset_volume','number_of_trades','taker_buy_base_volume','taker_buy_quote_volume']].values\n\nfeatures_test_scaled = scaler.transform(features_test)\n\nX_test_new = torch.tensor(features_test_scaled, dtype=torch.float32)\n\nX_test_new = X_test_new.unsqueeze(1) \n\nmodel.eval()\n\nwith torch.no_grad():\n    outputs = model(X_test_new)\n    \n    predictions = (outputs > 0.467).float()\n\npredicted_labels = predictions.numpy().flatten()\nprint(predicted_labels)  \n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T06:03:34.968198Z","iopub.execute_input":"2024-10-08T06:03:34.968609Z","iopub.status.idle":"2024-10-08T06:03:39.142762Z","shell.execute_reply.started":"2024-10-08T06:03:34.968551Z","shell.execute_reply":"2024-10-08T06:03:39.141675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission = pd.DataFrame(test_df['row_id'], columns = ['row_id'])\n#submission['target'] = predicted_labels\n#submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T06:03:39.144669Z","iopub.execute_input":"2024-10-08T06:03:39.145322Z","iopub.status.idle":"2024-10-08T06:03:40.650694Z","shell.execute_reply.started":"2024-10-08T06:03:39.145271Z","shell.execute_reply":"2024-10-08T06:03:40.649706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion(LB score of 0.515)\nIn this notebook, I explored three different approaches for predicting cryptocurrency price movement: Random Forest, LightGBM (LGBM), and Long Short-Term Memory (LSTM) neural networks.\n- LSTM performed the best among the models tested, indicating that incorporating temporal information is crucial for this type of time-series prediction task.\n- Random Forest performed decently as a baseline model, but it failed to capture the sequential patterns in the data.\n- LightGBM performed slightly worse than Random Forest\n\n\nFurther tuning of the LSTM model, particularly with hyperparameters such as sequence length, learning rate, or hidden layer sizes, could lead to even better results. Additionally, more advanced architectures, such as bidirectional LSTMs, GRUs, or even Transformers, could be explored to capture more complex temporal patterns in the data.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}