{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom pandas.api.types import is_numeric_dtype\nfrom pandas.core.base import PandasObject\npd.set_option('display.max_columns', 100)\n\nimport os\nimport matplotlib.pyplot as plt\n\npath_data = '/kaggle/input/anomaly-class-detection/'","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.021924,"end_time":"2023-06-28T19:44:42.244433","exception":false,"start_time":"2023-06-28T19:44:42.222509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:08:06.906500Z","iopub.execute_input":"2023-07-27T11:08:06.907426Z","iopub.status.idle":"2023-07-27T11:08:06.939104Z","shell.execute_reply.started":"2023-07-27T11:08:06.907391Z","shell.execute_reply":"2023-07-27T11:08:06.938052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(path_data+'train/train.csv')\ntest = pd.read_csv(path_data+'test.csv')\n\ndata.result.value_counts()","metadata":{"papermill":{"duration":6.272911,"end_time":"2023-06-28T19:44:48.522798","exception":false,"start_time":"2023-06-28T19:44:42.249887","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:08:06.941151Z","iopub.execute_input":"2023-07-27T11:08:06.941645Z","iopub.status.idle":"2023-07-27T11:09:02.325628Z","shell.execute_reply.started":"2023-07-27T11:08:06.941616Z","shell.execute_reply":"2023-07-27T11:09:02.324486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categ = ['exchange']\nsignal = ['RSI_signal','Stoch_O_signal','fibonacci_signal','Bollinger_signal','ichimoku_c_signal']\ntarget = ['result']\nvalues = [c for c in data.columns if c not in categ+target]\n\ndict_signal = {\n    'Buy':1,\n    'Hold':0,\n    'Sell':-1,\n }\n\ndata[signal] = data[signal].replace(dict_signal)","metadata":{"papermill":{"duration":1.85006,"end_time":"2023-06-28T19:44:50.378193","exception":false,"start_time":"2023-06-28T19:44:48.528133","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:09:02.329293Z","iopub.execute_input":"2023-07-27T11:09:02.329975Z","iopub.status.idle":"2023-07-27T11:09:19.896514Z","shell.execute_reply.started":"2023-07-27T11:09:02.329934Z","shell.execute_reply":"2023-07-27T11:09:19.895467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def better_describe(df):\n    \n    df_describe = df.describe(include='all')\n    df_describe.loc['dtype'] = df.dtypes\n    df_describe.loc['unique_count'] = df.nunique()\n    df_describe.loc['nan_count'] = df.isna().sum()\n    \n    df_describe.loc['constant_diff'] = np.nan\n    for c in df.columns:\n        if is_numeric_dtype(df[c]):\n            diffs = np.diff(np.unique(df[c]))\n            min_diffs = min(diffs)\n            if all([i.is_integer() for i in (diffs/min_diffs)]):\n                df_describe.loc['constant_diff',c] = min_diffs\n    \n    for value in [np.inf,-np.inf]:\n        df_describe.loc[f'{str(value)}_count'] = (df == value).sum()\n\n    order = ['dtype','count','unique_count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max', 'nan_count', 'inf_count','-inf_count']\n        \n    return df_describe.loc[order]\n\nPandasObject.better_describe = better_describe\n\ndesc = data.better_describe()\ndesc","metadata":{"papermill":{"duration":5.736604,"end_time":"2023-06-28T19:44:56.120076","exception":false,"start_time":"2023-06-28T19:44:50.383472","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:09:19.899296Z","iopub.execute_input":"2023-07-27T11:09:19.900163Z","iopub.status.idle":"2023-07-27T11:10:14.045994Z","shell.execute_reply.started":"2023-07-27T11:09:19.900119Z","shell.execute_reply":"2023-07-27T11:10:14.044925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA - relationship with target\n\nwe use optbinning package, my favorite solution to bin/explore data","metadata":{"papermill":{"duration":0.006106,"end_time":"2023-06-28T19:44:56.132555","exception":false,"start_time":"2023-06-28T19:44:56.126449","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip -q install optbinning","metadata":{"papermill":{"duration":35.12643,"end_time":"2023-06-28T19:45:31.265203","exception":false,"start_time":"2023-06-28T19:44:56.138773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:10:14.048455Z","iopub.execute_input":"2023-07-27T11:10:14.048797Z","iopub.status.idle":"2023-07-27T11:10:44.841904Z","shell.execute_reply.started":"2023-07-27T11:10:14.048767Z","shell.execute_reply":"2023-07-27T11:10:44.840935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optbinning\n\ndata_sample = data.sample(frac=0.01)","metadata":{"papermill":{"duration":2.171735,"end_time":"2023-06-28T19:45:33.443903","exception":false,"start_time":"2023-06-28T19:45:31.272168","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:10:44.843411Z","iopub.execute_input":"2023-07-27T11:10:44.843794Z","iopub.status.idle":"2023-07-27T11:10:46.757587Z","shell.execute_reply.started":"2023-07-27T11:10:44.843758Z","shell.execute_reply":"2023-07-27T11:10:46.755984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in data.columns:\n    if c not in target:\n            print(c)\n            dtype = 'numerical' if c not in categ else 'categorical'\n            \n            optb = optbinning.OptimalBinning(name=c, dtype=dtype, solver=\"cp\")\n            optb.fit(data_sample[c].values, data_sample[target].values.flatten())\n\n            binning_table = optb.binning_table\n            binning_table.build()\n            binning_table.plot(metric=\"event_rate\")","metadata":{"papermill":{"duration":29.093469,"end_time":"2023-06-28T19:46:02.544615","exception":false,"start_time":"2023-06-28T19:45:33.451146","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:10:46.760387Z","iopub.execute_input":"2023-07-27T11:10:46.761316Z","iopub.status.idle":"2023-07-27T11:11:17.050922Z","shell.execute_reply.started":"2023-07-27T11:10:46.761254Z","shell.execute_reply":"2023-07-27T11:11:17.049867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nfeature = 's&p500_move_15m'\ntarget = 'result'\ndf_sample= data.sample(frac=0.001)\n\nsns.regplot(x=np.abs(df_sample[feature]), \n            y=df_sample[target], \n            logistic = True,\n            x_jitter = 0.1,\n            y_jitter = 0.1,\n            scatter_kws={'s':1},\n            line_kws={\"color\": \"red\"}).set_title(f\"{target} / {feature}\");","metadata":{"papermill":{"duration":11.049225,"end_time":"2023-06-28T19:46:13.631425","exception":false,"start_time":"2023-06-28T19:46:02.582200","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:11:17.052459Z","iopub.execute_input":"2023-07-27T11:11:17.053027Z","iopub.status.idle":"2023-07-27T11:11:26.241551Z","shell.execute_reply.started":"2023-07-27T11:11:17.052995Z","shell.execute_reply":"2023-07-27T11:11:26.240479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline Logistic Regression","metadata":{"papermill":{"duration":0.033748,"end_time":"2023-06-28T19:46:13.699911","exception":false,"start_time":"2023-06-28T19:46:13.666163","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\n\nsss = StratifiedShuffleSplit(n_splits=5, test_size=0.3, random_state=0)\n\nX, y = np.abs(data[feature]), data[target]\n\nfor threshold in [0.175,0.25,0.425,0.5,0.675,0.75,0.825]:\n    all_score = []\n    all_conf = []\n    for  i, (index_train, index_test) in enumerate(sss.split(X.index,y)):\n\n        clf = LogisticRegression(random_state=0).fit(X.loc[index_train].values.reshape(-1, 1), (y.loc[index_train]>0).values.flatten())\n\n        prob = clf.predict_proba(X.loc[index_test].values.reshape(-1, 1))[:,1]\n\n        preds = (prob>threshold).astype(int)\n        score = f1_score(y.loc[index_test].values, preds, average='macro')\n\n        all_score.append(score)\n        all_conf.append(confusion_matrix(y.loc[index_test].values, preds))\n\n    print(f'------------------------')\n    print(f'Threshold {threshold:.2}')\n    print(f'Score: {np.mean(all_score):.2%}')\n    print(f'Confusion Matrix {threshold:.2}')\n    display(np.mean(all_conf,axis=0))\n    print(classification_report(y.loc[index_test].values, preds))","metadata":{"papermill":{"duration":11.732134,"end_time":"2023-06-28T19:46:25.465619","exception":false,"start_time":"2023-06-28T19:46:13.733485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:39:38.034382Z","iopub.execute_input":"2023-07-27T11:39:38.034783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = LogisticRegression(random_state=0).fit(X.values.reshape(-1, 1), y.values.flatten())\nprob = clf.predict_proba(test[feature].values.reshape(-1, 1))[:,1]\n\nthreshold = 0.75\n\npreds = (prob>threshold).astype(int)","metadata":{"papermill":{"duration":0.405915,"end_time":"2023-06-28T19:46:25.905917","exception":false,"start_time":"2023-06-28T19:46:25.500002","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:12:49.074656Z","iopub.execute_input":"2023-07-27T11:12:49.075089Z","iopub.status.idle":"2023-07-27T11:12:51.417051Z","shell.execute_reply.started":"2023-07-27T11:12:49.075035Z","shell.execute_reply":"2023-07-27T11:12:51.415093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.054864,"end_time":"2023-06-28T19:46:26.032055","exception":false,"start_time":"2023-06-28T19:46:25.977191","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.DataFrame({'id':test.index})\nsub['result'] = preds\nsub.to_csv('submission.csv',index=False)\nsub.head()","metadata":{"papermill":{"duration":0.059541,"end_time":"2023-06-28T19:46:26.125983","exception":false,"start_time":"2023-06-28T19:46:26.066442","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-27T11:12:51.422019Z","iopub.execute_input":"2023-07-27T11:12:51.422939Z","iopub.status.idle":"2023-07-27T11:12:51.745850Z","shell.execute_reply.started":"2023-07-27T11:12:51.422882Z","shell.execute_reply":"2023-07-27T11:12:51.744762Z"},"trusted":true},"execution_count":null,"outputs":[]}]}