{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":63054,"databundleVersionId":6872631,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns \nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-20T02:53:13.741919Z","iopub.execute_input":"2023-11-20T02:53:13.74239Z","iopub.status.idle":"2023-11-20T02:53:16.071021Z","shell.execute_reply.started":"2023-11-20T02:53:13.742354Z","shell.execute_reply":"2023-11-20T02:53:16.069393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load in the datasets","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/anyas-gojo-revival-bondman/trainData.csv\")\ntest_df = pd.read_csv(\"../input/anyas-gojo-revival-bondman/testData.csv\")\ntrain_df['train_test'] = 1\ntest_df['train_test'] = 0\ntest_df['is_fraud'] = np.NaN\ndata_df = pd.concat([train_df, test_df])\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:16.073771Z","iopub.execute_input":"2023-11-20T02:53:16.074723Z","iopub.status.idle":"2023-11-20T02:53:28.977385Z","shell.execute_reply.started":"2023-11-20T02:53:16.074662Z","shell.execute_reply":"2023-11-20T02:53:28.97642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Keep useful features\nuseful_features = ['trans_date_trans_time', 'merchant', 'category', 'amt', 'gender', 'state', 'zip', 'lat', 'long', 'city_pop', 'job', 'dob', 'merch_lat', 'merch_long', 'is_fraud', 'train_test']\ntrain_df = train_df[useful_features]\ntest_df = test_df[useful_features]\ndata_df = data_df[useful_features]","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:28.978786Z","iopub.execute_input":"2023-11-20T02:53:28.980427Z","iopub.status.idle":"2023-11-20T02:53:29.394729Z","shell.execute_reply.started":"2023-11-20T02:53:28.980378Z","shell.execute_reply":"2023-11-20T02:53:29.393688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_id = pd.read_csv('../input/anyas-gojo-revival-bondman/testData.csv').id","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:29.397594Z","iopub.execute_input":"2023-11-20T02:53:29.398978Z","iopub.status.idle":"2023-11-20T02:53:33.713065Z","shell.execute_reply.started":"2023-11-20T02:53:29.398923Z","shell.execute_reply":"2023-11-20T02:53:33.711725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analyze the data\n### Check the columns\nFind out categorical and continuous features\n##### Categorical Data:\n* is_fraud\n* gender\n* category\n* trans_date_trans_time\n* street\n* state\n* zip\n* job\n* dob\n* merchant\n\n##### Continuous Data:\n* amt\n* lat\n* long\n* city_pop\n* merch_lat\n* merch_long","metadata":{}},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:33.715033Z","iopub.execute_input":"2023-11-20T02:53:33.715394Z","iopub.status.idle":"2023-11-20T02:53:33.725978Z","shell.execute_reply.started":"2023-11-20T02:53:33.715363Z","shell.execute_reply":"2023-11-20T02:53:33.72425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview the data\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:33.727774Z","iopub.execute_input":"2023-11-20T02:53:33.728232Z","iopub.status.idle":"2023-11-20T02:53:33.760911Z","shell.execute_reply.started":"2023-11-20T02:53:33.728196Z","shell.execute_reply":"2023-11-20T02:53:33.759726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview the end\ntrain_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:33.762778Z","iopub.execute_input":"2023-11-20T02:53:33.763549Z","iopub.status.idle":"2023-11-20T02:53:33.786844Z","shell.execute_reply.started":"2023-11-20T02:53:33.763509Z","shell.execute_reply":"2023-11-20T02:53:33.785513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Luckily for this dataset, there are no null values","metadata":{}},{"cell_type":"code","source":"# checking for any null values\ntrain_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:33.788821Z","iopub.execute_input":"2023-11-20T02:53:33.789637Z","iopub.status.idle":"2023-11-20T02:53:34.201925Z","shell.execute_reply.started":"2023-11-20T02:53:33.789599Z","shell.execute_reply":"2023-11-20T02:53:34.200419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:34.203654Z","iopub.execute_input":"2023-11-20T02:53:34.204007Z","iopub.status.idle":"2023-11-20T02:53:34.436321Z","shell.execute_reply.started":"2023-11-20T02:53:34.203976Z","shell.execute_reply":"2023-11-20T02:53:34.435035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of amt\nplt.figure(figsize=(12, 6))\nsns.histplot(data=train_df, x=\"amt\", log_scale=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:34.440719Z","iopub.execute_input":"2023-11-20T02:53:34.44143Z","iopub.status.idle":"2023-11-20T02:53:35.818539Z","shell.execute_reply.started":"2023-11-20T02:53:34.441395Z","shell.execute_reply":"2023-11-20T02:53:35.81732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of city_pop\nplt.figure(figsize=(12, 6))\nsns.histplot(data=train_df, x=\"city_pop\", log_scale=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:35.82056Z","iopub.execute_input":"2023-11-20T02:53:35.821331Z","iopub.status.idle":"2023-11-20T02:53:37.189243Z","shell.execute_reply.started":"2023-11-20T02:53:35.821281Z","shell.execute_reply":"2023-11-20T02:53:37.187721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of location\nlocation = [[\"lat\", \"long\"], [\"merch_lat\", \"merch_long\"]]\nfor coords in location:\n    plt.figure(figsize=(12, 6))\n    sns.scatterplot(data=train_df, x=coords[0], y=coords[1], hue=\"is_fraud\")\n    plt.title(\"Location Distribution\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:53:37.191245Z","iopub.execute_input":"2023-11-20T02:53:37.191596Z","iopub.status.idle":"2023-11-20T02:54:13.036956Z","shell.execute_reply.started":"2023-11-20T02:53:37.191566Z","shell.execute_reply":"2023-11-20T02:54:13.035366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get categorical data to plot\ncategorical_col = train_df[[ 'category', 'gender', 'state', 'merchant', 'job']]","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:13.038572Z","iopub.execute_input":"2023-11-20T02:54:13.03892Z","iopub.status.idle":"2023-11-20T02:54:13.075463Z","shell.execute_reply.started":"2023-11-20T02:54:13.038889Z","shell.execute_reply":"2023-11-20T02:54:13.074211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n# get the top 30 values for a category\ndef top_30(category):\n    counter = Counter()\n    for i in category:\n        counter[i] += 1;\n    df = pd.DataFrame.from_dict(counter, orient='index').reset_index()\n    df = df.rename(columns={'index':'value', 0:'count'}).sort_values(by='count', ascending=False).reset_index(drop=True)\n    return df[:30]","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:13.077254Z","iopub.execute_input":"2023-11-20T02:54:13.077617Z","iopub.status.idle":"2023-11-20T02:54:13.085156Z","shell.execute_reply.started":"2023-11-20T02:54:13.077575Z","shell.execute_reply":"2023-11-20T02:54:13.084209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(train_df, x=\"is_fraud\")\nplt.title(\"is_fraud\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:13.086484Z","iopub.execute_input":"2023-11-20T02:54:13.086892Z","iopub.status.idle":"2023-11-20T02:54:13.372402Z","shell.execute_reply.started":"2023-11-20T02:54:13.086825Z","shell.execute_reply":"2023-11-20T02:54:13.37142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in categorical_col.columns:\n    plt.figure(figsize=(12, 10))\n    if i == 'merchant' or i == 'job':\n        top_30_df = top_30(categorical_col[i])\n        sns.barplot(x=top_30_df[\"count\"], y=top_30_df[\"value\"])\n    else:    \n        sns.barplot(y=categorical_col[i].value_counts().index, x=categorical_col[i].value_counts())\n    plt.title(i)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:13.37381Z","iopub.execute_input":"2023-11-20T02:54:13.375193Z","iopub.status.idle":"2023-11-20T02:54:17.170415Z","shell.execute_reply.started":"2023-11-20T02:54:13.375119Z","shell.execute_reply":"2023-11-20T02:54:17.168925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering\n#### Extract year from dob\n#### Add age as of 2023\n#### Get year, month, day, hour from transaction time","metadata":{}},{"cell_type":"code","source":"train_df['dob_year'] = train_df['dob'].apply(lambda x: x.split(\"-\")[0])\ntrain_df['trans_year'] = train_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[0])\ntrain_df['trans_month'] = train_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[1])\ntrain_df['trans_day'] = train_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[2].split(\" \")[0])\ntrain_df['trans_hour'] = train_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[2].split(\" \")[1].split(\":\")[0])\ntrain_df['age'] = train_df['dob_year'].apply(lambda x: 2023 - int(x))","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:17.171956Z","iopub.execute_input":"2023-11-20T02:54:17.172369Z","iopub.status.idle":"2023-11-20T02:54:19.847693Z","shell.execute_reply.started":"2023-11-20T02:54:17.172334Z","shell.execute_reply":"2023-11-20T02:54:19.846331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check new columns\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:19.849393Z","iopub.execute_input":"2023-11-20T02:54:19.849738Z","iopub.status.idle":"2023-11-20T02:54:19.878286Z","shell.execute_reply.started":"2023-11-20T02:54:19.849708Z","shell.execute_reply":"2023-11-20T02:54:19.876776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:19.880152Z","iopub.execute_input":"2023-11-20T02:54:19.880581Z","iopub.status.idle":"2023-11-20T02:54:19.89423Z","shell.execute_reply.started":"2023-11-20T02:54:19.880547Z","shell.execute_reply":"2023-11-20T02:54:19.893198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_int = ['trans_year', 'trans_month', 'trans_day', 'trans_hour']\nfor col in col_to_int:\n    train_df[col] = train_df[col].astype('int64')","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:19.896188Z","iopub.execute_input":"2023-11-20T02:54:19.897249Z","iopub.status.idle":"2023-11-20T02:54:20.221889Z","shell.execute_reply.started":"2023-11-20T02:54:19.897201Z","shell.execute_reply":"2023-11-20T02:54:20.220455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:20.223441Z","iopub.execute_input":"2023-11-20T02:54:20.223801Z","iopub.status.idle":"2023-11-20T02:54:20.233988Z","shell.execute_reply.started":"2023-11-20T02:54:20.223773Z","shell.execute_reply":"2023-11-20T02:54:20.232379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Find correlation to fraud","metadata":{}},{"cell_type":"markdown","source":"#### Fraud seems to occur in transactions with older people and if the amount is large. However, there does not seem to be much correlation between location. If population is large, fraud tends to happen more often which makes sense. A city with a larger population will more often have frauds than the smaller ones. ","metadata":{}},{"cell_type":"code","source":"# compare fraud rates across age, amt, lat, long, merch_lat, merch_long\npd.pivot_table(train_df, index = 'is_fraud', values = ['age', 'amt', 'lat', 'long', 'merch_lat', 'merch_long', 'city_pop', 'trans_year', 'trans_month', 'trans_day', 'trans_hour'])","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:20.23625Z","iopub.execute_input":"2023-11-20T02:54:20.236642Z","iopub.status.idle":"2023-11-20T02:54:20.343573Z","shell.execute_reply.started":"2023-11-20T02:54:20.23661Z","shell.execute_reply":"2023-11-20T02:54:20.342239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_graph = ['age', 'trans_year', 'trans_month', 'trans_day', 'trans_hour']\nfor col in col_to_graph:\n    sns.set()\n    sns.set_palette(\"pastel\")\n    plt.figure(figsize=(18,6))\n    sns.countplot(data=train_df[train_df['is_fraud'] == 1], x=col, color='blue')\n    plt.title(\"Fraud Distribution of \" + col)\n    plt.xlabel(col)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:20.344965Z","iopub.execute_input":"2023-11-20T02:54:20.345623Z","iopub.status.idle":"2023-11-20T02:54:24.444555Z","shell.execute_reply.started":"2023-11-20T02:54:20.345589Z","shell.execute_reply":"2023-11-20T02:54:24.443204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correlation between Numeric Features\n##### It seems that latitude, longitude, and zip are highly correlated with each other thus we can remove latitude and longitude. Obviously age and dob_year are correlated.","metadata":{}},{"cell_type":"code","source":"corr_mat = train_df[['age', 'amt', 'lat', 'long', 'merch_lat', 'merch_long', 'city_pop', 'dob_year', 'zip', 'trans_year', 'trans_month', 'trans_day', 'trans_hour']].corr()\n\nplt.figure(figsize=(12,12))\nplt.title(\"Correlation between Numeric Features\")\nsns.heatmap(corr_mat, cmap='RdBu', annot=True, center=0, vmin=-1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:24.446699Z","iopub.execute_input":"2023-11-20T02:54:24.447208Z","iopub.status.idle":"2023-11-20T02:54:26.086617Z","shell.execute_reply.started":"2023-11-20T02:54:24.447163Z","shell.execute_reply":"2023-11-20T02:54:26.085361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preprocessing for Models\n#### **1)** Create variables from above for both data sets\n#### **2)** Normalize amt and city_pop using log to get a normal distribution\n#### **3)** Transform categorical data\n#### **4)** Scale the numerical features","metadata":{}},{"cell_type":"code","source":"#create all variables that we did above for both training and test dataframes \ndata_df['dob_year'] = data_df['dob'].apply(lambda x: x.split(\"-\")[0])\ndata_df['trans_year'] = data_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[0])\ndata_df['trans_month'] = data_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[1])\ndata_df['trans_day'] = data_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[2].split(\" \")[0])\ndata_df['trans_hour'] = data_df['trans_date_trans_time'].apply(lambda x: x.split(\"-\")[2].split(\" \")[1].split(\":\")[0])\ndata_df['age'] = data_df['dob_year'].apply(lambda x: 2023 - int(x))\n\n# Set columns to ints\ncol_to_int = ['trans_year', 'trans_month', 'trans_day', 'trans_hour']\nfor col in col_to_int:\n    data_df[col] = data_df[col].astype('int64')\n    \n# log norm of amt\ndata_df['norm_amt'] = np.log(data_df.amt+1)\ndata_df['norm_amt'].hist()\n\n# log norm of city_pop\ndata_df['norm_city_pop'] = np.log(data_df.city_pop+1)\ndata_df['norm_city_pop'].hist()\n\n# keep important features\nfeatures = ['merchant', 'category', 'gender', 'state', 'lat', 'long', 'job', 'is_fraud', 'train_test', 'trans_year','trans_month', 'trans_day', 'trans_hour', 'age', 'norm_amt', 'norm_city_pop']\n\ndata_df = data_df[features]\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:26.088271Z","iopub.execute_input":"2023-11-20T02:54:26.089617Z","iopub.status.idle":"2023-11-20T02:54:33.332203Z","shell.execute_reply.started":"2023-11-20T02:54:26.089572Z","shell.execute_reply":"2023-11-20T02:54:33.330728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:33.333988Z","iopub.execute_input":"2023-11-20T02:54:33.334482Z","iopub.status.idle":"2023-11-20T02:54:33.422296Z","shell.execute_reply.started":"2023-11-20T02:54:33.334438Z","shell.execute_reply":"2023-11-20T02:54:33.420893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()\ndata_df[['trans_year','trans_month', 'trans_day', 'trans_hour']]= scaler.fit_transform(data_df[['trans_year','trans_month', 'trans_day', 'trans_hour']])\ndata_df","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:33.424316Z","iopub.execute_input":"2023-11-20T02:54:33.424684Z","iopub.status.idle":"2023-11-20T02:54:33.54496Z","shell.execute_reply.started":"2023-11-20T02:54:33.424652Z","shell.execute_reply":"2023-11-20T02:54:33.543761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder = LabelEncoder()\n\nlabel_df = data_df.copy().drop(columns='is_fraud')\n\nfor column in label_df.select_dtypes(include=['object']).columns:\n    label_df[column] = label_encoder.fit_transform(label_df[column])\n    \nX_train_labeled = label_df[label_df.train_test == 1].drop(['train_test'], axis =1)\nX_test_labeled = label_df[label_df.train_test == 0].drop(['train_test'], axis =1)\nX_train_labeled","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:33.550518Z","iopub.execute_input":"2023-11-20T02:54:33.550929Z","iopub.status.idle":"2023-11-20T02:54:36.173318Z","shell.execute_reply.started":"2023-11-20T02:54:33.550899Z","shell.execute_reply":"2023-11-20T02:54:36.172002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_mat = label_df[['merchant', 'category', 'gender', 'state', 'lat', 'long', 'job', 'trans_year', 'trans_month', 'trans_day', 'trans_hour', 'age', 'norm_amt','norm_city_pop']].corr()\n\nplt.figure(figsize=(12,12))\nplt.title(\"Correlation between All Features\")\nsns.heatmap(corr_mat, cmap='RdBu', annot=True, center=0, vmin=-1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:36.175437Z","iopub.execute_input":"2023-11-20T02:54:36.176321Z","iopub.status.idle":"2023-11-20T02:54:38.452074Z","shell.execute_reply.started":"2023-11-20T02:54:36.176276Z","shell.execute_reply":"2023-11-20T02:54:38.450575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_dummies = pd.get_dummies(data_df[['merchant','category','gender','state','lat','long','job','train_test','trans_year','trans_month','trans_day','trans_hour','age','norm_amt','norm_city_pop']])\nall_dummies","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:38.453873Z","iopub.execute_input":"2023-11-20T02:54:38.454303Z","iopub.status.idle":"2023-11-20T02:54:50.830765Z","shell.execute_reply.started":"2023-11-20T02:54:38.454262Z","shell.execute_reply":"2023-11-20T02:54:50.829536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Scale data \nfrom sklearn.preprocessing import StandardScaler\nnum_features = ['norm_amt', 'norm_city_pop', 'age', 'lat', 'long']\nscale = StandardScaler()\nall_dummies_scaled = all_dummies.copy()\nall_dummies_scaled[num_features]= scale.fit_transform(all_dummies_scaled[num_features])\nall_dummies_scaled\n\nX_train_scaled = all_dummies_scaled[all_dummies_scaled.train_test == 1].drop(['train_test'], axis =1)\nX_test_scaled = all_dummies_scaled[all_dummies_scaled.train_test == 0].drop(['train_test'], axis =1)\n\ny_train = data_df[data_df.train_test==1].is_fraud\nX_train_scaled","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:50.832911Z","iopub.execute_input":"2023-11-20T02:54:50.833793Z","iopub.status.idle":"2023-11-20T02:54:56.452045Z","shell.execute_reply.started":"2023-11-20T02:54:50.833747Z","shell.execute_reply":"2023-11-20T02:54:56.45065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The number of transactions that are fraud is way less than the number of transactions that aren't so I undersampled the number of transactions that aren't fraud","metadata":{}},{"cell_type":"code","source":"from imblearn.under_sampling import RandomUnderSampler\nundersampler = RandomUnderSampler(sampling_strategy='majority', random_state=1)\nX_resampled, y_resampled = undersampler.fit_resample(X_train_labeled, y_train)\nX_resampled","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:56.453568Z","iopub.execute_input":"2023-11-20T02:54:56.453907Z","iopub.status.idle":"2023-11-20T02:54:57.398694Z","shell.execute_reply.started":"2023-11-20T02:54:56.453877Z","shell.execute_reply":"2023-11-20T02:54:57.397481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.set_style()\nsns.countplot(x=y_resampled)\nplt.title(\"is_fraud Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:54:57.400328Z","iopub.execute_input":"2023-11-20T02:54:57.401843Z","iopub.status.idle":"2023-11-20T02:54:57.72124Z","shell.execute_reply.started":"2023-11-20T02:54:57.401797Z","shell.execute_reply":"2023-11-20T02:54:57.719708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Since our dataset with one hot encoding contains a lot of features, I use PCA to lower the dimensions","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\npca = PCA()\npca.fit(X_train_scaled)\n\nexplained_variance_ratios = pca.explained_variance_ratio_\ncum_evr = explained_variance_ratios.cumsum()\n\nplt.figure(figsize=(12, 6))\nplt.plot(range(1, len(cum_evr) + 1), cum_evr, marker='o', label='Cumulative Explained Variance Ratio')\n\nplt.title(\"Cumulative Explained Variance Ratio vs. Number of Components\")\nplt.xlabel(\"Number of Components\")\nplt.ylabel(\"Cumulative Explained Variance Ratio\")\n\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T01:11:18.25466Z","iopub.execute_input":"2023-11-19T01:11:18.255832Z","iopub.status.idle":"2023-11-19T01:15:08.304535Z","shell.execute_reply.started":"2023-11-19T01:11:18.255785Z","shell.execute_reply":"2023-11-19T01:15:08.302635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot on smaller range to see 80% cumulative explained variance ratio\nplt.figure(figsize=(12, 6))\nplt.plot(range(1, len(cum_evr) + 1 - 1200), cum_evr[:len(cum_evr) - 1200], marker='o', label='Cumulative Explained Variance Ratio')\n\nplt.title(\"Cumulative Explained Variance Ratio vs. Number of Components\")\nplt.xlabel(\"Number of Components\")\nplt.ylabel(\"Cumulative Explained Variance Ratio\")\n\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T01:15:08.306952Z","iopub.execute_input":"2023-11-19T01:15:08.307395Z","iopub.status.idle":"2023-11-19T01:15:08.823578Z","shell.execute_reply.started":"2023-11-19T01:15:08.307361Z","shell.execute_reply":"2023-11-19T01:15:08.822035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=60)\npca.fit(X_train_scaled)\nX_train_pca = pca.transform(X_train_scaled)\nX_test_pca = pca.transform(X_test_scaled)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T01:15:08.825503Z","iopub.execute_input":"2023-11-19T01:15:08.825985Z","iopub.status.idle":"2023-11-19T01:17:21.752234Z","shell.execute_reply.started":"2023-11-19T01:15:08.825947Z","shell.execute_reply":"2023-11-19T01:17:21.750353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Building (Baseline Validation Performance)","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn import tree\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier, VotingClassifier","metadata":{"execution":{"iopub.status.busy":"2023-11-20T02:56:53.343052Z","iopub.execute_input":"2023-11-20T02:56:53.343529Z","iopub.status.idle":"2023-11-20T02:56:53.37345Z","shell.execute_reply.started":"2023-11-20T02:56:53.343497Z","shell.execute_reply":"2023-11-20T02:56:53.372001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gnb = GaussianNB()\ncv = cross_val_score(gnb,X_train_labeled,y_train,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:15:38.388371Z","iopub.execute_input":"2023-11-19T03:15:38.38875Z","iopub.status.idle":"2023-11-19T03:15:39.341364Z","shell.execute_reply.started":"2023-11-19T03:15:38.388705Z","shell.execute_reply":"2023-11-19T03:15:39.339806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gnb = GaussianNB()\ncv = cross_val_score(gnb,X_resampled,y_resampled,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:15:40.860601Z","iopub.execute_input":"2023-11-19T03:15:40.860976Z","iopub.status.idle":"2023-11-19T03:15:40.930528Z","shell.execute_reply.started":"2023-11-19T03:15:40.860945Z","shell.execute_reply":"2023-11-19T03:15:40.929516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(max_iter=2000)\ncv = cross_val_score(lr,X_train_labeled,y_train,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:16:16.05709Z","iopub.execute_input":"2023-11-19T03:16:16.057488Z","iopub.status.idle":"2023-11-19T03:19:34.594928Z","shell.execute_reply.started":"2023-11-19T03:16:16.057453Z","shell.execute_reply":"2023-11-19T03:19:34.592767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(max_iter=2000)\ncv = cross_val_score(lr,X_resampled,y_resampled,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:19:34.596294Z","iopub.execute_input":"2023-11-19T03:19:34.596634Z","iopub.status.idle":"2023-11-19T03:19:42.196256Z","shell.execute_reply.started":"2023-11-19T03:19:34.596606Z","shell.execute_reply":"2023-11-19T03:19:42.195507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(max_iter=2000)\ncv = cross_val_score(lr, X_train_pca, y_train, cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-17T18:32:04.955526Z","iopub.execute_input":"2023-11-17T18:32:04.957247Z","iopub.status.idle":"2023-11-17T18:32:34.171006Z","shell.execute_reply.started":"2023-11-17T18:32:04.95717Z","shell.execute_reply":"2023-11-17T18:32:34.16984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = tree.DecisionTreeClassifier(random_state = 1)\ncv = cross_val_score(dt,X_train_labeled,y_train,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:19:42.197673Z","iopub.execute_input":"2023-11-19T03:19:42.198243Z","iopub.status.idle":"2023-11-19T03:20:09.344623Z","shell.execute_reply.started":"2023-11-19T03:19:42.19821Z","shell.execute_reply":"2023-11-19T03:20:09.343508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = tree.DecisionTreeClassifier(random_state = 1)\ncv = cross_val_score(dt,X_resampled,y_resampled,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:20:09.346206Z","iopub.execute_input":"2023-11-19T03:20:09.346437Z","iopub.status.idle":"2023-11-19T03:20:09.835773Z","shell.execute_reply.started":"2023-11-19T03:20:09.346417Z","shell.execute_reply":"2023-11-19T03:20:09.834346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = tree.DecisionTreeClassifier(random_state = 1)\ncv = cross_val_score(dt,X_train_pca,y_train,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-17T18:34:24.7029Z","iopub.execute_input":"2023-11-17T18:34:24.703289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = RandomForestClassifier(random_state = 1, n_estimators=10)\ncv = cross_val_score(rf,X_train_labeled,y_train,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:20:09.836827Z","iopub.execute_input":"2023-11-19T03:20:09.837239Z","iopub.status.idle":"2023-11-19T03:21:07.205306Z","shell.execute_reply.started":"2023-11-19T03:20:09.837217Z","shell.execute_reply":"2023-11-19T03:21:07.203844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = RandomForestClassifier(random_state = 1, n_estimators=10)\ncv = cross_val_score(rf,X_resampled,y_resampled,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:21:07.206809Z","iopub.execute_input":"2023-11-19T03:21:07.207152Z","iopub.status.idle":"2023-11-19T03:21:08.201417Z","shell.execute_reply.started":"2023-11-19T03:21:07.207123Z","shell.execute_reply":"2023-11-19T03:21:08.200267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = RandomForestClassifier(random_state = 1, n_estimators=10)\ncv = cross_val_score(rf,X_train_pca,y_train,cv=5,scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### After this point, I stick with just using X_resampled because it runs faster and has better cross validation scores using roc auc. It also just makes more sense to train on data taht contains an even proportion for each class","metadata":{}},{"cell_type":"code","source":"knn = KNeighborsClassifier()\ncv = cross_val_score(knn,X_resampled,y_resampled,cv=5,scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:22:30.362911Z","iopub.execute_input":"2023-11-19T03:22:30.363243Z","iopub.status.idle":"2023-11-19T03:22:30.649804Z","shell.execute_reply.started":"2023-11-19T03:22:30.363218Z","shell.execute_reply":"2023-11-19T03:22:30.648893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svc = SVC(probability = True)\ncv = cross_val_score(svc,X_resampled,y_resampled,cv=5,scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:19:46.826576Z","iopub.execute_input":"2023-11-19T08:19:46.8271Z","iopub.status.idle":"2023-11-19T08:22:28.14793Z","shell.execute_reply.started":"2023-11-19T08:19:46.827066Z","shell.execute_reply":"2023-11-19T08:22:28.145832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nxgb = XGBClassifier(random_state =1)\ncv = cross_val_score(xgb,X_resampled,y_resampled,cv=5,scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:22:53.772486Z","iopub.execute_input":"2023-11-19T03:22:53.772875Z","iopub.status.idle":"2023-11-19T03:22:59.646968Z","shell.execute_reply.started":"2023-11-19T03:22:53.772846Z","shell.execute_reply":"2023-11-19T03:22:59.645369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = tree.DecisionTreeClassifier(random_state = 1)\nrf = RandomForestClassifier(random_state = 1, n_estimators=10)\nknn = KNeighborsClassifier()\ngnb = GaussianNB()\n\nvoting_clf = VotingClassifier(estimators = [('knn',knn),('rf',rf),('xgb', xgb), ('dt', dt), ('gnb', gnb)], voting = 'soft') \ncv = cross_val_score(voting_clf,X_resampled,y_resampled,cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:23:23.02911Z","iopub.execute_input":"2023-11-19T03:23:23.029449Z","iopub.status.idle":"2023-11-19T03:23:31.00248Z","shell.execute_reply.started":"2023-11-19T03:23:23.029422Z","shell.execute_reply":"2023-11-19T03:23:31.001611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbc = GradientBoostingClassifier(random_state=1)\ncv = cross_val_score(gbc, X_resampled,y_resampled, cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:23:31.00403Z","iopub.execute_input":"2023-11-19T03:23:31.00428Z","iopub.status.idle":"2023-11-19T03:23:44.545964Z","shell.execute_reply.started":"2023-11-19T03:23:31.004242Z","shell.execute_reply":"2023-11-19T03:23:44.54519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"abc = AdaBoostClassifier(random_state=1)\ncv = cross_val_score(abc, X_resampled,y_resampled, cv=5, scoring='roc_auc')\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:23:44.547376Z","iopub.execute_input":"2023-11-19T03:23:44.547957Z","iopub.status.idle":"2023-11-19T03:23:48.305906Z","shell.execute_reply.started":"2023-11-19T03:23:44.547927Z","shell.execute_reply":"2023-11-19T03:23:48.304244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Tuned Performance \nAfter getting the baselines, I mainly used grid search to tune the models. I also used Randomized Search for the Random Forest to simplify testing time. \n\n|Model|Tuned Performance|\n|-----|-------|\n|Naive Bayes|NA|\n|Logistic Regression|79.2%|\n|Decision Tree| 94.8% |\n|Random Forest|98.8% |\n|K Nearest Neighbor|NA|\n|SVC| NA|\n|Xtreme Gradient Boosting|NA|\n|Ada Boosting| 95.0%\n|Gradient Boosting|98.8%\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV \nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:34:56.55695Z","iopub.execute_input":"2023-11-19T03:34:56.557334Z","iopub.status.idle":"2023-11-19T03:34:56.562232Z","shell.execute_reply.started":"2023-11-19T03:34:56.557305Z","shell.execute_reply":"2023-11-19T03:34:56.561192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clf_performance(classifier, model_name):\n    print(model_name)\n    print('Best Score: ' + str(classifier.best_score_))\n    print('Best Parameters: ' + str(classifier.best_params_))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:26:53.025302Z","iopub.execute_input":"2023-11-19T03:26:53.02564Z","iopub.status.idle":"2023-11-19T03:26:53.031337Z","shell.execute_reply.started":"2023-11-19T03:26:53.025613Z","shell.execute_reply":"2023-11-19T03:26:53.03019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression()\nparam_grid = {'max_iter' : [1000,2000,5000,10000],\n              'penalty' : ['l1', 'l2'],\n              'C' : np.logspace(-4, 4, 20),\n              'tol': [0.0001, 0.001, 0.01, 0.1, 0.5, 1.0],\n              'solver' : ['liblinear']}\n\nclf_lr = GridSearchCV(lr, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_lr = clf_lr.fit(X_resampled,y_resampled)\nclf_performance(best_clf_lr,'Logistic Regression')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:30:31.106424Z","iopub.execute_input":"2023-11-19T03:30:31.106811Z","iopub.status.idle":"2023-11-19T03:34:33.394228Z","shell.execute_reply.started":"2023-11-19T03:30:31.106771Z","shell.execute_reply":"2023-11-19T03:34:33.393023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc = roc_auc_score(y_train, best_clf_lr.predict(X_train_labeled))\nprint(f\"ROC_AUC Score: {roc_auc}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:34:59.425468Z","iopub.execute_input":"2023-11-19T03:34:59.425881Z","iopub.status.idle":"2023-11-19T03:34:59.580053Z","shell.execute_reply.started":"2023-11-19T03:34:59.425845Z","shell.execute_reply":"2023-11-19T03:34:59.578887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = tree.DecisionTreeClassifier(random_state = 1)\nparam_grid = {'criterion' : ['gini', 'entropy', 'log_loss'],\n              'splitter' : ['random', 'best'],\n              'max_depth': [3,5,10,20,50,75,100,None],\n              'max_features': [None, 'sqrt', 'log2']}\n\nclf_dt = GridSearchCV(dt, param_grid = param_grid, cv = 10, verbose = True, n_jobs = -1, scoring='roc_auc')\nbest_clf_dt = clf_dt.fit(X_resampled,y_resampled)\nclf_performance(best_clf_dt,'Decision Tree')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:35:12.236054Z","iopub.execute_input":"2023-11-19T03:35:12.236433Z","iopub.status.idle":"2023-11-19T03:35:31.374866Z","shell.execute_reply.started":"2023-11-19T03:35:12.236404Z","shell.execute_reply":"2023-11-19T03:35:31.373944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc = roc_auc_score(y_train, best_clf_dt.predict(X_train_labeled))\nprint(f\"ROC_AUC Score: {roc_auc}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-19T03:35:37.0328Z","iopub.execute_input":"2023-11-19T03:35:37.03318Z","iopub.status.idle":"2023-11-19T03:35:37.160705Z","shell.execute_reply.started":"2023-11-19T03:35:37.033146Z","shell.execute_reply":"2023-11-19T03:35:37.159237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"from sklearn.model_selection import RandomizedSearchCV \n\nrf = RandomForestClassifier(random_state = 1)\nparam_grid =  {'n_estimators': [5, 10, 20], \n                                  'bootstrap': [True,False],\n                                  'max_depth': [3,5,10,20,50,75,100,None],\n                                  'max_features': ['log2','sqrt'],\n                                  'min_samples_leaf': [1,2,4,10],\n                                  'min_samples_split': [2,5,10]}\n                                  \nclf_rf_rnd = RandomizedSearchCV(rf, param_distributions = param_grid, n_iter = 250, cv = 5, verbose = True, n_jobs = -1, scoring='roc_auc')\nbest_clf_rf_rnd = clf_rf_rnd.fit(X_resampled,y_resampled)\nclf_performance(best_clf_rf_rnd,'Random Forest')\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:22:57.000618Z","iopub.execute_input":"2023-11-19T08:22:57.001133Z","iopub.status.idle":"2023-11-19T08:24:41.242253Z","shell.execute_reply.started":"2023-11-19T08:22:57.001096Z","shell.execute_reply":"2023-11-19T08:24:41.240482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = RandomForestClassifier(random_state = 1)\nparam_grid =  {'n_estimators': [10,20,30,40],\n               'criterion':['gini','entropy'],\n                'bootstrap': [False],\n                'max_depth': [50, 100, 150, 200],\n                'max_features': ['sqrt'],\n                'min_samples_leaf': [1, 2, 3, 4],\n                'min_samples_split': [4, 5, 6]}\n                                  \nclf_rf = GridSearchCV(rf, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_rf = clf_rf.fit(X_resampled,y_resampled)\nclf_performance(best_clf_rf,'Random Forest')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:25:51.598017Z","iopub.execute_input":"2023-11-19T08:25:51.598463Z","iopub.status.idle":"2023-11-19T08:33:15.98164Z","shell.execute_reply.started":"2023-11-19T08:25:51.598419Z","shell.execute_reply":"2023-11-19T08:33:15.979941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc = roc_auc_score(y_train, best_clf_rf.best_estimator_.predict(X_train_labeled))\nprint(f\"ROC_AUC Score: {roc_auc}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:34:06.378407Z","iopub.execute_input":"2023-11-19T08:34:06.378889Z","iopub.status.idle":"2023-11-19T08:34:08.712504Z","shell.execute_reply.started":"2023-11-19T08:34:06.378855Z","shell.execute_reply":"2023-11-19T08:34:08.710928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ab = AdaBoostClassifier(random_state = 1)\nparam_grid =  {'n_estimators': [50, 100, 200, 500],\n               'algorithm':['SAMME','SAMME.R'],\n                'learning_rate': [0.01, 0.1, 0.5, 1.0]}\n                                  \nclf_ab = GridSearchCV(ab, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_ab = clf_ab.fit(X_resampled,y_resampled)\nclf_performance(best_clf_ab,'Ada Boost Classifier')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:37:57.769869Z","iopub.execute_input":"2023-11-19T08:37:57.770272Z","iopub.status.idle":"2023-11-19T08:40:51.187558Z","shell.execute_reply.started":"2023-11-19T08:37:57.770237Z","shell.execute_reply":"2023-11-19T08:40:51.185708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc = roc_auc_score(y_train, best_clf_ab.best_estimator_.predict(X_train_labeled))\nprint(f\"ROC_AUC Score: {roc_auc}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:41:21.374002Z","iopub.execute_input":"2023-11-19T08:41:21.374432Z","iopub.status.idle":"2023-11-19T08:41:48.519563Z","shell.execute_reply.started":"2023-11-19T08:41:21.374393Z","shell.execute_reply":"2023-11-19T08:41:48.518096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gb = GradientBoostingClassifier(random_state = 1)\nparam_grid =  {'n_estimators': [50, 100, 200, 500],\n               'loss':['log_loss','exponential'],\n               'learning_rate': [0.01, 0.1, 0.5, 1.0],\n               'criterion' : ['friedman_mse', 'squared_error']}\n                                  \nclf_gb = GridSearchCV(gb, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_gb = clf_gb.fit(X_resampled,y_resampled)\nclf_performance(best_clf_gb,'Gradient Boost Classifier')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:41:55.95556Z","iopub.execute_input":"2023-11-19T08:41:55.956049Z","iopub.status.idle":"2023-11-19T08:51:26.798966Z","shell.execute_reply.started":"2023-11-19T08:41:55.956011Z","shell.execute_reply":"2023-11-19T08:51:26.797757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc = roc_auc_score(y_train, best_clf_gb.best_estimator_.predict(X_train_labeled))\nprint(f\"ROC_AUC Score: {roc_auc}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:51:43.490906Z","iopub.execute_input":"2023-11-19T08:51:43.491438Z","iopub.status.idle":"2023-11-19T08:51:46.809094Z","shell.execute_reply.started":"2023-11-19T08:51:43.491396Z","shell.execute_reply":"2023-11-19T08:51:46.807332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stacking Models with Logistic Regression as the Meta-Model\n#### 1) Find all combinations of models with above 90%\n#### 2) Perform a stacking classifier on all and get roc scores\n#### 3) Pick the best score","metadata":{}},{"cell_type":"code","source":"from itertools import combinations, chain\n\nbest_rf = best_clf_rf.best_estimator_\nbest_dt = best_clf_dt.best_estimator_\nbest_gb = best_clf_gb.best_estimator_\nbest_ab = best_clf_ab.best_estimator_\n\nmodels = {\n    'gb': best_gb,\n    'ada': best_ab,\n    'rf': best_rf,\n    'dt': best_dt,\n    'xgb': XGBClassifier(random_state =1),\n}\n\n# Get all combinations of models\nall_model_combinations = []\n\nfor r in range(1, len(models) + 1):\n    model_combinations = combinations(models.keys(), r)\n    all_model_combinations.extend(model_combinations)\n    \nall_model_combinations = [list(combination) for combination in all_model_combinations][5:]\n\nprint(all_model_combinations)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:52:28.964494Z","iopub.execute_input":"2023-11-19T08:52:28.964939Z","iopub.status.idle":"2023-11-19T08:52:28.973149Z","shell.execute_reply.started":"2023-11-19T08:52:28.964907Z","shell.execute_reply":"2023-11-19T08:52:28.972091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import StackingClassifier","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:56:55.260095Z","iopub.execute_input":"2023-11-19T08:56:55.260473Z","iopub.status.idle":"2023-11-19T08:56:55.266983Z","shell.execute_reply.started":"2023-11-19T08:56:55.260442Z","shell.execute_reply":"2023-11-19T08:56:55.264563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"combination_scores = {}\nfor model_combination in all_model_combinations:\n    base_models = [(name, models[name]) for name in model_combination]\n    meta_model = LogisticRegression()\n\n    stacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model)\n    stacked_model.fit(X_resampled, y_resampled)\n    \n    roc_auc = roc_auc_score(y_train, stacked_model.predict(X_train_labeled))\n    combination_scores[tuple(model_combination)] = roc_auc\n    \n    print(f\"Combination: {model_combination}, Stacked Model Accuracy: {roc_auc}\")\n    \"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T08:56:58.342578Z","iopub.execute_input":"2023-11-19T08:56:58.342983Z","iopub.status.idle":"2023-11-19T09:41:28.148835Z","shell.execute_reply.started":"2023-11-19T08:56:58.342954Z","shell.execute_reply":"2023-11-19T09:41:28.147797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"sorted_combinations = sorted(combination_scores.items(), key=lambda x: x[1], reverse=True)\n\nfor combination, score in sorted_combinations:\n    print(f\"Combination: {combination}, Stacked Model AUC-ROC: {score}\")\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T09:42:27.250127Z","iopub.execute_input":"2023-11-19T09:42:27.250652Z","iopub.status.idle":"2023-11-19T09:42:27.259359Z","shell.execute_reply.started":"2023-11-19T09:42:27.250608Z","shell.execute_reply":"2023-11-19T09:42:27.257814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"combination_scores = {}\nfor model_combination in all_model_combinations:\n    base_models = [(name, models[name]) for name in model_combination]\n    meta_model = LogisticRegression()\n\n    voting_model = VotingClassifier(estimators=base_models, voting='soft')\n    voting_model.fit(X_resampled, y_resampled)\n    \n    roc_auc = roc_auc_score(y_train, voting_model.predict(X_train_labeled))\n    combination_scores[tuple(model_combination)] = roc_auc\n    \n    print(f\"Combination: {model_combination}, Stacked Model Accuracy: {roc_auc}\")\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T09:43:46.085748Z","iopub.execute_input":"2023-11-19T09:43:46.086199Z","iopub.status.idle":"2023-11-19T09:59:12.957282Z","shell.execute_reply.started":"2023-11-19T09:43:46.086169Z","shell.execute_reply":"2023-11-19T09:59:12.955978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"sorted_combinations = sorted(combination_scores.items(), key=lambda x: x[1], reverse=True)\n\nfor combination, score in sorted_combinations:\n    print(f\"Combination: {combination}, Stacked Model AUC-ROC: {score}\")\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T09:59:12.959131Z","iopub.execute_input":"2023-11-19T09:59:12.95948Z","iopub.status.idle":"2023-11-19T09:59:12.968609Z","shell.execute_reply.started":"2023-11-19T09:59:12.959449Z","shell.execute_reply":"2023-11-19T09:59:12.967108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_stacking_clf = StackingClassifier(estimators=[('dt', models['dt']), ('gb', models['gb']), ('rf', models['rf'])], final_estimator= LogisticRegression())\nbest_stacking_xgb_clf = StackingClassifier(estimators=[('gb', models['gb']), ('rf', models['rf']), ('dt', models['dt']),('xgb',models['xgb'])], final_estimator= LogisticRegression())\n\nbest_voting_clf = VotingClassifier(estimators=[('gb',models['gb']), ('rf', models['rf']), ('dt', models['dt'])], voting='soft')\nbest_voting_xgb_clf = VotingClassifier(estimators=[('ada',models['ada']),('gb',models['gb']),('xgb',models['xgb'])], voting='soft')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T10:14:31.107211Z","iopub.execute_input":"2023-11-19T10:14:31.107671Z","iopub.status.idle":"2023-11-19T10:14:31.11707Z","shell.execute_reply.started":"2023-11-19T10:14:31.107639Z","shell.execute_reply":"2023-11-19T10:14:31.115464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_stacking_clf.fit(X_resampled, y_resampled)\nbest_stacking_xgb_clf.fit(X_resampled, y_resampled)\nbest_voting_clf.fit(X_resampled, y_resampled)\nbest_voting_xgb_clf.fit(X_resampled, y_resampled)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T10:14:34.612589Z","iopub.execute_input":"2023-11-19T10:14:34.613543Z","iopub.status.idle":"2023-11-19T10:18:28.865535Z","shell.execute_reply.started":"2023-11-19T10:14:34.613482Z","shell.execute_reply":"2023-11-19T10:18:28.86467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_stacking_pred = best_stacking_clf.predict(X_test_labeled)\nbest_stacking_no_xgb_pred = best_stacking_xgb_clf.predict(X_test_labeled)\nbest_voting_pred = best_voting_clf.predict(X_test_labeled)\nbest_voting_xgb_pred = best_voting_xgb_clf.predict(X_test_labeled)\n\nbest_stacking_sub = pd.DataFrame({'is_fraud': best_stacking_pred})\nbest_stacking_no_xgb_sub = pd.DataFrame({'is_fraud': best_stacking_no_xgb_pred})\nbest_voting_sub = pd.DataFrame({'is_fraud': best_voting_pred})\nbest_voting_xgb_sub = pd.DataFrame({'is_fraud': best_voting_xgb_pred})\n\nbest_stacking_sub.index = df_test_id\nbest_stacking_no_xgb_sub.index = df_test_id\nbest_voting_sub.index = df_test_id\nbest_voting_xgb_sub.index = df_test_id\n\nbest_stacking_sub.index.name = 'id'\nbest_stacking_no_xgb_sub.index.name = 'id'\nbest_voting_sub.index.name = 'id'\nbest_voting_xgb_sub.index.name = 'id'\n\nbest_stacking_sub.to_csv('best_stacking_submission.csv') \nbest_stacking_no_xgb_sub.to_csv('no_xgb_stacking_submission.csv') \nbest_voting_sub.to_csv('best_voting_submission.csv')\nbest_voting_xgb_sub.to_csv('best_voting_xgb_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T10:19:13.437619Z","iopub.execute_input":"2023-11-19T10:19:13.438067Z","iopub.status.idle":"2023-11-19T10:20:04.810176Z","shell.execute_reply.started":"2023-11-19T10:19:13.438035Z","shell.execute_reply":"2023-11-19T10:20:04.809243Z"},"trusted":true},"execution_count":null,"outputs":[]}]}