{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":63054,"databundleVersionId":6872631,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-21T19:02:51.744192Z","iopub.execute_input":"2023-11-21T19:02:51.744551Z","iopub.status.idle":"2023-11-21T19:02:52.190803Z","shell.execute_reply.started":"2023-11-21T19:02:51.744523Z","shell.execute_reply":"2023-11-21T19:02:52.189206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nfrom datetime import datetime\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.ensemble import GradientBoostingClassifier, AdaBoostClassifier\nfrom imblearn.over_sampling import SMOTE","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:02:52.193402Z","iopub.execute_input":"2023-11-21T19:02:52.194182Z","iopub.status.idle":"2023-11-21T19:02:54.561217Z","shell.execute_reply.started":"2023-11-21T19:02:52.194138Z","shell.execute_reply":"2023-11-21T19:02:54.559911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/anyas-gojo-revival-bondman/trainData.csv')\ntest_data = pd.read_csv('/kaggle/input/anyas-gojo-revival-bondman/testData.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:02:54.562510Z","iopub.execute_input":"2023-11-21T19:02:54.563009Z","iopub.status.idle":"2023-11-21T19:03:05.117593Z","shell.execute_reply.started":"2023-11-21T19:02:54.562976Z","shell.execute_reply":"2023-11-21T19:03:05.116661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.120292Z","iopub.execute_input":"2023-11-21T19:03:05.120656Z","iopub.status.idle":"2023-11-21T19:03:05.164804Z","shell.execute_reply.started":"2023-11-21T19:03:05.120625Z","shell.execute_reply":"2023-11-21T19:03:05.163650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# plt.figure(figsize=(12, 8))\n# sns.heatmap(train_data.corr(),annot = True,fmt = '.2f')","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.166575Z","iopub.execute_input":"2023-11-21T19:03:05.167053Z","iopub.status.idle":"2023-11-21T19:03:05.171526Z","shell.execute_reply.started":"2023-11-21T19:03:05.167020Z","shell.execute_reply":"2023-11-21T19:03:05.170517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing id, cc_num, first, last, street, city, zip, city_pop, trans_num, unix_time, merch_lat, merch_long\ndrops = ['id', 'cc_num', 'first', 'last', 'street', 'city', 'zip', 'city_pop', 'trans_num', 'unix_time', 'merch_lat', 'merch_long']\ntrain_data = train_data.drop(drops, axis = 1)\ntest_data = test_data.drop(drops, axis = 1)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.173157Z","iopub.execute_input":"2023-11-21T19:03:05.173494Z","iopub.status.idle":"2023-11-21T19:03:05.315927Z","shell.execute_reply.started":"2023-11-21T19:03:05.173466Z","shell.execute_reply":"2023-11-21T19:03:05.315095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.317008Z","iopub.execute_input":"2023-11-21T19:03:05.317747Z","iopub.status.idle":"2023-11-21T19:03:05.336823Z","shell.execute_reply.started":"2023-11-21T19:03:05.317716Z","shell.execute_reply":"2023-11-21T19:03:05.335699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['male'] = np.where(train_data.gender == 'M', 1, 0)\ntest_data['male'] = np.where(test_data.gender == 'M', 1, 0)\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.338379Z","iopub.execute_input":"2023-11-21T19:03:05.339232Z","iopub.status.idle":"2023-11-21T19:03:05.552702Z","shell.execute_reply.started":"2023-11-21T19:03:05.339192Z","shell.execute_reply":"2023-11-21T19:03:05.551449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drops = ['gender']\ntrain_data = train_data.drop(drops, axis = 1)\ntest_data = test_data.drop(drops, axis = 1)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.554007Z","iopub.execute_input":"2023-11-21T19:03:05.554314Z","iopub.status.idle":"2023-11-21T19:03:05.627575Z","shell.execute_reply.started":"2023-11-21T19:03:05.554289Z","shell.execute_reply":"2023-11-21T19:03:05.626565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now looking at the trans_date_time column, we dont want the date, extracting the hour only , but we need to convert this column into date time\n\ntrain_data['trans_date_trans_time'] = pd.to_datetime(train_data['trans_date_trans_time'])\n# train_data.info()\ntest_data['trans_date_trans_time'] = pd.to_datetime(test_data['trans_date_trans_time'])\n# now extracting the time only\n\ntrain_data['time'] = train_data.trans_date_trans_time.dt.hour\ntest_data['time'] = test_data.trans_date_trans_time.dt.hour\n\ntrain_data.head()\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:05.629073Z","iopub.execute_input":"2023-11-21T19:03:05.629380Z","iopub.status.idle":"2023-11-21T19:03:06.255539Z","shell.execute_reply.started":"2023-11-21T19:03:05.629354Z","shell.execute_reply":"2023-11-21T19:03:06.254404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_data.groupby(['time']).agg({'is_fraud': 'mean'}).plot()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:06.259599Z","iopub.execute_input":"2023-11-21T19:03:06.259954Z","iopub.status.idle":"2023-11-21T19:03:06.625708Z","shell.execute_reply.started":"2023-11-21T19:03:06.259926Z","shell.execute_reply":"2023-11-21T19:03:06.624632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drops = ['trans_date_trans_time']\ntrain_data = train_data.drop(drops, axis = 1)\ntest_data = test_data.drop(drops, axis = 1)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:06.626962Z","iopub.execute_input":"2023-11-21T19:03:06.627290Z","iopub.status.idle":"2023-11-21T19:03:06.722408Z","shell.execute_reply.started":"2023-11-21T19:03:06.627261Z","shell.execute_reply":"2023-11-21T19:03:06.720935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:06.724313Z","iopub.execute_input":"2023-11-21T19:03:06.724763Z","iopub.status.idle":"2023-11-21T19:03:06.743014Z","shell.execute_reply.started":"2023-11-21T19:03:06.724720Z","shell.execute_reply":"2023-11-21T19:03:06.741956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting age using dob\ntrain_data['dob'] = pd.to_datetime(train_data['dob'], format='%Y-%m-%d')\n\n# Calculate age as of November 13, 2023\nreference_date = pd.to_datetime('2023-11-13')\ntrain_data['age'] = ((reference_date - train_data['dob']).dt.days / 365.25).astype(int)\n\n# getting age using dob\ntest_data['dob'] = pd.to_datetime(test_data['dob'], format='%Y-%m-%d')\n\n# Calculate age as of November 13, 2023\nreference_date = pd.to_datetime('2023-11-13')\ntest_data['age'] = ((reference_date - test_data['dob']).dt.days / 365.25).astype(int)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:06.744324Z","iopub.execute_input":"2023-11-21T19:03:06.744655Z","iopub.status.idle":"2023-11-21T19:03:07.178746Z","shell.execute_reply.started":"2023-11-21T19:03:06.744627Z","shell.execute_reply":"2023-11-21T19:03:07.177259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:07.180221Z","iopub.execute_input":"2023-11-21T19:03:07.180574Z","iopub.status.idle":"2023-11-21T19:03:07.201016Z","shell.execute_reply.started":"2023-11-21T19:03:07.180544Z","shell.execute_reply":"2023-11-21T19:03:07.199806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drops = ['dob']\ntrain_data = train_data.drop(drops, axis = 1)\ntest_data = test_data.drop(drops, axis = 1)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:07.202490Z","iopub.execute_input":"2023-11-21T19:03:07.203197Z","iopub.status.idle":"2023-11-21T19:03:07.298239Z","shell.execute_reply.started":"2023-11-21T19:03:07.203154Z","shell.execute_reply":"2023-11-21T19:03:07.297059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:07.299501Z","iopub.execute_input":"2023-11-21T19:03:07.302670Z","iopub.status.idle":"2023-11-21T19:03:07.320383Z","shell.execute_reply.started":"2023-11-21T19:03:07.302633Z","shell.execute_reply":"2023-11-21T19:03:07.319158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_data.merchant.value_counts()))\nprint(len(test_data.merchant.value_counts()))\n\ndef isEqual(data1, data2):\n  data1 = data1.unique()\n  data2 = data2.unique()\n  print('length of data 1',len(data1))\n  print('length of data 2',len(data2))\n  # print(data1[50])\n  if(sorted(data1) == sorted(data2)):\n    return \"yes\"\n  else:\n    return \"no\"\n\nprint(isEqual(train_data.merchant, test_data.merchant))\nprint(isEqual(train_data.state, test_data.state))\n\nprint(isEqual(train_data.category, test_data.category))\nprint(isEqual(train_data.job, test_data.job))\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:07.322358Z","iopub.execute_input":"2023-11-21T19:03:07.322788Z","iopub.status.idle":"2023-11-21T19:03:07.970164Z","shell.execute_reply.started":"2023-11-21T19:03:07.322750Z","shell.execute_reply":"2023-11-21T19:03:07.968845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing fraud_ from all the values of merchant column\ntrain_data['merchant'] = train_data['merchant'].str.replace('fraud_', '')\ntest_data['merchant'] = test_data['merchant'].str.replace('fraud_', '')\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:07.971773Z","iopub.execute_input":"2023-11-21T19:03:07.972378Z","iopub.status.idle":"2023-11-21T19:03:08.745432Z","shell.execute_reply.started":"2023-11-21T19:03:07.972343Z","shell.execute_reply":"2023-11-21T19:03:08.744325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport re\n\ndef preprocess_text(text):\n    # Convert to lowercase\n    text = text.lower()\n    # Remove non-alphabetic characters\n    text = re.sub(r'[^a-z ]', '', text)\n    return text\n\ndef vectorize_and_drop(df, column_name, max_features=50):\n    print(f'Vectorizing {column_name} column')\n    # Preprocess text column\n    df[column_name] = df[column_name].apply(preprocess_text)\n\n    # Initialize the CountVectorizer\n    vectorizer = CountVectorizer(max_features=max_features)\n\n    # Fit and transform the column\n    vectors = vectorizer.fit_transform(df[column_name])\n\n    # Convert the vectors to a DataFrame\n    vectors_df = pd.DataFrame(vectors.toarray(), columns=vectorizer.get_feature_names_out())\n\n    # Concatenate the original DataFrame with the new vectors\n    df = pd.concat([df, vectors_df], axis=1)\n\n    # Drop the original column\n    df = df.drop(column_name, axis=1)\n\n    return df\n\ndef vectorize_text_columns(df, text_columns, max_features=50):\n    for column_name in text_columns:\n        df = vectorize_and_drop(df, column_name, max_features)\n    return df\n\n# Columns to vectorize\ntext_columns_to_vectorize = ['merchant', 'state', 'category', 'job']\n\n# Apply vectorization to train_data\ntrain_data = vectorize_text_columns(train_data, text_columns_to_vectorize)\n\n# Apply vectorization to test_data\ntest_data = vectorize_text_columns(test_data, text_columns_to_vectorize)\n\n# Display the resulting DataFrames\nprint(\"Train Data Vectorized:\")\nprint(train_data.head())\n\nprint(\"\\nTest Data Vectorized:\")\nprint(test_data.head())\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:08.747353Z","iopub.execute_input":"2023-11-21T19:03:08.747790Z","iopub.status.idle":"2023-11-21T19:03:53.852311Z","shell.execute_reply.started":"2023-11-21T19:03:08.747751Z","shell.execute_reply":"2023-11-21T19:03:53.850959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape)\nprint(test_data.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:53.853655Z","iopub.execute_input":"2023-11-21T19:03:53.854000Z","iopub.status.idle":"2023-11-21T19:03:53.859984Z","shell.execute_reply.started":"2023-11-21T19:03:53.853972Z","shell.execute_reply":"2023-11-21T19:03:53.858750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find columns present in train_data but not in test_data\nmissing_columns_train = set(train_data.columns) - set(test_data.columns)\n\n# Find columns present in test_data but not in train_data\nmissing_columns_test = set(test_data.columns) - set(train_data.columns)\n\nprint(\"Columns present in train_data but not in test_data:\", missing_columns_train)\nprint(\"Columns present in test_data but not in train_data:\", missing_columns_test)\n\n# Exclude the target column 'is_fraud' from the columns to drop\ndrops_train = list(set(missing_columns_train) - {'is_fraud'})\ndrops_test = list(missing_columns_test)\n\n# drop uncommon columns in both train_data and test_data\ntrain_data = train_data.drop(drops_train, axis=1, errors='ignore')\ntest_data = test_data.drop(drops_test, axis=1, errors='ignore')\n\n# Display the resulting DataFrames\nprint(\"Train Data after dropping uncommon columns:\")\nprint(train_data.head())\n\nprint(\"\\nTest Data after dropping uncommon columns:\")\nprint(test_data.head())\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:53.861203Z","iopub.execute_input":"2023-11-21T19:03:53.861550Z","iopub.status.idle":"2023-11-21T19:03:54.400230Z","shell.execute_reply.started":"2023-11-21T19:03:53.861522Z","shell.execute_reply":"2023-11-21T19:03:54.397969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Separate features and target variable for training data\nX_train = train_data.drop('is_fraud', axis=1)\ny_train = train_data['is_fraud']\n\n# Apply SMOTE to the training data only\nsmote = SMOTE(sampling_strategy={1: 15000})  \nX_train_resampled, y_train_resampled = smote.fit_resample(X_train, y_train)\n\n# Concatenate the resampled data with the original non-fraud data\nX_train = pd.concat([X_train_resampled, X_train[y_train == 0]])\ny_train = pd.concat([y_train_resampled, y_train[y_train == 0]])\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:54.402117Z","iopub.execute_input":"2023-11-21T19:03:54.402550Z","iopub.status.idle":"2023-11-21T19:03:59.211065Z","shell.execute_reply.started":"2023-11-21T19:03:54.402518Z","shell.execute_reply":"2023-11-21T19:03:59.209854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Features and target variable in the training data\n# X_train = train_data.drop('is_fraud', axis=1)\n# y_train = train_data['is_fraud']\n\n# Initializing the Random Forest model\nrandom_forest_model = RandomForestClassifier(n_estimators=130, random_state=42)\n\n# Train the model on the full training data\nrandom_forest_model.fit(X_train, y_train)\n\n# Features in the test data\nX_test = test_data  # Replace with your actual test_data\n\n# Make predictions on the test data\ny_pred_proba = random_forest_model.predict_proba(X_test)[:, 1]  # Probability of positive class\n\n# Applying a threshold to convert probabilities to binary predictions\nthreshold = 0.4 # chose this thresh on the basis of ROC AUC curve and hit and trial\ny_pred_binary = (y_pred_proba >= threshold).astype(int)\n\n# Create a DataFrame for the predictions\npredictions_df = pd.DataFrame({\n    'id': range(1, len(y_pred_binary) + 1),\n    'is_fraud': y_pred_binary\n})\n\n# Save the predictions to a CSV file\npredictions_df.to_csv('predictions.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:03:59.212518Z","iopub.execute_input":"2023-11-21T19:03:59.212921Z","iopub.status.idle":"2023-11-21T19:13:20.794133Z","shell.execute_reply.started":"2023-11-21T19:03:59.212877Z","shell.execute_reply":"2023-11-21T19:13:20.792815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_proba[:10]","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:13:20.795356Z","iopub.execute_input":"2023-11-21T19:13:20.795689Z","iopub.status.idle":"2023-11-21T19:13:20.803075Z","shell.execute_reply.started":"2023-11-21T19:13:20.795661Z","shell.execute_reply":"2023-11-21T19:13:20.801949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.is_fraud.value_counts()\npredictions_df.is_fraud.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:13:20.804933Z","iopub.execute_input":"2023-11-21T19:13:20.805737Z","iopub.status.idle":"2023-11-21T19:13:20.830820Z","shell.execute_reply.started":"2023-11-21T19:13:20.805696Z","shell.execute_reply":"2023-11-21T19:13:20.829652Z"},"trusted":true},"execution_count":null,"outputs":[]}]}