{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Obtaining Data","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nplt.style.use('dark_background') # Remove if using light mode.\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegressionCV\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder\n\nfrom imblearn.over_sampling import SMOTE","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrainTransact = pd.read_csv('../input/frauddetect/train_transaction.csv')\ntestTransact = pd.read_csv('../input/frauddetect/test_transaction.csv')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrainIdentity = pd.read_csv('../input/frauddetect/train_identity.csv')\ntestIdentity = pd.read_csv('../input/frauddetect/test_identity.csv')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in trainTransact:\n    if col in trainIdentity:\n        print(col)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_full = pd.merge(trainTransact, trainIdentity, on = 'TransactionID', how = 'left', suffixes = (\"\", \"\"));\ntest_full = pd.merge(testTransact, testIdentity, on = 'TransactionID', how = 'left', suffixes = (\"\", \"\"))","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Memory Reduction**","metadata":{"heading_collapsed":true}},{"cell_type":"markdown","source":"Step 1 : Delete unused dataframes.","metadata":{"hidden":true}},{"cell_type":"code","source":"del trainTransact\ndel testTransact\ndel trainIdentity\ndel testIdentity","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = reduce_mem_usage(train_full)\ntest = reduce_mem_usage(test_full)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_full\ndel test_full","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Writing to csv files to save time for future EDA.","metadata":{"hidden":true}},{"cell_type":"code","source":"# train_reduce.to_csv('train.csv', index = False)\n# test_reduce.to_csv('test.csv', index = False)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Commonly used functions**","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"def cat_and_num_colNames(df):\n    categoricalCol = df.select_dtypes(include=['O']).columns\n    numCol = df.select_dtypes(include=[np.number]).columns\n    return categoricalCol, numCol\n\ndef understandData(df):\n    for col in df.columns:\n        datatype = df[col].dtype\n        print(\"Feature \", end = ' ')\n        print(col)\n        if datatype == 'object':\n            print(\"Categorical Variable, no of unique values = \", end = ' ')\n            print(len(df[col].unique()), end = ' ')\n            print(df[col].unique())\n        else:\n            print(\"Numerical col\")\n        print(\"Missing value percentage = \", end = ' ')\n        print(df[col].isna().sum()/len(df))\n        print()\n        \ndef facetGridPlotter(data, x, col):\n    sns.set(font_scale=1)\n    sns.set_style(\"white\")\n    p = sns.catplot(\n    data=data,\n    x=x,\n    col= col,\n    col_wrap=2,\n    kind=\"count\",\n    # hue=\"var\",\n    sharex=False,\n    dodge=False,\n    sharey=False,\n    )\n    for axes in p.axes.flat:\n        axes.set_xticklabels(\n            axes.get_xticklabels(), rotation=65, horizontalalignment=\"right\"\n        )\n        \ndef getNanPercentage(df, cols):\n    for col in cols:\n        print(col, end = ' ')\n        print(df[col].isna().sum()/len(df))\n        \ndef dropCol(df, cols):\n    for col in cols:\n        if col in df.columns:\n            df.drop(col, axis = 1, inplace = True)\n            \ndef removeColWithNan(df, cols, threshold):\n    removedCol = []\n    for col in cols:\n        l1 = df[col].isna().sum()\n        l2 = len(df)\n        nanPercentage = 100*(l1/l2)\n        if nanPercentage >= threshold:\n            if col in df.columns:\n                df = df.drop(col, axis = 1)\n                print(col + str(\" \") + str(nanPercentage) + \" Nan Percentage\")\n                removedCol.append(col)\n    return df, removedCol\n\ndef boxPlotter(data, compare_var, targetCol):\n    data = pd.concat([data[compare_var], data[targetCol]], axis = 1)\n    fig = sns.boxplot(x = compare_var, y = targetCol, data = data)\n    # fig.axis(ymin = -1, ymax = 150)\n    plt.show()\n\ndef updateNan(df, col, value):\n    df[col].fillna(value, inplace = True)\n    return df","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove_these = []\nlabel_encoded_cols = []\none_hot_encoded_cols = []\nskew_cols = []\nstandardize_columns = []\ntraining_col = []","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**","metadata":{}},{"cell_type":"markdown","source":"## Initial EDA","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"def initial_drop_columns(train, test, threshold):\n    majority_null_columns_train = []\n    huge_majority_rule_columns_train = []\n    one_or_none_value_columns_train = []\n    for i in train.columns:\n        if train[i].isnull().sum()/train.shape[0] > threshold:\n            majority_null_columns_train.append(i)\n        if train[i].dtype!='object':\n            if train[i].value_counts(dropna=False, normalize=True).values[0]>0.91:\n                huge_majority_rule_columns_train.append(i)\n        if train[i].nunique()<=1:\n            one_or_none_value_columns_train.append(i)\n\n    print(majority_null_columns_train)\n    print(len(majority_null_columns_train))\n    print()\n\n    print(huge_majority_rule_columns_train)\n    print(len(huge_majority_rule_columns_train))\n    print()\n\n    print(one_or_none_value_columns_train)\n    print(len(one_or_none_value_columns_train))\n    print()\n    \n    for i in huge_majority_rule_columns_train:\n        print(train[i].value_counts())\n        print()\n    \n    categorical_columns_train = ['isFraud', 'V104', 'V107', 'V108', 'V109', 'V110', 'V111', 'V112', 'V113', 'V114', 'V115', 'V116', 'V117', 'V118', 'V119', 'V120', 'V121', 'V122', 'V123', 'V124', 'V125', 'V281', 'V284', 'V286', 'V297', 'V300', 'V301', 'V305']\n\n    for i in categorical_columns_train:\n        huge_majority_rule_columns_train.remove(i)\n    \n    drop_train_columns = list(set(majority_null_columns_train + huge_majority_rule_columns_train + one_or_none_value_columns_train))\n#     print(drop_train_columns)\n#     print(len(drop_train_columns))\n#     print()\n    \n    majority_null_columns_test = []\n    huge_majority_rule_columns_test = []\n    one_or_none_value_columns_test = []\n    for i in test.columns:\n        if test[i].isnull().sum()/test.shape[0] > threshold:\n            majority_null_columns_test.append(i)\n        if test[i].dtype!='object':\n            if test[i].value_counts(dropna=False, normalize=True).values[0]>0.91:\n                huge_majority_rule_columns_test.append(i)\n        if test[i].nunique()<=1:\n            one_or_none_value_columns_test.append(i)\n\n#     print(majority_null_columns_test)\n#     print(len(majority_null_columns_test))\n#     print()\n\n#     print(huge_majority_rule_columns_test)\n#     print(len(huge_majority_rule_columns_test))\n#     print()\n\n#     print(one_or_none_value_columns_test)\n#     print(len(one_or_none_value_columns_test))\n#     print()\n    \n#     for i in huge_majority_rule_columns_train:\n#         print(train[i].value_counts())\n#         print()\n    \n    categorical_columns_test = ['V98', 'V284', 'V286']\n\n    for i in categorical_columns_test:\n        huge_majority_rule_columns_test.remove(i)\n    \n    drop_test_columns = list(set(majority_null_columns_test + huge_majority_rule_columns_test + one_or_none_value_columns_test))\n    print(drop_test_columns)\n    print(len(drop_test_columns))\n    print()\n    \n    drop_columns = list(set(drop_train_columns + drop_test_columns))\n    print(drop_columns)\n    print(len(drop_columns))\n    \n    return drop_columns","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_be_dropped = initial_drop_columns(train, test, 0.85)\nto_be_dropped","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(to_be_dropped)","metadata":{"hidden":true,"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dropCol(train, to_be_dropped)\ndropCol(test, to_be_dropped)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Class-wise analysis","metadata":{}},{"cell_type":"markdown","source":"### **Card columns analysis**","metadata":{}},{"cell_type":"code","source":"card_cols = [col for col in train if col.startswith('card')]\ncard_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_col = training_col + card_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"understandData(train[card_cols])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"card_cat, card_num = cat_and_num_colNames(train[card_cols])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"card_cat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in card_cat:\n    train[col].fillna('not_given', inplace = True)\n    test[col].fillna('not_given', inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in card_num:\n    train[col].fillna(train[col].median(), inplace = True)\n    test[col].fillna(test[col].median(), inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facetGridPlotter(train, \"card4\", \"isFraud\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facetGridPlotter(train, \"card6\", \"isFraud\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoded_cols.append('card4')\nlabel_encoded_cols.append('card6')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Analysis of D columns**\nThese columns give different data on time intervals","metadata":{}},{"cell_type":"code","source":"d_cols = [col for col in train if col.startswith('D')]\nd_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_cols.remove('DeviceType')\nd_cols.remove('DeviceInfo')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_col = training_col + d_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_cat, d_num = cat_and_num_colNames(train[d_cols])\nd_num","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"getNanPercentage(train, d_num)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nsns.heatmap(train[d_num].corr(), vmin=-1, cmap=\"coolwarm\", annot=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove_these.append('D2')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in d_num:\n    train[col].fillna(train[col].median(), inplace = True)\n    test[col].fillna(test[col].median(), inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Analysis of M columns**","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"m_cols = [col for col in train if col.startswith('M')]\nm_cols\ntraining_col = training_col + m_cols","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_cat, m_num = cat_and_num_colNames(train[m_cols])\nm_cat","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"understandData(train[m_cols])","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in m_cols:\n    train[col].fillna('not_given', inplace = True)\n    test[col].fillna('not_given', inplace = True)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_hot_encoded_cols = one_hot_encoded_cols + m_cols","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis of miscellaneous categorical columns","metadata":{"heading_collapsed":true}},{"cell_type":"markdown","source":" ### P_emaildomain","metadata":{"hidden":true}},{"cell_type":"code","source":"train['P_emaildomain'].isna().sum()/len(train)\ntraining_col.append('P_emaildomain')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['P_emaildomain'].fillna('not_given', inplace = True)\ntest['P_emaildomain'].fillna('not_given', inplace = True)\ntrain['P_emaildomain'].unique()","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groupedData = train[['P_emaildomain', 'isFraud']].groupby(by = 'P_emaildomain', as_index = False).mean().sort_values(by = 'isFraud', ascending = True)\ngroupedData","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 8))\nsns.barplot(x = groupedData['P_emaildomain'], y = groupedData['isFraud'])","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trusted_domains = []\nfor index, row in groupedData.iterrows():\n    if(row['isFraud'] == 0):\n        trusted_domains.append(row['P_emaildomain'])\n\nmapping = {}\nfor col in train['P_emaildomain'].unique():\n    if col in trusted_domains:\n        mapping[col] = \"trusted_domain.com\"\n    else:\n        mapping[col] = col\n\ntrain['P_emaildomain'] = train['P_emaildomain'].map(mapping)\ntest['P_emaildomain'] = test['P_emaildomain'].map(mapping)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groupedData = train[['P_emaildomain', 'isFraud']].groupby(by = 'P_emaildomain', as_index = False).mean().sort_values(by = 'isFraud', ascending = True)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 8))\nsns.barplot(x = groupedData['P_emaildomain'], y = groupedData['isFraud'])","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoded_cols.append('P_emaildomain')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### R_emaildomain","metadata":{"heading_collapsed":true,"hidden":true}},{"cell_type":"code","source":"train['R_emaildomain'].isna().sum()/len(train)\ntraining_col.append('R_emaildomain')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['R_emaildomain'].fillna('not_given', inplace = True)\ntest['R_emaildomain'].fillna('not_given', inplace = True)\ntrain['R_emaildomain'].unique()","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groupedData = train[['R_emaildomain', 'isFraud']].groupby(by = 'R_emaildomain', as_index = False).mean().sort_values(by = 'isFraud', ascending = True)\ngroupedData","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 8))\nsns.barplot(x = groupedData['R_emaildomain'], y = groupedData['isFraud'])","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trusted_domains = []\nfor index, row in groupedData.iterrows():\n    if(row['isFraud'] == 0):\n        trusted_domains.append(row['R_emaildomain'])","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping = {}\nfor col in train['R_emaildomain'].unique():\n    if col in trusted_domains:\n        mapping[col] = \"trusted_domain.com\"\n    else:\n        mapping[col] = col","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['R_emaildomain'] = train['R_emaildomain'].map(mapping)\ntest['R_emaildomain'] = test['R_emaildomain'].map(mapping)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groupedData = train[['R_emaildomain', 'isFraud']].groupby(by = 'R_emaildomain', as_index = False).mean().sort_values(by = 'isFraud', ascending = True)\ngroupedData","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 8))\nsns.barplot(x = groupedData['R_emaildomain'], y = groupedData['isFraud'])","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoded_cols.append('R_emaildomain')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **DeviceType**","metadata":{"heading_collapsed":true,"hidden":true}},{"cell_type":"code","source":"train['DeviceType'].unique()","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['DeviceType'].isna().sum()/len(train)\ntrain['DeviceType'].fillna('not_given', inplace = True)\ntest['DeviceType'].fillna('not_given', inplace = True)\ntraining_col.append('DeviceType')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facetGridPlotter(train, \"DeviceType\", \"isFraud\")","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_hot_encoded_cols.append('DeviceType')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **DeviceInfo**","metadata":{"heading_collapsed":true,"hidden":true}},{"cell_type":"code","source":"print(len(train['DeviceInfo'].unique()))\ntrain['DeviceInfo'].fillna('not_given', inplace = True)\ntraining_col.append('DeviceInfo')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groupedData = train[['DeviceInfo', 'isFraud']].groupby(by = 'DeviceInfo', as_index = False).mean().sort_values(by = 'isFraud', ascending = True)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groupedData","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trusted_device = []\nfraudulent_device = []\nfor index, row in groupedData.iterrows():\n    if(row['isFraud'] <= 0.1):\n        trusted_device.append(row['DeviceInfo'])\n    elif row['isFraud'] >= 0.9:\n        fraudulent_device.append(row['DeviceInfo'])\n\nmapping = {}\nfor col in train['DeviceInfo'].unique():\n    if col in trusted_device:\n        mapping[col] = \"trusted_device\"\n    elif col in fraudulent_device:\n        mapping[col] = \"fradulent_device\"\n    else:\n        mapping[col] = col\n\ntrain['DeviceInfo'] = train['DeviceInfo'].map(mapping)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train['DeviceInfo'].unique())","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoded_cols.append('DeviceInfo')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Analysis of numerical columns**","metadata":{"heading_collapsed":true,"hidden":true}},{"cell_type":"code","source":"boxPlotter(train, \"isFraud\", \"TransactionAmt\")","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(train['TransactionAmt'])\nplt.xlim(-10, 300)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def outlier_range(df, col):\n    q1 = df[col].quantile(q = 0.25)\n    q3 = df[col].quantile(q = 0.75)\n    iqr = q3 - q1\n    outlier_range = 1.5*iqr\n    r_whisker = q3 + outlier_range\n    l_whisker = q1 - outlier_range\n    return l_whisker, r_whisker","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l, r = outlier_range(train, 'TransactionAmt')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train.TransactionAmt.apply(lambda x : remove_outliers(train, 'TransactionAmt', x))\ntrain.TransactionAmt.loc[train.TransactionAmt > r] = r\ntrain.TransactionAmt.loc[train.TransactionAmt < l] = l","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['TransactionAmt'].describe()\ntraining_col.append('TransactionAmt')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skew_cols.append('TransactionAmt')","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"training_col","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dropCol(train, remove_these)\ndropCol(test, remove_these)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in remove_these:\n    if col in training_col:\n        training_col.remove(col)\n    if col in label_encoded_cols:\n        label_encoded_col.remove(col)\n    if col in one_hot_encoded_cols:\n        one_not_encoded_col.remove(col)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoded_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_hot_encoded_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encoding of categorical data and skew removal","metadata":{}},{"cell_type":"code","source":"tr = train[training_col]\nte = test[training_col]\ntr = pd.get_dummies(tr, columns = one_hot_encoded_cols)\nte = pd.get_dummies(te, columns = one_hot_encoded_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def labelEncode(df, labelCol):\n    labelEncoder = LabelEncoder()\n    for col in labelCol:\n        print(col)\n        df[col] = df[col].astype(str)\n        labelEncoder.fit(df[col])\n        df[col] = labelEncoder.transform(df[col])\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = tr.select_dtypes(include=['O']).columns\nprint(cat_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr = labelEncode(tr, cat_cols)\nte = labelEncode(te, cat_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def skewRemoval(df, cols):\n    for col in cols:\n        df[col] = np.log(df[col] + 1)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skew_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr = skewRemoval(tr, skew_cols)\nte = skewRemoval(te, skew_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Scaling of numerical data","metadata":{}},{"cell_type":"code","source":"tr.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\nV_data = pd.read_pickle('../input/pickle-files/V_data.pkl')\nID_data = pd.read_pickle('../input/pickle-files/ID_data.pkl')\nid_categorical_data = pd.read_pickle('../input/pickle-files/id_categorical_data.pkl')\n\nV_test_data = pd.read_pickle('../input/pickle-files-test/V_test_data.pkl')\nID_test_data = pd.read_pickle('../input/pickle-files-test/ID_test_data.pkl')\nid_categorical_test_data = pd.read_pickle('../input/pickle-files-test/id_categorical_test_data.pkl')\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\nid_columns = labelEncode(id_categorical_data, id_categorical_data.columns)\nid_columns_test = labelEncode(id_categorical_test_data, id_categorical_test_data.columns)\n\"\"\"\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\ntr[id_columns.columns] = id_columns.values\nte[id_columns_test.columns] = id_columns_test.values\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stdScalar = StandardScaler()\n# tr[training_col] = stdScalar.fit_transform(tr[training_col])\n# te[training_col] = stdScalar.fit_transform(te[training_col])\nfor col in tr:\n    tr[col] = (tr[col])/(tr[col].max() - tr[col].min())\n    te[col] = (te[col])/(te[col].max() - te[col].min())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\ntr[V_data.columns] = V_data.values\ntr[ID_data.columns] = ID_data.values\n\nte[V_test_data.columns] = V_test_data.values\nte[ID_test_data.columns] = ID_test_data.values\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"te.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D_data = pd.read_pickle(\"../input/pickle-files/D_data.pkl\")\n\nD_test_data = pd.read_pickle(\"../input/pickle-files/D_test_data.pkl\")\n\nD_columns = ['D1', 'D3', 'D4', 'D5', 'D10', 'D11', 'D15']\n\nprint(D_columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr = tr.drop(columns = D_columns)\nte = te.drop(columns = D_columns)\n\n\ntr[D_data.columns] = D_data.values\nte[D_test_data.columns] = D_test_data.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Handling imbalanced data.\n    Using Synthetic Minority Oversampling Technique","metadata":{}},{"cell_type":"code","source":"ytrain = train['isFraud']\nytrain = ytrain.to_frame()\nfinal_cols = tr.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain['isFraud'] = (ytrain['isFraud'] == True).astype(int)\nytrain['isFraud'].dtype","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sm = SMOTE(random_state=27, sampling_strategy=1.0)\nx_sampled, y_sampled = sm.fit_sample(tr, ytrain)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training","metadata":{"heading_collapsed":true}},{"cell_type":"markdown","source":"## LogisticRegressionCV()","metadata":{}},{"cell_type":"code","source":"log_reg = LogisticRegressionCV(cv = 5)\nlog_reg.fit(x_sampled, y_sampled)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_Prediction = log_reg.predict_proba(te)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_accuracy = log_reg.score(tr, ytrain)\nprint(train_accuracy)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_sampled[y_sampled['isFraud'] == 1])/len(y_sampled)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_Prediction[:,1]\nYY = pd.DataFrame(data = Y_Prediction, columns = [\"zero\", \"one\"])\nYY['one'].describe()","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n        \"TransactionID\": test[\"TransactionID\"],\n        \"IsFraud\": YY['one']\n    })\nsubmission.to_csv('submission.csv', index=False)","metadata":{"hidden":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## RandomForestClassifier()","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\nclf = RandomForestClassifier()\nclf.fit(x_sampled, y_sampled)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_output = clf.predict(te)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.DataFrame(y_output).value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ytrain.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\nn = 0\nfor i,j in zip(y_output, ytrain.values):\n    if(i==j):\n        n+=1\nprint(n/len(y_output)*100)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\nsubmission = pd.DataFrame({\n        \"TransactionID\": test[\"TransactionID\"],\n        \"IsFraud\": y_output\n    })\nsubmission.to_csv('submission.csv', index=False)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LGBM","metadata":{}},{"cell_type":"code","source":"import gc\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import accuracy_score\nfrom time import time\nimport datetime\nfrom tqdm import tqdm_notebook\nimport lightgbm as lgb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in tqdm_notebook(train.columns):\n    if train[col].dtype == 'object':\n        le = LabelEncoder()\n        le.fit(list(train[col].astype(str).values) + list(test[col].astype(str).values))\n        train[col] = le.transform(list(train[col].astype(str).values))\n        test[col] = le.transform(list(test[col].astype(str).values))   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train[\"isFraud\"]\ny_test=test[\"isFraud\"]\ntrain_data=lgb.Dataset(tr,label=y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'num_leaves': 491,\n          'min_child_weight': 0.03454472573214212,\n          'feature_fraction': 0.3797454081646243,\n          'bagging_fraction': 0.4181193142567742,\n          'min_data_in_leaf': 106,\n          'objective': 'binary',\n          'max_depth': -1,\n          'learning_rate': 0.006883242363721497,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"metric\": 'auc',\n          \"verbosity\": -1,\n          'reg_alpha': 0.3899927210061127,\n          'reg_lambda': 0.6485237330340494,\n          'random_state': 47\n         }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model= lgb.train(params, train_data, 100) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_output = clf.predict_proba(tr)\ny_output=model.predict(te)\nfor i in range(0,185):\n    if y_output[i]>=0.5: \n        y_output[i]=1\nelse: \n    y_output[i]=0\naccuracy_score(y_test, y_output)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = pd.DataFrame(data = y_output, columns = [\"idc\", \"prob_output\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n        \"TransactionID\": test[\"TransactionID\"],\n        \"IsFraud\": Y['prob_output']\n    })\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Future improvements.\n\n1. Priniciple Component Analysis on columns to understand dependency.\n2. Using better models ( decision trees)\n3. Ensembling techniques.\n4. Inclusion and analysis of all the columns","metadata":{"heading_collapsed":true}}]}