{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-12T05:12:55.409403Z","iopub.execute_input":"2021-08-12T05:12:55.409861Z","iopub.status.idle":"2021-08-12T05:12:55.421906Z","shell.execute_reply.started":"2021-08-12T05:12:55.409827Z","shell.execute_reply":"2021-08-12T05:12:55.420752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import all library\nimport warnings\nwarnings.simplefilter(action='ignore')\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn import model_selection\nfrom sklearn import preprocessing\nfrom sklearn import linear_model\nfrom sklearn import metrics\nfrom sklearn import tree","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:55.693913Z","iopub.execute_input":"2021-08-12T05:12:55.694311Z","iopub.status.idle":"2021-08-12T05:12:55.700698Z","shell.execute_reply.started":"2021-08-12T05:12:55.694277Z","shell.execute_reply":"2021-08-12T05:12:55.699373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_fold(data):\n    # create extra column\n    data['kfold'] = -1\n    \n    # randomize data\n    data = data.sample(frac = 1).reset_index(drop = True)\n    \n    # stratified k fold initialization\n    kf = model_selection.StratifiedKFold(n_splits = 6)\n    \n    # assign fold number to kfold\n    for fold , (t_ , v_) in enumerate(kf.split(X = data , y = data.target.values)):\n        data.loc[v_  , 'kfold'] = fold\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:55.733638Z","iopub.execute_input":"2021-08-12T05:12:55.735846Z","iopub.status.idle":"2021-08-12T05:12:55.742754Z","shell.execute_reply.started":"2021-08-12T05:12:55.735800Z","shell.execute_reply":"2021-08-12T05:12:55.741705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(data , fold):\n    \n    # except for target , id and kfold all are features\n    features = [ i for i in data.columns if i not in ('kfold' , 'id' , 'target') ]\n    \n    # first let's try simple linear model , for linear model we need one-hot encoder\n    # all the features are categorical so let's fill na with NONE\n    # i'm converting all columns into string cause everything is categorical so it doesn't matter\n    for col in features:\n        data.loc[:,col] = data[col].astype(str).fillna(\"NONE\")\n    \n    df_train = data[data.kfold != fold].reset_index(drop = True)\n    df_val = data[data.kfold == fold].reset_index(drop = True)\n    \n    # now let's do one hot\n    one_hot = preprocessing.OneHotEncoder(sparse = True)\n    \n    # reason behind this is to handle rare data at validation time\n    full_data = pd.concat([ df_train[features] , df_val[features] ] , axis = 0)\n    \n    one_hot.fit(full_data[features])\n    \n    X_train = one_hot.transform(df_train[features])\n    X_val = one_hot.transform(df_val[features])\n    \n    # Logistic \n    model = linear_model.LogisticRegression(solver = 'liblinear')\n    model.fit(X_train , df_train.target.values)\n    y_pred = model.predict_proba(X_val)[:,1]\n    \n    # we'll use AUC score cause data is skewed\n    auc = metrics.roc_auc_score(df_val.target.values , y_pred)\n    \n    print(f\"AUC score : {auc}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:56.258950Z","iopub.execute_input":"2021-08-12T05:12:56.259348Z","iopub.status.idle":"2021-08-12T05:12:56.269837Z","shell.execute_reply.started":"2021-08-12T05:12:56.259312Z","shell.execute_reply":"2021-08-12T05:12:56.268759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_visualize(data):\n    plt.figure(figsize = (5,5))\n    sns.countplot(data.target)\n    plt.xlabel('target' ,fontsize = 20)\n    plt.ylabel('count' , fontsize = 20)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:56.481259Z","iopub.execute_input":"2021-08-12T05:12:56.481761Z","iopub.status.idle":"2021-08-12T05:12:56.486511Z","shell.execute_reply.started":"2021-08-12T05:12:56.481728Z","shell.execute_reply":"2021-08-12T05:12:56.485126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if __name__ == '__main__':\n#     df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\n    \n#     data_visualize(df)\n    \n#     # data is skewed so we should go for stratified k-fold\n#     data = create_fold(df)\n    \n#     for fold_ in range(6):\n#         run(data , fold_)\n        \n    # df.kfold.value_counts()\n    # let's see target distribution in each folds , it's almost same\n    # for fold in range(6):\n    #     print(df[df.kfold == fold].target.value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:56.682409Z","iopub.execute_input":"2021-08-12T05:12:56.682835Z","iopub.status.idle":"2021-08-12T05:12:56.687464Z","shell.execute_reply.started":"2021-08-12T05:12:56.682798Z","shell.execute_reply":"2021-08-12T05:12:56.686435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now let's use some tree based algo. and select one which is better to evaluate test set\ndef run_decision(data , fold):\n    \n    features = [i for i in data.columns if i not in ('target' , 'kfold' , 'id')]\n    \n    for col in features:\n        data.loc[:,col] = data[col].astype(str).fillna(\"NONE\")\n    \n    for col in features:\n        lbl = preprocessing.LabelEncoder()\n        lbl.fit(data[col])\n        data.loc[:,col] = lbl.transform(data[col])\n    \n    data_train = data[data.kfold != fold].reset_index(drop = True)\n    data_val = data[data.kfold == fold].reset_index(drop = True)\n    \n    model = tree.DecisionTreeClassifier()\n    model.fit(data_train[features] , data_train.target.values)\n    pred = model.predict(data_val[features])\n    \n    AUC = metrics.roc_auc_score(data_val.target , pred)\n    print(f\"AUC : {AUC}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:56.899317Z","iopub.execute_input":"2021-08-12T05:12:56.899949Z","iopub.status.idle":"2021-08-12T05:12:56.910764Z","shell.execute_reply.started":"2021-08-12T05:12:56.899895Z","shell.execute_reply":"2021-08-12T05:12:56.909558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if __name__ == '__main__':\n#     df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\n    \n#     data_visualize(df)\n    \n#     # data is skewed so we should go for stratified k-fold\n#     data = create_fold(df)\n    \n#     for fold_ in range(6):\n#         run_decision(data , fold_)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:57.095519Z","iopub.execute_input":"2021-08-12T05:12:57.095880Z","iopub.status.idle":"2021-08-12T05:12:57.099492Z","shell.execute_reply.started":"2021-08-12T05:12:57.095849Z","shell.execute_reply":"2021-08-12T05:12:57.098709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv')\n# X.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:57.293766Z","iopub.execute_input":"2021-08-12T05:12:57.296068Z","iopub.status.idle":"2021-08-12T05:12:57.299461Z","shell.execute_reply.started":"2021-08-12T05:12:57.296015Z","shell.execute_reply":"2021-08-12T05:12:57.298747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# so after implementing 2 algorithm we can see that logistic has more accuracy then decisiontree\ndata_test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\ndata_train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\n\nfeatures = [i for i in data_train.columns if i not in ('id' , 'target')]\n\nfor col in features:\n    data_train.loc[:,col] = data_train[col].astype(str).fillna('NONE')\n    data_test.loc[:,col] = data_test[col].astype(str).fillna('NONE')\n\none_hot = preprocessing.OneHotEncoder(sparse = True)\n\nfull_data = pd.concat( [data_train[features] , data_test[features]] , axis = 0)\n\none_hot.fit(full_data[features])\n\nX_train = one_hot.transform(data_train[features])\nX_test = one_hot.transform(data_test[features])\n\nmodel = linear_model.LogisticRegression(solver = 'liblinear')\nmodel.fit(X_train , data_train.target)\nanswer = model.predict_proba(X_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:12:57.496773Z","iopub.execute_input":"2021-08-12T05:12:57.497509Z","iopub.status.idle":"2021-08-12T05:14:19.421181Z","shell.execute_reply.started":"2021-08-12T05:12:57.497460Z","shell.execute_reply":"2021-08-12T05:14:19.420017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def true_negative(y_true , y_pred):\n    cnt = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == 0 and y_p == 0:\n            cnt += 1\n    return cnt\n\ndef false_positive(y_true , y_pred):\n    cnt = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == 0 and y_p == 1:\n            cnt += 1\n    return cnt\n\ndef tpr(y_true , y_pred):\n    return metrics.recall_score(y_true , y_pred)\n\ndef fpr(y_true , y_pred):\n    FP = false_positive(y_true , y_pred)\n    TN = true_negative(y_true , y_pred)\n    return FP / (FP + TN)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:14:19.423115Z","iopub.execute_input":"2021-08-12T05:14:19.423511Z","iopub.status.idle":"2021-08-12T05:14:19.431545Z","shell.execute_reply.started":"2021-08-12T05:14:19.423478Z","shell.execute_reply":"2021-08-12T05:14:19.430334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ny_train = model.predict_proba(X_train)[:,1]\nthreshold = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.99, 1.0]\nfpr_l = []\ntpr_l = []\nfor th in threshold:\n    temp = [1 if i>=th else 0 for i in y_train]\n    fpr_l.append(fpr(data_train.target , temp))\n    tpr_l.append(tpr(data_train.target , temp))\n    \n# y_train = y_train >= 0.5\n# print(metrics.roc_auc_score(data_train.target , y_train))\n\nplt.figure(figsize = (10,10))\nplt.fill_between(fpr_l , tpr_l , alpha = 0.4)\nplt.xlim(0,1.0)\nplt.ylim(0,1.0)\nplt.xlabel('FPR' , fontsize=15)\nplt.ylabel('TPR' , fontsize=15)\nplt.plot(fpr_l,tpr_l)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:14:19.433976Z","iopub.execute_input":"2021-08-12T05:14:19.434460Z","iopub.status.idle":"2021-08-12T05:14:40.229570Z","shell.execute_reply.started":"2021-08-12T05:14:19.434419Z","shell.execute_reply":"2021-08-12T05:14:40.227297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_train = model.predict_proba(X_train)[:,1]\n\n# y_train = y_train >= 0.4\n\n# print(metrics.accuracy_score(data_train.target , y_train))","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:14:40.232270Z","iopub.execute_input":"2021-08-12T05:14:40.232861Z","iopub.status.idle":"2021-08-12T05:14:40.237003Z","shell.execute_reply.started":"2021-08-12T05:14:40.232791Z","shell.execute_reply":"2021-08-12T05:14:40.236151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(threshold)\n# print(fpr_l)\n# print(tpr_l)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:14:40.238076Z","iopub.execute_input":"2021-08-12T05:14:40.238467Z","iopub.status.idle":"2021-08-12T05:14:40.261500Z","shell.execute_reply.started":"2021-08-12T05:14:40.238436Z","shell.execute_reply":"2021-08-12T05:14:40.260191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ans = answer >= 0.2\n# print(ans.astype(int))\n\nmy_sub = pd.DataFrame({\n    'id' : data_test.id,\n    'target' : ans.astype(int)\n})","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:14:40.263165Z","iopub.execute_input":"2021-08-12T05:14:40.263494Z","iopub.status.idle":"2021-08-12T05:14:40.283687Z","shell.execute_reply.started":"2021-08-12T05:14:40.263462Z","shell.execute_reply":"2021-08-12T05:14:40.282218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_sub.to_csv('my_submission.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T05:14:40.285435Z","iopub.execute_input":"2021-08-12T05:14:40.285808Z","iopub.status.idle":"2021-08-12T05:14:41.056955Z","shell.execute_reply.started":"2021-08-12T05:14:40.285771Z","shell.execute_reply":"2021-08-12T05:14:41.055660Z"},"trusted":true},"execution_count":null,"outputs":[]}]}