{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **I'm writing this notebook to emphasize some of the getting started tips for ML, taken from AAAMLP written by Abhishek Thakur. You'll find the link of the book at the end (i highly recommend you to read it)**\n\n# **This Notebook contains some of the important topics like , cross validation , evaluation metrics , feature engineering and handling of categorical values...**","metadata":{}},{"cell_type":"code","source":"# importing almost all libraris that we need to do the preprocessing and for modelling..\nimport numpy as np\nimport pandas as pd\n\n# for visualization\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport seaborn as sns\n\n# for modelling \nfrom sklearn import datasets\nfrom sklearn import tree # all tree based model\nfrom sklearn import metrics # all evaluation metrics\nfrom sklearn import model_selection # for validation\nfrom sklearn import preprocessing # for encoding and preprocessing\nfrom sklearn import impute # to fill missing values\n\n# for feature engineering\nfrom tsfresh.feature_extraction import feature_calculators as fc\n\n# just to block warnings...\nimport warnings\nwarnings.simplefilter(action = 'ignore')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.089353Z","iopub.execute_input":"2021-08-16T05:26:54.090166Z","iopub.status.idle":"2021-08-16T05:26:54.119642Z","shell.execute_reply.started":"2021-08-16T05:26:54.090113Z","shell.execute_reply":"2021-08-16T05:26:54.118392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CROSS VALIDATION**","metadata":{}},{"cell_type":"code","source":"# cross validation : cross-validation is a step in the process of building a machine learning model which\n#                    helps us ensure that our models fit the data accurately and also ensures that we do not overfit.\n\n# thera are many types of cross-validation's availabel :\n\n# 1. hold-out based validation\n# 2. k-fold cross validation\n# 3. stratified k-fold cross validation\n# 4. leave-one-out cross validation\n# 5. group k-fold cross validation\n\n# Let's implement some of the cross validation","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.120849Z","iopub.execute_input":"2021-08-16T05:26:54.121137Z","iopub.status.idle":"2021-08-16T05:26:54.125007Z","shell.execute_reply.started":"2021-08-16T05:26:54.121111Z","shell.execute_reply":"2021-08-16T05:26:54.123682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import CSV file\ndf = pd.read_csv('/kaggle/input/red-wine-quality-cortez-et-al-2009/winequality-red.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.126395Z","iopub.execute_input":"2021-08-16T05:26:54.126746Z","iopub.status.idle":"2021-08-16T05:26:54.198826Z","shell.execute_reply.started":"2021-08-16T05:26:54.126714Z","shell.execute_reply":"2021-08-16T05:26:54.197466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check how many unique values are present in quality column\ndf['quality'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.202707Z","iopub.execute_input":"2021-08-16T05:26:54.203196Z","iopub.status.idle":"2021-08-16T05:26:54.218214Z","shell.execute_reply.started":"2021-08-16T05:26:54.203151Z","shell.execute_reply":"2021-08-16T05:26:54.216925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just did mapping\nquality_map = {\n    3:0,\n    4:1,\n    5:2,\n    6:3,\n    7:4,\n    8:5\n}\n\ndf.loc[:,'quality'] = df['quality'].map(quality_map)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.221392Z","iopub.execute_input":"2021-08-16T05:26:54.221872Z","iopub.status.idle":"2021-08-16T05:26:54.232973Z","shell.execute_reply.started":"2021-08-16T05:26:54.221823Z","shell.execute_reply":"2021-08-16T05:26:54.231068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hold-out based validation\n\ndf = df.sample(frac=1).reset_index(drop=True) # for randomization\ndf_train = df.head(1000)\ndf_test = df.tail(599)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.234478Z","iopub.execute_input":"2021-08-16T05:26:54.234814Z","iopub.status.idle":"2021-08-16T05:26:54.249736Z","shell.execute_reply.started":"2021-08-16T05:26:54.234781Z","shell.execute_reply":"2021-08-16T05:26:54.248698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dec_tree = tree.DecisionTreeClassifier(max_depth = 7)\n\ncols = [ 'fixed acidity' , 'volatile acidity' , 'citric acid' , 'residual sugar' , 'chlorides' , 'free sulfur dioxide' , 'total sulfur dioxide' , 'density' , 'pH' , 'sulphates' , 'alcohol']\n\ndec_tree.fit(df_train[cols] , df_train['quality'])","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.251185Z","iopub.execute_input":"2021-08-16T05:26:54.251722Z","iopub.status.idle":"2021-08-16T05:26:54.282459Z","shell.execute_reply.started":"2021-08-16T05:26:54.251687Z","shell.execute_reply":"2021-08-16T05:26:54.280234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pred = dec_tree.predict(df_train[cols])\ntest_pred = dec_tree.predict(df_test[cols])\n\ntrain_acc = metrics.accuracy_score(df_train.quality , train_pred)\ntest_acc = metrics.accuracy_score(df_test.quality , test_pred)\n\nprint('train_accuracy : ' , train_acc , ' test_accuracy : ' , test_acc)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.284084Z","iopub.execute_input":"2021-08-16T05:26:54.284571Z","iopub.status.idle":"2021-08-16T05:26:54.301536Z","shell.execute_reply.started":"2021-08-16T05:26:54.284526Z","shell.execute_reply":"2021-08-16T05:26:54.300317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_accuracy = [50]\ntest_accuracy = [50]\n\n# check all the depth and select one which is more suitable\n# if you don't know anything about decision Tree then \n# have look at this : https://www.kaggle.com/karad1818/all-about-decision-tree-from-scratch\n\nfor depth in range(1,25):\n    dec_tree = tree.DecisionTreeClassifier(max_depth = depth)\n    dec_tree.fit(df_train[cols] , df_train['quality'])\n    \n    train_pred = dec_tree.predict(df_train[cols])\n    test_pred = dec_tree.predict(df_test[cols])\n\n    train_acc = metrics.accuracy_score(df_train.quality , train_pred)\n    test_acc = metrics.accuracy_score(df_test.quality , test_pred)\n    \n    train_accuracy.append(train_acc*100)\n    test_accuracy.append(test_acc*100)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.302837Z","iopub.execute_input":"2021-08-16T05:26:54.303259Z","iopub.status.idle":"2021-08-16T05:26:54.679534Z","shell.execute_reply.started":"2021-08-16T05:26:54.303199Z","shell.execute_reply":"2021-08-16T05:26:54.678315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10,5))\nplt.plot(train_accuracy , label = 'Train')\nplt.plot(test_accuracy , label = 'Test')\nplt.legend(loc=\"upper left\", prop={'size': 15})\nplt.xlabel(\"max_depth\", size=20)\nplt.ylabel(\"accuracy in %\", size=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.681037Z","iopub.execute_input":"2021-08-16T05:26:54.681342Z","iopub.status.idle":"2021-08-16T05:26:54.888383Z","shell.execute_reply.started":"2021-08-16T05:26:54.681315Z","shell.execute_reply":"2021-08-16T05:26:54.887352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    df = pd.read_csv('/kaggle/input/red-wine-quality-cortez-et-al-2009/winequality-red.csv')\n\n    # we'll create 1 more column and fill it with -1\n    df['kfold'] = -1\n\n    df = df.sample(frac = 1).reset_index(drop = True)\n    kf = model_selection.KFold(n_splits = 5)\n\n    for fold , (train , val) in enumerate(kf.split(X = df)):\n        df.loc[val , 'kfold'] = fold\n\n    # df.to_csv('train_fold.csv' , index=False)\n    # df.tail()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.889681Z","iopub.execute_input":"2021-08-16T05:26:54.890001Z","iopub.status.idle":"2021-08-16T05:26:54.90747Z","shell.execute_reply.started":"2021-08-16T05:26:54.889971Z","shell.execute_reply":"2021-08-16T05:26:54.90628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# in simple k-fold if we have 90% positive example and 10% negative example then it might possible that one fold has all negative and rest have all postive\n# so to avoid that we'll use stratified k-fold in which ratio will be maintained in all folds\n\nif __name__ == '__main__':\n    df = pd.read_csv('/kaggle/input/red-wine-quality-cortez-et-al-2009/winequality-red.csv')\n\n    # we'll create 1 more column and fill it with -1\n    df['kfold'] = -1\n\n    df = df.sample(frac = 1).reset_index(drop = True)\n    kf = model_selection.StratifiedKFold(n_splits = 5)\n    \n    y = df.quality.values\n    for fold , (train , val) in enumerate(kf.split(X = df , y=y)):\n        df.loc[val , 'kfold'] = fold\n\n    # df.to_csv('train_fold.csv' , index=False)\n    # df.tail()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.908966Z","iopub.execute_input":"2021-08-16T05:26:54.909273Z","iopub.status.idle":"2021-08-16T05:26:54.930291Z","shell.execute_reply.started":"2021-08-16T05:26:54.909237Z","shell.execute_reply":"2021-08-16T05:26:54.929493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rule : if it's standard classification problem then use stratified K-fold\n# But if we have a larger amount of data (1M) then we can go for hold-out based validation","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.931612Z","iopub.execute_input":"2021-08-16T05:26:54.932203Z","iopub.status.idle":"2021-08-16T05:26:54.936763Z","shell.execute_reply.started":"2021-08-16T05:26:54.932156Z","shell.execute_reply":"2021-08-16T05:26:54.935768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For Regression we can use all the strategy that we've discussed earlier except stratified k-fold\n# to use stratified k-fold we have to divide target into bins and then we can use...\n# how many bins should we create ? --> if we have 10K, 100K data then we can go for 10,20,.. bins but if we have smaller amount of data then we should follow\n# Sturge's Rule : number of bins = 1 + log2(N)\ndef create_fold(data):\n    data['kfold'] = -1\n    data = data.sample(frac = 1).reset_index(drop = True)\n    num_bin = int(np.floor(1 + np.log2(len(data))))\n    \n    data.loc[:,'bins'] = pd.cut(data['target'] , bins = num_bin , labels = False)\n    kf = model_selection.StratifiedKFold(n_splits = 5)\n    \n    for fold , (train , value) in enumerate(kf.split(X = data , y = data.bins.values)):\n        data.loc[value , 'kfold'] = fold\n        \n    data = data.drop('bins',axis=1)\n    \n    return data\n\nif __name__ == '__main__':\n    X , y = datasets.make_regression(n_samples = 10000 , n_features = 100 , n_targets = 1)\n    df = pd.DataFrame(X , columns = [f\"col_{i}\" for i in range(X.shape[1])])\n    df.loc[:,'target'] = y\n    df = create_fold(df)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:54.938601Z","iopub.execute_input":"2021-08-16T05:26:54.939198Z","iopub.status.idle":"2021-08-16T05:26:55.070809Z","shell.execute_reply.started":"2021-08-16T05:26:54.939153Z","shell.execute_reply":"2021-08-16T05:26:55.069529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's implement cross validation in MNIST : https://www.kaggle.com/karad1818/mnist","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **ALL METRICS FOR EVALUATION**","metadata":{}},{"cell_type":"code","source":"# Evaluation Metrics : We can check how good our model is, using evaluation metrics..\n\n# classification Metrics :\n# 1. Accuracy\n# 2. Precision\n# 3. Recall\n# 4. F1 Score\n# 5. Area under the ROC (Receiver Operating Characteristic) curve or simply AUC\n# 6. log loss\n# 7. Precesion at k\n# 8. Average Precision at k\n# 9. Mean Average Precision at k\n\n# Regression Metrics:\n# 1. Mean absolute error\n# 2. Mean square error\n# 3. Root mean square error\n# 4. Root mean squared logarithmic error\n# 5. Mean percentage error\n# 6. Mean absolute percentage error\n# 7. R^2\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.076297Z","iopub.execute_input":"2021-08-16T05:26:55.076843Z","iopub.status.idle":"2021-08-16T05:26:55.0868Z","shell.execute_reply.started":"2021-08-16T05:26:55.076787Z","shell.execute_reply":"2021-08-16T05:26:55.085481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In binary classification, when we have equal number of positive and equal number of negative then we'll use Accuracy , Precision , Recall and f1 score\n\n# accuracy is just simple if 88 images got correct prediction then accuracy will be 88%\n\ndef accuracy(y_true , y_pred):\n    correct = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == y_p:\n            correct += 1\n    return correct / len(y_true)\n\nl1 = [0,1,1,1,0,0,0,1]\nl2 = [0,1,0,1,0,1,0,0]\n\nprint(accuracy(l1,l2))\nprint(metrics.accuracy_score(l1,l2))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.092773Z","iopub.execute_input":"2021-08-16T05:26:55.093333Z","iopub.status.idle":"2021-08-16T05:26:55.113147Z","shell.execute_reply.started":"2021-08-16T05:26:55.093284Z","shell.execute_reply":"2021-08-16T05:26:55.111996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if your dataset is skewed(number of positive is much higher then number of negative or vice versa) then it's not recommanded to use accuracy \n# In this case it's better to use precision\n# Before that let's learn some terminology :\n\n# True positive (TP) : if model predict positive and correct value is also positive , it is considered as True Positive\n# True negative (TN) : if model predict negative and correct value is also negative , it is considered as True Negative\n# False positive (FP) : if model predict positive and correct value is negative , it is considered as False Positive\n# False negative (FN) : if model predict negative and correct value is positive , it is considered as False Negative","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.114553Z","iopub.execute_input":"2021-08-16T05:26:55.114881Z","iopub.status.idle":"2021-08-16T05:26:55.127638Z","shell.execute_reply.started":"2021-08-16T05:26:55.114849Z","shell.execute_reply":"2021-08-16T05:26:55.126737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def true_positive(y_true , y_pred):\n    cnt = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == 1 and y_p == 1:\n            cnt += 1\n    return cnt\n\ndef true_negative(y_true , y_pred):\n    cnt = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == 0 and y_p == 0:\n            cnt += 1\n    return cnt\n\ndef false_positive(y_true , y_pred):\n    cnt = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == 0 and y_p == 1:\n            cnt += 1\n    return cnt\n        \ndef false_negative(y_true , y_pred):\n    cnt = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        if y_t == 1 and y_p == 0:\n            cnt += 1\n    return cnt\n\n# accuarcy score = (TP + TN) / (TP + TN + FP + FN)\n\ndef accuracy_v2(y_true , y_pred):\n    TP = true_positive(y_true , y_pred)\n    TN = true_negative(y_true ,y_pred)\n    FP = false_positive(y_true , y_pred)\n    FN = false_negative(y_true , y_pred)\n    \n    return (TP + TN) / (TP + TN + FN + FP)\n\nl1 = [0,1,1,1,0,0,0,1]\nl2 = [0,1,0,1,0,1,0,0]\n\nprint(accuracy(l1,l2))\nprint(accuracy_v2(l1,l2))\nprint(metrics.accuracy_score(l1,l2))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.128995Z","iopub.execute_input":"2021-08-16T05:26:55.129425Z","iopub.status.idle":"2021-08-16T05:26:55.146219Z","shell.execute_reply.started":"2021-08-16T05:26:55.129393Z","shell.execute_reply":"2021-08-16T05:26:55.144943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Precision : TP / (TP + FP)\n# let's say in skewed data 80 out of 90 negative classified correctly and 8 out of 10 positive classified correctly\n# thus our accuracy will be 88% but precision will be 8 / (8 + 10) = 44%\n\ndef precision(y_true  ,y_pred):\n    TP = true_positive(y_true , y_pred)\n    FP = false_positive(y_true , y_pred)\n    return TP / (TP + FP)\n\nprint(precision(l1,l2))\nprint(metrics.precision_score(l1,l2))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.149775Z","iopub.execute_input":"2021-08-16T05:26:55.150104Z","iopub.status.idle":"2021-08-16T05:26:55.167766Z","shell.execute_reply.started":"2021-08-16T05:26:55.150075Z","shell.execute_reply":"2021-08-16T05:26:55.166291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Recall = TP / (TP + FN)\n# in above example recall = 8/ (8 + 2)\n\ndef recall(y_true , y_pred):\n    TP = true_positive(y_true , y_pred)\n    FN = false_negative(y_true , y_pred)\n    return TP / (TP + FN)\n\nprint(recall(l1,l2))\nprint(metrics.recall_score(l1,l2))\n\n# In both recall and precision we want that FN and FP should be low cause sometimes it is more penalized that some positive should clasify as negative and vice versa\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.169801Z","iopub.execute_input":"2021-08-16T05:26:55.170264Z","iopub.status.idle":"2021-08-16T05:26:55.185547Z","shell.execute_reply.started":"2021-08-16T05:26:55.170216Z","shell.execute_reply":"2021-08-16T05:26:55.184255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# precision - recall curve :\n\n# Most of the models predict probability of getting 1 or 0 in this we always choose threshold like 0.5 in most cases but it's not always useful to choose\n# 0.5 as a threshold so depending on this threshold, your value of precision and recall can change drastically.\n\n# lets's look at one example here y_pred is a probability of getting 1\ny_true = [0, 0, 0, 1, 0, 0, 0, 0, 0, 0,1, 0, 0, 0, 0, 0, 0, 0, 1, 0]\ny_pred = [0.02638412, 0.11114267, 0.31620708, 0.0490937, 0.0191491, 0.17554844, 0.15952202, 0.03819563, 0.11639273, 0.079377, 0.08584789, 0.39095342, 0.27259048, 0.03447096, 0.04644807, 0.03543574, 0.18521942, 0.05934905, .61977213, 0.33056815]\n\nthresholds = [0.0490937 , 0.05934905, 0.079377,0.08584789, 0.11114267, 0.11639273, 0.15952202, 0.17554844, 0.18521942, 0.27259048, 0.31620708, 0.33056815, 0.39095342, 0.61977213 ]\n\nrecall_l = []\nprecision_l = []\nfor threshold in thresholds:\n    temp_l = [1 if i >= threshold else 0 for i in y_pred]\n    recall_l.append(metrics.recall_score(y_true , temp_l))\n    precision_l.append(metrics.precision_score(y_true , temp_l))\n    \nplt.figure(figsize = (7,7))\nplt.plot(recall_l , precision_l)\nplt.xlabel('Recall' , fontsize=15)\nplt.ylabel('Precision' , fontsize=15)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.187114Z","iopub.execute_input":"2021-08-16T05:26:55.187672Z","iopub.status.idle":"2021-08-16T05:26:55.390659Z","shell.execute_reply.started":"2021-08-16T05:26:55.187627Z","shell.execute_reply":"2021-08-16T05:26:55.389378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# You will notice that it’s challenging to choose a value of threshold that gives both\n# good precision and recall values. If the threshold is too high, you have a smaller\n# number of true positives and a high number of false negatives. This decreases your\n# recall; however, your precision score will be high. If you reduce the threshold too\n# low, false positives will increase a lot, and precision will be less.\n\n# F1 score : it combines precision and recall together , it's just harmonic mean of both\n\n# F1 : 2PR / (P + R)\n# so it will converted to F1 = 2*TP / (2TP + FP + FN)\n\ndef f1(y_true , y_pred):\n    p = precision(y_true , y_pred)\n    r = recall(y_true , y_pred)\n    return 2*p*r / (p + r)\n\nprint(f1(l1,l2))\nprint(metrics.f1_score(l1,l2))\n\n# Instead of looking at precision and recall individually, you can also just look at F1\n# score. Same as for precision, recall and accuracy, F1 score also ranges from 0 to 1,\n# and a perfect prediction model has an F1 of 1. When dealing with datasets that have\n# skewed targets, we should look at F1 (or precision and recall) instead of accuracy","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.392166Z","iopub.execute_input":"2021-08-16T05:26:55.392589Z","iopub.status.idle":"2021-08-16T05:26:55.402564Z","shell.execute_reply.started":"2021-08-16T05:26:55.392545Z","shell.execute_reply":"2021-08-16T05:26:55.401268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there are other metrics as well :\n# TPR (true positive rate) which is same as recall , it is also known as sensitivity\n\ndef tpr(y_true , y_pred):\n    return recall(y_true , y_pred)\n\n# FPR (false positive rate)  = FP / (FP + TN)\n\ndef fpr(y_true , y_pred):\n    FP = false_positive(y_true , y_pred)\n    TN = true_negative(y_true , y_pred)\n    return FP / (FP + TN)\n\n# 1-FPR is also known as specificity or True negative rate or TNR\n\ny_true = [0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1]\n\n# predicted probabilities of a sample being 1\ny_pred = [0.1, 0.3, 0.2, 0.6, 0.8, 0.05, 0.9, 0.5, 0.3, 0.66, 0.3, 0.2, 0.85, 0.15, 0.99]\n\n# threshold\nthresholds = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.99, 1.0]\n\nfpr_l = []\ntpr_l = []\nfor threshold in thresholds:\n    temp = [1 if i >= threshold else 0 for i in y_pred]\n    fpr_l.append(fpr(y_true , temp))\n    tpr_l.append(tpr(y_true , temp))\n\nplt.figure(figsize = (10,10))\nplt.fill_between(fpr_l , tpr_l , alpha = 0.4)\nplt.xlim(0,1.0)\nplt.ylim(0,1.0)\nplt.xlabel('FPR' , fontsize=15)\nplt.ylabel('TPR' , fontsize=15)\nplt.plot(fpr_l,tpr_l)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.404076Z","iopub.execute_input":"2021-08-16T05:26:55.404562Z","iopub.status.idle":"2021-08-16T05:26:55.562425Z","shell.execute_reply.started":"2021-08-16T05:26:55.40451Z","shell.execute_reply":"2021-08-16T05:26:55.56126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This TPR vs FPR curve is also known as receiver operating characteristic(ROC) and if we calculate area under ROC curve then it is another metrics\n# known as AUC (area under curve or area under ROC curve) which is used very often when you have a dataset which has skewed binary targets. \ny_true = [0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1]\ny_pred = [0.1, 0.3, 0.2, 0.6, 0.8, 0.05, 0.9, 0.5, 0.3, 0.66, 0.3, 0.2, 0.85, 0.15, 0.99]\n\nprint(metrics.roc_auc_score(y_true  ,y_pred))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.563699Z","iopub.execute_input":"2021-08-16T05:26:55.564032Z","iopub.status.idle":"2021-08-16T05:26:55.574376Z","shell.execute_reply.started":"2021-08-16T05:26:55.564Z","shell.execute_reply":"2021-08-16T05:26:55.573318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# AUC = 1 implies you have a perfect model. Most of the time, it means that\n# you made some mistake with validation and should revisit data processing\n# and validation pipeline of yours. If you didn’t make any mistakes, then\n# congratulations, you have the best model one can have for the dataset you\n# built it on.\n\n# AUC = 0 implies that your model is very bad (or very good!). Try inverting\n# the probabilities for the predictions, for example, if your probability for the\n# positive class is p, try substituting it with 1-p. This kind of AUC may also\n# mean that there is some problem with your validation or data processing.\n\n# AUC = 0.5 implies that your predictions are random. So, for any binary\n# classification problem, if I predict all targets as 0.5, I will get an AUC of\n# 0.5.\n\n# what does AUC say about your model ? \n# let's say of AUC is 0.85 that means if you randomely select positive sample and negative sample then positive sample will rank higher then negative sample with probability of 0.85\n\n# you can use the ROC curve to choose this threshold!\ny_true = [0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1]\n\n# predicted probabilities of a sample being 1\ny_pred = [0.1, 0.3, 0.2, 0.6, 0.8, 0.05, 0.9, 0.5, 0.3, 0.66, 0.3, 0.2, 0.85, 0.15, 0.99]\n\n# threshold\nthresholds = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.99, 1.0]\n\ntp_l = []\nfp_l = []\nfor threshold in thresholds:\n    temp = [1 if i>=threshold else 0 for i in y_pred]\n    tp_l.append(true_positive(y_true , temp))\n    fp_l.append(false_positive(y_true , temp))\nprint(tp_l)\nprint(fp_l)\nprint(thresholds)\n\n# it's always better to choose top-left threshold from ROC curve\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.576064Z","iopub.execute_input":"2021-08-16T05:26:55.576405Z","iopub.status.idle":"2021-08-16T05:26:55.588859Z","shell.execute_reply.started":"2021-08-16T05:26:55.57636Z","shell.execute_reply":"2021-08-16T05:26:55.587914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://i.ibb.co/8XTp67v/aaaaa.png)\n","metadata":{}},{"cell_type":"code","source":"# log loss metric :\n# log loss = -1 * (target * log(prediction) + (1-target)*log(1-prediction))  in logistic known as negative log likelihood\n\n#  One thing to remember is that log loss penalizes quite high for an incorrect or a far-off prediction\n\ndef log_loss(y_true , y_pred):\n    epsilon = 1e-15\n    loss = []\n    for y_t , y_p in zip(y_true , y_pred):\n        y_p = np.clip(y_p , epsilon , 1-epsilon)\n        \n        temp_loss = -1.0 * (y_t * np.log(y_p) + (1 - y_t) * np.log(1 - y_p))\n        loss.append(temp_loss)\n    return np.mean(loss)\n\ny_true = [0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1]\ny_pred = [0.1, 0.3, 0.2, 0.6, 0.8, 0.05, 0.9, 0.5, 0.3, 0.66, 0.3, 0.2, 0.85, 0.15, 0.99]\n\nprint(log_loss(y_true , y_pred))\nprint(metrics.log_loss(y_true , y_pred))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.59007Z","iopub.execute_input":"2021-08-16T05:26:55.590411Z","iopub.status.idle":"2021-08-16T05:26:55.612879Z","shell.execute_reply.started":"2021-08-16T05:26:55.590374Z","shell.execute_reply":"2021-08-16T05:26:55.611332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Most of the metrics that we calculated can be generalized for multiclass classification as well.\n# Let's do this for precision , \n# There are three different ways of doing it :\n# 1. Macro averaged precision : calculate for each class and then take average of it\n# 2. Micro averaged precision : calculate class wise TP and FP and then use that to calculate overall precision\n# 3. Wighted precision : same as Macro but here we'll take weighted average depending upon number of items in each class\nfrom collections import Counter\ndef macro_precision(y_true , y_pred):\n    num_class = len(np.unique(y_true))\n    pre = 0\n    for class_ in range(num_class):\n        temp_true = [1 if i == class_ else 0 for i in y_true]\n        temp_pred = [1 if i == class_ else 0 for i in y_pred]\n        pre += precision(temp_true , temp_pred)\n    return pre / num_class\n\ndef micro_precision(y_true , y_pred):\n    num_class = len(np.unique(y_true))\n    TP = 0\n    FP = 0\n    for class_ in range(num_class):\n        temp_true = [1 if i == class_ else 0 for i in y_true]\n        temp_pred = [1 if i == class_ else 0 for i in y_pred]\n        TP += true_positive(temp_true , temp_pred)\n        FP += false_positive(temp_true , temp_pred)\n    return TP / (TP + FP)\n        \ndef weighted_precision(y_true , y_pred):\n    num_class = len(np.unique(y_true))\n    class_count = Counter(y_true)\n    pre = 0\n    for class_ in range(num_class):\n        temp_true = [1 if i == class_ else 0 for i in y_true]\n        temp_pred = [1 if i == class_ else 0 for i in y_pred]\n        pre += class_count[class_] * precision(temp_true , temp_pred)\n    return pre / len(y_true)\n    \n    \ny_true = [0, 1, 2, 0, 1, 2, 0, 2, 2]\ny_pred = [0, 2, 1, 0, 2, 1, 0, 0, 2]\nprint(macro_precision(y_true , y_pred))\nprint(metrics.precision_score(y_true , y_pred , average = 'macro'))\nprint(micro_precision(y_true , y_pred))\nprint(metrics.precision_score(y_true , y_pred , average = 'micro'))\nprint(weighted_precision(y_true , y_pred))\nprint(metrics.precision_score(y_true , y_pred , average = 'weighted'))\n\n# similarly we can also implement recall and f1-score as well\n\n# let's just implement weighted f1\n\ndef weighted_f1(y_true , y_pred):\n    num_class = len(np.unique(y_true))\n    class_count = Counter(y_true)\n    f1_score = 0\n    for class_ in range(num_class):\n        temp_true = [1 if i == class_ else 0 for i in y_true]\n        temp_pred = [1 if i == class_ else 0 for i in y_pred]\n        \n        p = precision(temp_true , temp_pred)\n        r = recall(temp_true , temp_pred)\n        \n        temp_f1 = 0\n        if p + r != 0:\n            temp_f1 = 2*p*r / (p + r)\n        \n        f1_score += class_count[class_] * temp_f1\n    return f1_score / len(y_true)\n\nprint(weighted_f1(y_true , y_pred))\nprint(metrics.f1_score(y_true , y_pred , average = 'weighted'))\n\n# same we can do with AUC and log-loss this conversion is known as one-vs-all \n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.614593Z","iopub.execute_input":"2021-08-16T05:26:55.615182Z","iopub.status.idle":"2021-08-16T05:26:55.754981Z","shell.execute_reply.started":"2021-08-16T05:26:55.615094Z","shell.execute_reply":"2021-08-16T05:26:55.753514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In binary or multiclass classification there is another popular thing known as Confusion Matrix\n# FP is also known as type - I error and FN as type - II error\n\n# we can also expand the binary confusion matric to multiclass as well\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.757283Z","iopub.execute_input":"2021-08-16T05:26:55.757864Z","iopub.status.idle":"2021-08-16T05:26:55.765969Z","shell.execute_reply.started":"2021-08-16T05:26:55.757817Z","shell.execute_reply":"2021-08-16T05:26:55.764507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](http://i.ibb.co/ZX1BnTn/confused.png)","metadata":{}},{"cell_type":"code","source":"# A perfect confusion matrix should only be filled diagonally from left to right. \n# Note that confusion matrix that we have in scikit-learn is a transpose of what we have drawn here.\n\ny_true = [0, 1, 2, 0, 1, 2, 0, 2, 2]\ny_pred = [0, 2, 1, 0, 2, 1, 0, 0, 2]\ncm = metrics.confusion_matrix(y_true , y_pred)\nprint(cm)\nplt.figure(figsize = (5,5))\ncmap = sns.cubehelix_palette(n_colors = 50 , hue = 0.05 , rot = 0 , light = 0.9 , dark = 0 , as_cmap = True)\nsns.set(font_scale = 2.5)\nsns.heatmap(cm , annot = True , cmap = cmap , cbar = False)\nplt.xlabel('Prediction' ,fontsize=20)\nplt.ylabel('Actual' , fontsize = 20)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:55.767892Z","iopub.execute_input":"2021-08-16T05:26:55.768403Z","iopub.status.idle":"2021-08-16T05:26:56.018666Z","shell.execute_reply.started":"2021-08-16T05:26:55.768354Z","shell.execute_reply":"2021-08-16T05:26:56.017544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Multilabel classification : it's like one image can have multiple label associated with it\n\n# Some of the metrics for multilabel classifications are :\n# 1. Precision at k (P@k)\n# 2. Average precision at k (AP@k)\n# 3. Mean average precision at k (MAP@k)\n# 4. log loss\n\n# 1. Precision at k (P@k) :\n\n# If you have a list of original classes for a given\n# sample and list of predicted classes for the same, precision is defined as the number\n# of hits in the predicted list considering only top-k predictions, divided by k.\n\ndef pk(y_true , y_pred , k):\n    \"\"\" This function will calculate precision for only one sample\"\"\"\n    if k == 0:\n        return 0\n    # top k prediction\n    y_pred = y_pred[:k]\n    \n    pred_set = set(y_pred)\n    true_set = set(y_true)\n    comman_values = pred_set.intersection(true_set)\n    return len(comman_values) / len(y_pred[:k])\n\n# Now , Average precision at k can be calculated by takeing average of P@k\n\ndef apk(y_true , y_pred , k):\n    \"\"\" This function also calculate apk for only one sample\"\"\"\n    \n    pk_value = []\n    for i in range(1,k+1):\n        pk_value.append(pk(y_true , y_pred , i))\n    \n    if len(pk_value) == 0:\n        return 0\n    return sum(pk_value) / len(pk_value)\n\ny_true = [ [1, 2, 3], [0, 2], [1], [2, 3], [1, 0], [] ]\ny_pred = [ [0, 1, 2], [1], [0, 2, 3], [2, 3, 4, 0], [0, 1, 2], [0] ]\n\n# for i in range(len(y_true)):\n#     for j in range(1,4):\n#         print(f\"y_true[{i}] = {y_true[i]}\\ny_pred[{i}] = {y_pred[i]}\\nAP@{j} = {apk(y_true[i] , y_pred[i] , j)}\")\n\n# In ML , we're interested in all samples that's why we have MAP@k\n\ndef mapk(y_true , y_pred , k):\n    \"\"\"This function will calculate mean average precision for all samples\"\"\"\n    apk_value = []\n    for i in range(len(y_true)):\n        apk_value.append(apk(y_true[i] , y_pred[i] , k))\n    return sum(apk_value) / len(apk_value)\n\nfor k in range(1,5):\n    print(f\"map@{k} = {mapk(y_true , y_pred , k)}\")\n    \n# Please note that sometimes you might see different implementations of P@k and AP@k on the internet. ","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.022374Z","iopub.execute_input":"2021-08-16T05:26:56.022741Z","iopub.status.idle":"2021-08-16T05:26:56.037006Z","shell.execute_reply.started":"2021-08-16T05:26:56.022708Z","shell.execute_reply":"2021-08-16T05:26:56.035745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's move forward with regression metrics\n# 1. Mean absolute error(MAE) :\n\ndef mean_absolute_error(y_true , y_pred):\n    error = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        error += np.abs(y_t , y_p)\n    return error / len(y_true)\n\n# 2. Mean squared error(MSE) :\n\ndef mean_squared_error(y_true , y_pred):\n    error = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        error += (y_t - y_p) ** 2\n    return error / len(y_true)\n\n# 3. Root mean squared error(RMSE) : most popular one  = sqrt(MSE)\n\n# 4. Mean squared logarithmic error (MSLE) :\n\ndef mean_squared_logarithmic_error(y_true , y_pred):\n    error = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        error += (np.log(1 + y_t) - np.log(1 + y_p) ** 2)\n    return error / len(y_true)\n\n# 5. Root mean squared logarithmic error(RMSLE) : sqrt(MSLE)\n\n# 6. Mean percentage error :\n\ndef mean_percentage_error(y_true , y_pred):\n    error = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        error += (y_t - y_p) / y_t\n    return error / len(y_true)\n\n# 7. Mean absolute percentage error :\n\ndef mean_absolute_percentage_error(y_true , y_pred):\n    error = 0\n    for y_t , y_p in zip(y_true , y_pred):\n        error += np.abs(y_t - y_p) / y_t\n    return error / len(y_true)\n\n# 8. R^2 (R-squared / coefficient of determination): \n\n# In simple words, R-squared says how good your model fits the data. R-squared\n# closer to 1.0 says that the model fits the data quite well, whereas closer 0 means\n# that model isn’t that good. R-squared can also be negative when the model just\n# makes absurd predictions.\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.039585Z","iopub.execute_input":"2021-08-16T05:26:56.040041Z","iopub.status.idle":"2021-08-16T05:26:56.057173Z","shell.execute_reply.started":"2021-08-16T05:26:56.039994Z","shell.execute_reply":"2021-08-16T05:26:56.055895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](http://i.ibb.co/xKZNx9D/rr.png)","metadata":{}},{"cell_type":"code","source":"def r2(y_true , y_pred):\n    y_true_mean = np.mean(y_true)\n    numerator = 0\n    denominator = 0\n    \n    for y_t , y_p in zip(y_true , y_pred):\n        numerator += (y_t - y_p) ** 2\n        denominator += (y_t - y_true_mean) ** 2\n    ratio = numerator / denominator\n    return 1 - ratio\n\n# So we have implemented all metrics in straightforward manner , that means they are not efficient enough .. but is's easy to understand\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.058686Z","iopub.execute_input":"2021-08-16T05:26:56.059044Z","iopub.status.idle":"2021-08-16T05:26:56.075506Z","shell.execute_reply.started":"2021-08-16T05:26:56.059011Z","shell.execute_reply":"2021-08-16T05:26:56.074581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are some advanced metrics availabel :\n\n# 1. Quadratic weighted kappa (QWK / Cohen's kappa) :\n\n# QWK measures the “agreement” between two “ratings”. The ratings can be any real numbers in 0 to N. And\n# predictions are also in the same range. An agreement can be defined as how close\n# these ratings are to each other. So, it’s suitable for a classification problem with N\n# different categories/classes. If the agreement is high, the score is closer towards 1.0.\n# In the case of low agreement, the score is close to 0\n\ny_true = [1, 2, 3, 1, 2, 3, 1, 2, 3]\ny_pred = [2, 1, 3, 1, 2, 3, 3, 1, 2]\n\nprint(metrics.cohen_kappa_score(y_true , y_pred , weights = 'quadratic'))\n\n# A QWK greater than 0.85 is considered to be very good!\n\n# 2. Mathew's correlation coefficient (MCC) :\n\n# MCC ranges from -1 to 1. 1 is perfect prediction, -1 is imperfect prediction, and 0 is random prediction.","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.076536Z","iopub.execute_input":"2021-08-16T05:26:56.07709Z","iopub.status.idle":"2021-08-16T05:26:56.092842Z","shell.execute_reply.started":"2021-08-16T05:26:56.077057Z","shell.execute_reply":"2021-08-16T05:26:56.091661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](http://i.ibb.co/f1PQpYp/mathew.png)","metadata":{}},{"cell_type":"code","source":"def mcc(y_true , y_pred):\n    TP = true_positive(y_true , y_pred)\n    FP = false_positive(y_true , y_pred)\n    TN = true_negative(y_true , y_pred)\n    FN = false_negative(y_true , y_pred)\n    \n    numerator = TP*TN - FP*FN\n    denominator = ((TP + FP) * (FN + TN) * (FP + TN) * (TP + FN)) ** 0.5\n    \n    return numerator / denominator","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.094074Z","iopub.execute_input":"2021-08-16T05:26:56.094512Z","iopub.status.idle":"2021-08-16T05:26:56.108829Z","shell.execute_reply.started":"2021-08-16T05:26:56.094479Z","shell.execute_reply":"2021-08-16T05:26:56.108012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  **APPROACHING CATEGORICAL VARIABLE**","metadata":{}},{"cell_type":"code","source":"# two major types of categorical variable's are there\n# Nominal : order is not associated for e.g. gender\n# Ordinal : order is associated for e.g. something related to low,medium,high kinda level\n# binary : only two category\n# cyclic : days in week , sunday ,monday .... sunday ..","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.109821Z","iopub.execute_input":"2021-08-16T05:26:56.110218Z","iopub.status.idle":"2021-08-16T05:26:56.123696Z","shell.execute_reply.started":"2021-08-16T05:26:56.110191Z","shell.execute_reply":"2021-08-16T05:26:56.122572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if __name__ == '__main__':\ndf = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:26:56.125346Z","iopub.execute_input":"2021-08-16T05:26:56.12591Z","iopub.status.idle":"2021-08-16T05:27:00.715157Z","shell.execute_reply.started":"2021-08-16T05:26:56.125834Z","shell.execute_reply":"2021-08-16T05:27:00.713954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x = df.target)\n\n# After seeing , we can say that data is too skewed so we'll use AUC as a metric\n\ndf.ord_2.unique()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:00.716839Z","iopub.execute_input":"2021-08-16T05:27:00.717238Z","iopub.status.idle":"2021-08-16T05:27:00.966956Z","shell.execute_reply.started":"2021-08-16T05:27:00.717204Z","shell.execute_reply":"2021-08-16T05:27:00.965825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now we know that ord_2 column has six different values and we know computer can not understand text so we need to convert it into numeric\n# so one idea is to map it with 0,1,2...\ndef label(df):\n    mapping = {\n        'Hot' : 0,\n        'Warm' : 1,\n        'Freezing' : 2,\n        'Lava Hot' : 3,\n        'Cold' : 4,\n        'Boiling Hot' : 5\n    }\n\n    df.loc[:,'ord_2'] = df['ord_2'].map(mapping)\n\n# this type of encoding known as label encoding , same thing we can do with sklearn","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:00.968289Z","iopub.execute_input":"2021-08-16T05:27:00.968639Z","iopub.status.idle":"2021-08-16T05:27:00.975188Z","shell.execute_reply.started":"2021-08-16T05:27:00.968607Z","shell.execute_reply":"2021-08-16T05:27:00.973964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef mapping(df):\n    # fill NaN with NONE\n    df.loc[:,'ord_2'] = df['ord_2'].fillna(\"NONE\")\n\n    label_enc = preprocessing.LabelEncoder()\n\n    # P.S: do not use this directly. fit first, then transform\n    df.loc[:,'ord_2'] = label_enc.fit_transform(df['ord_2'].values)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:00.976665Z","iopub.execute_input":"2021-08-16T05:27:00.976967Z","iopub.status.idle":"2021-08-16T05:27:00.994588Z","shell.execute_reply.started":"2021-08-16T05:27:00.97694Z","shell.execute_reply":"2021-08-16T05:27:00.993534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we can use this type of encoding in many tree based algo. e.g. decision tree , random forest , extra trees , any boosted tree model\n# but it's not used in linear model , SVM , neural nets as they expect data to be normalized\n# for this type of model we can binarize the data\n# e.g. freezing : 0 0 1\n#      warm     : 0 1 0\n#      cold     : 0 1 1 ...\n\n# It becomes easy to store lots of binarized variables like this if we store them in a\n# sparse format. A sparse format is nothing but a representation or way of storing\n# data in memory in which you do not store all the values but only the values that\n# matter. In the case of binary variables described above, all that matters is where we have ones (1s). \n\n# if we use simple binary things then we're using more space, e.g.\n\ndata = np.array([\n    [0,0,1],\n    [1,0,0],\n    [1,0,1]\n])\n# 3*3*8\nprint(data.nbytes)\n\n# another way to do this by using only 1s position\n# for e.g. (0,2) , (1,0) , (2,0) , (2,2) so this will only store 4*8 memory\n# in numpy we can do this , \nfrom scipy import sparse\nsparse_data = sparse.csr_matrix(data)\nprint(sparse_data.data.nbytes)\n\n# it will print 32 which is so less then dense array\n# total size of sparse matrix :\nprint(sparse_data.data.nbytes + sparse_data.indptr.nbytes + sparse_data.indices.nbytes)\n# which is 64 but it's still less then dense array , this difference become vast when data becomes larger that's why we'll prefer sparse array over dense array","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:00.995999Z","iopub.execute_input":"2021-08-16T05:27:00.996521Z","iopub.status.idle":"2021-08-16T05:27:01.013944Z","shell.execute_reply.started":"2021-08-16T05:27:00.996485Z","shell.execute_reply":"2021-08-16T05:27:01.012838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There is another transformation which takes much less memory then sparse matrix as well , and that is one-hot encoding\n# just look at below image\n\ndata = np.array([\n    [0,0,0,0,1,0],\n    [0,1,0,0,0,0],\n    [1,0,0,0,0,0]\n])\n\nsparse_d = sparse.csr_matrix(data)\nprint(sparse_d.data.nbytes + sparse_d.indptr.nbytes + sparse_d.indices.nbytes) # this is just 52\n\n# let's implement it on big data\ndata = np.random.randint(1000,size=1000000)\n\none_hot = preprocessing.OneHotEncoder(sparse = False)\noh_data = one_hot.fit_transform(data.reshape(-1,1))\nprint(f\"size of dense array : {oh_data.nbytes}\")\n\none_hot = preprocessing.OneHotEncoder(sparse = True)\noh_data = one_hot.fit_transform(data.reshape(-1,1))\nprint(f\"size of full sparse array : {oh_data.data.nbytes + oh_data.indptr.nbytes + oh_data.indices.nbytes}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:01.015376Z","iopub.execute_input":"2021-08-16T05:27:01.015984Z","iopub.status.idle":"2021-08-16T05:27:06.777202Z","shell.execute_reply.started":"2021-08-16T05:27:01.015937Z","shell.execute_reply":"2021-08-16T05:27:06.775915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](http://i.ibb.co/2YdTSDL/one-hot.png)","metadata":{}},{"cell_type":"code","source":"# those 3 methods are most important ways to handle categorical data\n# there exist some other methods as well like converting data into numerical values but it doesn't make sense sometime\n# df[df['ord_2'] == 'Lava Hot'].shape\n\n# what we can do is that we can fill value with it's count\ndf.groupby(['ord_2'])['id'].count()\n\n# now we can transform those counts into columns\ndf.groupby(['ord_2'])['id'].transform('count') # we can also group by with 2 columns and then give some count\n\n# there is also one trick that we can combine 2 columns and create new feature\n\ndf['new_featue'] = (df['ord_1'].astype(str) + '_' + df['ord_2'].astype(str))\n\ndf.new_featue\n# Not that NaN will also be converted as string and we can count it as another category\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:06.778589Z","iopub.execute_input":"2021-08-16T05:27:06.779014Z","iopub.status.idle":"2021-08-16T05:27:07.238051Z","shell.execute_reply.started":"2021-08-16T05:27:06.778962Z","shell.execute_reply":"2021-08-16T05:27:07.236329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# so in summary , whenever we get categorical data \n# 1. fill NaN values (it's important)\n# 2. convert them into labels by LabelEncoder or may be using mapping\n# 3. use one-hot if needed\n# 4. now go for modelling..","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:07.239516Z","iopub.execute_input":"2021-08-16T05:27:07.239824Z","iopub.status.idle":"2021-08-16T05:27:07.24491Z","shell.execute_reply.started":"2021-08-16T05:27:07.239796Z","shell.execute_reply":"2021-08-16T05:27:07.24337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling NaN values\n# 1. simply drop that row (it's simple but not ideal)\n# 2. Another way is to use it as a new category (this is most preferred way) :\n# df.ord_2.unique()\ndf.ord_2.fillna(\"NONE\").value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:07.2464Z","iopub.execute_input":"2021-08-16T05:27:07.246761Z","iopub.status.idle":"2021-08-16T05:27:07.476104Z","shell.execute_reply.started":"2021-08-16T05:27:07.246718Z","shell.execute_reply":"2021-08-16T05:27:07.475347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rare category :\n# category which appear as very less percentage of total number of sample\n# Now let's assume that we deployed our model and we get category that's not present in our training set ,\n# so in this case our model will throw an error\n\n# so to handle this , \n# Let's say we have f1,f2,f3 and f4 feature and we know that f3 can have rare category then we'll train our model on all category except f3\n# Thus, you will be creating a model that predicts “f3” when it’s not known or not available in training\n\n# If you have a fixed test set, you can add your test data to training to know about the\n# categories in a given feature. This is very similar to semi-supervised learning in\n# which you use data which is not available for training to improve your model. This\n# will also take care of rare values that appear very less number of times in training\n# data but are in abundance in test data. Your model will be more robust. \n\n# another way is that if we have NONE in training set and when we test the model and we get some unknown then we'll convert it into NONE..\n\n# df.ord_4.value_counts()\ndf.ord_4 = df.ord_4.fillna('NONE')\n# here we can see that J and L exist <2000 times\n# so we might want that this two category will be used as RARE category\n\ndf.loc[ df['ord_4'].value_counts()[df['ord_4']].values < 2000 , 'ord_4'] = 'RARE'\ndf.ord_4.value_counts()\n\n# We say that wherever the value count for a certain category is less than 2000,\n# replace it with rare. So, now, when it comes to test data, all the new, unseen\n# categories will be mapped to “RARE”, and all missing values will be mapped to “NONE”.","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:07.477389Z","iopub.execute_input":"2021-08-16T05:27:07.477944Z","iopub.status.idle":"2021-08-16T05:27:08.085851Z","shell.execute_reply.started":"2021-08-16T05:27:07.4779Z","shell.execute_reply":"2021-08-16T05:27:08.084914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# so now we're done with categorical data , let's train model now..\n# so here is one notebook for categorical data : https://www.kaggle.com/karad1818/categorical-data-encoding-aaamlp","metadata":{"execution":{"iopub.status.busy":"2021-08-16T05:27:08.087113Z","iopub.execute_input":"2021-08-16T05:27:08.087675Z","iopub.status.idle":"2021-08-16T05:27:08.090841Z","shell.execute_reply.started":"2021-08-16T05:27:08.08764Z","shell.execute_reply":"2021-08-16T05:27:08.090037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **FEATURE ENGINEERING**","metadata":{}},{"cell_type":"code","source":"# We must keep in mind that feature engineering is something that is done in the best possible manner only when you\n# have some knowledge about the domain of the problem and depends a lot on the data in concern.\n\n# Feature engineering is not just about creating new features from data but also includes different types of normalization and transformations.\n\n# if we have a data with date time column for e.g.\ns = pd.date_range('2020-01-06' , '2020-01-10' , freq = '10H').to_series()\n# print(s)\ndf = pd.DataFrame()\n# create feature :\nfeature = {\n    'dayofweek' : s.dt.dayofweek.values,\n    'dayofyear': s.dt.dayofyear.values,\n    'month' : s.dt.month.values,\n    'hour': s.dt.hour.values,\n    'is_leap_year': s.dt.is_leap_year.values,\n    'quarter': s.dt.quarter.values,\n    'weekofyear': s.dt.weekofyear.values\n}\n\nfor key , value in feature.items():\n    df.loc[: , key] = value\ndf","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:07:41.972503Z","iopub.execute_input":"2021-08-16T06:07:41.972867Z","iopub.status.idle":"2021-08-16T06:07:42.004397Z","shell.execute_reply.started":"2021-08-16T06:07:41.972837Z","shell.execute_reply":"2021-08-16T06:07:42.003219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How to use aggregation :\n\n# so let's say we want to claculate in which month customer is most active? or what's mean for some category for particular customer?\nidx = [1,1,1,2,2,3,3,3,3,4]\nmonth = [5,6,6,5,6,7,6,7,8,9]\nnum1 = [2,2,3,2,4,5,6,7,8,5]\n\ndata = pd.DataFrame({\n    'id' : idx,\n    'month' : month,\n    'num1' : num1\n})\n\naggs = {}\naggs['month'] = ['nunique' , 'mean']\naggs['num1'] = ['sum' , 'max' , 'mean']\naggs['id'] = ['nunique']\naggs['id'] = ['size']\n\nagg_data = data.groupby('id').agg(aggs)\nagg_data = agg_data.reset_index()\n\nagg_data # we can also merge this data frame with original as well and do modelling....","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:32:33.442636Z","iopub.execute_input":"2021-08-16T06:32:33.443011Z","iopub.status.idle":"2021-08-16T06:32:33.47974Z","shell.execute_reply.started":"2021-08-16T06:32:33.44298Z","shell.execute_reply":"2021-08-16T06:32:33.478507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some of the statistical feature :\ndata = np.array([1,2,3,3,3,4,4,5,5,6,6,6,7,7,7,7])\n\nfeature_dict = {\n    'mean' : np.mean(data),\n    'max' : np.max(data),\n    'min' : np.min(data),\n    'std' : np.std(data),\n    'var' : np.var(data),\n    'peak-to-peak' : np.ptp(data),\n    'percentile_10' : np.percentile(data,10),\n    'percentile_60' : np.percentile(data,60),\n    'quantile_5' : np.quantile(data , 0.05),\n    'quantile_95' : np.quantile(data , 0.95),\n    'quantile_99' : np.quantile(data , 0.99)\n}\nprint(feature_dict)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:41:51.238186Z","iopub.execute_input":"2021-08-16T06:41:51.238601Z","iopub.status.idle":"2021-08-16T06:41:51.24914Z","shell.execute_reply.started":"2021-08-16T06:41:51.238567Z","shell.execute_reply":"2021-08-16T06:41:51.248034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# time series data can be converted to a lot of features\n# A python library called tsfresh is instrumental in this case.\n\nfeature_dict['abs_energy'] = fc.abs_energy(data)\nfeature_dict['count_above_mean'] = fc.count_above_mean(data)\nfeature_dict['count_below_mean'] = fc.count_below_mean(data)\nfeature_dict['mean_abs_change'] = fc.mean_abs_change(data)\nfeature_dict['mean_change'] = fc.mean_change(data)\nprint(feature_dict)\n\n# that's not it.. there is more...","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:49:39.610663Z","iopub.execute_input":"2021-08-16T06:49:39.611033Z","iopub.status.idle":"2021-08-16T06:49:39.619109Z","shell.execute_reply.started":"2021-08-16T06:49:39.611002Z","shell.execute_reply":"2021-08-16T06:49:39.617781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# polynomial basis transformation : generally used in polynomial regression\ndata = pd.DataFrame(\n    np.random.rand(100,2),\n    columns = [f\"f_{x}\" for x in range(2)]\n)\n\np_data = preprocessing.PolynomialFeatures(\n    degree = 2,\n    interaction_only = False, # if True this will produce feature like : x*y , x*y*z but it will not produce x2*y, x2 ,...\n    include_bias = False # true then add column with all 1\n)\np_data.fit(data)\npoly_feature = p_data.transform(data)\n\nnum_feature = poly_feature.shape[1]\ndata_poly = pd.DataFrame(\n    poly_feature,\n    columns = [f\"f_{x}\" for x in range(num_feature)]\n)\ndata_poly","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:25:29.206687Z","iopub.execute_input":"2021-08-16T07:25:29.207094Z","iopub.status.idle":"2021-08-16T07:25:29.232593Z","shell.execute_reply.started":"2021-08-16T07:25:29.207057Z","shell.execute_reply":"2021-08-16T07:25:29.231469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Another very famous feature conversion is : binning\n# so in this approach we'll divide data in some parts(bins)\ndata['bin_10'] = pd.cut(data['f_0'] , bins=10 , labels = False)\ndata\n\n# Binning also enables you to treat numerical features as categorical.","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:20:24.085951Z","iopub.execute_input":"2021-08-16T07:20:24.086403Z","iopub.status.idle":"2021-08-16T07:20:24.106154Z","shell.execute_reply.started":"2021-08-16T07:20:24.086361Z","shell.execute_reply":"2021-08-16T07:20:24.104923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we can also apply any kind of functional transformation on data\ndata['f_0'] = data['f_0'] * 1000\n\nprint(f\"var : {np.var(data.f_0)}\") # so it has a very high variance\n\n# so to decrease the variance we'll apply log transformation\ndata['f_0'] = data['f_0'].apply(lambda x : np.log(1+x))\n\nprint(f\"var : {np.var(data.f_0)}\") ","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:27:17.686831Z","iopub.execute_input":"2021-08-16T07:27:17.687234Z","iopub.status.idle":"2021-08-16T07:27:17.69565Z","shell.execute_reply.started":"2021-08-16T07:27:17.687194Z","shell.execute_reply":"2021-08-16T07:27:17.694691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one way to fill missing values in numerical feature is to fill it with mean \n\n# A fancy way of filling in the missing values would be to use a k-nearest neighbour method. You can select a sample with missing values and find the nearest\n# neighbours utilising some kind of distance metric, for example, Euclidean distance. Then you can take the mean of all nearest neighbours and fill up the missing value.\n\nX = np.random.randint(1,15,(10,6)).astype(float)\n\n# randomely assign NaN\nX.ravel()[np.random.choice(X.size , 10 , replace = False)] = np.nan\n\nknn_imputer = impute.KNNImputer(n_neighbors = 2)\nknn_imputer.fit(X)\nX = knn_imputer.transform(X)\nX","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:36:35.206833Z","iopub.execute_input":"2021-08-16T07:36:35.207189Z","iopub.status.idle":"2021-08-16T07:36:35.229256Z","shell.execute_reply.started":"2021-08-16T07:36:35.207161Z","shell.execute_reply":"2021-08-16T07:36:35.228192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Another way of imputing missing values in a column would be to train a regression\n# model that tries to predict missing values in a column based on other columns.\n\n# Always remember that imputing values for tree-based models is unnecessary as they\n# can handle it themselves.\n\n# And always remember to scale or normalize your\n# features if you are using linear models like logistic regression or a model like SVM.\n# Tree-based models will always work fine without any normalization of features.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **This whole notebook is based on Ahishek thakur's AAAMLP (Approching almost any machine learning problem..) here is a [link](https://github.com/abhishekkrthakur/approachingalmost/blob/master/AAAMLP.pdf) , i think you should look at it.**\n# **This Notebook contains Half of the AAAMLP (up to 154 pages).** \n# **Thanks for reading and forking..**","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}