{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing the libraries\nimport pandas as pd\nimport numpy as np\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import classification_report\nfrom sklearn.preprocessing import scale\n\n# hide warnings\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.getcwd()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Understanding"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"round(df.isnull().sum()/len(df.index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.describe(percentiles=[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.96,0.97,0.98,0.99,1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Splitting Data , Scaling"},{"metadata":{"trusted":true},"cell_type":"code","source":"# splitting into X and y\nX = df.drop(\"label\", axis = 1)\ny = df['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Scaling data\nX_scaled = scale(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train test split\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size = 0.8, random_state = 101)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Building"},{"metadata":{"trusted":true},"cell_type":"code","source":"# linear model\n\nmodel_linear = SVC(kernel='linear')\nmodel_linear.fit(X_train, y_train)\n\n# predict\ny_pred = model_linear.predict(X_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# confusion matrix and accuracy\n\n# accuracy\nprint(\"accuracy:\", metrics.accuracy_score(y_true=y_test, y_pred=y_pred), \"\\n\")\n\n# confusion matrix\nprint(metrics.confusion_matrix(y_true=y_test, y_pred=y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#printing the classification matrix\nfrom sklearn.metrics import classification_report\nprint(classification_report(y_test, y_pred))\n\n#Checking the precison and recall values\nprint(\"Precision Score :: \",metrics.precision_score(y_test,y_pred,pos_label='positive',average='micro'),\"\\n\")\n\nprint(\"Recall Score :: \",metrics.recall_score(y_test,y_pred,pos_label='positive',average='micro'),\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### The linear model gives approximately 91% accuracy. Let's look at a sufficiently non-linear model with randomly chosen hyperparameters."},{"metadata":{"trusted":true},"cell_type":"code","source":"# non-linear model\n# using rbf kernel, C=1, default value of gamma\n\n# model\nnon_linear_model = SVC(kernel='rbf')\n\n# fit\nnon_linear_model.fit(X_train, y_train)\n\n# predict\ny_pred = non_linear_model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Checking the below values for the non-linear model\n\n# accuracy\nprint(\"accuracy:\", metrics.accuracy_score(y_true=y_test, y_pred=y_pred), \"\\n\")\n\n# cm\nprint(metrics.confusion_matrix(y_true=y_test, y_pred=y_pred))\n\n#Checking the precison and recall values\nprint(\"Precision Score :: \",metrics.precision_score(y_test,y_pred,pos_label='positive',average='micro'),\"\\n\")\n\nprint(\"Recall Score :: \",metrics.recall_score(y_test,y_pred,pos_label='positive',average='micro'),\"\\n\")\n\n#printing the classification matrix\nfrom sklearn.metrics import classification_report\nprint(classification_report(y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Grid Search : Hyperparameter Tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating a KFold object with 5 splits \nfolds = KFold(n_splits = 5, shuffle = True, random_state = 101)\n\n# specify range of hyperparameters\n# Set the parameters by cross-validation\nhyper_params = [ {'gamma': [1e-2, 1e-3, 1e-4],\n                     'C': [1, 10, 100, 1000]}]\n\n\n# specify model\nmodel = SVC(kernel=\"rbf\")\n\n# set up GridSearchCV()\nmodel_cv = GridSearchCV(estimator = model, \n                        param_grid = hyper_params, \n                        scoring= 'accuracy', \n                        cv = folds, \n                        verbose = 1,\n                        return_train_score=True)      \n\n# fit the model\nmodel_cv.fit(X_train, y_train) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cv results\ncv_results = pd.DataFrame(model_cv.cv_results_)\ncv_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# converting C to numeric type for plotting on x-axis\ncv_results['param_C'] = cv_results['param_C'].astype('int')\n\n# # plotting\nplt.figure(figsize=(16,6))\n\n# subplot 1/3\nplt.subplot(131)\ngamma_01 = cv_results[cv_results['param_gamma']==0.01]\n\nplt.plot(gamma_01[\"param_C\"], gamma_01[\"mean_test_score\"])\nplt.plot(gamma_01[\"param_C\"], gamma_01[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.01\")\nplt.ylim([0.60, 1])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\n# subplot 2/3\nplt.subplot(132)\ngamma_001 = cv_results[cv_results['param_gamma']==0.001]\n\nplt.plot(gamma_001[\"param_C\"], gamma_001[\"mean_test_score\"])\nplt.plot(gamma_001[\"param_C\"], gamma_001[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.001\")\nplt.ylim([0.60, 1])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\n\n# subplot 3/3\nplt.subplot(133)\ngamma_0001 = cv_results[cv_results['param_gamma']==0.0001]\n\nplt.plot(gamma_0001[\"param_C\"], gamma_0001[\"mean_test_score\"])\nplt.plot(gamma_0001[\"param_C\"], gamma_0001[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.0001\")\nplt.ylim([0.60, 1])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The plots above show some useful insights:\n\nNon-linear models perform  better than the linear model.\nAt any value of gamma, a high value of C leads to better performance None of the models tend to overfit (even the complex ones), since the training and test accuracies closely follow each other This suggests that the problem and the data is inherently non-linear in nature, and a complex model will outperform simple, linear models in this case"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Choosing the best hyperparameters\n# printing the optimal accuracy score and hyperparameters\nbest_score = model_cv.best_score_\nbest_hyperparams = model_cv.best_params_\n\nprint(\"The best test score is {0} corresponding to hyperparameters {1}\".format(best_score, best_hyperparams))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model with optimal hyperparameters\n\n# model\nmodel = SVC(C=10, gamma=0.001, kernel=\"rbf\")\n\nmodel.fit(X_train, y_train)\ny_pred = model.predict(X_test)\n\n# metrics\nprint(\"accuracy\", metrics.accuracy_score(y_test, y_pred), \"\\n\")\n\nprint(\"Precision Score :: \",metrics.precision_score(y_test,y_pred,pos_label='positive',average='micro'))\n\nprint(\"Recall Score :: \",metrics.recall_score(y_test,y_pred,pos_label='positive',average='micro'))\n\nfrom sklearn.metrics import classification_report\nprint(classification_report(y_test, y_pred))\n\nprint(metrics.confusion_matrix(y_test, y_pred), \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Conclusion\nThe accuracy achieved using a non-linear kernel (0.947) is higher than that of a linear one (0.91). We can conclude that the problem is highly non-linear in nature."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"}},"nbformat":4,"nbformat_minor":1}