{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Digit Recognizer"},{"metadata":{},"cell_type":"markdown","source":"## There will be six steps\n\n## 1. Data Understanding\n## 2. Data Preparation\n## 3. Feature Selection\n## 4. Model Building\n## 5. Hyperparameter Tuning\n## 6. Final Result"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing libraries\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV\nfrom sklearn.svm import SVC\nfrom sklearn import metrics\nfrom sklearn.preprocessing import scale\nfrom sklearn.decomposition import PCA\nfrom sklearn.pipeline import Pipeline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading dataset\ntrain_dataset = pd.read_csv(\"../input/train.csv\")\ntest_dataset = pd.read_csv(\"../input/test.csv\")\ncontainer = [train_dataset, test_dataset]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1. Data Understanding"},{"metadata":{"trusted":true},"cell_type":"code","source":"container[0].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"container[1].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking for row having null value\n\nprint(container[0][container[0].isnull().sum(axis = 1) > 0])\nprint(\"\\n\\n\\n\")\nprint(container[1][container[1].isnull().sum(axis = 1) > 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Info of datasets\n\nprint(container[0].info())\nprint(\"\\n\\n\\n\")\nprint(container[1].info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Info of datasets\n\nprint(container[0].describe())\nprint(\"\\n\\n\\n\")\nprint(container[1].describe())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking the shape of train and test data\n\nprint(container[0].shape)\nprint(container[1].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Seeing the unique values in column data to be predicted\n\nnum = container[0]['label'].unique()\nnum.sort()\nnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Seeing the distribution of each label\n\nplt.figure(figsize = (10,8))\nsns.countplot(x = 'label', data = container[0], order = num)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking the count of each pixel for a given number\n\nnum_mean = container[0].groupby('label').sum()\nnum_mean","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Data Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dividing data in X and y\n\nX = container[0].drop(['label'], axis = 1)\ny = container[0]['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scaling X as pixel value ranges from 0 to 254\n\nX = scale(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dividing in train and test data\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, train_size = 0.9, test_size = 0.1, random_state = 101)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Feature Selection"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating an instance of PCA\n\npca = PCA(svd_solver = 'randomized', random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.explained_variance_ratio_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plotting cummulative variance to get better understanding\n\nplt.figure(figsize = (10,8))\nplt.plot(np.cumsum(pca.explained_variance_ratio_))\nplt.xlabel(\"Number of Components\")\nplt.ylabel(\"Cummulative explained variance\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plotting cummulative variance to get better understanding (Zooming in)\n\nplt.figure(figsize = (10,8))\nplt.plot(np.cumsum(pca.explained_variance_ratio_))\nplt.xlabel(\"Number of Components\")\nplt.ylabel(\"Cummulative explained variance\")\nplt.ylim(0.8, 1.025)\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plotting variance of each component to get better understanding\n\nplt.figure(figsize = (10,8))\nplt.plot(pca.explained_variance_ratio_)\nplt.xlabel(\"Number of Components\")\nplt.ylabel(\"Cummulative explained variance\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plotting variance of each component to get better understanding (Zooming in)\n\nplt.figure(figsize = (10,8))\nplt.plot(pca.explained_variance_ratio_)\nplt.xlabel(\"Number of Components\")\nplt.ylabel(\"Cummulative explained variance\")\nplt.xlim(-5,100)\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Tuning hyperparameters\n\n## Creating instance of model and pca\n#model = SVC(kernel = 'rbf')\n#pca = PCA()\n\n## Creating pipeline instance\n#pipe = Pipeline(steps = [('pca', pca), ('model', model)])\n\n\n#folds = KFold(n_splits = 3, shuffle = True, random_state = 4)\n\n#param_grid = [{'pca__n_components' : [35, 50],\n#              'model__C' : [1, 10, 100, 1000], \n#              'model__gamma' : [0.0001, 0.001, 0.01]\n#              }]\n\n#model_cv = GridSearchCV(estimator = pipe,\n#                        param_grid = param_grid,\n#                        scoring = 'accuracy',\n#                        cv = folds,\n#                        verbose = 1,\n#                        return_train_score = True)\n\n#model_cv.fit(X_train, y_train)\n\n#model_cv.best_params_\n\n### Used the above code and got the best parameters as n_components = 35, C = 100, gamma = 0.001\n### Therefore we will select 35 features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Building final model and checking\n\npca = PCA(n_components = 35)\n\nmodel = SVC(C = 20, gamma = 0.001, kernel = 'rbf')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_tr = pca.transform(X_train)\nX_test_tr = pca.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_train_tr, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(X_test_tr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.score(X_test_tr, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.score(X_train_tr, y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. Model Building"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now we will build our final model with the entire dataset\n# For final model dividing dataset in train and test\n\nX_train_f = container[0].drop(['label'], axis = 1)\nX_test_f = container[1]\ny_train_f = container[0]['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scaling the data\nX_train_f = scale(X_train_f)\nX_test_f = scale(X_test_f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using PCA to reduce dimunsions\n\npca_f = PCA(n_components = 35)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_ft = pca_f.fit_transform(X_train_f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_ft.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_ft = pca_f.transform(X_test_f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_ft.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. Hyperparameter Tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyperparameter tuning\n# Here again running the hypermater to plot graph with results\n\n# Instantiating SVC model\nmodel = SVC(kernel = 'rbf')\n\n# Instantiating flods\nfolds = KFold(n_splits = 3, shuffle = True, random_state = 4)\n\nparam_grid = [{'C' : [1, 10, 100, 1000],\n               'gamma' : [0.0001, 0.001, 0.01]}]\n\n\nmodel_cv = GridSearchCV(estimator = model,\n                        param_grid = param_grid,\n                        scoring = 'accuracy',\n                        cv = folds,\n                        verbose = 1,\n                        return_train_score = True)\n\nmodel_cv.fit(X_train_ft, y_train_f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv_results = pd.DataFrame(model_cv.cv_results_)\n\ncv_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plotting the results\n\nplt.figure(figsize = (16,6))\n\nplt.subplot(1,3,1)\ngamma_01 = cv_results[cv_results['param_gamma'] == 0.0001]\n\nplt.plot(gamma_01['param_C'], gamma_01['mean_train_score'])\nplt.plot(gamma_01['param_C'], gamma_01['mean_test_score'])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title('Gamma = 0.0001')\nplt.ylim([0.00, 1.2])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\nplt.subplot(1,3,2)\ngamma_02 = cv_results[cv_results['param_gamma'] == 0.001]\n\nplt.plot(gamma_01['param_C'], gamma_02['mean_train_score'])\nplt.plot(gamma_01['param_C'], gamma_02['mean_test_score'])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title('Gamma = 0.001')\nplt.ylim([0.00, 1.2])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\nplt.subplot(1,3,3)\ngamma_03 = cv_results[cv_results['param_gamma'] == 0.01]\n\nplt.plot(gamma_01['param_C'], gamma_03['mean_train_score'])\nplt.plot(gamma_01['param_C'], gamma_03['mean_test_score'])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title('Gamma = 0.01')\nplt.ylim([0.00, 1.2])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_cv.best_params_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Again tuened with the below hyperparameter\n\n#param_grid = [{'C' : [10, 15, 20, 30, 40, 50, 100, 500],\n#               'gamma' : [0.0005, 0.001, 0.005]}]\n\n# Got best paramenter as C = 20, gamma = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Building final model\n\nmodel = SVC(C = 20, gamma = 0.001, kernel = 'rbf')\n\nmodel.fit(X_train_ft, y_train_f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. Final Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Final predictions\n\ny_pred = model.predict(X_test_ft)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = pd.DataFrame({'ImageId': np.arange(1,len(y_pred) + 1), 'Label': y_pred})\nd.set_index('ImageId', inplace = True)\nd.to_csv('predictions.csv', sep=\",\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}