{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import library","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\nfrom collections import Counter\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost.sklearn import XGBClassifier\nfrom sklearn.metrics import classification_report\nfrom sklearn.decomposition import PCA\n\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"digit = pd.read_csv('../input/digit-recognizer/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check the five rows\ndigit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# printing the shape of the database\ndigit.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print some imformation about database\ndigit.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print descriptive statistic on numerical columns\ndigit.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Target variable \ndigit.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Separating the label values and column values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"y = digit[\"label\"]\nx = digit.loc[:, digit.columns != \"label\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature scaling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_std = StandardScaler().fit_transform(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# split into training and test set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_std, y, test_size = 0.25, random_state = 42, stratify = y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"X_train: \", X_train.shape)\nprint(\"X_test: \", X_test.shape)\nprint(\"y_train: \", y_train.shape)\nprint(\"y_test: \", y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Logistic regression ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"log  = LogisticRegression(random_state = 42, multi_class=\"multinomial\", solver=\"saga\", max_iter=200)\n\nstart_time = time.time()\nlog.fit(X_train, y_train)\nend_time = time.time()\n\ntime1 = end_time-start_time\nprint(\"Time elapsed: \",time1)\n\ny_pred = log.predict(X_test)\n\n# Accuracy Estimation\nprint('Accuracy Score (Train Data):', np.round(log.score(X_train, y_train), decimals = 3))\nprint('Accuracy Score (Test Data):', np.round(log.score(X_test, y_test), decimals = 3))\n\n# Classification Report\nlogistic_report = classification_report(y_test, y_pred)\nprint(logistic_report)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PCA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here we will aim to explain 98% of the variance with PCA. We could reduce or increase it as per the needs of our project\npca = PCA(.96)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lower_dimensional_data = pca.fit_transform(X_train)\n\napproximation = pca.inverse_transform(lower_dimensional_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.n_components_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"var=np.cumsum(np.round(pca.explained_variance_ratio_, decimals=4)* 100)\nplt.ylabel('% Variance Explained')\nplt.xlabel('Number of Features')\nplt.title('PCA Analysis')\nplt.ylim(30,100.5)\nplt.style.context('seaborn-whitegrid')\nplt.plot(var)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8,4));\n\n# Original Image\nplt.subplot(1, 2, 1);\nplt.imshow(x.values[1].reshape(28,28),\n              cmap = plt.cm.gray, interpolation='nearest',\n              clim=(0, 255));\nplt.xlabel('784 components', fontsize = 14)\nplt.title('Original Image', fontsize = 20);\n\n# 154 principal components\nplt.subplot(1, 2, 2);\nplt.imshow(approximation[1].reshape(28, 28),\n              cmap = plt.cm.gray, interpolation='nearest',\n              clim=(0, 255));\nplt.xlabel('270 components', fontsize = 14)\nplt.title('95% of Explained Variance', fontsize = 20);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit and transform  the data\npca = PCA(n_components=443, random_state = 0)\nX_pca_t = pca.fit_transform(X_train)\nprint(X_pca_t.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transform  the data\nX_std_t = pca.transform(X_std)\nprint(X_std_t.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_std_t, y, test_size=0.25, random_state=42, stratify = y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"X_train: \", X_train.shape)\nprint(\"X_test: \", X_test.shape)\nprint(\"y_train: \", y_train.shape)\nprint(\"y_test: \", y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Logistic Regression on PCA reduced dataset\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"log  = LogisticRegression(random_state = 42, multi_class=\"multinomial\", solver=\"saga\", max_iter=200)\n\nstart_time = time.time()\n\nlog.fit(X_train, y_train)\n\nend_time = time.time()\ntime1 = end_time-start_time\nprint(\"Time elapsed: \",time1)\n\ny_pred = log.predict(X_test)\n\n# Accuracy Estimation\nprint('Accuracy Score (Train Data):', np.round(log.score(X_train, y_train), decimals = 3))\nprint('Accuracy Score (Test Data):', np.round(log.score(X_test, y_test), decimals = 3))\n\n# Classification Report\nlogistic_report = classification_report(y_test, y_pred)\nprint(logistic_report)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://github.com/mGalarnyk/Python_Tutorials/blob/master/Sklearn/PCA/PCA_Image_Reconstruction_and_such.ipynb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}