{"cells":[{"metadata":{"_uuid":"02bae3855dc0851ef0713eef6e27a52c184b16b1"},"cell_type":"markdown","source":"# Machine Learning on MNIST dataset (Handwritten digits recognition)\n*Upload the notebook to the Kaggle competition: https://www.kaggle.com/c/digit-recognizer/*\n\n**Datasets**\n\n- train.csv: training set containing the label and the 784 pixel values (28x28 images), between 0-255\n- test.csv : the set to make predictions over\n- sample_submission.csv : example of submission csv format for the competition\n\n**Metric**: accuracy of the classification (% of right predictions)"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f1a2ea193246e607378d2a597dfffd802b5f856e"},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom scipy.sparse import csr_matrix","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"df51c2acc765bf2c67d89b10647b4c677f434081"},"cell_type":"markdown","source":"## EDA"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"714034c94e2f18da0954871ef426b71f9df8c151"},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\nprint(train.head())\nprint(train.info())\nprint(\"\\n SHape of the dataset:\", train.shape)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f715290ccf10f8f437982656f4f899a15a6b3d2f"},"cell_type":"code","source":"#NaN values in the dataset ?\nnan = train.isnull().sum()\nprint(nan[nan != 0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66327085760e205867881237137d27a688a28130"},"cell_type":"markdown","source":"*--> No missing values*"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"d2c790e920f989c4ea369cc8c07115f7af3472b6"},"cell_type":"code","source":"#Displays 4 handwritten digit images\ndef display_digits(N):\n    \"\"\"Picks-up randomly N images within the \n    train dataset between 0 and 41999 and displays the images\n    with 4 images/row\"\"\"\n    \n    train = pd.read_csv('../input/digit-recognizer/train.csv')\n    images = np.random.randint(low=0, high=42001, size=N).tolist()\n    \n    subset_images = train.iloc[images,:]\n    subset_images.index = range(1, N+1)\n    print(\"Handwritten picked-up digits: \", subset_images['label'].values)\n    subset_images.drop(columns=['label'], inplace=True)\n\n    for i, row in subset_images.iterrows():\n        plt.subplot((N//8)+1, 8, i)\n        pixels = row.values.reshape((28,28))\n        plt.imshow(pixels, cmap='gray')\n        plt.xticks([])\n        plt.yticks([])\n    plt.title('Randomly picked-up images from the training dataset')\n    plt.show()\n\n    return \"\"","execution_count":7,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"88c020ae87ae250032ff283d20072162aea3f2bc"},"cell_type":"code","source":"display_digits(40)","execution_count":8,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f5ef23176a63d273dbaff2f8226e93e36bb6ae56"},"cell_type":"code","source":"#Analyse the pixels intensity values\nsubset_pixels = train.iloc[:, 1:]\nsubset_pixels.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd28b4b35ee4e5fe58b3b4654a958f176f0caf80"},"cell_type":"markdown","source":"*--> Some pixels have always an intensity of 0 (max=0) or 255 (min=255)*"},{"metadata":{"trusted":false,"_uuid":"f64be758b43c593805930edebe677be026a39fd3"},"cell_type":"code","source":"#Distribution of the digits in the dataset\n_ = train['label'].value_counts().plot(kind='bar')\nplt.show()","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"378e257c9d63e06ff38368668887c8152da47a6d"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"1df6620e5234a3030c153cd702bfc3cf2c1e0ae1"},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"_uuid":"81144a8da7e155e8da10a95b1227d5da88a62950"},"cell_type":"markdown","source":"### Pre-traitment\n#### Remove pixels with constant intensity\nThe removal of constant pixels (smae intensity in the entire training set) allow to reduce the number of features and compute faster"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"1ef53fc508c621d75cc759df522272c53aa6f4b0"},"cell_type":"code","source":"def remove_constant_pixels(pixels_df):\n    \"\"\"Removes from the images the pixels that have a constant intensity value,\n    either always black (0) or white (255)\n    Returns the cleared dataset & the list of the removed pixels (columns)\"\"\"\n\n    #Remove the pixels that are always black to compute faster\n    changing_pixels_df = pixels_df.loc[:]\n    dropped_pixels_b = []\n\n    #Pixels with max value =0 are pixels that never change\n    for col in pixels_df:\n        if changing_pixels_df[col].max() == 0:\n            changing_pixels_df.drop(columns=[col], inplace=True)\n            dropped_pixels_b.append(col)\n    print(\"Constantly black pixels that have been dropped: {}\".format(dropped_pixels_b))\n\n\n    #Same with pixels with min=255 (white pixels)\n    dropped_pixels_w = []\n    for col in changing_pixels_df:\n        if changing_pixels_df[col].min() == 255:\n            changing_pixels_df.drop(columns=[col], inplace=True)\n            dropped_pixel_w.append(col)\n    print(\"\\n Constantly white pixels that have been dropped: {}\".format(dropped_pixels_b))\n\n    print(changing_pixels_df.head())\n    print(\"Remaining pixels: {}\".format(len(changing_pixels_df.columns)))\n    print(\"Pixels removed: {}\".format(784-len(changing_pixels_df.columns)))\n    \n    return changing_pixels_df, dropped_pixels_b + dropped_pixels_w","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"09e4e243bda4fe7b33514d9122a9e379bba968bc"},"cell_type":"code","source":"train_pixels_df = pd.read_csv('../input/digit-recognizer/train.csv').drop(columns=['label'])\ntrain_changing_pixels_df, dropped_pixels = remove_constant_pixels(train_pixels_df)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"d3c67f70158559d9a174863953f7a86c16ddf073"},"cell_type":"code","source":"#To save time and not have to run the entire function\nDROPPED_PIX = ['pixel0', 'pixel1', 'pixel2', 'pixel3', 'pixel4', 'pixel5', 'pixel6', 'pixel7', 'pixel8', 'pixel9', \n               'pixel10', 'pixel11', 'pixel16', 'pixel17', 'pixel18', 'pixel19', 'pixel20', 'pixel21', 'pixel22', \n               'pixel23', 'pixel24', 'pixel25', 'pixel26', 'pixel27', 'pixel28', 'pixel29', 'pixel30', 'pixel31', \n               'pixel52', 'pixel53', 'pixel54', 'pixel55', 'pixel56', 'pixel57', 'pixel82', 'pixel83', 'pixel84', \n               'pixel85', 'pixel111', 'pixel112', 'pixel139', 'pixel140', 'pixel141', 'pixel168', 'pixel196', \n               'pixel392', 'pixel420', 'pixel421', 'pixel448', 'pixel476', 'pixel532', 'pixel560', 'pixel644', \n               'pixel645', 'pixel671', 'pixel672', 'pixel673', 'pixel699', 'pixel700', 'pixel701', 'pixel727', \n               'pixel728', 'pixel729', 'pixel730', 'pixel731', 'pixel754', 'pixel755', 'pixel756', 'pixel757', \n               'pixel758', 'pixel759', 'pixel760', 'pixel780', 'pixel781', 'pixel782', 'pixel783']\ntrain_changing_pixels_df = pd.read_csv('../input/changing-pixels/train_changing_pixels_DB.csv', index_col=0)\nprint(train_changing_pixels_df.head())\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\nprint(train.head())","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"9aa0e94bbf2291461fc4534a76bb2bc35821c87b"},"cell_type":"markdown","source":"#### Pictures intensity rescaling\n- Accentuation of contrasts by resclaling\n- changing pixels values to only 0 or 255 (blakc or white)"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"ceb9cba049c13207e8a46bd3759b2ca120d08d48"},"cell_type":"code","source":"#Pick-up one random image from original training set\ni = np.random.randint(low=0, high=42001, size=1).tolist()[0]\npixels = train.iloc[i, 1:]\nimage = train.iloc[i, 1:].values.reshape((28,28))\n\n#Pixel intensity hstogram\nplt.hist(pixels, bins=256, range=(0,256), normed=True)\nplt.title('original image - pixel intensity distribution')\nplt.show()\n\n#Rescaling the intensity\npmin, pmax = image.min(), image.max()\nrescaled_image = 255*(image-pmin) / (pmax - pmin)\nrescaled_pixels = rescaled_image.flatten()\n\n#Only black or white pixels\nbw_pixels = pixels.apply(lambda x: 0 if x<128 else 255)\nbw_image = bw_pixels.values.reshape((28,28))\n\n\n#Visual comparison of images\nplt.subplot(1, 3, 1)\nplt.imshow(image, cmap='gray')\nplt.title('original image')\nplt.subplot(1, 3, 2)\nplt.imshow(rescaled_image, cmap='gray')\nplt.title('rescaled image')\nplt.subplot(1, 3, 3)\nplt.imshow(bw_image, cmap='gray')\nplt.title('black&wite only image')\nplt.show()\n\n\n#Visual Histogram comparison\nplt.subplot(1, 2, 1)\nplt.hist(pixels, bins=256, range=(0,256), normed=True)\nplt.title('original image')\nplt.subplot(1, 2, 2)\nplt.hist(rescaled_pixels, bins=256, range=(0,256), normed=True)\nplt.title('rescaled image')\nplt.show()","execution_count":116,"outputs":[]},{"metadata":{"_uuid":"8bfe36ee227a0c38d74944b46a723d5fb6e6fc36"},"cell_type":"markdown","source":"*Image rescaling is not applicable here as the intensity distribution is not srpead enough*"},{"metadata":{"_uuid":"29d9ef6f9ed367ebeb9ec0890b91304bf1327af2"},"cell_type":"markdown","source":"#### Dummy variables and target ?"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c5530fea8d220e334e0c81852054745fc0a77173"},"cell_type":"code","source":"#Dummy target ?\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b281451b2e1fc7450051a64f48d37a8fcac5ce6"},"cell_type":"markdown","source":"### Dimension Reduction\n#### Principal Components Analysis (PCA)"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f99778191a9125d93250a620b23a4fc710c0d138"},"cell_type":"code","source":"#Preparing samples and labels arrays\n#s = np.random.randint(low=0, high=42001, size=1050).tolist()\nsamples = train_changing_pixels_df.values  #.iloc[s, :]\ndigits = train['label'].tolist()  #.iloc[s, :]\nprint(samples.shape)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"5337f4cfab33e2b6740f81b32f39692bacaeb650"},"cell_type":"code","source":"#PCA model\npca = PCA()\npca.fit(samples)\n\n#PCA features variance visualization\npca_features = range(pca.n_components_)\n_ = plt.figure(figsize=(30,20))\n_ = plt.bar(pca_features, pca.explained_variance_)\n_ = plt.xticks(pca_features)\n_ = plt.title('Principal Components Analysis for Dimension Reduction')\n_ = plt.xlabel('PCA features')\n_ = plt.ylabel('Variance of the PCA feature')\n_ = plt.savefig('visualizations/PCA features variance.png')\nplt.show()\n\n#PCA features variance visualization - ZOOM in\nl= 100\nx = range(l)\n_ = plt.figure(figsize=(30,20))\n_ = plt.bar(x, pca.explained_variance_[:l])\n_ = plt.xticks(x)\n_ = plt.title('Principal Components Analysis for Dimension Reduction - Zoom In {} first features'.format(l))\n_ = plt.xlabel('PCA features')\n_ = plt.ylabel('Variance of the PCA feature')\n_ = plt.savefig('visualizations/PCA features variance_zoom.png')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62aef5e7f65ac79fcadb1c714368211d4f1d6e66"},"cell_type":"markdown","source":"*The best number of PCA features to keep (reduced dimension) is <80. The 1st PCA feature represents a lot of the variance, as the 5 following ones.*"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"faf54b7aeb664cda6bc2f747b77421e09a1fd868"},"cell_type":"code","source":"#Visualization of the variance of the data carried by the number of PCA features\nn_components = np.array([1,2,3,4,5,6, 10, 30, 60, 80, 100, 200, 400, 700])\ncumul_variance = np.empty(len(n_components))\nfor i, n in enumerate(n_components):\n    pca = PCA(n_components=n)\n    pca.fit(samples)\n    cumul_variance[i] = np.sum(pca.explained_variance_ratio_)\n\nprint(cumul_variance)\n\n_ = plt.figure(figsize=(30,20))\n_ = plt.grid(which='both')\n_ = plt.plot(n_components, cumul_variance, color='red')\n_ = plt.xscale('log')\n_ = plt.xlabel('Number of PCA features', size=20)\n_ = plt.ylabel('Cumulated variance of data (%)', size=20)\n_ = plt.title('Data variance cumulated vs number of PCA features', size=20)\nplt.savefig('visualizations/cumulated variance_pca features.png')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"45bcd4ef1fad87159509a0ad1bc0760a3d7384ff"},"cell_type":"markdown","source":"*50% of the data variance is carried by 10 PCA features, and 90% of the data variance is carried by 80 PCA features (vs 708 features initially). The reduction is huge !*\n\n*However, the PCE features have no interpretable meaning.*\n\n*--> The use of PCA() in the pipleine along with StandardScaler() and SVC() increased the accuracy up to 0.97 trained on half of the samples.*"},{"metadata":{"_uuid":"77fc2aeaae345f77830a2862d1a73f7906b46505"},"cell_type":"markdown","source":"#### Non-Negative MAtrix Factorization (NMF)"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a162a82833ec2c5630daf424618770f956c33065"},"cell_type":"code","source":"#Preparing samples and labels arrays\ns = np.random.randint(low=0, high=42001, size=8200).tolist()\nsamples = train.drop(columns='label').values  #.iloc[s, :]\ndigitsa = train['label'].tolist()  #.iloc[s, :]\nprint(samples.shape)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"736485b0c1207033c7962e89e47f152d15951a20"},"cell_type":"code","source":"#Creating the NMF model and the features & components\nnmf = NMF(n_components=16)\nnmf_features = nmf.fit_transform(samples)\nnmf_components = nmf.components_\nprint(\"Shape of NMF features: {}, shape of NMF components: {}\".format(nmf_features.shape, nmf_components.shape))\n\n#Visualization of the features\nfor i, component in enumerate(nmf_components):\n    N = nmf_components.shape[0]\n    ax = plt.subplot((N//3)+1, 3, i+1)\n    bitmap = component.reshape((28,28))\n    plt.imshow(bitmap, cmap='gray')\n    plt.xticks([])\n    plt.yticks([])\nplt.title('NMF Components from the original images')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae390131193aadb79050ecdfdb42889a3002e892"},"cell_type":"markdown","source":"*The samples have been split between NMF features and components, in a way that Features x Components = Samples (with matrices product)*\n\n*The images have been decomposed into combinations of the NMF components displaed above.*"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c797ed3b8fd13ee145e651f2c1cb44edacfcd4fa"},"cell_type":"code","source":"%%time\n\n#Sample randomly the dataset using discrete_uniform pick-up to reduce the amount of data\n#sample = np.random.randint(low=0, high=42001, size=8400).tolist()\nX = train_changing_pixels_df.values  #.iloc[sample, :]\nX = X / 255.0\ny = train['label'].values  #.iloc[sample, :]\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=4, stratify=y)\n\n#Yielding the scores according to number of NMF components\ncomponents = np.arange(1, 100)\nscores = np.empty(len(components))\nfor n in components:\n    pipeline = make_pipeline(NMF(n_components=n), SVC(kernel='rbf', cache_size=1000))\n    pipeline.fit(X_train, y_train)\n    scores[n-1] = pipeline.score(X_test, y_test)\n\n#Plotting of the scores evlution\n_ = plt.figure(figsize=(30,20))\n_ = plt.grid(which='both')\n_ = plt.plot(components, scores)\n_ = plt.xlabel('Number of NMF components', size=20)\n_ = plt.ylabel('Score obtained', size=20)\n_ = plt.title('Evolution of SVC classification score (samples={})'.format(len(y)), \n              size=30)\nplt.savefig('visualizations/Score vs components NMF.png')\nplt.show()\n\nprint(\"Best score {} obtained for {} components\".format(scores.max(), scores.argmax()+1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89ad8ff6ba0a1305f807eef94fe9570c8eb37bc6"},"cell_type":"markdown","source":"*Highest score of 0.83 obtained for 16 components*"},{"metadata":{"_uuid":"72202c361df18bbace3a505fc5144e3ca4a28c6d"},"cell_type":"markdown","source":"#### Sparse Matrix\nAs the pixel inteisties present a lot of 0 values, we can turn the train 2D array into a sparse matrix to reduce the memory size\n\nhttps://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html\n- TruncatedSVD for the equivalent of PCA()\n- NMF() directly with the csr_matrix"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"b3a959f7f3c1cffbd919384e7c7bf706e104e5ed"},"cell_type":"code","source":"#Sparsity visuaization & sparse matrix creation\nsamples = train_changing_pixels_df.values\n_ = plt.figure(figsize=(10,100))\n_ = plt.spy(samples)\nplt.show()\n\nsparse_samples = csr_matrix(samples)\n\n#Memory Size comparison\ndense_size = samples.nbytes/1e6\nsparse_size = (sparse_samples.data.nbytes + \n               sparse_samples.indptr.nbytes + sparse_samples.indices.nbytes)/1e6\nprint(\"From {} to {} Mo in memory usage with the sparse matrix\".format(dense_size, sparse_size))\n\n#Dimension reduction using PCA equivalent for sparse matrix\nmodel = TruncatedSVD(n_components=10)\nmodel.fit(sparse_samples)\nreduced_sparse_samples = model.transform(sparse_samples)\nprint(reduced_sparse_samples.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cc23bc514ec80b5267a4e96675079f16b451cac"},"cell_type":"markdown","source":"*The sparsity visualization confirms the relevance of the use of a csr matrix for the samples*"},{"metadata":{"_uuid":"8d2e84bcb2454b29c9f4521baaf3fe9b8dd4ad15"},"cell_type":"markdown","source":"## Simple Classifiers\n- training datasets\n    - changing pixels only dataset\n    - randomly sampled\n- Classifier to try: \n    - k-nn, \n    - LDA/QDA,\n    - SVC & Linear SVC, \n    - Decision tree,\n    - LVQ"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c0dfbfe3712b1ab61f479e53d401821b8c2328b9"},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, Normalizer, MaxAbsScaler\nfrom sklearn.decomposition import PCA, NMF, TruncatedSVD\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\nfrom sklearn.tree import DecisionTreeClassifier, export_graphviz\nimport graphviz\n\nfrom sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score, RandomizedSearchCV","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"5e69ad5afcc738b8645dcf17418bc204bbeae94b"},"cell_type":"markdown","source":"#### K-NN"},{"metadata":{"trusted":false,"_uuid":"3edb25e5c583beccfefba56289a8aa6e72747b62"},"cell_type":"code","source":"#Sample randomly the dataset using discrete_uniform pick-up\nsample = np.random.randint(low=0, high=42001, size=2100).tolist()\n\n#Prepare the X (features) and y (label) arrays for the sampled images\nX = train_changing_pixels_df.iloc[sample, :].values\ny = train.loc[sample, 'label'].values#.reshape(-1,1)\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4, stratify=y)\n\n#Fine tune the k value\nparam_grid = {'n_neighbors': np.arange(1,10)}\nknn_cv = GridSearchCV(KNeighborsClassifier(), param_grid, cv=5)\nknn_cv.fit(X_train, y_train)\n\n#Best k parameter\nbest_k = knn_cv.best_params_\nbest_accuracy = knn_cv.best_score_\nprint(\"Best accuracy on test set during training is {} obtained for {}\".format(best_accuracy, best_k))","execution_count":14,"outputs":[]},{"metadata":{"_uuid":"b2ba503f13adabe3acf4a1708194302d82ef13d4"},"cell_type":"markdown","source":"*Obtained score on Kaggle: 0.94300 trianed on entire original dataset*"},{"metadata":{"_uuid":"ea74b9effc2c385754543e494d5948aa11c247e3"},"cell_type":"markdown","source":"#### Linear Discriminant Analysis"},{"metadata":{"trusted":false,"_uuid":"ed7fdbc088b6cae66916e06441ff0f1e91458aa4"},"cell_type":"code","source":"%%time\n\n#Sample randomly the dataset using discrete_uniform pick-up\n#sample = np.random.randint(low=0, high=42001, size=4200).tolist()\n\n#Prepare the X (features) and y (label) arrays for 4000 images\nX = train_simplified_pixels_df.values #.iloc[sample, 1:]\ny = train.loc[:, 'label'].values#.reshape(-1,1)\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=4, stratify=y)\n\n#Fit the model (no hyperparameter tuning for this model)\nlda = LinearDiscriminantAnalysis(solver='lsqr', shrinkage='auto') #best results with these arguments\n#lda = QuadraticDiscriminantAnalysis()#reg_parameter=?\nlda.fit(X_train, y_train)\nscore = lda.score(X_test, y_test)\nprint(\"Accuracy on test set: {}\".format(score))\n\n#Best cv scores\nlda_cv_scores = cross_val_score(lda, X_train, y_train, cv=5)\nbest_accuracy = lda_cv_scores.max()\nprint(\"Best accuracy during CV is {}\".format(best_accuracy))","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"4d462b37f3f9b513d2a72a00a5bc145565c3e179"},"cell_type":"markdown","source":"*--> Linear Discriminant Analysis returns a score around 87%, better than the Quadratic Discriminant Analysis, within less than 2min execution on the whole training set (which is pretty fast !)*\n\n*--> Kaggle submission score:  0.86971*"},{"metadata":{"_uuid":"0f42d4d2ab42dcb3416586a92170cacffd99fe6e"},"cell_type":"markdown","source":"#### Support Vector Machines\n##### SVC()\n- test different kernels of SVC (linear, polynomial, rbf, sigmoid)\n- RandomizedSearchCV to fine tune the hyperparameters (depending on the kernel)"},{"metadata":{"trusted":false,"_uuid":"12817064a2caf50366b508500a746dab0ffd30f8"},"cell_type":"code","source":"%%time\n\n#Sample randomly the dataset using discrete_uniform pick-up\nsample = np.random.randint(low=0, high=42001, size=2100).tolist()\n\n#Prepare the X (features) and y (label) arrays for the sampled images\nX = train_simplified_pixels_df.iloc[sample, :].values\ny = train.loc[sample, 'label'].values#.reshape(-1,1)\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=4, stratify=y)\n\n#Fine tune the hyperparameters using RandomizeSearchCV rather than GridSearchCV (too expensive with more than 1 hyperparameter)\nparam_grid = {'C': np.logspace(0, 3, 20),\n              'gamma':np.logspace(0, -4, 20)#, not for linear kernel\n              #'degree': [2,3,4,5]  #only for poly kernel\n              #'coef0': []  #only for poly & sigmoid kernels\n             }\nsvm_cv = RandomizedSearchCV(SVC(kernel='rbf', cache_size=3000), \n                            param_grid, cv=5)\nsvm_cv.fit(X_train, y_train)\n\n#Best k parameter\nbest_k = svm_cv.best_params_\nbest_accuracy = svm_cv.best_score_\nprint(\"Best accuracy on test set during training is {} obtained for {}\".format(best_accuracy, best_k))","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"4dd743ca2e03863780a6d496ed81c0e1034866e7"},"cell_type":"markdown","source":"**Scores of submission, SVC() fitted on simplified dataset: --> scaled data required! **\n- rbf, C=4.4984, gamma=0.01842: 0.9606 on test set, 0.95642 on submission\n- poly, C=39.069, gamma = 0.01098, degre=3  : 0.937 on test, 0.93 on submission\n- sigmoid, C=754.3, gamma=0.00095  : 0.899 on test*"},{"metadata":{"_uuid":"a364c3e862f8942adfada42366deb973c7ef8c46"},"cell_type":"markdown","source":"##### LinearSVC()"},{"metadata":{"trusted":false,"_uuid":"eb51fd13ca678a56437a91b89d0da500f0e480c5"},"cell_type":"code","source":"%%time\n\n#Sample randomly the dataset using discrete_uniform pick-up\nsample = np.random.randint(low=0, high=42001, size=4100).tolist()\n\n#Prepare the X (features) and y (label) arrays for the sampled images\nX = train_simplified_pixels_df.iloc[sample, :].values\ny = train.loc[sample, 'label'].values#.reshape(-1,1)\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=4, stratify=y)\n\n#Fine tune the hyperparameters using RandomizeSearchCV rather than GridSearchCV (too expensive with more than 1 hyperparameter)\nparam_grid = {'multi_class': ['ovr', 'crammer_singer'],\n              'penalty': ['l1', 'l2'],\n              'C': np.logspace(0, 4, 50)}\n\nlinsvc_cv = GridSearchCV(LinearSVC(dual=False), param_grid, cv=5)\n#linsvc_cv = RandomizedSearchCV(LinearSVC(dual=False), param_grid, cv=5)\nlinsvc_cv.fit(X_train, y_train)\n\n#Best k parameter\nbest_k = linsvc_cv.best_params_\nbest_accuracy = linsvc_cv.best_score_\nprint(\"Best accuracy on test set during training is {} obtained for {}\".format(best_accuracy, best_k))","execution_count":5,"outputs":[]},{"metadata":{"_uuid":"f5257db06ccf23adea57955a0a2d6a693583eb2c"},"cell_type":"markdown","source":"*--> C=1, multi-class, crammer_singer, penatly=l1, result on test = 0.845*"},{"metadata":{"_uuid":"431cf18bcb8083e075f13f1782a6f72bf9ee871c"},"cell_type":"markdown","source":"#### Decision Tree classifier"},{"metadata":{"trusted":false,"_uuid":"0fc5e7d39a3bc78ccb20495866154406e7806587"},"cell_type":"code","source":"%%time\n\n#Sample randomly the dataset using discrete_uniform pick-up\n#sample = np.random.randint(low=0, high=42001, size=4200).tolist()\n\n#Prepare the X (features) and y (label) arrays for 4000 images\nX = train_changing_pixels_df.values  #.iloc[sample, :]\ny = train.loc[:, 'label'].values#.reshape(-1,1)\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=4, stratify=y)\n\n#Fine tune the hyperparameters\nparam_grid = {'max_depth': np.arange(3, 50),\n              'min_samples_leaf': np.arange(5, 50, 1),\n              'min_samples_split': np.arange(2,50, 1)\n             }\ntree_cv = RandomizedSearchCV(DecisionTreeClassifier(),\n                       param_grid, cv=5)\ntree_cv.fit(X_train, y_train)\n\n#Best k parameter\nbest_k = tree_cv.best_params_\nbest_accuracy = tree_cv.best_score_\nprint(\"Best accuracy on test set during training is {} obtained for {}\".format(best_accuracy, best_k))","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"406fc2a364fb1d171bc402e639722384df145bdb"},"cell_type":"markdown","source":"*--> 0.85 on test*"},{"metadata":{"_uuid":"0f69a607522051246f42a6c66eb370f592005ef3"},"cell_type":"markdown","source":"## Pipeline\n- Using the changing pixels dataset\n- Rescaling intensity between 0 and 1 (StandardScaler() (pixels with different variances) or the custom function)\n- Normalize the samples ?\n- no missing values --> no need for an Imputer\n- DImension Reduction (PCA / NMF/ sparce matrix + TruncatedSVD)\n- Hyperparameters Fine tune (GriSearchCV/RandomisedSearchCV)\n- simple classifiers performances: SVC(rbf kernel) > k-nn > LDA > LinearSVC > DecisionTree,...\n- Bagging, Boost"},{"metadata":{"trusted":false,"_uuid":"cde3ca3b70a8d56d3f3c135271a3a32d2bb89747"},"cell_type":"code","source":"#To save time and not have to run the entire function\nDROPPED_PIX = ['pixel0', 'pixel1', 'pixel2', 'pixel3', 'pixel4', 'pixel5', 'pixel6', 'pixel7', 'pixel8', 'pixel9', \n               'pixel10', 'pixel11', 'pixel16', 'pixel17', 'pixel18', 'pixel19', 'pixel20', 'pixel21', 'pixel22', \n               'pixel23', 'pixel24', 'pixel25', 'pixel26', 'pixel27', 'pixel28', 'pixel29', 'pixel30', 'pixel31', \n               'pixel52', 'pixel53', 'pixel54', 'pixel55', 'pixel56', 'pixel57', 'pixel82', 'pixel83', 'pixel84', \n               'pixel85', 'pixel111', 'pixel112', 'pixel139', 'pixel140', 'pixel141', 'pixel168', 'pixel196', \n               'pixel392', 'pixel420', 'pixel421', 'pixel448', 'pixel476', 'pixel532', 'pixel560', 'pixel644', \n               'pixel645', 'pixel671', 'pixel672', 'pixel673', 'pixel699', 'pixel700', 'pixel701', 'pixel727', \n               'pixel728', 'pixel729', 'pixel730', 'pixel731', 'pixel754', 'pixel755', 'pixel756', 'pixel757', \n               'pixel758', 'pixel759', 'pixel760', 'pixel780', 'pixel781', 'pixel782', 'pixel783']\ntrain_changing_pixels_df = pd.read_csv('../input/changing-pixels/train_changing_pixels_DB.csv', index_col=0)\nprint(train_changing_pixels_df.head())\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\nprint(train.head())","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"d74cb6125f3892f33b205d612dc3cebe61da9828"},"cell_type":"code","source":"%%time\n\n#Sample randomly the dataset using discrete_uniform pick-up to reduce the amount of data\n#sample = np.random.randint(low=0, high=42001, size=21000).tolist()\n\n#Prepare the X (features) and y (label) arrays for the images\nX = csr_matrix(train_changing_pixels_df.values)  #use .iloc[sample, :] for reduced sample\n#X = X / 255.0  #intensities recaled between 0 and 1, NMF don't take negative values\ny = train['label'].values#.reshape(-1,1)   #idem if using sample\nprint(\"Shape of X and Y arrays: {}\".format((X.shape, y.shape)))\n\n#Split the training set\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=4, stratify=y)\n\n#Pipeline with fine tune\npipeline = Pipeline([#('scaler', StandardScaler()), \n                     ('pca', TruncatedSVD()),\n                     ('svm', SVC(kernel='rbf', cache_size=3000))\n                    ])\nparam_grid = {'pca__n_components': np.arange(5, 80),\n              'svm__C': np.logspace(0, 4, 50),\n              'svm__gamma':np.logspace(0, -4, 50)}\npipeline_cv = RandomizedSearchCV(pipeline, param_grid, cv=5)\n\n#fitting\npipeline_cv.fit(X_train, y_train)\n\n#Best k parameter\nbest_k = pipeline_cv.best_params_\nbest_accuracy = pipeline_cv.best_score_\nprint(\"Best accuracy on test set during training is {} obtained for {}\".format(best_accuracy, best_k))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1d80e721a3a8500382dc86489f260a543f3e3c8a"},"cell_type":"code","source":"%%time\n\n#Predict on the test dataset (holdout) that MUST contain as many columns (ie pixels) than in the training set\nholdout = pd.read_csv('test.csv').drop(columns=DROPPED_PIX)\nX_holdout = holdout.values\nprint(X_holdout.shape)\n\npredictions = pipeline_cv.predict(X_holdout)\nsubmission_df = pd.DataFrame({'ImageId': range(1,28001), 'Label': predictions})\nprint(\"Overview of the obtained predictions :\\n\", submission_df.head())\n\n#Save as submission file for competition\nsubmission_df.to_csv('submission_pca_svc_DB.csv', index=False)","execution_count":7,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"f442f4c394d62603932146e10757c7dff8da9ca1"},"cell_type":"markdown","source":"- PCA, SVM: (no scaler, no normalizer) *0.9747 obtained for {gamma: 0.1, C: 345.5107, n_components: 40}*"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"47c26c956f82ef083e0149f1e80b5d152b7d0813"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}