{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\nfrom matplotlib import style\nstyle.use('dark_background')\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_dir='/kaggle/input/digit-recognizer/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets read the data sets into train and test  "},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv(data_dir+'train.csv')\ntest=pd.read_csv(data_dir+'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## all classes are present in almost equal poportions, so no problem of imbalanced data"},{"metadata":{},"cell_type":"markdown","source":"# Lets check for nan in train and test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# not much of help as feature set is very large, this is not a good method to check for nan's"},{"metadata":{"trusted":true},"cell_type":"code","source":"features=[i for i in train.columns]\nk=0\nfor feature in features:\n    if train[feature].isnull().sum()==0:\n        k=k+1\n    \n    else:\n         \n         print('{0} has {1} null values'.format(feature,train[feature].isnull().sum()))\n            \nif(k==train.shape[1]):\n    print('no nan\\'s in the train dataset,so  proceed')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features=[i for i in test.columns]\nk=0\nfor feature in features:\n    if test[feature].isnull().sum()==0:\n        k=k+1\n    \n    else:\n        \n         print('{0} has {1} null values'.format(feature,test[feature].isnull().sum()))\n            \nif(k==test.shape[1]):\n    print('no nan\\'s in the test dataset,so  proceed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# separate feature matrix and target vector"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y=train['label']\ntrain_x=train.drop('label',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_x.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets visualize a sample point, which label does it belong"},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_printer(i,train_x):\n    idx=i\n    grid_data=train_x.iloc[idx].to_numpy().reshape(28,28).astype('uint8')\n    plt.imshow(grid_data)\n    \n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,10))\nfor i in range(100):\n    plt.subplot(10, 10, i+1)\n    image_printer(i, train_x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pca without Sklearn"},{"metadata":{},"cell_type":"markdown","source":"## training 42k points will take too long so slice for a 20k points"},{"metadata":{"trusted":true},"cell_type":"code","source":"data=train_x[0:20000]\nlabel=train_y[0:20000]\nprint(data.shape,label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets standardize it"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nstd_data=StandardScaler().fit_transform(data)\nprint(np.mean(std_data))\nprint(np.std(std_data))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets generate our covariance matrix of dXd because it will help us to get top 2 eigen vectors for our 2d visualization "},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_data=std_data\ncovar_matrix=np.matmul(temp_data.T,temp_data)\nprint(f'shape of covar matrix is {covar_matrix.shape}')\nprint(f'shape of my data is {temp_data.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# fetching the top two eigen values and eigen vectors because for top two principal axis we need top two eigen vectors"},{"metadata":{},"cell_type":"markdown","source":"# as eigen vect starts from 0 so the last eigen vector is 783 and second last is 782 which are two largest eigen vectors among 784 eigen vectors "},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.linalg import eigh\n\n\neigh_values,eigh_vectors=eigh(covar_matrix,eigvals=(782,783))\n\nprint(f'shape of eigen vectors {eigh_vectors.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets have our pca_points for top two axis by matrix multiplication of our sample data with top two eigen vectors"},{"metadata":{"trusted":true},"cell_type":"code","source":"pca_points=np.matmul(temp_data,eigh_vectors)\nprint(f'shape of my pca points is {pca_points.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# this is our final pca dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"pca_data=np.vstack((pca_points.T,label)).T\nprint(f'shape of my pca data is {pca_data.shape}')\n\npca_dataframe=pd.DataFrame(data=pca_data,columns=('1st Principal comp','2nd Principal comp','label'))\nprint(pca_dataframe.head(10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Time to visualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.FacetGrid(pca_dataframe,hue='label',size=10).map(plt.scatter,'1st Principal comp','2nd Principal comp').add_legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Producing the same using pca from sklearn"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import decomposition\npca=decomposition.PCA()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_data1=std_data\n\npca.n_components=2\npca_data=pca.fit_transform(temp_data1)\n\npca_data=np.vstack((pca_data.T,label)).T\npca_df=pd.DataFrame(data=pca_data,columns=('1st principal','2nd principal','label'))\n\nsns.FacetGrid(pca_df,hue='label',size=10).map(plt.scatter,'1st principal','2nd principal').add_legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# conclusion both of the methods with sklearn and without sklearn produces visually same o/p (rotaion is not a problem so ignore it). so we can say we are successful in our task"},{"metadata":{},"cell_type":"markdown","source":"# lets find the no of top best features that can best explain variance in our dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.n_components=784\npca_data1=pca.fit_transform(temp_data1)\n\npercent_variance_explained=pca.explained_variance_/np.sum(pca.explained_variance_)\ncummulative_variance_explained=np.cumsum(percent_variance_explained)\n\nplt.plot(cummulative_variance_explained,linewidth=2)\n\n\nplt.xlabel('no of top features')\nplt.ylabel('cummulative explained variance')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# top 450 principal components that can explain almost 99% of total variance so no point of taking more features than 450 because it wilI l only lead to curse of dimensionality"},{"metadata":{},"cell_type":"markdown","source":"# Now applying tsne to visualize better"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.manifold import TSNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=TSNE(n_components=2,perplexity=30,n_iter=5000,random_state=0,n_jobs=-1)\ntsne_data=model.fit_transform(std_data)\ntsne_data=np.vstack((tsne_data.T,label)).T\ntsne_df=pd.DataFrame(data=tsne_data,columns=(\"Dimension 1\",\"Dimension 2\",\"label\"))\nsns.FacetGrid(tsne_df,hue='label',size=9).map(plt.scatter,\"Dimension 1\",\"Dimension 2\").add_legend()\nplt.title(f'with perplexity {30} and iterations {5000}')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=TSNE(n_components=2,perplexity=50,n_iter=5000,random_state=0,n_jobs=-1)\ntsne_data1=model.fit_transform(std_data)\ntsne_data1=np.vstack((tsne_data1.T,label)).T\ntsne_df1=pd.DataFrame(data=tsne_data1,columns=(\"Dimension 1\",\"Dimension 2\",\"label\"))\nsns.FacetGrid(tsne_df1,hue='label',size=9).map(plt.scatter,\"Dimension 1\",\"Dimension 2\").add_legend()\nplt.title(f'with perplexity {50} and iterations {5000}')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# well we all can now clearly see the clusters that are formed...tsne is time consuming but it's worth the effort"},{"metadata":{},"cell_type":"markdown","source":"# Building new dataset with top 450 features"},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_data1=std_data\n\npca.n_components=450\npca_data=pca.fit_transform(temp_data1)\n\npca_data_new=np.vstack((pca_data1.T,label)).T\nnew_data=pca_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets train some models with different algos"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.model_selection import cross_val_score,cross_val_predict\nfrom sklearn.metrics import accuracy_score,confusion_matrix,classification_report\nfrom scipy.stats import uniform,truncnorm,randint","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying multiclass Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf=LogisticRegression(multi_class='multinomial',n_jobs=-1)\ncv_scores=cross_val_score(clf,new_data,label,cv=10)\nprint(f'we can expect accuracy between {cv_scores.min()} and {cv_scores.max()} with avg accuracy of {cv_scores.mean()}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generating classification Report"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_pred=cross_val_predict(clf,new_data,label,cv=10)\naccLR=accuracy_score(label,class_pred)\nprint(f'accuracy of our model is {accLR}')\nprint(f' confusion matrix is  ')\nprint(confusion_matrix(label,class_pred))\nprint(f'classification report is ')\nprint(classification_report(label,class_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Note: I could've used gridsearch and randomizedsearch but they are taking sooo long, i waited for 3-4 hours but still they still they didnt produced anything and also i donot want to redeuce my train pts.....so i modified my code to find the best params...here at least i can observe my progress "},{"metadata":{},"cell_type":"markdown","source":"# Applying SVM"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import svm\n\nfor kernel in ('poly','rbf'):\n    clf=svm.SVC(kernel=kernel)\n    cv_scores=cross_val_score(clf,new_data,label,n_jobs=-1,cv=10)\n    print(f'accuracy of {kernel} kernel varies between {cv_scores.min()} and {cv_scores.max()} with mean {cv_scores.mean()}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for C in (1,10,100):\n    clf=svm.SVC(kernel='rbf',C=C)\n    cv_scores=cross_val_score(clf,new_data,label,n_jobs=-1,cv=10)\n    print(f'accuracy of rbf kernel with C={C} varies between {cv_scores.min()} and {cv_scores.max()} with mean {cv_scores.mean()}')\n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for gamma in (0.001,0.01):\n    clf=svm.SVC(kernel='rbf',C=10,gamma=gamma)\n    cv_scores=cross_val_score(clf,new_data,label,n_jobs=-1,cv=10)\n    print(f'accuracy of rbf kernel with C=10 and gamma={gamma} varies between {cv_scores.min()} and {cv_scores.max()} with mean {cv_scores.mean()}')\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# it seems like rbf kernel with C=10 and gamma=0.001 is the best hyperparams set for my svm so lets train with the final model"},{"metadata":{"trusted":true},"cell_type":"code","source":"clf=svm.SVC(kernel='rbf',C=10,gamma=0.001)\ncv_scores=cross_val_score(clf,new_data,label,n_jobs=-1,cv=10)\nprint(f'accuracy of rbf kernel with C=10 and gamma= 0.001 varies between {cv_scores.min()} and {cv_scores.max()} with mean {cv_scores.mean()}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating classification report for final svm model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_pred=cross_val_predict(clf,new_data,label,cv=10)\naccSVM=accuracy_score(label,class_pred)\nprint(f'accuracy of our model is {accSVM}')\nprint(f' confusion matrix is  ')\nprint(confusion_matrix(label,class_pred))\nprint(f'classification report is ')\nprint(classification_report(label,class_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying KNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom sklearn.neighbors import KNeighborsClassifier\n\nfrom sklearn import tree\nlst=list(range(0,20))\nneighbor=[i for i in lst if i%2!=0]\ncv_scores=[]\n\nfor k in neighbor:\n    print(k,end='')\n    clf=KNeighborsClassifier(n_neighbors=k,n_jobs=-1)\n    scores=cross_val_score(clf,new_data,label,cv=10,scoring='accuracy',n_jobs=-1)\n    cv_scores.append(scores.mean())\n    \n    \nMSE=[1-x for x in cv_scores]    \nbest_k=neighbor[MSE.index(min(MSE))]\nprint(f'best k here for knn is {best_k}')\nplt.plot(neighbor,MSE)\nplt.xlabel('K of KNN--->')\nplt.ylabel('Mean Squared Error')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Best k for our knn is 5 so lets build our final model with k=5"},{"metadata":{"trusted":true},"cell_type":"code","source":"clf=KNeighborsClassifier(n_neighbors=5,n_jobs=-1)\ncv_score=cross_val_score(clf,new_data,label,cv=10)\nprint(f' we can expect accuracy bwetween  {cv_score.min()} and {cv_score.max()} with a mean of {cv_score.mean()}')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating Classification Report"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_pred=cross_val_predict(clf,new_data,label,cv=10)\naccKNN=accuracy_score(label,class_pred)\nprint(f'accuracy of our model is {accKNN}')\nprint(f' confusion matrix is  ')\nprint(confusion_matrix(label,class_pred))\nprint(f'classification report is ')\nprint(classification_report(label,class_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying Decision Tree"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nmax_depth_range=list(range(1,20))\ncv_score=[]\nfor depth in max_depth_range:\n    print(depth,end='')\n    clf=DecisionTreeClassifier(max_depth=depth,random_state=0)\n    score=cross_val_score(clf,new_data,label,cv=10,scoring='accuracy',n_jobs=-1)\n    cv_score.append(score.mean())\n    \nMSE=[1-x for x in cv_score]\nbest_n=max_depth_range[MSE.index(min(MSE))]\nprint(f'optimal no of tree is {best_n}')\nplt.plot(max_depth_range,MSE)\nplt.xlabel('Depth of Tree')\nplt.ylabel('Mean Squared Error')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf=DecisionTreeClassifier(max_depth=best_n,random_state=0)\ncv_score=cross_val_score(clf,new_data,label,cv=10,scoring='accuracy',n_jobs=-1)\nprint(f' with {best_n} as depth of trees we can expect accuracy bwetween  {cv_score.min()} and {cv_score.max()} with a mean of {cv_score.mean()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf=DecisionTreeClassifier(max_depth=best_n,random_state=0)\nclf.fit(new_data,label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import graphviz\nimport pylab\n\ncn=['0','1','2','3','4','5','6','7','8','9']\ndot_data = tree.export_graphviz(clf, out_file=None, \n                                \n                                class_names=cn,\n                                filled=True)\n\n\ngraph = graphviz.Source(dot_data,format=\"png\") \ngraph\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# tree is very large to visualize...just scroll down left and right to visualize the tree.... i  failed to scale down the size...... if you know the trick then pls share with me"},{"metadata":{},"cell_type":"markdown","source":"# Generating Classification Report"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_pred=cross_val_predict(clf,new_data,label,cv=10)\naccDT=accuracy_score(label,class_pred)\nprint(f'accuracy of our model is {accDT}')\nprint(f' confusion matrix is  ')\nprint(confusion_matrix(label,class_pred))\nprint(f'classification report is ')\nprint(classification_report(label,class_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying Random Forest "},{"metadata":{},"cell_type":"markdown","source":"# Note: i precomputed the best no of trees for randomforest is 600 and it took me 4.5 hours to realize this so i am not again running this coz it would take insane amount of time........code is below if you are born with iron patience you can use the code :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n# max_trees_range=list(range(100,1000))\n# lst=[i for i in max_trees_range if i%100==0]\n# cv_score=[]\n# for ntree in lst: \n#     print(ntree,end='')\n#     clf=RandomForestClassifier(n_estimators=ntree,max_depth=best_n,n_jobs=-1)\n#     score=cross_val_score(clf,new_data,label,cv=10,n_jobs=-1)\n#     cv_score.append(score.mean())\n    \n# MSE=[1-x for x in cv_score]\n# best_ntree=lst[MSE.index(min(MSE))]\n# print(f' best no of tree is {best_ntree}')\n# plt.plot(lst,MSE)\n# plt.xlabel('no of trees')\n# plt.ylabel('Mean Squared Error')\n# plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf=RandomForestClassifier(n_estimators=600,max_depth=best_n,random_state=0,n_jobs=-1)\ncv_score=cross_val_score(clf,new_data,label,cv=10,scoring='accuracy',n_jobs=-1)\nprint(f' with {best_n} as depth of trees and {600} as no of trees we can expect accuracy bwetween  {cv_score.min()} and {cv_score.max()} with a mean of {cv_score.mean()}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating Classification Report"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_pred=cross_val_predict(clf,new_data,label,cv=10)\naccRF=accuracy_score(label,class_pred)\nprint(f'accuracy of our model is {accRF}')\nprint(f' confusion matrix is  ')\nprint(confusion_matrix(label,class_pred))\nprint(f'classification report is ')\nprint(classification_report(label,class_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Finally lets call the Navy Seals i.e CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras import backend as k\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Activation, BatchNormalization\nfrom keras.layers.convolutional import Conv2D, MaxPooling2D\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import plot_model\nfrom keras.callbacks import EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing the data for CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x=test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_x.shape)\nprint(test.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_cols=28\nimg_rows=28","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reshaping and normalizing the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"if k.image_data_format=='channels_first':\n    train_x = train_x.values.reshape(train_x.shape[0], 1,img_cols,img_rows)\n    test = test.values.reshape(test.shape[0], 1,img_cols,img_rows)\n    train_x=train_x/255.0\n    test=test/255.0\n    input_shape = (1,img_cols,img_rows)\nelse:\n    train_x=train_x.values.reshape(train_x.shape[0],img_cols,img_rows,1)\n    test=test.values.reshape(test.shape[0],img_cols,img_rows,1)\n    train_x=train_x/255.0\n    test=test/255.0\n    input_shape = (img_cols,img_rows,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"earlystopping = EarlyStopping(monitor =\"val_accuracy\",\n                              mode = 'auto', patience = 10,\n                              restore_best_weights = True)\n\n\nmodelacc = []\n\nnfilters = [32, 64, 128,256]\nconv_layers = [1, 2, 3,4]\ndense_layers = [0, 1, 2,3]\n#dropouts \ndp=0.5\n\nfor filters in nfilters:\n    for conv_layer in conv_layers:\n        for dense_layer in dense_layers:\n           \n                #lets iterate for all the combinations\n                cnnsays = 'No of filters/feature maps: {} conv_layers: {} dense_layers: {} dropout: {}'.format(filters, conv_layer, dense_layer, dp)\n                print(cnnsays)\n\n                model = Sequential()\n\n                model.add(Conv2D(filters, (3, 3), input_shape = input_shape))\n                model.add(BatchNormalization())\n                model.add(Activation(\"relu\"))\n                model.add(MaxPooling2D(pool_size=(2, 2)))\n                model.add(Dropout(dp))\n\n                for i in range(conv_layer-1):\n\n                    model.add(Conv2D(filters, (3, 3),padding=\"same\"))\n                    model.add(BatchNormalization())\n                    model.add(Activation('relu'))\n                    model.add(MaxPooling2D(pool_size=(2, 2)))\n                    model.add(Dropout(dp))\n\n\n                model.add(Flatten())\n\n                for i in range(dense_layer):\n                    model.add(Dense(256))\n                    model.add(BatchNormalization())\n                    model.add(Activation('relu'))\n                    model.add(Dropout(dp))\n\n\n                model.add(Dense(10, activation='softmax'))\n\n\n                model.compile(optimizer = 'adam',\n                  loss = 'sparse_categorical_crossentropy',\n                  metrics = ['accuracy'])\n\n                EPOCHS = 30\n\n                history = model.fit(train_x, train_y, epochs=EPOCHS, batch_size=32, validation_split=0.2, callbacks=[earlystopping])\n\n                modelacc.append([round(100*max(history.history['val_accuracy']), 2), cnnsays])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Highest Validation Accuracy so far : {}%\".format(round(100*max(history.history['val_accuracy']), 2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelacc.sort(reverse=True)\nmodelacc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_train = history.history['loss']\nloss_val = history.history['val_loss']\nepochs = range(1,len(loss_val)+1)\nplt.plot(epochs, loss_train, 'g', label='Training loss')\nplt.plot(epochs, loss_val, 'b', label='Validation loss')\nplt.title('Training & Validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"acc_train = history.history['accuracy']\nacc_val = history.history['val_accuracy']\nepochs = range(1,len(acc_val)+1)\nplt.plot(epochs, acc_train, 'g', label='Training accuracy')\nplt.plot(epochs, acc_val, 'b', label='Validation accuracy')\nplt.title('Training & Validation accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=model.predict([test])\nsoln=[]\nfor i in range(len(pred)):\n    soln.append(np.argmax(pred[i]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# o/p"},{"metadata":{"trusted":true},"cell_type":"code","source":"final = pd.DataFrame()\nfinal['ImageId']=[i+1 for i in test_x.index]\nfinal['Label']=soln\nfinal.to_csv('mnistcnn.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}