{"cells":[{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n# hide warnings\nimport warnings\nwarnings.simplefilter('ignore')\n\n# Any results you write to the current directory are saved as output.\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\n\n# Graph\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nfrom sklearn.metrics import confusion_matrix\nfrom mpl_toolkits.mplot3d import axes3d\nimport plotly.express as px\n\n#Dimensional Reduction\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From this competition, I learned that CNN is extremely powerful technique to classify MNIST datasets. I guess someone who achieved above 98% definitely used CNN. I think more than 95% accuracy means model is well generalizing the situation but does not discriminate outliers(looking really strange or should not be there).\nTo check my hypothsis, I look into what letters are misclassified and how it looks like."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Original Label\ntrain_label=train['label'].copy()\ntrain_label=train_label.values\n\n#Original\ntrain_data = train.loc[:,'pixel0':].values\ntrain_data = train_data/255.0\ntrain_data=train_data.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I cited CNN model from [here](https://www.kaggle.com/cdeotte/25-million-images-0-99757-mnist) which performs above 99.7%, and I reduced 15  to 3 with 3 epochs since my laptop is not bearable to computing all, but it will achieve above 99% accuracy on the trainset! This will be enough to detect the outliers."},{"metadata":{"trusted":true},"cell_type":"code","source":"# BUILD CONVOLUTIONAL NEURAL NETWORKS\nnets = 3\nmodel = [0] *nets\nfor j in range(nets):\n    model[j] = Sequential()\n\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1)))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n\n    model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Flatten())\n    model[j].add(Dropout(0.4))\n    model[j].add(Dense(10, activation='softmax'))\n\n    #Compile\n    model[j].compile(optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# TRAIN NETWORKS\nhistory = [0] * nets\nepochs = 3\nfor j in range(nets):\n    history[j] = model[j].fit(train_data,train_label,\n        epochs = epochs, batch_size = 64,  \n         verbose=0)\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}\".format(\n        j+1,epochs,max(history[j].history['accuracy'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ENSEMBLE PREDICTIONS\nresults = np.zeros( (train_data.shape[0],10) ) \nresult=[0]*nets\nfor j in range(nets):\n    output = model[j].predict(train_data)\n    results = results + output\n    result[j] = output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def labelConcat(results):\n    choice = np.argmax(results,axis=1)\n    choice = pd.DataFrame({'predict_label':choice})\n    labels = pd.DataFrame({'true_label':train['label']})\n    labels = pd.concat([labels,choice], axis=1)\n    return labels\nlabels = labelConcat(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in len(result):\n    \n    if i ==0:\n        choice = result.copy()\n        choice = np.argmax(choice,axis=1)\n        labels = pd.DataFrame({'label':train['label']})\n        choice = pd.DataFrame({'predict_label':choice})\n        labels = pd.concat([labels,choice], axis=1)\n    else:\n        \n        choice = [choice,labelConcat(result[i])]\n    \n    return choice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Took a long time to compute...\nLet's check the letters which deceive 10 stacked CNN."},{"metadata":{},"cell_type":"markdown","source":"I aggregated all discrepancies between true label and predicted label from the models and visualized it.\nAccroding to the chart, some letters are confused with some specific letters or each other. \n\n0, 1, and 2 are not distinguishable. 3, 6 and 7 are not distinguishable. We will see how these looks like."},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"def unmatchPlot(labels):\n    idx = labels['true_label']!=labels['predict_label']\n    unmatch = labels[idx]\n    fig = sns.swarmplot(x=\"true_label\", y=\"predict_label\", hue=\"predict_label\",\n             alpha=.5\n            , data=unmatch,size=7)\n    plt.ylim(-1, 9)\n    plt.legend(bbox_to_anchor=(1, 0.95))\n    \n    \nunmatchPlot(labels)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"def UnmatchImages(labels,imageset,num):\n    \n    if num>10:\n        \n        print('out of bounds!')\n        \n    else:\n        \n        img = [0]*num\n        pred = [0]*num\n        for i in range(num):\n            idx = (labels['true_label']==i)&(labels['true_label']!=labels['predict_label'])\n            img[i] = imageset.loc[idx]\n            img[i] = img[i].loc[:,'pixel0':].values\n            img[i]=img[i]#.reshape(-1,28,28,1)\n            pred[i] = labels['predict_label'].loc[idx].values\n            \n    return img , pred\n\n\ndef Randomdiplayoutliers(Images,pred):\n    \n    width=10\n    height=4\n    \n    fig, ax = plt.subplots(height\n                           ,width\n                           ,figsize=(10,5)\n                           ,subplot_kw = {'xticks':[], 'yticks':[]})\n    \n    for i , ax in enumerate(ax.flat):\n        \n        num = i%width\n        a = np.arange(len(Images[num]))\n        np.random.shuffle(a)\n        \n\n        if len(a[:1])!=0:\n            ax.imshow(Images[num][a[:1]].reshape((28,28))\n                              ,cmap='binary'\n                              ,clim = (0,16))\n            ax.title.set_text('predict%d' % pred[num][a[:1]])\n            Images[num] = Images[num][a[1:]]\n            \n        else:\n            ax.title.set_text('DNE')\n            ax.imshow(np.zeros((28,28))\n                              ,cmap='binary'\n                              ,clim = (0,16))\n            \nImages, prediction = UnmatchImages(labels,train,10)\nRandomdiplayoutliers(Images,prediction)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Please check that 0,1,2 group and 3,6,7 group. each groups are similar in it. 9 to 6 also. \nIt seems difficult to clarify if it is truly labeled in my opinion."},{"metadata":{"trusted":true},"cell_type":"code","source":"def labelExtract(datasets,labels,compare,comp_num,method):\n    \n    dataset = datasets.copy()\n    \n    k=0\n    idx=[]\n    idx1=[]\n    idx2=[]\n    for i in compare:\n        \n        if k==0:\n            idx = labels['true_label'] == i\n            dataset['label'][idx] = 'true_label{}'.format(i)\n            idx1 = labels['true_label']!=labels['predict_label']\n            dataset['label'][idx1] = 'outlier{}'.format(i)\n        else:\n            idx2 = labels['true_label'] == i\n            dataset['label'][idx2] = 'label{}'.format(i)   \n        k = k+1\n    \n    if len(idx2)!=0:\n        idx3 = (idx2 | idx)\n    else:\n        idx3 = idx\n        \n    dataset = dataset.loc[idx3]\n    dataset_val = dataset.loc[:,'pixel0':].values\n    \n    \n    if method=='PCA':\n        \n        comp = ['PC1','PC2','PC3']\n        \n        scaler = StandardScaler()\n        scaler.fit(dataset_val)\n        dataset_val = StandardScaler().fit_transform(dataset_val)\n        pca = PCA(n_components=comp_num)\n        dataset_PC = pca.fit_transform(dataset_val)\n        \n        if comp_num ==2:\n        \n            dataset_PC = pd.DataFrame(data = dataset_PC\n                                  , columns = comp[:2])\n            dataset_PC['label'] = dataset['label'].values\n            \n        elif comp_num == 3:\n            \n            dataset_PC = pd.DataFrame(data = dataset_PC\n                                  , columns = comp)\n            dataset_PC['label'] = dataset['label'].values\n        \n        dataset = dataset_PC\n            \n    elif method == 'tSNE':\n        \n        comp = ['Axis1','Axis2','Axis3']\n        \n        tsne = TSNE(random_state = 42, n_components=comp_num\n                , verbose=0, perplexity=40\n                , n_iter=300).fit_transform(dataset_val)\n        \n            \n        if comp_num == 2:\n            \n            dataset_tSNE = pd.DataFrame(data = tsne\n                                  , columns = comp[:2])\n            \n            dataset_tSNE['label'] = dataset['label'].values\n        \n        elif comp_num == 3:\n            \n            dataset_tSNE = pd.DataFrame(data = tsne\n                                  , columns = comp)\n        \n            dataset_tSNE['label'] = dataset['label'].values\n            \n        dataset = dataset_tSNE\n            \n    \n    return dataset\n\ndef labelPCAplot(datasets):\n    \n    dim = len(datasets.T)-1\n    \n    if dim == 2:\n        \n        fig=px.scatter(datasets\n                       , x ='PC1'\n                       ,y ='PC2'\n                       ,color='label')\n        fig.show()\n        \n    \n    elif dim == 3:\n        \n        \n        fig = px.scatter_3d(datasets\n                            , x='PC1'\n                            , y='PC2'\n                            , z='PC3'\n                            ,color=\"label\")\n        \n        fig.show()\n        \n    else:\n        \n        print('>3D dimensional plot???')\n\ndef labeltSNEplot(datasets):\n    \n    dim = len(datasets.T)-1\n    \n    if dim == 2:\n        \n        \n        fig=px.scatter(datasets\n                       ,x ='Axis1'\n                       ,y ='Axis2'\n                       ,color='label')\n        fig.show()\n        \n    elif dim == 3:\n        \n        \n        fig = px.scatter_3d(datasets\n                            , x='Axis1'\n                            , y='Axis2'\n                            , z='Axis3'\n                            ,color=\"label\")\n        fig.show()\n        \n    else:\n        print('>3D dimensional plot???')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[9],3,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some dimensional reduction techniques are good to check similarities among classes, so I will use it to compare them with the outliers detected.\n\nPCA is the one of classical and famous dimensonal reduction techniques\nIntuitively, It finds new coordinates(basis) in which every points behave independently in each axis. and principal components are usually choosen from some axis in which points are broadely disributed in that axis.\nI draw 2D,3D scatter plots about 2 and 3 principal components using plotly and see where these outliers places in the cluster , and results were not so much distinctive as opposed to what I expected at first. One of the reasons might be it losses a lot of information from original one since it reduces the dimension from 28*28(784) dim to 2 or 3 dim. \nYou might check the proportion of 2 or 3 principal components from the total(It is not big enough). \n\nHowever, here something intresting. One can see that the outliers are clustered in some area, so this implies that outliers are simliar to each other. From this, I have suspicion that the area where outliers clustered might be overlapped with another cluster. \n\nLet's check it"},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[6,9],3,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[3],2,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[7,3],3,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[7,3],2,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[2,0],2,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[2,0],3,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[2],2,'PCA')\nlabelPCAplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Outliers from 6 are placed middle of two clusters as you can check"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[9],2,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is tSNE. I do not know this technique well, but it seems to try to reduce Kullerback-Leiber divergence between target dimension(reduced dimension) and original dimension by converting its metric distance to probability-looking formula. if these metric distance is similar, then KL-divergence also reduced. Am I right? :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[6,9],2,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[3],2,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[7,3],2,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[7,3],3,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[3],3,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[2,0],2,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Reduced = labelExtract(train,labels,[2],2,'tSNE')\nlabeltSNEplot(Reduced)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are more interested things I want to check, but I do not have much time :)\nFor examplem, I want to see average distributions drawn from the actual output(softmax values) and would like to check these outliers equally or unequally distributed and so on."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}