{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf # Import tensorflow library\nfrom tensorflow import keras # Import Keras Library","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv(\"../input/Kannada-MNIST/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test=pd.read_csv(\"../input/Kannada-MNIST/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = train['label']\nX_train = train.drop(columns=['label'])\nX_test = test.drop(columns=['id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"x_train shape:\", X_train.shape, \"y_train shape:\", Y_train.shape) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train=np.array(X_train)\n#Y_train=np.array(Y_train)\n#X_test=np.array(X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Visulaize the input data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.values[100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(X_train.values[100].reshape(28,28), cmap = plt.cm.binary, interpolation = 'nearest') #plt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"(a) Distribution of training data\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"digit_train, counts_train = np.unique(Y_train, return_counts = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.bar(digit_train,counts_train,width =0.6)\nplt.title('Distribution of Y_train')\nplt.xlabel('Digit Number')\nplt.ylabel('Counts')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Equal distribution of training data"},{"metadata":{},"cell_type":"markdown","source":"Feature Scaling or Standardization"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Using Standardization Scaler method\nfrom sklearn.preprocessing import StandardScaler \nscaler=StandardScaler()\n#from sklearn.preprocessing import MinMaxScaler\n#scaler=MinMaxScaler()\nX_train_scaled = pd.DataFrame(scaler.fit_transform(X_train))\n\n#X_train_scaled=scaler.fit_transform(X_train.values)\nX_test_scaled=pd.DataFrame(scaler.transform(X_test.values.reshape(len(X_test),784)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(1,2)\nax[0].imshow(X_train.values[100].reshape(28,28), cmap = plt.cm.binary, interpolation = 'nearest') #plt.axis(\"off\")\nax[0].set_title('Unscaled')\nax[1].imshow(X_train_scaled.values[100].reshape(28,28), cmap = plt.cm.binary, interpolation = 'nearest') #plt.axis(\"off\")\nax[1].set_title('Scaled')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig_object, ax_object = plt.subplots(1, 10, figsize=(12,5))\nax_object = ax_object.reshape(10,)\n    \nfor i in range(len(ax_object)):\n    ax = ax_object[i]\n    idx=np.argwhere(Y_train.values==i)[0]\n    ax.imshow(X_train.values[idx].reshape(28,28), cmap = plt.cm.binary, interpolation = 'nearest')\n    ax.set_xlabel(Y_train.values[i])\n    ax.set_title(i)\n    \nplt.show()\n\nfig_object, ax_object = plt.subplots(1, 10, figsize=(12,5))\nax_object = ax_object.reshape(10,)\n      \nfor i in range(len(ax_object)):\n    ax = ax_object[i]\n    idx=np.argwhere(Y_train==i)[0]\n    ax.imshow(X_train_scaled.values[idx].reshape(28,28), cmap = plt.cm.binary, interpolation = 'nearest')\n    ax.set_xlabel(Y_train[i])\n    plt.xlabel(Y_train[i])\n    ax.set_title(i)       \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_val,Y_train,Y_val = train_test_split(X_train_scaled,Y_train,test_size=0.25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## AutoEncoder"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\" #model will be trained on GPU 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport gzip\n%matplotlib inline\nfrom keras.layers import Input,Conv2D,MaxPooling2D,UpSampling2D\nfrom keras.models import Model\nfrom keras.optimizers import RMSprop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.reshape(-1, 28,28, 1)\nX_test = X_test.reshape(-1, 28,28, 1)\nX_train.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\nepochs = 128\ninChannel = 1\nx, y = 28, 28\ninput_img = Input(shape = (x, y, inChannel))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def autoencoder(input_img):\n    #encoder\n    #input = 28 x 28 x 1 (wide and thin)\n    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img) #28 x 28 x 32\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) #14 x 14 x 32\n    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1) #14 x 14 x 64\n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) #7 x 7 x 64\n    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2) #7 x 7 x 128 (small and thick)\n\n    #decoder\n    conv4 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3) #7 x 7 x 128\n    up1 = UpSampling2D((2,2))(conv4) # 14 x 14 x 128\n    conv5 = Conv2D(64, (3, 3), activation='relu', padding='same')(up1) # 14 x 14 x 64\n    up2 = UpSampling2D((2,2))(conv5) # 28 x 28 x 64\n    decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(up2) # 28 x 28 x 1\n    return decoded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder = Model(input_img, autoencoder(input_img))\nautoencoder.compile(loss='mean_squared_error', optimizer = RMSprop())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder_train = autoencoder.fit(X_train, Y_train, batch_size=batch_size,epochs=epochs,verbose=1,validation_data=(X_val, Y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Building Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential\nfrom keras.layers import Convolution2D\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import Flatten\nfrom keras.layers import Dense\n#Import libraries to build the model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=Sequential()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Convolution2D(64,3,3,input_shape=(28,28,1),activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Convolution2D(128,3,3,input_shape=(28,28,1),activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Flatten())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Dense(output_dim=256,activation='relu'))\nmodel.add(Dense(output_dim=128,activation='relu'))\nmodel.add(Dense(output_dim=10,activation='sigmoid'))\nmodel.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train=X_train.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs=30\n\n# fits the model on batches with real-time data augmentation:\nhistory=model.fit(x=X_train, y=Y_train, epochs=epochs, batch_size=64, validation_data=(X_val.values.reshape(len(X_val),28,28,1), Y_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Evaluate the Accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = model.evaluate(X_val.values.reshape(len(X_val),28,28,1),  Y_val,verbose=2)\nprint('\\nTest accuracy:', test_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(range(epochs),history.history['accuracy'])\nplt.xlabel('No. of Epochs')\nplt.ylabel('Accuracy')\nplt.title('Training Accuracy')\nplt.show()\nplt.plot(range(epochs),history.history['loss'])\nplt.xlabel('No. of Epochs')\nplt.ylabel('Loss Value')\nplt.title('Loss Function')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = model.predict(X_val.values.reshape(len(X_val),28,28,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_pred=[]\nfor i in range(len(predict)):\n    Y_pred.append(np.argmax(predict[i,:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Perfomance Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(Y_val,Y_pred)\nprint(cm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc=0\nfor j in range(len(cm)):\n    acc=acc+cm[j,j]\nprint(acc/15000*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predict = model.predict(X_test_scaled.values.reshape(len(X_test),28,28,1))\nY_predict=np.argmax(test_predict,axis=1)\nsubmission['label']=Y_predict\nsubmission['id']=range(0,len(X_test_scaled))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.inception_v3 import InceptionV3\nfrom keras.preprocessing import image\nfrom keras.models import Model\nfrom keras.layers import Dense, GlobalAveragePooling2D\nfrom keras import backend as K\n\n# create the base pre-trained model\nbase_model = InceptionV3(weights='imagenet', include_top=False)\n\n# add a global spatial average pooling layer\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\n# let's add a fully-connected layer\nx = Dense(1024, activation='relu')(x)\n# and a logistic layer -- let's say we have 200 classes\npredictions = Dense(200, activation='softmax')(x)\n\n# this is the model we will train\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\n# first: train only the top layers (which were randomly initialized)\n# i.e. freeze all convolutional InceptionV3 layers\nfor layer in base_model.layers:\n    layer.trainable = False\n\n# compile the model (should be done *after* setting layers to non-trainable)\nmodel.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n\n# train the model on the new data for a few epochs\nmodel.fit_generator(...)\n\n# at this point, the top layers are well trained and we can start fine-tuning\n# convolutional layers from inception V3. We will freeze the bottom N layers\n# and train the remaining top layers.\n\n# let's visualize layer names and layer indices to see how many layers\n# we should freeze:\nfor i, layer in enumerate(base_model.layers):\n    print(i, layer.name)\n\n# we chose to train the top 2 inception blocks, i.e. we will freeze\n# the first 249 layers and unfreeze the rest:\nfor layer in model.layers[:249]:\n    layer.trainable = False\nfor layer in model.layers[249:]:\n    layer.trainable = True\n\n# we need to recompile the model for these modifications to take effect\n# we use SGD with a low learning rate\nfrom keras.optimizers import SGD\nmodel.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy')\n\n# we train our model again (this time fine-tuning the top 2 inception blocks\n# alongside the top Dense layers\nmodel.fit_generator(...)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shape_x = 28\nshape_y = 28\nnRows,nCols,nDims = X_train_scaled.shape[1:]\ninput_shape = (nRows, nCols, nDims)\nclasses = np.unique(Y_train)\nnClasses = len(classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}