{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import BatchNormalization\n\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils.np_utils import to_categorical\n\nfrom tensorflow.keras.layers import Dense, Input, Flatten, Conv2D, MaxPooling2D, Dropout\nfrom tensorflow.keras.models import Model","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:50:54.978472Z","iopub.execute_input":"2023-02-28T03:50:54.979327Z","iopub.status.idle":"2023-02-28T03:51:04.503719Z","shell.execute_reply.started":"2023-02-28T03:50:54.979276Z","shell.execute_reply":"2023-02-28T03:51:04.502577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir('../input/digit-recognizer'))","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:51:50.010976Z","iopub.execute_input":"2023-02-28T03:51:50.011844Z","iopub.status.idle":"2023-02-28T03:51:50.019644Z","shell.execute_reply.started":"2023-02-28T03:51:50.011805Z","shell.execute_reply":"2023-02-28T03:51:50.018474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\nsub = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:51:51.637280Z","iopub.execute_input":"2023-02-28T03:51:51.638020Z","iopub.status.idle":"2023-02-28T03:51:56.900919Z","shell.execute_reply.started":"2023-02-28T03:51:51.637978Z","shell.execute_reply":"2023-02-28T03:51:56.899857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training data size is {train.shape}\\nTesting data size is {test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:02.251007Z","iopub.execute_input":"2023-02-28T03:52:02.251690Z","iopub.status.idle":"2023-02-28T03:52:02.257994Z","shell.execute_reply.started":"2023-02-28T03:52:02.251652Z","shell.execute_reply":"2023-02-28T03:52:02.256423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(['label'], axis = 1).values\ny = train['label'].values","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:04.133200Z","iopub.execute_input":"2023-02-28T03:52:04.133618Z","iopub.status.idle":"2023-02-28T03:52:04.226334Z","shell.execute_reply.started":"2023-02-28T03:52:04.133584Z","shell.execute_reply":"2023-02-28T03:52:04.225041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X / 255.0","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:05.973683Z","iopub.execute_input":"2023-02-28T03:52:05.974798Z","iopub.status.idle":"2023-02-28T03:52:06.081508Z","shell.execute_reply.started":"2023-02-28T03:52:05.974713Z","shell.execute_reply":"2023-02-28T03:52:06.080415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:08.431604Z","iopub.execute_input":"2023-02-28T03:52:08.432669Z","iopub.status.idle":"2023-02-28T03:52:08.438456Z","shell.execute_reply.started":"2023-02-28T03:52:08.432618Z","shell.execute_reply":"2023-02-28T03:52:08.437024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = to_categorical(y)\n\nprint(f\"Label size {y.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:11.196043Z","iopub.execute_input":"2023-02-28T03:52:11.196850Z","iopub.status.idle":"2023-02-28T03:52:11.205958Z","shell.execute_reply.started":"2023-02-28T03:52:11.196811Z","shell.execute_reply":"2023-02-28T03:52:11.204200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, random_state = 0)\n\nX_train = tf.image.resize(X_train, [227, 227]) # if we want to resize \nX_test = tf.image.resize(X_test, [227, 227]) # if we want to resize ","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:12.735091Z","iopub.execute_input":"2023-02-28T03:52:12.736233Z","iopub.status.idle":"2023-02-28T03:52:16.003933Z","shell.execute_reply.started":"2023-02-28T03:52:12.736181Z","shell.execute_reply":"2023-02-28T03:52:16.002850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_test.shape, y_train.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:52:16.008199Z","iopub.execute_input":"2023-02-28T03:52:16.008524Z","iopub.status.idle":"2023-02-28T03:52:16.020387Z","shell.execute_reply.started":"2023-02-28T03:52:16.008492Z","shell.execute_reply":"2023-02-28T03:52:16.019299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train__ = tf.reshape(X_train, [X_train.shape[0], 227, 227])\n\nfig, axis = plt.subplots(1, 4, figsize=(20, 10))\nfor i, ax in enumerate(axis.flat):\n    ax.imshow(X_train__[i], cmap='binary')\n    digit = y_train[i].argmax()\n    ax.set(title = f\"Real Number is {digit}\");","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:53:06.817683Z","iopub.execute_input":"2023-02-28T03:53:06.818063Z","iopub.status.idle":"2023-02-28T03:53:07.506454Z","shell.execute_reply.started":"2023-02-28T03:53:06.818029Z","shell.execute_reply":"2023-02-28T03:53:07.505446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AlexNet(tf.keras.Model):\n    \n    def __init__(self, num_classes = 10):\n        super(AlexNet, self).__init__()\n        self.conv1 = Conv2D(filters = 96, kernel_size=(11,11), strides=(4,4), activation='relu')\n        self.pool1 = MaxPooling2D(pool_size=(3,3), strides=(2,2))\n        self.conv2 = Conv2D(filters = 256, kernel_size=(5,5), strides=(1,1), padding='same', activation='relu')\n        self.pool2 = MaxPooling2D(pool_size=(3,3), strides=(2,2))\n        self.conv3 = Conv2D(filters = 384, kernel_size=(3,3), strides=(1,1), padding='same', activation='relu')\n        self.conv4 = Conv2D(filters = 384, kernel_size=(3,3), strides=(1,1), padding='same', activation='relu')\n        self.conv5 = Conv2D(filters = 256, kernel_size=(3,3), strides=(1,1), padding='same', activation='relu')\n        self.pool3 = MaxPooling2D(pool_size=(3,3), strides=(2,2))\n        self.flatten = Flatten()\n        self.fc1 = Dense(units = 4096, activation='relu')\n        self.dropout1 = Dropout(0.5)\n        self.fc2 = Dense(units = 4096, activation='relu')\n        self.dropout2 = Dropout(0.5)\n        self.fc3 = Dense(units = num_classes, activation='softmax')\n\n    def call(self, input_tensor):\n        x = self.conv1(input_tensor)\n        x = self.pool1(x)\n        x = self.conv2(x)\n        x = self.pool2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n        x = self.conv5(x)\n        x = self.pool3(x)\n        x = self.flatten(x)\n        x = self.fc1(x)\n        x = self.dropout1(x)\n        x = self.fc2(x)\n        x = self.dropout2(x)\n        x = self.fc3(x)\n        \n        return x\n\ninput_layer = Input(shape  = (227, 227, 1))\nx = AlexNet()(input_layer)\nmodel = Model(inputs = input_layer, outputs = x)\n\nprint(model.summary(expand_nested = True))","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:53:22.503519Z","iopub.execute_input":"2023-02-28T03:53:22.504168Z","iopub.status.idle":"2023-02-28T03:53:23.182029Z","shell.execute_reply.started":"2023-02-28T03:53:22.504112Z","shell.execute_reply":"2023-02-28T03:53:23.181101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    loss = tf.keras.losses.CategoricalCrossentropy(),\n    metrics = tf.keras.metrics.CategoricalAccuracy(),\n    optimizer = tf.keras.optimizers.Adam()\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:53:32.197201Z","iopub.execute_input":"2023-02-28T03:53:32.197616Z","iopub.status.idle":"2023-02-28T03:53:32.220691Z","shell.execute_reply.started":"2023-02-28T03:53:32.197581Z","shell.execute_reply":"2023-02-28T03:53:32.219719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n    featurewise_center = False,  # set input mean to 0 over the dataset\n    samplewise_center = False,  # set each sample mean to 0\n    featurewise_std_normalization = False,  # divide inputs by std of the dataset\n    samplewise_std_normalization = False,  # divide each input by its std\n    zca_whitening = False,  # apply ZCA whitening\n    rotation_range = 15,  # randomly rotate images in the range (degrees, 0 to 180)\n    zoom_range = 0.01, # Randomly zoom image \n    width_shift_range = 0.1,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range = 0.1,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip = False,  # randomly flip images\n    vertical_flip = False\n)  # randomly flip images\n\n\n#datagen.fit(X_train)\ntrain_gen = datagen.flow(X_train, y_train, batch_size = 128)\ntest_gen = datagen.flow(X_test, y_test, batch_size = 128)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:53:37.419139Z","iopub.execute_input":"2023-02-28T03:53:37.419535Z","iopub.status.idle":"2023-02-28T03:53:44.754512Z","shell.execute_reply.started":"2023-02-28T03:53:37.419480Z","shell.execute_reply":"2023-02-28T03:53:44.752740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2\nbatch_size = 128\ntrain_steps = X_train.shape[0] // batch_size\nvalid_steps = X_test.shape[0] // batch_size\n\nes = tf.keras.callbacks.EarlyStopping(\n        monitor = \"val_categorical_acc\", # metrics to monitor\n        patience = 2, # how many epochs before stop\n        verbose = 1,\n        mode = \"max\", # we need the maximum accuracy.\n        restore_best_weights = True, # \n     )","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:53:51.244018Z","iopub.execute_input":"2023-02-28T03:53:51.244401Z","iopub.status.idle":"2023-02-28T03:53:51.250569Z","shell.execute_reply.started":"2023-02-28T03:53:51.244368Z","shell.execute_reply":"2023-02-28T03:53:51.249534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_gen, \n                    epochs = epochs, \n                    steps_per_epoch = train_steps,\n                    validation_data = test_gen,\n                    validation_steps = valid_steps, \n                    callbacks = [es])","metadata":{"execution":{"iopub.status.busy":"2023-02-28T03:53:54.143302Z","iopub.execute_input":"2023-02-28T03:53:54.144111Z","iopub.status.idle":"2023-02-28T04:00:03.121035Z","shell.execute_reply.started":"2023-02-28T03:53:54.144070Z","shell.execute_reply":"2023-02-28T04:00:03.119897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(1,1, figsize=(20, 5))\nax.plot(history.history['loss'], color='b', label=\"Training loss\")\nax.plot(history.history['val_loss'], color='r', label=\"validation loss\",axes = ax)\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nlegend = ax.legend(loc='best', shadow=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T04:18:22.761087Z","iopub.execute_input":"2023-02-28T04:18:22.761477Z","iopub.status.idle":"2023-02-28T04:18:23.048645Z","shell.execute_reply.started":"2023-02-28T04:18:22.761444Z","shell.execute_reply":"2023-02-28T04:18:23.047505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(20, 5))\nax.plot(history.history['categorical_accuracy'], color='b', label=\"Training accuracy\")\nax.plot(history.history['val_categorical_accuracy'], color='r',label=\"Validation accuracy\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"categorical_accuracy\")\nlegend = ax.legend(loc = 'best', shadow = True)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T04:18:29.724435Z","iopub.execute_input":"2023-02-28T04:18:29.725017Z","iopub.status.idle":"2023-02-28T04:18:30.095168Z","shell.execute_reply.started":"2023-02-28T04:18:29.724975Z","shell.execute_reply":"2023-02-28T04:18:30.093437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nplt.plot(history.history['val_loss'], color='b', label=\"validation loss\")\nplt.title(\"Test Loss\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T04:04:16.256423Z","iopub.execute_input":"2023-02-28T04:04:16.257046Z","iopub.status.idle":"2023-02-28T04:04:16.479907Z","shell.execute_reply.started":"2023-02-28T04:04:16.257006Z","shell.execute_reply":"2023-02-28T04:04:16.478846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10)) # Set Figure\n\ny_pred = model.predict(X_test) # Predict class probabilities as 2 => [0.1, 0, 0.9, 0, 0, 0, 0, 0, 0, 0]\n\nY_pred = np.argmax(y_pred, 1) # Decode Predicted labels\nY_test = np.argmax(y_test, 1) # Decode labels\n\nmat = confusion_matrix(Y_test, Y_pred) # Confusion matrix\n\n# Plot Confusion matrix\nsns.heatmap(mat.T, square=True, annot=True, cbar=False, cmap=plt.cm.Blues, fmt='.0f')\nplt.xlabel('Predicted Values')\nplt.ylabel('True Values');\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2023-02-28T04:04:04.627419Z","iopub.execute_input":"2023-02-28T04:04:04.628383Z","iopub.status.idle":"2023-02-28T04:04:07.935510Z","shell.execute_reply.started":"2023-02-28T04:04:04.628344Z","shell.execute_reply":"2023-02-28T04:04:07.934452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)\nX_test__ = tf.reshape(X_test, [X_test.shape[0], 227, 227])\n\nfig, axis = plt.subplots(4, 4, figsize=(12, 14))\nfor i, ax in enumerate(axis.flat):\n    ax.imshow(X_test__[i], cmap='binary')\n    ax.set(title = f\"Real Number is {y_test[i].argmax()}\\nPredict Number is {y_pred[i].argmax()}\");","metadata":{"execution":{"iopub.status.busy":"2023-02-28T04:05:29.522629Z","iopub.execute_input":"2023-02-28T04:05:29.523242Z","iopub.status.idle":"2023-02-28T04:05:32.561613Z","shell.execute_reply.started":"2023-02-28T04:05:29.523196Z","shell.execute_reply":"2023-02-28T04:05:32.560412Z"},"trusted":true},"execution_count":null,"outputs":[]}]}