{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import to_categorical\nfrom keras.optimizers import Adam\n\nimport keras_tuner as kt\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-27T16:25:58.822028Z","iopub.execute_input":"2023-07-27T16:25:58.822759Z","iopub.status.idle":"2023-07-27T16:26:10.907250Z","shell.execute_reply.started":"2023-07-27T16:25:58.822723Z","shell.execute_reply":"2023-07-27T16:26:10.906085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Getting Data","metadata":{}},{"cell_type":"code","source":"train_data_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntrain_data_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:10.909507Z","iopub.execute_input":"2023-07-27T16:26:10.910445Z","iopub.status.idle":"2023-07-27T16:26:15.315391Z","shell.execute_reply.started":"2023-07-27T16:26:10.910409Z","shell.execute_reply":"2023-07-27T16:26:15.314215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntest_data_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:15.316719Z","iopub.execute_input":"2023-07-27T16:26:15.317053Z","iopub.status.idle":"2023-07-27T16:26:17.848789Z","shell.execute_reply.started":"2023-07-27T16:26:15.317018Z","shell.execute_reply":"2023-07-27T16:26:17.847343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = test_data_df.to_numpy()\nprint(f'Type of the test_data container: {type(test_data)}')\nprint(f'Shape of the test_data array: {test_data.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:17.850502Z","iopub.execute_input":"2023-07-27T16:26:17.851397Z","iopub.status.idle":"2023-07-27T16:26:17.859049Z","shell.execute_reply.started":"2023-07-27T16:26:17.851354Z","shell.execute_reply":"2023-07-27T16:26:17.857645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Understanding","metadata":{}},{"cell_type":"code","source":"train_data_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:17.863619Z","iopub.execute_input":"2023-07-27T16:26:17.864145Z","iopub.status.idle":"2023-07-27T16:26:17.923606Z","shell.execute_reply.started":"2023-07-27T16:26:17.864101Z","shell.execute_reply":"2023-07-27T16:26:17.922316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract labels","metadata":{}},{"cell_type":"code","source":"train_data_labels = train_data_df.iloc[:,0].to_numpy()\nprint(f'Type of the labels container: {type(train_data_labels)}')\nprint(f'Shape of the labels array: {train_data_labels.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:17.925157Z","iopub.execute_input":"2023-07-27T16:26:17.925649Z","iopub.status.idle":"2023-07-27T16:26:17.933895Z","shell.execute_reply.started":"2023-07-27T16:26:17.925592Z","shell.execute_reply":"2023-07-27T16:26:17.932597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract features","metadata":{}},{"cell_type":"code","source":"train_data = train_data_df.iloc[:, 1:].to_numpy()\nprint(f'Type of the features container: {type(train_data)}')\nprint(f'Shape of the features array: {train_data.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:17.935968Z","iopub.execute_input":"2023-07-27T16:26:17.936354Z","iopub.status.idle":"2023-07-27T16:26:17.947703Z","shell.execute_reply.started":"2023-07-27T16:26:17.936323Z","shell.execute_reply":"2023-07-27T16:26:17.945399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Display image for better intuition about data","metadata":{}},{"cell_type":"code","source":"def plot_digit(image_data):\n    image = image_data.reshape(28, 28)\n    plt.imshow(image, cmap='binary')\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:17.949318Z","iopub.execute_input":"2023-07-27T16:26:17.949804Z","iopub.status.idle":"2023-07-27T16:26:17.962220Z","shell.execute_reply.started":"2023-07-27T16:26:17.949761Z","shell.execute_reply":"2023-07-27T16:26:17.961019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"digit = train_data[10]\nprint(f'Digit - {train_data_labels[10]}')\n\nplot_digit(digit)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:17.963912Z","iopub.execute_input":"2023-07-27T16:26:17.964309Z","iopub.status.idle":"2023-07-27T16:26:18.103688Z","shell.execute_reply.started":"2023-07-27T16:26:17.964277Z","shell.execute_reply":"2023-07-27T16:26:18.101900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Display first 100 digits from the training dataset.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(9,9))\nfor i, image_data in enumerate(test_data[:100]):\n    plt.subplot(10, 10, i+1)\n    plot_digit(image_data)\nplt.subplots_adjust(wspace=0, hspace=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:18.106068Z","iopub.execute_input":"2023-07-27T16:26:18.106528Z","iopub.status.idle":"2023-07-27T16:26:21.884715Z","shell.execute_reply.started":"2023-07-27T16:26:18.106464Z","shell.execute_reply":"2023-07-27T16:26:21.883627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reshape and normalize data\n\nWe need to reshape numpy array of the data to have the next shape - (batch_size, image_rows, image_cols, color_bytes).\n\n**color_bytes** dimension represents RGB color. Black/white in our case.\nSo, for the current dataset it should be (42000, 28, 28, 1)","metadata":{}},{"cell_type":"code","source":"def reshape_and_normalize(data):\n    data = np.reshape(data, (data.shape[0], 28, 28, 1))\n    data = np.divide(data, 255)\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:21.886357Z","iopub.execute_input":"2023-07-27T16:26:21.886761Z","iopub.status.idle":"2023-07-27T16:26:21.894303Z","shell.execute_reply.started":"2023-07-27T16:26:21.886727Z","shell.execute_reply":"2023-07-27T16:26:21.891955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transform test dataset\ntest_data = reshape_and_normalize(test_data)\nprint(f'Shape of the test features: {test_data.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:21.895951Z","iopub.execute_input":"2023-07-27T16:26:21.896347Z","iopub.status.idle":"2023-07-27T16:26:22.002230Z","shell.execute_reply.started":"2023-07-27T16:26:21.896315Z","shell.execute_reply":"2023-07-27T16:26:22.000976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split train dataset into train and validation","metadata":{}},{"cell_type":"code","source":"VALIDATION_SIZE = 0.1","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:22.003839Z","iopub.execute_input":"2023-07-27T16:26:22.006126Z","iopub.status.idle":"2023-07-27T16:26:22.012403Z","shell.execute_reply.started":"2023-07-27T16:26:22.006074Z","shell.execute_reply":"2023-07-27T16:26:22.010470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_data, train_data_labels, test_size=VALIDATION_SIZE, random_state=42)\n\nprint(f'Train data shape: {X_train.shape}')\nprint(f'Validation data shape: {X_val.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:22.017448Z","iopub.execute_input":"2023-07-27T16:26:22.017924Z","iopub.status.idle":"2023-07-27T16:26:22.476296Z","shell.execute_reply.started":"2023-07-27T16:26:22.017891Z","shell.execute_reply":"2023-07-27T16:26:22.475026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Data Generators","metadata":{}},{"cell_type":"code","source":"def train_val_generators(training_images, training_labels, validation_images, validation_labels, classes_num):\n    training_images = np.reshape(training_images, (training_images.shape[0], 28, 28, 1))\n    validation_images = np.reshape(validation_images, (validation_images.shape[0], 28, 28, 1))\n\n    # train data generator\n    train_datagen = ImageDataGenerator(rescale=1/255,\n                                       rotation_range=10,\n                                       zoom_range=0.1,\n                                       width_shift_range=0.1,\n                                       height_shift_range=0.1)\n\n    train_labels_cat = to_categorical(training_labels, classes_num)\n    train_generator = train_datagen.flow(x=training_images,\n                                       y=train_labels_cat,\n                                       batch_size=128)\n    \n    # validation data generator\n    validation_datagen = ImageDataGenerator(rescale=1/255)\n    validation_labels_cat = to_categorical(validation_labels, classes_num)\n    validation_generator = validation_datagen.flow(x=validation_images,\n                                                 y=validation_labels_cat,\n                                                 batch_size=128)\n\n    return train_generator, validation_generator","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:22.477781Z","iopub.execute_input":"2023-07-27T16:26:22.478133Z","iopub.status.idle":"2023-07-27T16:26:22.489434Z","shell.execute_reply.started":"2023-07-27T16:26:22.478096Z","shell.execute_reply":"2023-07-27T16:26:22.487135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#generators\ntrain_generator, validation_generator = train_val_generators(X_train, y_train, X_val, y_val, 10)\n\nprint(f\"Images of training generator have shape: {train_generator.x.shape}\")\nprint(f\"Labels of training generator have shape: {train_generator.y.shape}\")\nprint(f\"Images of validation generator have shape: {validation_generator.x.shape}\")\nprint(f\"Labels of validation generator have shape: {validation_generator.y.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:22.491546Z","iopub.execute_input":"2023-07-27T16:26:22.492088Z","iopub.status.idle":"2023-07-27T16:26:22.576358Z","shell.execute_reply.started":"2023-07-27T16:26:22.492030Z","shell.execute_reply":"2023-07-27T16:26:22.575057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"code","source":"def plot_model_history(history):\n    # Plot the chart for accuracy and loss on both training and validation\n    acc = history.history['accuracy']\n    val_acc = history.history['val_accuracy']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    epochs = range(len(acc))\n\n    plt.plot(epochs, acc, 'r', label='Training accuracy')\n    plt.plot(epochs, val_acc, 'b', label='Validation accuracy')\n    plt.title('Training and validation accuracy')\n    plt.legend()\n    plt.figure()\n\n    plt.plot(epochs, loss, 'r', label='Training Loss')\n    plt.plot(epochs, val_loss, 'b', label='Validation Loss')\n    plt.title('Training and validation loss')\n    plt.legend()\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:22.577915Z","iopub.execute_input":"2023-07-27T16:26:22.578868Z","iopub.status.idle":"2023-07-27T16:26:22.588137Z","shell.execute_reply.started":"2023-07-27T16:26:22.578830Z","shell.execute_reply":"2023-07-27T16:26:22.586659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    model = Sequential()\n    model.add(Conv2D(64, (5,5), input_shape=(28,28,1), activation='relu', padding='same')),\n    model.add(BatchNormalization()),\n    \n    model.add(Conv2D(64, (5,5), activation='relu')),\n    model.add(BatchNormalization()),\n    model.add(MaxPooling2D(2,2)),\n    model.add(Dropout(0.1)),\n    \n    model.add(Conv2D(64, (3,3), input_shape=(28,28,1), activation='relu')),\n    model.add(BatchNormalization()),\n    \n    model.add(Conv2D(64, (3,3), input_shape=(28,28,1), activation='relu')),\n    model.add(BatchNormalization()),\n    model.add(MaxPooling2D(2,2)),\n    model.add(Dropout(0.1)),\n    \n    model.add(Flatten()),\n    model.add(Dense(256, activation=\"relu\")),\n    model.add(Dropout(0.1)),\n    \n    model.add(Dense(10, activation=\"softmax\"))\n    \n    model.compile(optimizer=Adam(learning_rate=1e-3),\n              loss = 'categorical_crossentropy',\n              metrics=['accuracy'])\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:19:56.418154Z","iopub.execute_input":"2023-07-27T19:19:56.418610Z","iopub.status.idle":"2023-07-27T19:19:56.431785Z","shell.execute_reply.started":"2023-07-27T19:19:56.418577Z","shell.execute_reply":"2023-07-27T19:19:56.430407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:35:49.046678Z","iopub.execute_input":"2023-07-27T16:35:49.047116Z","iopub.status.idle":"2023-07-27T16:35:49.291527Z","shell.execute_reply.started":"2023-07-27T16:35:49.047079Z","shell.execute_reply":"2023-07-27T16:35:49.290098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator, validation_data = validation_generator, epochs=30)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:35:52.222539Z","iopub.execute_input":"2023-07-27T16:35:52.223131Z","iopub.status.idle":"2023-07-27T19:14:12.419733Z","shell.execute_reply.started":"2023-07-27T16:35:52.223092Z","shell.execute_reply":"2023-07-27T19:14:12.418628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_history(history)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:16:46.430765Z","iopub.execute_input":"2023-07-27T19:16:46.431211Z","iopub.status.idle":"2023-07-27T19:16:47.049879Z","shell.execute_reply.started":"2023-07-27T19:16:46.431178Z","shell.execute_reply":"2023-07-27T19:16:47.048643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test data predictions","metadata":{}},{"cell_type":"code","source":"test_data[0].shape\nzero_test = test_data[0]\nzero_test =np.expand_dims(zero_test, axis=0)\nzero_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:16:50.402178Z","iopub.execute_input":"2023-07-27T19:16:50.402606Z","iopub.status.idle":"2023-07-27T19:16:50.411691Z","shell.execute_reply.started":"2023-07-27T19:16:50.402572Z","shell.execute_reply":"2023-07-27T19:16:50.410396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prob = model.predict(zero_test)\nnp.argmax(prob)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:16:54.865458Z","iopub.execute_input":"2023-07-27T19:16:54.868148Z","iopub.status.idle":"2023-07-27T19:16:55.158078Z","shell.execute_reply.started":"2023-07-27T19:16:54.868101Z","shell.execute_reply":"2023-07-27T19:16:55.156895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities = model.predict(test_data)\npredictions = np.argmax(probabilities, axis=1)\nids = np.arange(1, len(predictions) + 1)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:16:59.768686Z","iopub.execute_input":"2023-07-27T19:16:59.769094Z","iopub.status.idle":"2023-07-27T19:17:44.720456Z","shell.execute_reply.started":"2023-07-27T19:16:59.769063Z","shell.execute_reply":"2023-07-27T19:17:44.719287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame({'ImageId': ids, 'Label': predictions})\noutput","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:17:44.722361Z","iopub.execute_input":"2023-07-27T19:17:44.723006Z","iopub.status.idle":"2023-07-27T19:17:44.737811Z","shell.execute_reply.started":"2023-07-27T19:17:44.722971Z","shell.execute_reply":"2023-07-27T19:17:44.736556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T16:26:23.211922Z","iopub.status.idle":"2023-07-27T16:26:23.212357Z","shell.execute_reply.started":"2023-07-27T16:26:23.212145Z","shell.execute_reply":"2023-07-27T16:26:23.212164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save the model","metadata":{}},{"cell_type":"code","source":"model.save(\"my_model.keras\")","metadata":{"execution":{"iopub.status.busy":"2023-07-27T19:19:05.124167Z","iopub.execute_input":"2023-07-27T19:19:05.124658Z","iopub.status.idle":"2023-07-27T19:19:05.212287Z","shell.execute_reply.started":"2023-07-27T19:19:05.124623Z","shell.execute_reply":"2023-07-27T19:19:05.210971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}