{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### IMPORTS","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\npd.set_option('display.max_columns', None)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.rcParams['figure.figsize'] = (15, 7)\n\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import cross_val_score, KFold\n\nfrom sklearn.decomposition import PCA\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nimport random\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-08-23T13:45:41.883158Z","iopub.execute_input":"2023-08-23T13:45:41.883531Z","iopub.status.idle":"2023-08-23T13:45:41.891651Z","shell.execute_reply.started":"2023-08-23T13:45:41.883500Z","shell.execute_reply":"2023-08-23T13:45:41.890309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{}},{"cell_type":"code","source":"tr = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nts = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:41.903806Z","iopub.execute_input":"2023-08-23T13:45:41.904582Z","iopub.status.idle":"2023-08-23T13:45:46.775806Z","shell.execute_reply.started":"2023-08-23T13:45:41.904555Z","shell.execute_reply":"2023-08-23T13:45:46.774697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:46.778076Z","iopub.execute_input":"2023-08-23T13:45:46.778423Z","iopub.status.idle":"2023-08-23T13:45:47.045960Z","shell.execute_reply.started":"2023-08-23T13:45:46.778389Z","shell.execute_reply":"2023-08-23T13:45:47.044885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = tr.drop('label', axis = 1)\ny = tr.label","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:47.047378Z","iopub.execute_input":"2023-08-23T13:45:47.047785Z","iopub.status.idle":"2023-08-23T13:45:47.412581Z","shell.execute_reply.started":"2023-08-23T13:45:47.047753Z","shell.execute_reply":"2023-08-23T13:45:47.410251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows = 3, ncols = 3)\n\nfor ax in axes.ravel():\n    idx = random.randint(0, len(X))\n    ax.imshow(X.iloc[idx].to_numpy().reshape(28, 28))\n    label_x = -0.30\n    ax.text(label_x, 0.35, r\"Label:\", color='black', rotation='vertical', transform=ax.transAxes, fontsize = 12)\n    ax.text(label_x, 0.7, y.iloc[idx], color='blue', rotation='vertical', transform=ax.transAxes, fontsize = 12)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:47.415374Z","iopub.execute_input":"2023-08-23T13:45:47.416314Z","iopub.status.idle":"2023-08-23T13:45:48.745708Z","shell.execute_reply.started":"2023-08-23T13:45:47.416277Z","shell.execute_reply":"2023-08-23T13:45:48.744723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(y.value_counts() / len(y) * 100).to_frame().style.background_gradient()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:48.748831Z","iopub.execute_input":"2023-08-23T13:45:48.749395Z","iopub.status.idle":"2023-08-23T13:45:48.823766Z","shell.execute_reply.started":"2023-08-23T13:45:48.749363Z","shell.execute_reply":"2023-08-23T13:45:48.822865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = tr, x = 'label')","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:48.825287Z","iopub.execute_input":"2023-08-23T13:45:48.825607Z","iopub.status.idle":"2023-08-23T13:45:49.126019Z","shell.execute_reply.started":"2023-08-23T13:45:48.825576Z","shell.execute_reply":"2023-08-23T13:45:49.125098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **1ST MILESTONE** : baseline model (96%)","metadata":{}},{"cell_type":"code","source":"X_normalized = X / 255.0\nts_normalized = ts / 255.0","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:49.127197Z","iopub.execute_input":"2023-08-23T13:45:49.128150Z","iopub.status.idle":"2023-08-23T13:45:49.791474Z","shell.execute_reply.started":"2023-08-23T13:45:49.128114Z","shell.execute_reply":"2023-08-23T13:45:49.790436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = KNeighborsClassifier()\n\nkf = KFold(n_splits = 10)\nscores = cross_val_score(clf, X_normalized, y, scoring = 'accuracy', cv = kf)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:45:49.792835Z","iopub.execute_input":"2023-08-23T13:45:49.794209Z","iopub.status.idle":"2023-08-23T13:46:44.765244Z","shell.execute_reply.started":"2023-08-23T13:45:49.794171Z","shell.execute_reply":"2023-08-23T13:46:44.764154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores.mean() * 100","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:46:44.766750Z","iopub.execute_input":"2023-08-23T13:46:44.767203Z","iopub.status.idle":"2023-08-23T13:46:44.774515Z","shell.execute_reply.started":"2023-08-23T13:46:44.767168Z","shell.execute_reply":"2023-08-23T13:46:44.773456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf.fit(X_normalized, y)\npredictions = clf.predict(X_normalized)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:46:44.780096Z","iopub.execute_input":"2023-08-23T13:46:44.780479Z","iopub.status.idle":"2023-08-23T13:47:44.062188Z","shell.execute_reply.started":"2023-08-23T13:46:44.780444Z","shell.execute_reply":"2023-08-23T13:47:44.061087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize Predictions\n\nfig, axes = plt.subplots(nrows = 3, ncols = 3)\n\nfor ax in axes.ravel():\n    idx = random.randint(0, len(X)) # generate random number \n    ax.imshow(X.iloc[idx].to_numpy().reshape(28, 28)) # display choosen image\n    \n    # Change prediction color if the prediction differs from the actual value\n    pred_color = 'green' if predictions[idx] == y.iloc[idx] else 'red'\n    \n    label_x = -0.4\n    ax.text(label_x - 0.2, 0.35, r\"Actual:\", color='black', rotation='vertical', transform=ax.transAxes, fontsize = 10)\n    ax.text(label_x - 0.2, 0.7, y.iloc[idx], color='blue', rotation='vertical', transform=ax.transAxes, fontsize = 10)\n    ax.text(label_x, 0.3, r\"Predicted:\", color='black', rotation='vertical', transform=ax.transAxes, fontsize = 10)\n    ax.text(label_x, 0.85, predictions[idx], color=pred_color, rotation='vertical', transform=ax.transAxes, fontsize = 10)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:47:44.065800Z","iopub.execute_input":"2023-08-23T13:47:44.066113Z","iopub.status.idle":"2023-08-23T13:47:45.275231Z","shell.execute_reply.started":"2023-08-23T13:47:44.066085Z","shell.execute_reply":"2023-08-23T13:47:45.273928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv', usecols = ['ImageId'])\nsubmission['label'] = clf.predict(ts_normalized)\n\nsubmission.to_csv('000_submission.csv', index = False) #LB SCORE: 0.967","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:47:45.276866Z","iopub.execute_input":"2023-08-23T13:47:45.277586Z","iopub.status.idle":"2023-08-23T13:48:22.283624Z","shell.execute_reply.started":"2023-08-23T13:47:45.277539Z","shell.execute_reply":"2023-08-23T13:48:22.282589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **2ND MILESTONE** : dimetionality reduction (97.5%)","metadata":{}},{"cell_type":"code","source":"X_features = X_normalized.copy()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:22.285151Z","iopub.execute_input":"2023-08-23T13:48:22.285492Z","iopub.status.idle":"2023-08-23T13:48:22.441531Z","shell.execute_reply.started":"2023-08-23T13:48:22.285458Z","shell.execute_reply":"2023-08-23T13:48:22.440524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA()\npca.fit_transform(X_features)\npca_variance = pca.explained_variance_","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:22.443206Z","iopub.execute_input":"2023-08-23T13:48:22.443581Z","iopub.status.idle":"2023-08-23T13:48:29.806434Z","shell.execute_reply.started":"2023-08-23T13:48:22.443545Z","shell.execute_reply":"2023-08-23T13:48:29.803403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Note: less than 100 features explain 99% of the variance\nplt.figure(figsize = (15, 7))\nplt.bar(range(len(X_features.columns)), pca_variance, alpha=0.5, align='center', label='individual variance')\nplt.legend()\nplt.ylabel('Variance ratio')\nplt.xlabel('Principal components')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.807779Z","iopub.status.idle":"2023-08-23T13:48:29.808287Z","shell.execute_reply.started":"2023-08-23T13:48:29.808043Z","shell.execute_reply":"2023-08-23T13:48:29.808067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply princple component analysis\npca = PCA(n_components = 40)\nX_transformed = pca.fit_transform(X_features)\nts_transformed = pca.transform(ts_normalized)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:38.489525Z","iopub.execute_input":"2023-08-23T13:48:38.489889Z","iopub.status.idle":"2023-08-23T13:48:41.327198Z","shell.execute_reply.started":"2023-08-23T13:48:38.489857Z","shell.execute_reply":"2023-08-23T13:48:41.325621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = KNeighborsClassifier()\n\nkf = KFold(n_splits = 10)\nscores = cross_val_score(clf, X_transformed, y, scoring = 'accuracy', cv = kf)\nscores.mean() * 100","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:41.329385Z","iopub.execute_input":"2023-08-23T13:48:41.329747Z","iopub.status.idle":"2023-08-23T13:48:51.541858Z","shell.execute_reply.started":"2023-08-23T13:48:41.329712Z","shell.execute_reply":"2023-08-23T13:48:51.540878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf.fit(X_transformed, y)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:51.543371Z","iopub.execute_input":"2023-08-23T13:48:51.543714Z","iopub.status.idle":"2023-08-23T13:48:51.563467Z","shell.execute_reply.started":"2023-08-23T13:48:51.543680Z","shell.execute_reply":"2023-08-23T13:48:51.561928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['label'] = clf.predict(ts_transformed)\n\nsubmission.to_csv('001_submission.csv', index = False) #LB SCORE: 0.97478","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:51.565604Z","iopub.execute_input":"2023-08-23T13:48:51.565975Z","iopub.status.idle":"2023-08-23T13:48:58.183495Z","shell.execute_reply.started":"2023-08-23T13:48:51.565928Z","shell.execute_reply":"2023-08-23T13:48:58.182454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **3RD MILESTONE** : let's dive deep (97%)","metadata":{}},{"cell_type":"code","source":"X_array = np.array(X_normalized)\ny_array = tf.keras.utils.to_categorical(y)\n\nts_array = np.array(ts_normalized)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.821211Z","iopub.status.idle":"2023-08-23T13:48:29.822009Z","shell.execute_reply.started":"2023-08-23T13:48:29.821735Z","shell.execute_reply":"2023-08-23T13:48:29.821760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_model = tf.keras.Sequential([\n    tf.keras.layers.Dense(120, activation = 'relu'),\n    tf.keras.layers.Dense(10, activation = 'softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.823530Z","iopub.status.idle":"2023-08-23T13:48:29.824325Z","shell.execute_reply.started":"2023-08-23T13:48:29.824075Z","shell.execute_reply":"2023-08-23T13:48:29.824100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_model.compile(\n    loss = 'categorical_crossentropy',\n    optimizer = tf.keras.optimizers.Adam(),\n    metrics = ['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.825733Z","iopub.status.idle":"2023-08-23T13:48:29.826533Z","shell.execute_reply.started":"2023-08-23T13:48:29.826285Z","shell.execute_reply":"2023-08-23T13:48:29.826310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"earlystopping_cb = tf.keras.callbacks.EarlyStopping(patience = 3)\n\nhistory = nn_model.fit(\n    X_array, \n    y_array, \n    validation_split = 0.2, \n    epochs = 25, \n    callbacks = [earlystopping_cb]\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.827926Z","iopub.status.idle":"2023-08-23T13:48:29.828716Z","shell.execute_reply.started":"2023-08-23T13:48:29.828471Z","shell.execute_reply":"2023-08-23T13:48:29.828496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Neural Network model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.830185Z","iopub.status.idle":"2023-08-23T13:48:29.831129Z","shell.execute_reply.started":"2023-08-23T13:48:29.830805Z","shell.execute_reply":"2023-08-23T13:48:29.830833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Neural Network model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.832573Z","iopub.status.idle":"2023-08-23T13:48:29.833381Z","shell.execute_reply.started":"2023-08-23T13:48:29.833133Z","shell.execute_reply":"2023-08-23T13:48:29.833157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.argmax(nn_model.predict(ts_array), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.834803Z","iopub.status.idle":"2023-08-23T13:48:29.835627Z","shell.execute_reply.started":"2023-08-23T13:48:29.835373Z","shell.execute_reply":"2023-08-23T13:48:29.835398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['label'] = predictions\n\nsubmission.to_csv('002_submission.csv', index = False) #LB SCORE: 0.97271","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.837056Z","iopub.status.idle":"2023-08-23T13:48:29.837844Z","shell.execute_reply.started":"2023-08-23T13:48:29.837589Z","shell.execute_reply":"2023-08-23T13:48:29.837613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **4TH MILESTONE** : CNNs (98%)","metadata":{}},{"cell_type":"code","source":"X_reshaped = X_normalized.to_numpy().reshape(-1, 28, 28, 1)\nts_reshaped = ts_normalized.to_numpy().reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.839506Z","iopub.status.idle":"2023-08-23T13:48:29.840310Z","shell.execute_reply.started":"2023-08-23T13:48:29.840061Z","shell.execute_reply":"2023-08-23T13:48:29.840085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model = tf.keras.Sequential([\n    \n    tf.keras.layers.Conv2D(32, kernel_size = (5, 5), activation = 'relu', input_shape = (28, 28, 1)),\n    tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides = (2, 2)),\n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(120, activation = 'relu'),\n    tf.keras.layers.Dense(10, activation = 'softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.841729Z","iopub.status.idle":"2023-08-23T13:48:29.842537Z","shell.execute_reply.started":"2023-08-23T13:48:29.842288Z","shell.execute_reply":"2023-08-23T13:48:29.842312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model.compile(\n    loss = 'categorical_crossentropy',\n    optimizer = tf.keras.optimizers.Adam(), \n    metrics = ['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.843973Z","iopub.status.idle":"2023-08-23T13:48:29.844762Z","shell.execute_reply.started":"2023-08-23T13:48:29.844514Z","shell.execute_reply":"2023-08-23T13:48:29.844540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = cnn_model.fit(\n    X_reshaped,\n    y_array, \n    validation_split = 0.2,\n    epochs = 25, \n    callbacks = [earlystopping_cb]\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.846195Z","iopub.status.idle":"2023-08-23T13:48:29.846993Z","shell.execute_reply.started":"2023-08-23T13:48:29.846712Z","shell.execute_reply":"2023-08-23T13:48:29.846737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['label'] = np.argmax(cnn_model.predict(ts_reshaped), axis = 1)\n\nsubmission.to_csv('003_submission.csv', index = False) #LB SCORE 0.98928","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.848404Z","iopub.status.idle":"2023-08-23T13:48:29.849205Z","shell.execute_reply.started":"2023-08-23T13:48:29.848957Z","shell.execute_reply":"2023-08-23T13:48:29.848983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <center> <span style=\"color:yellow\">Upvote if you found this notebook helpful😄</center>","metadata":{}},{"cell_type":"markdown","source":"## **FINAL DISTINATION** : GOING EVEN DEEPER + DATA AUGMENTAION (99%)","metadata":{}},{"cell_type":"code","source":"final_model = tf.keras.Sequential([\n    \n    tf.keras.layers.Conv2D(64, kernel_size = (5, 5), activation = 'relu', padding = 'same', kernel_initializer='he_normal', input_shape = (28, 28, 1)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(64, kernel_size = (5, 5), activation = 'relu', padding = 'same', kernel_initializer='he_normal'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same', kernel_initializer='he_normal'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same', kernel_initializer='he_normal'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n    tf.keras.layers.Dropout(0.25),\n    \n    \n    tf.keras.layers.Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same', kernel_initializer='he_normal'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dropout(0.25),\n    \n    \n    tf.keras.layers.Flatten(),\n    \n    tf.keras.layers.Dense(256, activation = 'relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Dense(10, activation = 'softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.850757Z","iopub.status.idle":"2023-08-23T13:48:29.851646Z","shell.execute_reply.started":"2023-08-23T13:48:29.851403Z","shell.execute_reply":"2023-08-23T13:48:29.851428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model.compile(\n    loss = 'categorical_crossentropy',\n    optimizer = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08),\n    metrics = ['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.853112Z","iopub.status.idle":"2023-08-23T13:48:29.853895Z","shell.execute_reply.started":"2023-08-23T13:48:29.853647Z","shell.execute_reply":"2023-08-23T13:48:29.853671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        featurewise_center=False,  \n        samplewise_center=False,  \n        featurewise_std_normalization=False,  \n        samplewise_std_normalization=False, \n        zca_whitening=False,  \n        rotation_range=10,  \n        zoom_range = 0.1, \n        width_shift_range=0.1,  \n        height_shift_range=0.1,  \n        horizontal_flip=False,  \n        vertical_flip=False,\n        validation_split = 0.2) ","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.855330Z","iopub.status.idle":"2023-08-23T13:48:29.856131Z","shell.execute_reply.started":"2023-08-23T13:48:29.855856Z","shell.execute_reply":"2023-08-23T13:48:29.855880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen.fit(X_reshaped)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.857579Z","iopub.status.idle":"2023-08-23T13:48:29.858481Z","shell.execute_reply.started":"2023-08-23T13:48:29.858220Z","shell.execute_reply":"2023-08-23T13:48:29.858245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.859967Z","iopub.status.idle":"2023-08-23T13:48:29.860749Z","shell.execute_reply.started":"2023-08-23T13:48:29.860499Z","shell.execute_reply":"2023-08-23T13:48:29.860524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model.fit(\n    datagen.flow(X_reshaped, y_array, subset='training'),\n    validation_data = datagen.flow(X_reshaped, y_array, batch_size=8, subset='validation'),\n    epochs=25,\n    callbacks = [learning_rate_reduction]\n         )","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.862174Z","iopub.status.idle":"2023-08-23T13:48:29.862972Z","shell.execute_reply.started":"2023-08-23T13:48:29.862691Z","shell.execute_reply":"2023-08-23T13:48:29.862716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['label'] = np.argmax(final_model.predict(ts_reshaped), axis = 1)\n \nsubmission.to_csv('submission.csv', index = False) #LB SCORE 0.99389","metadata":{"execution":{"iopub.status.busy":"2023-08-23T13:48:29.864386Z","iopub.status.idle":"2023-08-23T13:48:29.865194Z","shell.execute_reply.started":"2023-08-23T13:48:29.864930Z","shell.execute_reply":"2023-08-23T13:48:29.864971Z"},"trusted":true},"execution_count":null,"outputs":[]}]}