{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n#Here I use random seed bcz I need it in future, when I'll divide train_dataset \n#into validation and train sets. THis will help me to split into same datasets\nnp.random.seed(69)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-08T10:35:34.775864Z","iopub.execute_input":"2023-05-08T10:35:34.776368Z","iopub.status.idle":"2023-05-08T10:35:34.785882Z","shell.execute_reply.started":"2023-05-08T10:35:34.776324Z","shell.execute_reply":"2023-05-08T10:35:34.784677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras import optimizers","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:35:37.314501Z","iopub.execute_input":"2023-05-08T10:35:37.315731Z","iopub.status.idle":"2023-05-08T10:35:37.321144Z","shell.execute_reply.started":"2023-05-08T10:35:37.315682Z","shell.execute_reply":"2023-05-08T10:35:37.319854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_set = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:35:39.104614Z","iopub.execute_input":"2023-05-08T10:35:39.105023Z","iopub.status.idle":"2023-05-08T10:35:43.073429Z","shell.execute_reply.started":"2023-05-08T10:35:39.104987Z","shell.execute_reply":"2023-05-08T10:35:43.072401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set.head() #Here we see the 'label' column","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:34:59.618886Z","iopub.execute_input":"2023-05-08T10:34:59.619324Z","iopub.status.idle":"2023-05-08T10:34:59.655190Z","shell.execute_reply.started":"2023-05-08T10:34:59.619284Z","shell.execute_reply":"2023-05-08T10:34:59.653902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set.head() #Here we need to predict the 'label' column","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:36:02.154198Z","iopub.execute_input":"2023-05-08T10:36:02.154646Z","iopub.status.idle":"2023-05-08T10:36:02.177008Z","shell.execute_reply.started":"2023-05-08T10:36:02.154608Z","shell.execute_reply":"2023-05-08T10:36:02.175357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I'm splitting my train dataset into df_label - which I'll try to predict and df_features - the features on which I'm going to predict","metadata":{}},{"cell_type":"code","source":"df_features = train_set.iloc[:, 1:785]\ndf_label = train_set.iloc[:, 0]\n\n\nX_test = test_set.iloc[:, 0:784]\n\nprint(X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:36:56.279773Z","iopub.execute_input":"2023-05-08T10:36:56.280234Z","iopub.status.idle":"2023-05-08T10:36:56.288787Z","shell.execute_reply.started":"2023-05-08T10:36:56.280196Z","shell.execute_reply":"2023-05-08T10:36:56.287417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I split my datasets into training and validation sets","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_cv, y_train, y_cv = train_test_split(df_features, df_label, \n                                                test_size = 0.2,\n                                                random_state = 69)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:37:45.359736Z","iopub.execute_input":"2023-05-08T10:37:45.360174Z","iopub.status.idle":"2023-05-08T10:37:45.998428Z","shell.execute_reply.started":"2023-05-08T10:37:45.360133Z","shell.execute_reply":"2023-05-08T10:37:45.997180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nAs the pixel intensities are currently between the range of 0 and 255, we have to normalize the features. In addition, we convert our labels from vectors to one-hot encoding binary data\n","metadata":{}},{"cell_type":"code","source":"# Feature Normalization \nX_train = X_train.astype('float32'); X_cv= X_cv.astype('float32'); X_test = X_test.astype('float32')\nX_train =  X_train / 255; X_cv = X_cv / 255; X_test = X_test / 255\n\n# Convert labels to One Hot Encoded\nnum_digits = 10\ny_train = keras.utils.to_categorical(y_train, num_digits)\ny_cv = keras.utils.to_categorical(y_cv, num_digits)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:41:22.216179Z","iopub.execute_input":"2023-05-08T10:41:22.216636Z","iopub.status.idle":"2023-05-08T10:41:22.419554Z","shell.execute_reply.started":"2023-05-08T10:41:22.216599Z","shell.execute_reply":"2023-05-08T10:41:22.418399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"markdown","source":"**MODEL FITTING PART**\nThe first model is a simple NN of Dense layers, consisting of 4 (300, 100, 100, 200) layers. I use the Dense class from the Keras lib to build layers. There are ReLu activation functions in hidden layers, and Softmax in the Output layer. The combination of ReLu in hidden layers and Softmax in the Output layer is common and advised bcz of its simplicity. ","metadata":{}},{"cell_type":"code","source":"\n# Input Parameters\nn_input = 784 # number of features\nn_hidden_1 = 300\nn_hidden_2 = 100\nn_hidden_3 = 100\nn_hidden_4 = 200\nnum_digits = 10","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:49:37.274259Z","iopub.execute_input":"2023-05-08T10:49:37.274688Z","iopub.status.idle":"2023-05-08T10:49:37.281420Z","shell.execute_reply.started":"2023-05-08T10:49:37.274643Z","shell.execute_reply":"2023-05-08T10:49:37.279749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Inp = Input(shape=(784,))\nx = Dense(n_hidden_1, activation='relu', name = \"Hidden_Layer_1\")(Inp)\nx = Dense(n_hidden_2, activation='relu', name = \"Hidden_Layer_2\")(x)\nx = Dense(n_hidden_3, activation='relu', name = \"Hidden_Layer_3\")(x)\nx = Dense(n_hidden_4, activation='relu', name = \"Hidden_Layer_4\")(x)\noutput = Dense(num_digits, activation='softmax', name = \"Output_Layer\")(x)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:49:54.556565Z","iopub.execute_input":"2023-05-08T10:49:54.556974Z","iopub.status.idle":"2023-05-08T10:49:54.785600Z","shell.execute_reply.started":"2023-05-08T10:49:54.556939Z","shell.execute_reply":"2023-05-08T10:49:54.784374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The model has 6 layers: input layer, 4 hidden layers, and 1 output layer\nmodel = Model(Inp, output)\nmodel.summary() #We have 297,910 parameters to estimate","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:50:04.814806Z","iopub.execute_input":"2023-05-08T10:50:04.815260Z","iopub.status.idle":"2023-05-08T10:50:04.851961Z","shell.execute_reply.started":"2023-05-08T10:50:04.815212Z","shell.execute_reply":"2023-05-08T10:50:04.850686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I try with some hyperparameters through compile Keras method","metadata":{}},{"cell_type":"code","source":"learning_rate = 0.1\ntraining_epochs = 22\nbatch_size = 100\nsgd = optimizers.SGD(lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:52:10.709036Z","iopub.execute_input":"2023-05-08T10:52:10.709523Z","iopub.status.idle":"2023-05-08T10:52:10.717582Z","shell.execute_reply.started":"2023-05-08T10:52:10.709482Z","shell.execute_reply":"2023-05-08T10:52:10.716255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I compile with a simple SGD optimizer, use categorical_crossentropy, bcz we're predicting a categorical variable (label).\n\nAlso I use simple accuracy metric to get some evaluation for the model","metadata":{}},{"cell_type":"code","source":"\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='sgd',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:55:57.438891Z","iopub.execute_input":"2023-05-08T10:55:57.440338Z","iopub.status.idle":"2023-05-08T10:55:57.463217Z","shell.execute_reply.started":"2023-05-08T10:55:57.440240Z","shell.execute_reply":"2023-05-08T10:55:57.461766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fitting a model\nmoDel1 = model.fit(X_train, y_train,\n                     batch_size = batch_size,\n                     epochs = training_epochs,\n                     verbose = 1,\n                     validation_data=(X_cv, y_cv))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:56:01.355544Z","iopub.execute_input":"2023-05-08T10:56:01.356736Z","iopub.status.idle":"2023-05-08T10:56:48.047523Z","shell.execute_reply.started":"2023-05-08T10:56:01.356684Z","shell.execute_reply":"2023-05-08T10:56:48.046058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I build another NN, using ADAM as optimizer, with the same learning rate, batch size, and number of epochs.","metadata":{}},{"cell_type":"code","source":"Inp = Input(shape=(784,))\nx = Dense(n_hidden_1, activation='relu', name = \"Hidden_Layer_1\")(Inp)\nx = Dense(n_hidden_2, activation='relu', name = \"Hidden_Layer_2\")(x)\nx = Dense(n_hidden_3, activation='relu', name = \"Hidden_Layer_3\")(x)\nx = Dense(n_hidden_4, activation='relu', name = \"Hidden_Layer_4\")(x)\noutput = Dense(num_digits, activation='softmax', name = \"Output_Layer\")(x)\n\n# We rely on ADAM as our optimizing methodology\nadam = keras.optimizers.Adam(lr=learning_rate)\nmodel2 = Model(Inp, output)\n\nmodel2.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T10:59:38.632632Z","iopub.execute_input":"2023-05-08T10:59:38.633113Z","iopub.status.idle":"2023-05-08T10:59:38.719296Z","shell.execute_reply.started":"2023-05-08T10:59:38.633062Z","shell.execute_reply":"2023-05-08T10:59:38.717973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fitting a model\nmoDel2 = model2.fit(X_train, y_train,\n                      batch_size = batch_size,\n                      epochs = training_epochs,\n                      verbose = 2,\n                      validation_data=(X_cv, y_cv))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:00:31.393969Z","iopub.execute_input":"2023-05-08T11:00:31.394432Z","iopub.status.idle":"2023-05-08T11:01:22.261968Z","shell.execute_reply.started":"2023-05-08T11:00:31.394390Z","shell.execute_reply":"2023-05-08T11:01:22.260694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In addition to ADAM optimizer, now I change learning rate from 0.1 to 0.01","metadata":{}},{"cell_type":"code","source":"Inp = Input(shape=(784,))\nx = Dense(n_hidden_1, activation='relu', name = \"Hidden_Layer_1\")(Inp)\nx = Dense(n_hidden_2, activation='relu', name = \"Hidden_Layer_2\")(x)\nx = Dense(n_hidden_3, activation='relu', name = \"Hidden_Layer_3\")(x)\nx = Dense(n_hidden_4, activation='relu', name = \"Hidden_Layer_4\")(x)\noutput = Dense(num_digits, activation='softmax', name = \"Output_Layer\")(x)\n\nlearning_rate = 0.01\nadam = keras.optimizers.Adam(lr=learning_rate)\nmodel2a = Model(Inp, output)\n\nmodel2a.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:03:39.941361Z","iopub.execute_input":"2023-05-08T11:03:39.941808Z","iopub.status.idle":"2023-05-08T11:03:40.025005Z","shell.execute_reply.started":"2023-05-08T11:03:39.941771Z","shell.execute_reply":"2023-05-08T11:03:40.023770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fitting a model\nmodel2A= model2a.fit(X_train, y_train,\n                        batch_size = batch_size,\n                        epochs = training_epochs,\n                        verbose = 2,\n                        validation_data=(X_cv, y_cv))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:03:53.625414Z","iopub.execute_input":"2023-05-08T11:03:53.625835Z","iopub.status.idle":"2023-05-08T11:04:43.844650Z","shell.execute_reply.started":"2023-05-08T11:03:53.625796Z","shell.execute_reply":"2023-05-08T11:04:43.843626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I change learning rate again, from 0.01 to 0.5\n","metadata":{}},{"cell_type":"code","source":"Inp = Input(shape=(784,))\nx = Dense(n_hidden_1, activation='relu', name = \"Hidden_Layer_1\")(Inp)\nx = Dense(n_hidden_2, activation='relu', name = \"Hidden_Layer_2\")(x)\nx = Dense(n_hidden_3, activation='relu', name = \"Hidden_Layer_3\")(x)\nx = Dense(n_hidden_4, activation='relu', name = \"Hidden_Layer_4\")(x)\noutput = Dense(num_digits, activation='softmax', name = \"Output_Layer\")(x)\n\nlearning_rate = 0.5\nadam = keras.optimizers.Adam(lr=learning_rate)\nmodel2b = Model(Inp, output)\n\nmodel2b.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:06:46.511628Z","iopub.execute_input":"2023-05-08T11:06:46.512413Z","iopub.status.idle":"2023-05-08T11:06:46.592833Z","shell.execute_reply.started":"2023-05-08T11:06:46.512363Z","shell.execute_reply":"2023-05-08T11:06:46.591443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fitting a model\nmoDel2b = model2b.fit(X_train, y_train,\n                        batch_size = batch_size,\n                        epochs = training_epochs,\n                        validation_data=(X_cv, y_cv))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:07:22.367330Z","iopub.execute_input":"2023-05-08T11:07:22.367731Z","iopub.status.idle":"2023-05-08T11:08:15.416140Z","shell.execute_reply.started":"2023-05-08T11:07:22.367697Z","shell.execute_reply":"2023-05-08T11:08:15.414785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What do we get if we add one more hidden layer? Let's do 1 input, 4 + 1 = 5 hidden, 1 output layers","metadata":{}},{"cell_type":"code","source":"# Input Parameters\nn_input = 784 # number of features\nn_hidden_1 = 300\nn_hidden_2 = 100\nn_hidden_3 = 100\nn_hidden_4 = 100\nn_hidden_5 = 200\nnum_digits = 10","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:11:33.682358Z","iopub.execute_input":"2023-05-08T11:11:33.682826Z","iopub.status.idle":"2023-05-08T11:11:33.688876Z","shell.execute_reply.started":"2023-05-08T11:11:33.682783Z","shell.execute_reply":"2023-05-08T11:11:33.687601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Inp = Input(shape=(784,))\nx = Dense(n_hidden_1, activation='relu', name = \"Hidden_Layer_1\")(Inp)\nx = Dense(n_hidden_2, activation='relu', name = \"Hidden_Layer_2\")(x)\nx = Dense(n_hidden_3, activation='relu', name = \"Hidden_Layer_3\")(x)\nx = Dense(n_hidden_4, activation='relu', name = \"Hidden_Layer_4\")(x)\nx = Dense(n_hidden_5, activation='relu', name = \"Hidden_Layer_5\")(x)\noutput = Dense(num_digits, activation='softmax', name = \"Output_Layer\")(x)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:11:53.978398Z","iopub.execute_input":"2023-05-08T11:11:53.978830Z","iopub.status.idle":"2023-05-08T11:11:54.058964Z","shell.execute_reply.started":"2023-05-08T11:11:53.978790Z","shell.execute_reply":"2023-05-08T11:11:54.057516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Our model would have '7' layers - input layer, 5 hidden layer and 1 output layer\nmodel3 = Model(Inp, output)\nmodel3.summary() # We have 308,010 parameters to estimate","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:12:10.960833Z","iopub.execute_input":"2023-05-08T11:12:10.962023Z","iopub.status.idle":"2023-05-08T11:12:11.001649Z","shell.execute_reply.started":"2023-05-08T11:12:10.961974Z","shell.execute_reply":"2023-05-08T11:12:11.000259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we will do it with ADAM optimizer","metadata":{}},{"cell_type":"code","source":"adam = keras.optimizers.Adam(lr=0.1)\n\nmodel3.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:14:08.496993Z","iopub.execute_input":"2023-05-08T11:14:08.497466Z","iopub.status.idle":"2023-05-08T11:14:08.514683Z","shell.execute_reply.started":"2023-05-08T11:14:08.497422Z","shell.execute_reply":"2023-05-08T11:14:08.513237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"moDel3 = model3.fit(X_train, y_train,\n                      batch_size = batch_size,\n                      epochs = training_epochs,\n                      validation_data=(X_cv, y_cv))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:14:22.496306Z","iopub.execute_input":"2023-05-08T11:14:22.496739Z","iopub.status.idle":"2023-05-08T11:15:19.760197Z","shell.execute_reply.started":"2023-05-08T11:14:22.496701Z","shell.execute_reply":"2023-05-08T11:15:19.759042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, having no significant influence of adding an extra layer to the accuracy, we can state that having 4 layers is enough for our model.","metadata":{}},{"cell_type":"code","source":"# Input Parameters\nn_input = 784 # number of features\nn_hidden_1 = 300\nn_hidden_2 = 100\nn_hidden_3 = 100\nn_hidden_4 = 200\nnum_digits = 10","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:20:03.878962Z","iopub.execute_input":"2023-05-08T11:20:03.879506Z","iopub.status.idle":"2023-05-08T11:20:03.886886Z","shell.execute_reply.started":"2023-05-08T11:20:03.879459Z","shell.execute_reply":"2023-05-08T11:20:03.885226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I use the dropout technique to randomly deactivate some neurons of our NN","metadata":{}},{"cell_type":"code","source":"Inp = Input(shape=(784,))\nx = Dense(n_hidden_1, activation='relu', name = \"Hidden_Layer_1\")(Inp)\nx = Dropout(0.3)(x)\nx = Dense(n_hidden_2, activation='relu', name = \"Hidden_Layer_2\")(x)\nx = Dropout(0.3)(x)\nx = Dense(n_hidden_3, activation='relu', name = \"Hidden_Layer_3\")(x)\nx = Dropout(0.3)(x)\nx = Dense(n_hidden_4, activation='relu', name = \"Hidden_Layer_4\")(x)\noutput = Dense(num_digits, activation='softmax', name = \"Output_Layer\")(x)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:21:48.568528Z","iopub.execute_input":"2023-05-08T11:21:48.569019Z","iopub.status.idle":"2023-05-08T11:21:48.649675Z","shell.execute_reply.started":"2023-05-08T11:21:48.568976Z","shell.execute_reply":"2023-05-08T11:21:48.648703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Our model would have '6' layers - input layer, 4 hidden layer and 1 output layer\nmodel4 = Model(Inp, output)\nmodel4.summary() # We have 297,910 parameters to estimate","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:21:58.062069Z","iopub.execute_input":"2023-05-08T11:21:58.062515Z","iopub.status.idle":"2023-05-08T11:21:58.102323Z","shell.execute_reply.started":"2023-05-08T11:21:58.062478Z","shell.execute_reply":"2023-05-08T11:21:58.100826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Having dropout\nmodel4.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:22:25.219458Z","iopub.execute_input":"2023-05-08T11:22:25.219886Z","iopub.status.idle":"2023-05-08T11:22:25.235406Z","shell.execute_reply.started":"2023-05-08T11:22:25.219850Z","shell.execute_reply":"2023-05-08T11:22:25.234068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Having dropout\ntraining_epochs = 22\nmoDel = model4.fit(X_train, y_train,\n                    batch_size = batch_size,\n                    epochs = training_epochs,\n                    validation_data=(X_cv, y_cv))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:22:38.893868Z","iopub.execute_input":"2023-05-08T11:22:38.894318Z","iopub.status.idle":"2023-05-08T11:23:41.367417Z","shell.execute_reply.started":"2023-05-08T11:22:38.894279Z","shell.execute_reply":"2023-05-08T11:23:41.366169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we predict, put results into a DataFrame, reorganize it to the form which is required for sending to the competition.","metadata":{}},{"cell_type":"code","source":"test_pred = pd.DataFrame(model4.predict(X_test, batch_size=200))\ntest_pred = pd.DataFrame(test_pred.idxmax(axis = 1))\ntest_pred.index.name = 'ImageId'\ntest_pred = test_pred.rename(columns = {0: 'Label'}).reset_index()\ntest_pred['ImageId'] = test_pred['ImageId'] + 1\n\ntest_pred.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:28:14.323507Z","iopub.execute_input":"2023-05-08T11:28:14.323919Z","iopub.status.idle":"2023-05-08T11:28:15.312098Z","shell.execute_reply.started":"2023-05-08T11:28:14.323883Z","shell.execute_reply":"2023-05-08T11:28:15.310624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred.to_csv('mnist_test.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T11:28:29.560054Z","iopub.execute_input":"2023-05-08T11:28:29.560566Z","iopub.status.idle":"2023-05-08T11:28:29.602221Z","shell.execute_reply.started":"2023-05-08T11:28:29.560523Z","shell.execute_reply":"2023-05-08T11:28:29.600675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we can get around 97.5% accuracy in Kaggle metrics of the competition. However, if we need to receive more accuracy points in the competition, there are techniques of data augmentation can be used. Also we can play with hyperparameters, suh as learning rate, optimizers, batch size, etc. ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}