{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### In this is notebook i used simple NeuralNetwork provided by Keras and some cheat for getting more data without further data processing ","metadata":{}},{"cell_type":"markdown","source":"## Read and normalize data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.utils import to_categorical \nimport time","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:19.989729Z","iopub.execute_input":"2022-02-02T14:14:19.99008Z","iopub.status.idle":"2022-02-02T14:14:19.995436Z","shell.execute_reply.started":"2022-02-02T14:14:19.990033Z","shell.execute_reply":"2022-02-02T14:14:19.994733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:19.997061Z","iopub.execute_input":"2022-02-02T14:14:19.997742Z","iopub.status.idle":"2022-02-02T14:14:23.612195Z","shell.execute_reply.started":"2022-02-02T14:14:19.997704Z","shell.execute_reply":"2022-02-02T14:14:23.610966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train shape:', train.shape)\nprint('Test shape:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:23.613676Z","iopub.execute_input":"2022-02-02T14:14:23.613926Z","iopub.status.idle":"2022-02-02T14:14:23.619693Z","shell.execute_reply.started":"2022-02-02T14:14:23.613896Z","shell.execute_reply":"2022-02-02T14:14:23.618874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train['label']\ny = to_categorical(y, num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:23.620729Z","iopub.execute_input":"2022-02-02T14:14:23.620924Z","iopub.status.idle":"2022-02-02T14:14:23.634375Z","shell.execute_reply.started":"2022-02-02T14:14:23.6209Z","shell.execute_reply":"2022-02-02T14:14:23.63348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.iloc[:,1:].values.reshape(-1,28,28,1)\ntest = test.iloc[:,:].values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:23.636268Z","iopub.execute_input":"2022-02-02T14:14:23.636518Z","iopub.status.idle":"2022-02-02T14:14:23.648555Z","shell.execute_reply.started":"2022-02-02T14:14:23.636487Z","shell.execute_reply":"2022-02-02T14:14:23.64762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(9):\n    plt.subplot(330 + 1 + i)\n    plt.imshow(train[i], cmap=plt.get_cmap('gray'))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:23.650663Z","iopub.execute_input":"2022-02-02T14:14:23.650923Z","iopub.status.idle":"2022-02-02T14:14:24.619852Z","shell.execute_reply.started":"2022-02-02T14:14:23.650894Z","shell.execute_reply":"2022-02-02T14:14:24.618693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.reshape((train.shape[0], 28*28)).astype('float64') / 255\ntest = test.reshape((test.shape[0], 28*28)).astype('float64') / 255","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:24.621517Z","iopub.execute_input":"2022-02-02T14:14:24.621758Z","iopub.status.idle":"2022-02-02T14:14:24.865332Z","shell.execute_reply.started":"2022-02-02T14:14:24.62173Z","shell.execute_reply":"2022-02-02T14:14:24.864313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train shape:', train.shape)\nprint('Test shape:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:24.866617Z","iopub.execute_input":"2022-02-02T14:14:24.866851Z","iopub.status.idle":"2022-02-02T14:14:24.87342Z","shell.execute_reply.started":"2022-02-02T14:14:24.866823Z","shell.execute_reply":"2022-02-02T14:14:24.872468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split data","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_val , y_train, y_val = train_test_split(train, y, test_size=0.1, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:24.87463Z","iopub.execute_input":"2022-02-02T14:14:24.874877Z","iopub.status.idle":"2022-02-02T14:14:25.31371Z","shell.execute_reply.started":"2022-02-02T14:14:24.874849Z","shell.execute_reply":"2022-02-02T14:14:25.313028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building NN architecture","metadata":{}},{"cell_type":"markdown","source":"#### Three layer NN with Dropout for regularization and Batch Normalization for optimizing algorithm. ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import Sequential\nfrom keras.layers import Dense, BatchNormalization, Dropout\nimport tensorflow as tf\nmodel = Sequential()\nmodel.add(Dense(256, input_dim=784, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\nmodel.add(Dense(64, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\nmodel.add(Dense(10, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:25.314685Z","iopub.execute_input":"2022-02-02T14:14:25.315479Z","iopub.status.idle":"2022-02-02T14:14:25.394303Z","shell.execute_reply.started":"2022-02-02T14:14:25.315444Z","shell.execute_reply":"2022-02-02T14:14:25.392405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"markdown","source":"### For training i chose Adam with lr=0.001, because it gave best results with my NN architecture.","metadata":{}},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam(0.001)\nmodel.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\nprint(optimizer.learning_rate)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:25.395553Z","iopub.execute_input":"2022-02-02T14:14:25.395761Z","iopub.status.idle":"2022-02-02T14:14:25.410619Z","shell.execute_reply.started":"2022-02-02T14:14:25.395736Z","shell.execute_reply":"2022-02-02T14:14:25.409666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\nhistory = model.fit(x_train, y_train, epochs=45, batch_size=64, validation_data = (x_val,y_val))\nend = time.time()\nprint(\"Algorithm time is {} s:\".format(round(end-start)))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:14:25.412071Z","iopub.execute_input":"2022-02-02T14:14:25.412729Z","iopub.status.idle":"2022-02-02T14:16:15.46491Z","shell.execute_reply.started":"2022-02-02T14:14:25.412683Z","shell.execute_reply":"2022-02-02T14:16:15.463852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.rcParams[\"figure.figsize\"] = (10,10)\nfig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:15.46622Z","iopub.execute_input":"2022-02-02T14:16:15.466467Z","iopub.status.idle":"2022-02-02T14:16:15.872292Z","shell.execute_reply.started":"2022-02-02T14:16:15.466435Z","shell.execute_reply":"2022-02-02T14:16:15.871611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test)\npredictions = np.argmax(predictions , axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:15.874625Z","iopub.execute_input":"2022-02-02T14:16:15.87554Z","iopub.status.idle":"2022-02-02T14:16:17.265782Z","shell.execute_reply.started":"2022-02-02T14:16:15.875487Z","shell.execute_reply":"2022-02-02T14:16:17.26477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### If we upload this predictions we will get something like ~0.979 accuracy and as we see in our plots, our model is quiet overfitted. Let's fix it and give our model more data, cheat data :)","metadata":{}},{"cell_type":"markdown","source":"## Load more data","metadata":{}},{"cell_type":"code","source":"from keras.datasets import mnist\nfrom keras.utils.np_utils import to_categorical\n(x_train_1, y_train_1), (x_val_1, y_val_1) = mnist.load_data()\nprint('Train: X=%s, y=%s' % (x_train_1.shape, y_train_1.shape))\nprint('Test: X=%s, y=%s' % (x_val_1.shape, y_val_1.shape))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:17.267185Z","iopub.execute_input":"2022-02-02T14:16:17.267423Z","iopub.status.idle":"2022-02-02T14:16:17.595856Z","shell.execute_reply.started":"2022-02-02T14:16:17.267395Z","shell.execute_reply":"2022-02-02T14:16:17.594759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_1 = x_train_1.reshape((x_train_1.shape[0], 28*28)).astype('float64') / 255\nx_val_1 = x_val_1.reshape((x_val_1.shape[0], 28*28)).astype('float64') / 255\ny_train_1 = to_categorical(y_train_1)\ny_val_1 = to_categorical(y_val_1)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:17.597174Z","iopub.execute_input":"2022-02-02T14:16:17.597437Z","iopub.status.idle":"2022-02-02T14:16:17.755721Z","shell.execute_reply.started":"2022-02-02T14:16:17.597405Z","shell.execute_reply":"2022-02-02T14:16:17.754802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#x_train, x_val_1 , y_train, y_val_1 = train_test_split(train, y, test_size=0.8, random_state=42)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_x_train = np.concatenate((x_train_1, x_val_1), axis=0)\nnew_y_train = np.concatenate((y_train_1, y_val_1), axis=0)\n#new_x_train = np.concatenate((new_x_train, x_train), axis=0)\n#new_y_train = np.concatenate((new_y_train, y_train), axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:17.757091Z","iopub.execute_input":"2022-02-02T14:16:17.757495Z","iopub.status.idle":"2022-02-02T14:16:17.95922Z","shell.execute_reply.started":"2022-02-02T14:16:17.757449Z","shell.execute_reply":"2022-02-02T14:16:17.95826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('new_x_train shape:', new_x_train.shape)\nprint('new_y_train shape:', new_y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:17.960635Z","iopub.execute_input":"2022-02-02T14:16:17.961345Z","iopub.status.idle":"2022-02-02T14:16:17.967253Z","shell.execute_reply.started":"2022-02-02T14:16:17.961306Z","shell.execute_reply":"2022-02-02T14:16:17.966327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now for splitting data we will give our mnist dataset from keras to train and old data from this competition to validate.","metadata":{}},{"cell_type":"code","source":"#x_val, x_test , y_val, y_test = train_test_split(train, y, test_size=0.15, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:17.968711Z","iopub.execute_input":"2022-02-02T14:16:17.968945Z","iopub.status.idle":"2022-02-02T14:16:18.380991Z","shell.execute_reply.started":"2022-02-02T14:16:17.968912Z","shell.execute_reply":"2022-02-02T14:16:18.380056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reduce lr\noptimizer = tf.keras.optimizers.Adam(0.0005)\nmodel.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\nprint(optimizer.learning_rate)\nstart = time.time()\nhistory = model.fit(new_x_train, new_y_train, epochs=60, batch_size=64, validation_data = (train,y))\nend = time.time()\nprint(\"Algorithm time is {} s:\".format(round(end-start)))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:16:18.382223Z","iopub.execute_input":"2022-02-02T14:16:18.382546Z","iopub.status.idle":"2022-02-02T14:20:41.723533Z","shell.execute_reply.started":"2022-02-02T14:16:18.382505Z","shell.execute_reply":"2022-02-02T14:20:41.722341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As wee see our accuracy and loss are very good, the same thing with validation data","metadata":{}},{"cell_type":"code","source":"#model.evaluate(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:20:41.724869Z","iopub.execute_input":"2022-02-02T14:20:41.725484Z","iopub.status.idle":"2022-02-02T14:20:42.239075Z","shell.execute_reply.started":"2022-02-02T14:20:41.725444Z","shell.execute_reply":"2022-02-02T14:20:42.238099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#result = model.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:20:42.240396Z","iopub.execute_input":"2022-02-02T14:20:42.240668Z","iopub.status.idle":"2022-02-02T14:20:42.679156Z","shell.execute_reply.started":"2022-02-02T14:20:42.240636Z","shell.execute_reply":"2022-02-02T14:20:42.678234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#result = np.argmax(result , axis=1)\n#y_test = np.argmax(y_test , axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:20:42.680513Z","iopub.execute_input":"2022-02-02T14:20:42.680823Z","iopub.status.idle":"2022-02-02T14:20:42.686323Z","shell.execute_reply.started":"2022-02-02T14:20:42.680781Z","shell.execute_reply":"2022-02-02T14:20:42.685507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Errors in images","metadata":{}},{"cell_type":"code","source":"\n# for result, y_test in zip(result, y_test):\n#     if result != y_test:\n#         print(y_test, 'has been classified as ', result)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:29:11.813535Z","iopub.execute_input":"2022-02-02T14:29:11.814299Z","iopub.status.idle":"2022-02-02T14:29:11.820677Z","shell.execute_reply.started":"2022-02-02T14:29:11.814237Z","shell.execute_reply":"2022-02-02T14:29:11.819761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test)\npredictions = np.argmax(predictions , axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:20:42.687689Z","iopub.execute_input":"2022-02-02T14:20:42.687956Z","iopub.status.idle":"2022-02-02T14:20:44.626689Z","shell.execute_reply.started":"2022-02-02T14:20:42.687922Z","shell.execute_reply":"2022-02-02T14:20:44.625725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmission['Label']=predictions\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:20:44.628418Z","iopub.execute_input":"2022-02-02T14:20:44.628745Z","iopub.status.idle":"2022-02-02T14:20:44.676359Z","shell.execute_reply.started":"2022-02-02T14:20:44.628699Z","shell.execute_reply":"2022-02-02T14:20:44.6753Z"},"trusted":true},"execution_count":null,"outputs":[]}]}