{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://i.ibb.co/3vF9yD8/Screenshot-from-2019-05-29-21-23-47.png)\n\n**MNIST (\"Modified National Institute of Standards and Technology\")** is the de facto “hello world” dataset of computer vision. Since its release in 1999, this classic dataset of handwritten images has served as the basis for benchmarking classification algorithms. As new machine learning techniques emerge, MNIST remains a reliable resource for researchers and learners alike.\n\n\nUsing CNN Keras and Tensorflow (v1) DNN Baseline with GPU. You can try to add more layers, you can try to change learning rate. Thank you for reading\n","metadata":{"id":"NuVpktiq32xp"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n\nfrom tensorflow.keras.optimizers import Nadam\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Flatten, Dense, AvgPool2D, BatchNormalization, Dropout\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom sklearn.model_selection import train_test_split\nfrom datetime import datetime\n\nimport tensorflow.compat.v1 as tf\ntf.disable_v2_behavior()\n\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\nsns.set_style ('darkgrid')\n%matplotlib inline\nnp.random.seed(42)","metadata":{"id":"2s6jw5FM4rHC","execution":{"iopub.status.busy":"2021-09-05T14:50:23.479572Z","iopub.execute_input":"2021-09-05T14:50:23.479874Z","iopub.status.idle":"2021-09-05T14:50:28.438153Z","shell.execute_reply.started":"2021-09-05T14:50:23.479803Z","shell.execute_reply":"2021-09-05T14:50:28.437336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","metadata":{"execution":{"iopub.status.busy":"2021-09-05T14:50:28.439459Z","iopub.execute_input":"2021-09-05T14:50:28.439795Z","iopub.status.idle":"2021-09-05T14:50:30.124338Z","shell.execute_reply.started":"2021-09-05T14:50:28.439762Z","shell.execute_reply":"2021-09-05T14:50:30.123368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')","metadata":{"id":"kCHi32Z25Frn","execution":{"iopub.status.busy":"2021-09-05T14:50:30.127176Z","iopub.execute_input":"2021-09-05T14:50:30.127647Z","iopub.status.idle":"2021-09-05T14:50:33.169139Z","shell.execute_reply.started":"2021-09-05T14:50:30.127607Z","shell.execute_reply":"2021-09-05T14:50:33.168296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"id":"dt86FzML5JyH","outputId":"b44a5799-a593-4a21-b756-5447d1ba9a93","execution":{"iopub.status.busy":"2021-09-05T14:50:33.170582Z","iopub.execute_input":"2021-09-05T14:50:33.170908Z","iopub.status.idle":"2021-09-05T14:50:33.200814Z","shell.execute_reply.started":"2021-09-05T14:50:33.170872Z","shell.execute_reply":"2021-09-05T14:50:33.199908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"id":"fTIJF85PGDjX","execution":{"iopub.status.busy":"2021-09-05T14:50:33.202148Z","iopub.execute_input":"2021-09-05T14:50:33.202490Z","iopub.status.idle":"2021-09-05T14:50:35.001958Z","shell.execute_reply.started":"2021-09-05T14:50:33.202456Z","shell.execute_reply":"2021-09-05T14:50:35.001063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"id":"k5VKfIQLGQPp","outputId":"cb74998b-ac55-49b3-e8a4-de6bf51e2194","execution":{"iopub.status.busy":"2021-09-05T14:50:35.005144Z","iopub.execute_input":"2021-09-05T14:50:35.005411Z","iopub.status.idle":"2021-09-05T14:50:35.022348Z","shell.execute_reply.started":"2021-09-05T14:50:35.005384Z","shell.execute_reply":"2021-09-05T14:50:35.020870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train.drop('label', axis=1)","metadata":{"id":"KHSbuQM65ebj","execution":{"iopub.status.busy":"2021-09-05T14:50:35.023583Z","iopub.execute_input":"2021-09-05T14:50:35.024116Z","iopub.status.idle":"2021-09-05T14:50:35.109478Z","shell.execute_reply.started":"2021-09-05T14:50:35.024056Z","shell.execute_reply":"2021-09-05T14:50:35.108631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train['label']","metadata":{"id":"EeFl2sDq5wID","execution":{"iopub.status.busy":"2021-09-05T14:50:35.112164Z","iopub.execute_input":"2021-09-05T14:50:35.112540Z","iopub.status.idle":"2021-09-05T14:50:35.116549Z","shell.execute_reply.started":"2021-09-05T14:50:35.112502Z","shell.execute_reply":"2021-09-05T14:50:35.115544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_ = np.array(features)","metadata":{"id":"eSGtCwA55z97","execution":{"iopub.status.busy":"2021-09-05T14:50:35.118681Z","iopub.execute_input":"2021-09-05T14:50:35.119045Z","iopub.status.idle":"2021-09-05T14:50:35.220441Z","shell.execute_reply.started":"2021-09-05T14:50:35.119009Z","shell.execute_reply":"2021-09-05T14:50:35.219632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = np.array(test)","metadata":{"id":"3zMbsYFIGdol","execution":{"iopub.status.busy":"2021-09-05T14:50:35.221705Z","iopub.execute_input":"2021-09-05T14:50:35.222058Z","iopub.status.idle":"2021-09-05T14:50:35.289471Z","shell.execute_reply.started":"2021-09-05T14:50:35.222023Z","shell.execute_reply":"2021-09-05T14:50:35.288672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_.reshape(X_.shape[0], 28, 28)","metadata":{"id":"wwcFQQWV55ia","execution":{"iopub.status.busy":"2021-09-05T14:50:35.290726Z","iopub.execute_input":"2021-09-05T14:50:35.291083Z","iopub.status.idle":"2021-09-05T14:50:35.295175Z","shell.execute_reply.started":"2021-09-05T14:50:35.291046Z","shell.execute_reply":"2021-09-05T14:50:35.294375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"id":"AxnbTCvJBb6X","outputId":"a25eafd6-01df-49fc-cfcf-73b2ea9823f9","execution":{"iopub.status.busy":"2021-09-05T14:50:35.296408Z","iopub.execute_input":"2021-09-05T14:50:35.296941Z","iopub.status.idle":"2021-09-05T14:50:35.307369Z","shell.execute_reply.started":"2021-09-05T14:50:35.296904Z","shell.execute_reply":"2021-09-05T14:50:35.306561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us take a look to out objects","metadata":{"id":"iFMndL56QNxD"}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,5))\n\nfor i in range(16):\n    fig.add_subplot(4, 4, i+1)\n    \n    plt.imshow(X_train[i], cmap='gray')\n    \n    plt.xticks([])\n    plt.yticks([])\n    plt.tight_layout()\n    plt.title('Digit: ' + str(target[i]))\n","metadata":{"id":"COj0MVAU6Fct","outputId":"0bb69e92-dc5b-4ad6-b1eb-c8edd0833279","execution":{"iopub.status.busy":"2021-09-05T14:50:35.308773Z","iopub.execute_input":"2021-09-05T14:50:35.309152Z","iopub.status.idle":"2021-09-05T14:50:36.708134Z","shell.execute_reply.started":"2021-09-05T14:50:35.309117Z","shell.execute_reply":"2021-09-05T14:50:36.707324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we have to check the count of values for our output layer","metadata":{"id":"n_YR_CdoQTHH"}},{"cell_type":"code","source":"target.value_counts(normalize=True)","metadata":{"id":"z29i5vCGZ_Wy","outputId":"236c6a87-0f66-4b40-c736-6df72a2bea7f","execution":{"iopub.status.busy":"2021-09-05T14:50:36.709547Z","iopub.execute_input":"2021-09-05T14:50:36.710047Z","iopub.status.idle":"2021-09-05T14:50:36.731843Z","shell.execute_reply.started":"2021-09-05T14:50:36.710009Z","shell.execute_reply":"2021-09-05T14:50:36.731117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(target.value_counts())","metadata":{"id":"ixPBsDyDAPhT","outputId":"e5d829fc-f65e-42d4-e2be-c6192585dcc3","execution":{"iopub.status.busy":"2021-09-05T14:50:36.733052Z","iopub.execute_input":"2021-09-05T14:50:36.733372Z","iopub.status.idle":"2021-09-05T14:50:36.740805Z","shell.execute_reply.started":"2021-09-05T14:50:36.733338Z","shell.execute_reply":"2021-09-05T14:50:36.739771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN\n","metadata":{"id":"L35xcC1Spgx4"}},{"cell_type":"markdown","source":"We have to reshape our data","metadata":{"id":"W128l9NXQdvN"}},{"cell_type":"code","source":"X_train = X_.reshape(X_.shape[0], 28, 28, 1)\nX_test_reshape = X_test.reshape(X_test.shape[0], 28, 28, 1)\nX_tr, X_val, y_tr, y_val = train_test_split(X_train, target, test_size=0.25, \n                                            random_state=42)","metadata":{"id":"pCTOQgjPHe2T","execution":{"iopub.status.busy":"2021-09-05T14:50:36.742461Z","iopub.execute_input":"2021-09-05T14:50:36.742884Z","iopub.status.idle":"2021-09-05T14:50:37.115914Z","shell.execute_reply.started":"2021-09-05T14:50:36.742849Z","shell.execute_reply":"2021-09-05T14:50:37.115063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us build our CNN. We do not need millions of params","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n\n\nmodel.add(Conv2D(filters = 64, \n                 kernel_size = (5,5), \n                 padding = 'same', \n                 activation ='elu', \n                 input_shape = (28,28,1)))\n\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\n\nmodel.add(Conv2D(filters = 64, \n                 kernel_size = (5,5), \n                 padding = 'same', \n                 activation ='elu', \n                 input_shape = (28,28,1)))\n\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\n\nmodel.add(AvgPool2D(pool_size=(2,2)))\n\nmodel.add(Conv2D(filters = 64, \n                 kernel_size = (5,5), \n                 padding = 'valid', \n                 activation ='elu'))\n\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\n\nmodel.add(Conv2D(filters = 32, \n                 kernel_size = (3,3), \n                 padding = 'valid', \n                 activation ='elu'))\n\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\n\nmodel.add(AvgPool2D(pool_size=(2,2)))\n\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(300, activation = \"elu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\nmodel.add(Dense(200, activation = \"elu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\nmodel.add(Dense(100, activation = \"elu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.3))\n\nmodel.add(Dense(10, activation = \"softmax\"))\n\n\nmodel.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', \n              metrics=['acc'])","metadata":{"id":"FEsje0Em6S3M","outputId":"ed16d356-c2a7-4384-f9ed-4792a8991e08","execution":{"iopub.status.busy":"2021-09-05T14:50:37.117240Z","iopub.execute_input":"2021-09-05T14:50:37.117588Z","iopub.status.idle":"2021-09-05T14:50:37.675577Z","shell.execute_reply.started":"2021-09-05T14:50:37.117550Z","shell.execute_reply":"2021-09-05T14:50:37.674799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"id":"Gl5PORhX8wtn","outputId":"b6d64307-a555-4edc-c081-3059fbd69b05","execution":{"iopub.status.busy":"2021-09-05T14:50:37.677550Z","iopub.execute_input":"2021-09-05T14:50:37.678041Z","iopub.status.idle":"2021-09-05T14:50:37.697262Z","shell.execute_reply.started":"2021-09-05T14:50:37.678003Z","shell.execute_reply":"2021-09-05T14:50:37.696523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model, show_shapes=True, show_layer_names=False)","metadata":{"id":"ZBlXSF3s938-","outputId":"ebf09f54-1ec2-4a87-cf30-54262faf47ef","execution":{"iopub.status.busy":"2021-09-05T14:50:37.700006Z","iopub.execute_input":"2021-09-05T14:50:37.700241Z","iopub.status.idle":"2021-09-05T14:50:38.169791Z","shell.execute_reply.started":"2021-09-05T14:50:37.700218Z","shell.execute_reply":"2021-09-05T14:50:38.168675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = EarlyStopping(\n    min_delta=0.0002,\n    mode='min', \n    patience=20,\n    restore_best_weights=True,\n)","metadata":{"id":"spsuGXlRFMee","execution":{"iopub.status.busy":"2021-09-05T14:50:38.171149Z","iopub.execute_input":"2021-09-05T14:50:38.171492Z","iopub.status.idle":"2021-09-05T14:50:38.177051Z","shell.execute_reply.started":"2021-09-05T14:50:38.171451Z","shell.execute_reply":"2021-09-05T14:50:38.176222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2021-09-05T14:50:38.178495Z","iopub.execute_input":"2021-09-05T14:50:38.179118Z","iopub.status.idle":"2021-09-05T14:50:38.192142Z","shell.execute_reply.started":"2021-09-05T14:50:38.179080Z","shell.execute_reply":"2021-09-05T14:50:38.191084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_tr, y_tr, \n          validation_data=(X_val, y_val),\n          verbose=1, epochs=75, batch_size=16,\n          callbacks=[early_stopping])\n\nhistory_df = pd.DataFrame(history.history)\nhistory_df.loc[:, ['loss', 'val_loss']].plot()\nprint(\"Minimum Validation Loss: {:0.4f}\".format(history_df['val_loss'].min()))","metadata":{"id":"L6-phU-g9-ae","outputId":"7dd9d817-6842-4419-ae01-51a49b47c82e","execution":{"iopub.status.busy":"2021-09-05T14:50:38.193522Z","iopub.execute_input":"2021-09-05T14:50:38.193996Z","iopub.status.idle":"2021-09-05T15:06:22.997714Z","shell.execute_reply.started":"2021-09-05T14:50:38.193960Z","shell.execute_reply":"2021-09-05T15:06:22.996868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_reshape.shape","metadata":{"id":"BMNbj2ceG57g","outputId":"ede95c56-5ed9-49fe-e6be-b7b55276b5b0","execution":{"iopub.status.busy":"2021-09-05T15:06:23.001951Z","iopub.execute_input":"2021-09-05T15:06:23.002213Z","iopub.status.idle":"2021-09-05T15:06:23.007986Z","shell.execute_reply.started":"2021-09-05T15:06:23.002186Z","shell.execute_reply":"2021-09-05T15:06:23.007047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.argmax(model.predict(X_test_reshape), axis=1)","metadata":{"id":"Kpc0DJF2ER-U","execution":{"iopub.status.busy":"2021-09-05T15:06:23.009996Z","iopub.execute_input":"2021-09-05T15:06:23.010336Z","iopub.status.idle":"2021-09-05T15:06:25.014185Z","shell.execute_reply.started":"2021-09-05T15:06:23.010302Z","shell.execute_reply":"2021-09-05T15:06:25.013340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"id":"SULIx1Y7IqVg","execution":{"iopub.status.busy":"2021-09-05T15:06:25.017363Z","iopub.execute_input":"2021-09-05T15:06:25.017631Z","iopub.status.idle":"2021-09-05T15:06:25.037209Z","shell.execute_reply.started":"2021-09-05T15:06:25.017605Z","shell.execute_reply":"2021-09-05T15:06:25.036430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.shape","metadata":{"id":"DPN-FNrHItCh","outputId":"8c5e2576-9e51-4b40-b28d-3fb3a617d03a","execution":{"iopub.status.busy":"2021-09-05T15:06:25.038378Z","iopub.execute_input":"2021-09-05T15:06:25.038738Z","iopub.status.idle":"2021-09-05T15:06:25.047198Z","shell.execute_reply.started":"2021-09-05T15:06:25.038702Z","shell.execute_reply":"2021-09-05T15:06:25.046363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['Label'] = preds\nsubmission.to_csv('my_submission_keras.csv',index=False)\n\n\nsubmission.head()","metadata":{"id":"WueEra3oGrpQ","outputId":"2920e2c9-2e5d-43a7-b992-94ee5a843488","execution":{"iopub.status.busy":"2021-09-05T15:06:25.048708Z","iopub.execute_input":"2021-09-05T15:06:25.049289Z","iopub.status.idle":"2021-09-05T15:06:25.106576Z","shell.execute_reply.started":"2021-09-05T15:06:25.049253Z","shell.execute_reply":"2021-09-05T15:06:25.105658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TensorFlow","metadata":{"id":"nvlh-PYnplXn"}},{"cell_type":"markdown","source":"Let us try to do it via only Tensorflow. Just for practice - want to share some hints.  \n\nThanks to my favourite book: Hands-On Machine Learning with Scikit-Learn and TensorFlow - I love this book","metadata":{"id":"oX371wtctSpW"}},{"cell_type":"markdown","source":"We indicate the number of neurons in layers, inputs and outputs","metadata":{"id":"XEIOf_rZqP72"}},{"cell_type":"code","source":"tf.reset_default_graph()","metadata":{"id":"9J4ADT3FFs-_","execution":{"iopub.status.busy":"2021-09-05T15:06:25.107868Z","iopub.execute_input":"2021-09-05T15:06:25.108201Z","iopub.status.idle":"2021-09-05T15:06:25.112206Z","shell.execute_reply.started":"2021-09-05T15:06:25.108167Z","shell.execute_reply":"2021-09-05T15:06:25.111211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2021-09-05T15:06:25.113395Z","iopub.execute_input":"2021-09-05T15:06:25.113875Z","iopub.status.idle":"2021-09-05T15:06:25.124170Z","shell.execute_reply.started":"2021-09-05T15:06:25.113841Z","shell.execute_reply":"2021-09-05T15:06:25.123438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUTS = 28 * 28 # MNIST size\n\nHIDDEN_1 = 300\nHIDDEN_2 = 100\nOUTPUTS = 10","metadata":{"id":"660sfSFRpnmB","execution":{"iopub.status.busy":"2021-09-05T15:06:25.125380Z","iopub.execute_input":"2021-09-05T15:06:25.125769Z","iopub.status.idle":"2021-09-05T15:06:25.135882Z","shell.execute_reply.started":"2021-09-05T15:06:25.125709Z","shell.execute_reply":"2021-09-05T15:06:25.135102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create placeholder nodes","metadata":{"id":"m3i7njWwqXcn"}},{"cell_type":"code","source":"X = tf.placeholder(tf.float32, shape=(None, INPUTS), name='X')\ny = tf.placeholder(tf.int32, shape=(None), name='y')","metadata":{"id":"9-cFCOKnp0xb","execution":{"iopub.status.busy":"2021-09-05T15:06:25.137076Z","iopub.execute_input":"2021-09-05T15:06:25.137444Z","iopub.status.idle":"2021-09-05T15:06:25.146885Z","shell.execute_reply.started":"2021-09-05T15:06:25.137395Z","shell.execute_reply":"2021-09-05T15:06:25.146042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we can create layers","metadata":{"id":"bs9KT0_asCMk"}},{"cell_type":"code","source":"with tf.name_scope('DNN'):\n  hidden1 = tf.layers.dense(X, HIDDEN_1, name='hidden_1', \n                    activation = tf.nn.leaky_relu)\n  hidden2 = tf.layers.dense(hidden1, HIDDEN_2, name='hidden_2', \n                    activation = tf.nn.leaky_relu)\n  logits = tf.layers.dense(hidden2, OUTPUTS, name='outputs')","metadata":{"id":"3K-fGm1Drhu7","execution":{"iopub.status.busy":"2021-09-05T15:06:25.148547Z","iopub.execute_input":"2021-09-05T15:06:25.149163Z","iopub.status.idle":"2021-09-05T15:06:25.194210Z","shell.execute_reply.started":"2021-09-05T15:06:25.149126Z","shell.execute_reply":"2021-09-05T15:06:25.193487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"logits - output of neurons before passing through softmax","metadata":{"id":"NJz2r_ZqtMQt"}},{"cell_type":"code","source":"with tf.name_scope('loss_func'):\n  cross_entr = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, \n                                                              logits=logits)\n  loss = tf.reduce_mean(tf.cast(cross_entr, tf.float32))","metadata":{"id":"OWt6fIWesciY","execution":{"iopub.status.busy":"2021-09-05T15:06:25.196659Z","iopub.execute_input":"2021-09-05T15:06:25.196909Z","iopub.status.idle":"2021-09-05T15:06:25.203524Z","shell.execute_reply.started":"2021-09-05T15:06:25.196884Z","shell.execute_reply":"2021-09-05T15:06:25.202686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.name_scope('train'):\n  optimizer = tf.train.RMSPropOptimizer(learning_rate=0.01)\n  training = optimizer.minimize(loss)","metadata":{"id":"hNR6wAZXuLsz","outputId":"35293e1a-99c7-4e9d-c3ab-80d1ee44e466","execution":{"iopub.status.busy":"2021-09-05T15:06:25.204779Z","iopub.execute_input":"2021-09-05T15:06:25.205339Z","iopub.status.idle":"2021-09-05T15:06:25.283296Z","shell.execute_reply.started":"2021-09-05T15:06:25.205304Z","shell.execute_reply":"2021-09-05T15:06:25.282579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.name_scope('eval'):\n  valid = tf.nn.in_top_k(logits, y, 1)\n  accuracy_score = tf.reduce_mean(tf.cast(valid, tf.float32))","metadata":{"id":"8s6PFKfMuyNu","execution":{"iopub.status.busy":"2021-09-05T15:06:25.284421Z","iopub.execute_input":"2021-09-05T15:06:25.284782Z","iopub.status.idle":"2021-09-05T15:06:25.291927Z","shell.execute_reply.started":"2021-09-05T15:06:25.284735Z","shell.execute_reply":"2021-09-05T15:06:25.291132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training (with GPU) (let us use only 5 epochs to compute result faster)","metadata":{"id":"wD4BIXMzvPu2"}},{"cell_type":"code","source":"epochs = 5\nbatch_size = 50","metadata":{"id":"ZoQCJcpYvKj9","execution":{"iopub.status.busy":"2021-09-05T15:06:25.293318Z","iopub.execute_input":"2021-09-05T15:06:25.293671Z","iopub.status.idle":"2021-09-05T15:06:25.300474Z","shell.execute_reply.started":"2021-09-05T15:06:25.293636Z","shell.execute_reply":"2021-09-05T15:06:25.299605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = features.to_numpy().astype(np.float32).reshape(-1, 28*28) / 255.0\nX_test = test.to_numpy().astype(np.float32).reshape(-1, 28*28) / 255.0\ny_train = target.to_numpy().astype(np.int32)\nX_train.shape, y_train.shape","metadata":{"id":"ZClrmY96Bzwj","outputId":"b29e315b-a7ef-42a6-c4da-6d96090f2fc8","execution":{"iopub.status.busy":"2021-09-05T15:06:25.305280Z","iopub.execute_input":"2021-09-05T15:06:25.305602Z","iopub.status.idle":"2021-09-05T15:06:25.506505Z","shell.execute_reply.started":"2021-09-05T15:06:25.305570Z","shell.execute_reply":"2021-09-05T15:06:25.505700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid, X_train = X_train[:5000], X_train[5000:]\ny_valid, y_train = y_train[:5000], y_train[5000:]","metadata":{"id":"Dj6Bd0jhDvuX","execution":{"iopub.status.busy":"2021-09-05T15:06:25.507674Z","iopub.execute_input":"2021-09-05T15:06:25.508035Z","iopub.status.idle":"2021-09-05T15:06:25.513575Z","shell.execute_reply.started":"2021-09-05T15:06:25.508004Z","shell.execute_reply":"2021-09-05T15:06:25.512642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid.shape","metadata":{"id":"Ip2OZbw-CF8C","outputId":"e6adf5a1-49b0-456f-cb8a-16639d65cdf0","execution":{"iopub.status.busy":"2021-09-05T15:06:25.515119Z","iopub.execute_input":"2021-09-05T15:06:25.515706Z","iopub.status.idle":"2021-09-05T15:06:25.527386Z","shell.execute_reply.started":"2021-09-05T15:06:25.515670Z","shell.execute_reply":"2021-09-05T15:06:25.526523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This func will help us to make batches for training","metadata":{"id":"_C8lkDRGPsdB"}},{"cell_type":"code","source":"def shuffle(X, y, batch_size):\n    idx = np.random.permutation(len(X))\n    rnd_batches = len(X) // batch_size\n    for batch_idx in np.array_split(idx, rnd_batches):\n        X_batch, y_batch = X[idx], y[idx]\n        yield X_batch, y_batch","metadata":{"id":"P3LDWk1fD_xD","execution":{"iopub.status.busy":"2021-09-05T15:06:25.528601Z","iopub.execute_input":"2021-09-05T15:06:25.529041Z","iopub.status.idle":"2021-09-05T15:06:25.536979Z","shell.execute_reply.started":"2021-09-05T15:06:25.529014Z","shell.execute_reply":"2021-09-05T15:06:25.536120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"init = tf.global_variables_initializer()\nsaver = tf.train.Saver()","metadata":{"id":"6ReC9Cw-xxlZ","execution":{"iopub.status.busy":"2021-09-05T15:06:25.538484Z","iopub.execute_input":"2021-09-05T15:06:25.538901Z","iopub.status.idle":"2021-09-05T15:06:25.565793Z","shell.execute_reply.started":"2021-09-05T15:06:25.538862Z","shell.execute_reply":"2021-09-05T15:06:25.565029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"to make sure we use GPU","metadata":{"id":"LeHyYjnTdDaQ"}},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2021-09-05T15:06:25.568340Z","iopub.execute_input":"2021-09-05T15:06:25.568602Z","iopub.status.idle":"2021-09-05T15:06:25.573815Z","shell.execute_reply.started":"2021-09-05T15:06:25.568577Z","shell.execute_reply":"2021-09-05T15:06:25.572852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nnow = datetime.utcnow().strftime('Y%m%d %H%M%S')\nroot_logdir = 'tf_logs'\nlogdir = '{}/run-{}/'.format(root_logdir, now)\n\ntf.debugging.set_log_device_placement(True)\nfile_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())\n\nwith tf.Session() as sess:\n\n  init.run()\n\n  for epoch in range(epochs):\n\n      for X_batch, y_batch in shuffle(X_train, y_train, batch_size):\n        sess.run(training, feed_dict={X: X_batch, y: y_batch})\n\n      if epoch % 1 == 0:\n\n        acc_train = accuracy_score.eval(feed_dict={X: X_batch, y: y_batch})\n        acc_valid = accuracy_score.eval(feed_dict={X: X_valid, y: y_valid})\n\n        print(epoch, 'Accuracy on training', acc_train, \n                     'Accuracy on validation', acc_valid)\n\n  save_path = saver.save(sess, './model.ckpt')","metadata":{"id":"wXGXSoZFvtXu","outputId":"44b1c19c-c7d3-4147-ef7a-cdd62a2cd9b7","execution":{"iopub.status.busy":"2021-09-05T15:06:25.575222Z","iopub.execute_input":"2021-09-05T15:06:25.575825Z","iopub.status.idle":"2021-09-05T15:30:51.719957Z","shell.execute_reply.started":"2021-09-05T15:06:25.575787Z","shell.execute_reply":"2021-09-05T15:30:51.719037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Results from Google Collab:\n\n![](https://i.ibb.co/4s4K6Xf/2021-09-03-23-04-10.png)","metadata":{}},{"cell_type":"code","source":"file_writer.close()","metadata":{"execution":{"iopub.status.busy":"2021-09-05T15:30:51.724806Z","iopub.execute_input":"2021-09-05T15:30:51.726766Z","iopub.status.idle":"2021-09-05T15:30:51.733025Z","shell.execute_reply.started":"2021-09-05T15:30:51.726710Z","shell.execute_reply":"2021-09-05T15:30:51.732221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is for runnig tensorboard. But now support is disabled","metadata":{}},{"cell_type":"code","source":"# %load_ext tensorboard\n# %tensorboard --logdir logs --bind_all","metadata":{"execution":{"iopub.status.busy":"2021-09-05T15:30:51.736839Z","iopub.execute_input":"2021-09-05T15:30:51.739092Z","iopub.status.idle":"2021-09-05T15:30:51.747817Z","shell.execute_reply.started":"2021-09-05T15:30:51.739054Z","shell.execute_reply":"2021-09-05T15:30:51.746910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Restore our model and predict our Test","metadata":{}},{"cell_type":"code","source":"with tf.Session() as sess:\n    saver.restore(sess, './model.ckpt')\n    Z = logits.eval(feed_dict={X: X_test})\n    y_pred = np.argmax(Z, axis=1)","metadata":{"id":"cLVe3449LSH-","outputId":"e60ab348-cda8-424e-f0d9-a0c045f18f36","execution":{"iopub.status.busy":"2021-09-05T15:30:51.751924Z","iopub.execute_input":"2021-09-05T15:30:51.754266Z","iopub.status.idle":"2021-09-05T15:30:51.981534Z","shell.execute_reply.started":"2021-09-05T15:30:51.754230Z","shell.execute_reply":"2021-09-05T15:30:51.980644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"id":"m8Kp-tJJLz8C","execution":{"iopub.status.busy":"2021-09-05T15:30:51.983197Z","iopub.execute_input":"2021-09-05T15:30:51.983509Z","iopub.status.idle":"2021-09-05T15:30:51.999548Z","shell.execute_reply.started":"2021-09-05T15:30:51.983475Z","shell.execute_reply":"2021-09-05T15:30:51.998795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['Label'] = y_pred\nsubmission.to_csv('my_submission_tensorflow.csv',index=False)\n\n\nsubmission.head()","metadata":{"id":"-Pz7ZvNiLvFF","outputId":"a23d7d4d-0629-412e-8fe9-d79979d94cd1","execution":{"iopub.status.busy":"2021-09-05T15:30:52.000825Z","iopub.execute_input":"2021-09-05T15:30:52.001298Z","iopub.status.idle":"2021-09-05T15:30:52.056621Z","shell.execute_reply.started":"2021-09-05T15:30:52.001260Z","shell.execute_reply":"2021-09-05T15:30:52.055721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thank you for reading. Good luck with learning. You can add some layers to CNN and it will improve score","metadata":{}}]}