{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-08-29T10:53:59.041093Z","iopub.status.busy":"2020-08-29T10:53:59.040393Z","iopub.status.idle":"2020-08-29T10:54:04.358965Z","shell.execute_reply":"2020-08-29T10:54:04.357762Z"},"papermill":{"duration":5.337305,"end_time":"2020-08-29T10:54:04.359097","exception":false,"start_time":"2020-08-29T10:53:59.021792","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:04.392766Z","iopub.status.busy":"2020-08-29T10:54:04.391998Z","iopub.status.idle":"2020-08-29T10:54:04.396131Z","shell.execute_reply":"2020-08-29T10:54:04.395635Z"},"papermill":{"duration":0.022105,"end_time":"2020-08-29T10:54:04.396231","exception":false,"start_time":"2020-08-29T10:54:04.374126","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:04.429805Z","iopub.status.busy":"2020-08-29T10:54:04.429092Z","iopub.status.idle":"2020-08-29T10:54:04.432259Z","shell.execute_reply":"2020-08-29T10:54:04.431783Z"},"papermill":{"duration":0.022295,"end_time":"2020-08-29T10:54:04.432353","exception":false,"start_time":"2020-08-29T10:54:04.410058","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tf.random.set_seed(89)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:04.467879Z","iopub.status.busy":"2020-08-29T10:54:04.467199Z","iopub.status.idle":"2020-08-29T10:54:07.674075Z","shell.execute_reply":"2020-08-29T10:54:07.672889Z"},"papermill":{"duration":3.228067,"end_time":"2020-08-29T10:54:07.674207","exception":false,"start_time":"2020-08-29T10:54:04.446140","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds=pd.read_csv(\"../input/digit-recognizer/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:07.711794Z","iopub.status.busy":"2020-08-29T10:54:07.709928Z","iopub.status.idle":"2020-08-29T10:54:07.714299Z","shell.execute_reply":"2020-08-29T10:54:07.713793Z"},"papermill":{"duration":0.02528,"end_time":"2020-08-29T10:54:07.714394","exception":false,"start_time":"2020-08-29T10:54:07.689114","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds.shape","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:07.751091Z","iopub.status.busy":"2020-08-29T10:54:07.750483Z","iopub.status.idle":"2020-08-29T10:54:07.770296Z","shell.execute_reply":"2020-08-29T10:54:07.769416Z"},"papermill":{"duration":0.041676,"end_time":"2020-08-29T10:54:07.770394","exception":false,"start_time":"2020-08-29T10:54:07.728718","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:07.809458Z","iopub.status.busy":"2020-08-29T10:54:07.807643Z","iopub.status.idle":"2020-08-29T10:54:07.810180Z","shell.execute_reply":"2020-08-29T10:54:07.810742Z"},"papermill":{"duration":0.023461,"end_time":"2020-08-29T10:54:07.810862","exception":false,"start_time":"2020-08-29T10:54:07.787401","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"X=ds.iloc[:,1:].values\ny=ds.iloc[:,0].values","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:07.846437Z","iopub.status.busy":"2020-08-29T10:54:07.845410Z","iopub.status.idle":"2020-08-29T10:54:07.949525Z","shell.execute_reply":"2020-08-29T10:54:07.949021Z"},"papermill":{"duration":0.123005,"end_time":"2020-08-29T10:54:07.949647","exception":false,"start_time":"2020-08-29T10:54:07.826642","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"#normalizing the data\nX_scaled=X/255.","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:07.990567Z","iopub.status.busy":"2020-08-29T10:54:07.989910Z","iopub.status.idle":"2020-08-29T10:54:10.718222Z","shell.execute_reply":"2020-08-29T10:54:10.717290Z"},"papermill":{"duration":2.752762,"end_time":"2020-08-29T10:54:10.718350","exception":false,"start_time":"2020-08-29T10:54:07.965588","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def Network():\n    model=tf.keras.models.Sequential()\n    model.add(tf.keras.Input(shape=(784)))\n    model.add(tf.keras.layers.Dense(256,activation=\"relu\"))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(tf.keras.layers.Dropout(0.2))\n    model.add(tf.keras.layers.Dense(128,activation=\"relu\"))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(tf.keras.layers.Dropout(0.4))\n    model.add(tf.keras.layers.Dense(10,activation=\"softmax\"))\n    \n    return model\nmodel=Network()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:10.755893Z","iopub.status.busy":"2020-08-29T10:54:10.755155Z","iopub.status.idle":"2020-08-29T10:54:10.815430Z","shell.execute_reply":"2020-08-29T10:54:10.814960Z"},"papermill":{"duration":0.080967,"end_time":"2020-08-29T10:54:10.815536","exception":false,"start_time":"2020-08-29T10:54:10.734569","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from keras import optimizers\noptim = optimizers.Adam(lr = 0.0005)\nmodel.compile(loss = 'SparseCategoricalCrossentropy', optimizer = optim, metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:10.852211Z","iopub.status.busy":"2020-08-29T10:54:10.851590Z","iopub.status.idle":"2020-08-29T10:54:10.855479Z","shell.execute_reply":"2020-08-29T10:54:10.855006Z"},"papermill":{"duration":0.024106,"end_time":"2020-08-29T10:54:10.855576","exception":false,"start_time":"2020-08-29T10:54:10.831470","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# defining scheduler\ndef scheduler(epoch, lr):\n      if epoch < 10:\n        return lr\n      else:\n        return lr * tf.math.exp(-0.1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:10.893047Z","iopub.status.busy":"2020-08-29T10:54:10.892402Z","iopub.status.idle":"2020-08-29T10:54:10.896538Z","shell.execute_reply":"2020-08-29T10:54:10.895971Z"},"papermill":{"duration":0.024837,"end_time":"2020-08-29T10:54:10.896632","exception":false,"start_time":"2020-08-29T10:54:10.871795","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Bundling all types of callbacks\nmy_callbacks=([\n    tf.keras.callbacks.LearningRateScheduler(scheduler),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",patience=2),\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\",factor=0.05,patience=1)\n    \n    ])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:54:10.933478Z","iopub.status.busy":"2020-08-29T10:54:10.932706Z","iopub.status.idle":"2020-08-29T10:55:22.371963Z","shell.execute_reply":"2020-08-29T10:55:22.371443Z"},"papermill":{"duration":71.459462,"end_time":"2020-08-29T10:55:22.372074","exception":false,"start_time":"2020-08-29T10:54:10.912612","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Training\ntrainer = model.fit(X_scaled, y,validation_split=0.3, epochs = 50, batch_size = 32, verbose = 1,callbacks=my_callbacks)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:23.478017Z","iopub.status.busy":"2020-08-29T10:55:23.477232Z","iopub.status.idle":"2020-08-29T10:55:23.706943Z","shell.execute_reply":"2020-08-29T10:55:23.707548Z"},"papermill":{"duration":0.735429,"end_time":"2020-08-29T10:55:23.707754","exception":false,"start_time":"2020-08-29T10:55:22.972325","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Plot loss per iteration\nplt.plot(trainer.history['loss'], label='loss')\nplt.plot(trainer.history['val_loss'], label='val_loss')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:24.530709Z","iopub.status.busy":"2020-08-29T10:55:24.529370Z","iopub.status.idle":"2020-08-29T10:55:24.725075Z","shell.execute_reply":"2020-08-29T10:55:24.725636Z"},"papermill":{"duration":0.607473,"end_time":"2020-08-29T10:55:24.725813","exception":false,"start_time":"2020-08-29T10:55:24.118340","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Plot accuracy per iteration\nplt.plot(trainer.history['accuracy'], label='acc')\nplt.plot(trainer.history['val_accuracy'], label='val_acc')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:25.632848Z","iopub.status.busy":"2020-08-29T10:55:25.632085Z","iopub.status.idle":"2020-08-29T10:55:27.698906Z","shell.execute_reply":"2020-08-29T10:55:27.697304Z"},"papermill":{"duration":2.523115,"end_time":"2020-08-29T10:55:27.699032","exception":false,"start_time":"2020-08-29T10:55:25.175917","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test=pd.read_csv(\"../input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:28.547280Z","iopub.status.busy":"2020-08-29T10:55:28.545436Z","iopub.status.idle":"2020-08-29T10:55:28.547939Z","shell.execute_reply":"2020-08-29T10:55:28.548396Z"},"papermill":{"duration":0.420977,"end_time":"2020-08-29T10:55:28.548520","exception":false,"start_time":"2020-08-29T10:55:28.127543","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test=test.iloc[:,:].values","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:29.380914Z","iopub.status.busy":"2020-08-29T10:55:29.379856Z","iopub.status.idle":"2020-08-29T10:55:29.450314Z","shell.execute_reply":"2020-08-29T10:55:29.449769Z"},"papermill":{"duration":0.508215,"end_time":"2020-08-29T10:55:29.450429","exception":false,"start_time":"2020-08-29T10:55:28.942214","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_scaled=test/255.","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:30.252647Z","iopub.status.busy":"2020-08-29T10:55:30.252052Z","iopub.status.idle":"2020-08-29T10:55:31.230184Z","shell.execute_reply":"2020-08-29T10:55:31.230749Z"},"papermill":{"duration":1.385189,"end_time":"2020-08-29T10:55:31.230910","exception":false,"start_time":"2020-08-29T10:55:29.845721","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"pred = model.predict_classes(test_scaled)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-29T10:55:32.046852Z","iopub.status.busy":"2020-08-29T10:55:32.046234Z","iopub.status.idle":"2020-08-29T10:55:32.303925Z","shell.execute_reply":"2020-08-29T10:55:32.302715Z"},"papermill":{"duration":0.662763,"end_time":"2020-08-29T10:55:32.304064","exception":false,"start_time":"2020-08-29T10:55:31.641301","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# create submission file\nfinal_submission = pd.DataFrame({'ImageId': np.array(range(1,28001)), 'Label': pred})\nfinal_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Just the 1st part\n### Next, needed to ensemble different models","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}