{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom sklearn.model_selection import train_test_split\n\nimport numpy as np\nimport pandas as pd\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_raw = train.drop(columns=['label']).to_numpy()\ny_train_raw = train['label'].to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train_raw.shape)\nprint(y_train_raw.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_mean = np.mean(X_train_raw)\nx_std = np.std(X_train_raw)\n\ndef standarize(X):\n    return (X - x_mean) / x_std\n\nX_train = np.reshape(X_train_raw,(-1,28,28,1))\nX_train = standarize(X_train)\n\ny_train_cat = keras.utils.to_categorical(y_train_raw)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_cat.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator = keras.preprocessing.image.ImageDataGenerator(\n    rotation_range=10,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    zoom_range=0.1\n)\n\ndata_generator.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model():\n    model = keras.models.Sequential()\n    model.add(keras.layers.Conv2D(32, kernel_size=3,activation='relu',kernel_initializer='he_normal',input_shape=(28, 28, 1)))\n    model.add(keras.layers.BatchNormalization()) \n    model.add(keras.layers.Conv2D(32, kernel_size=3,activation='relu'))\n    model.add(keras.layers.BatchNormalization()) \n    model.add(keras.layers.Conv2D(32, kernel_size=5, strides=2, padding='same',activation='relu'))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Dropout(0.4))\n    \n    model.add(keras.layers.Conv2D(64, kernel_size=3, activation ='relu'))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Conv2D(64, kernel_size=3, activation ='relu'))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Conv2D(64, kernel_size=5, strides=2, padding='same',activation='relu'))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Dropout(0.4))\n    \n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(128, activation = \"relu\"))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Dropout(0.4))\n    model.add(keras.layers.Dense(10, activation = \"softmax\"))\n    \n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ens_size=16\nmodel = [0]*ens_size\n\nfor i in range(ens_size):\n    model[i] = make_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nhistory = [0]*ens_size\n\n\nstart = time.perf_counter()\n\nfor i in range(ens_size):\n    callbacks = [keras.callbacks.ModelCheckpoint('/kaggle/working/mdl-{}-of-{}.hdf5'\n                                                 .format(i,ens_size-1),save_best_only=True, monitor='val_accuracy', mode='max')]\n    \n    X_train_ens, X_valid_ens, y_train_ens, y_valid_ens = train_test_split(X_train, y_train_cat, train_size=0.9)\n    train_aug = data_generator.flow(X_train_ens, y_train_ens)\n    \n    history[i] = model[i].fit_generator(train_aug, epochs=40,\n                              validation_data=(X_valid_ens, y_valid_ens),\n                              callbacks=callbacks, verbose = 0)\n    \n    print(\"CNN {}: Train accuracy={:0.5f}, Validation accuracy={:0.5f}\"\n      .format(i,max(history[i].history['accuracy']),max(history[i].history['val_accuracy']) ))\n    \nstop = time.perf_counter()\nprint(f\"{stop - start:0.4f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef visualize_accuracy(history):\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper right')\n    plt.show()\n\ndef visualize_loss(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper right')\n    plt.show()\n    \nvisualize_accuracy(history[0])\nvisualize_loss(history[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = np.reshape(test.to_numpy(),(-1,28,28,1))\nX_test = standarize(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = np.zeros((X_test.shape[0],10))\n\nfor i in range(ens_size):\n    mdl = keras.models.load_model('/kaggle/working/mdl-{}-of-{}.hdf5'.format(i,ens_size-1))\n    preds = preds + mdl.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_pred = np.argmax(preds,axis=1)\nsubmit_pred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submition = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\nsubmition['Label'] = submit_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submition.to_csv('/kaggle/working/submition_ens3.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}