{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.shape)\nprint(df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Root of 784 is 28. We have 28x28 images."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df_train['label']\ndf = pd.concat([df_train.drop(columns='label'), df_test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = df.to_numpy().reshape((70000, 28, 28, 1))\nX = data[:df_train.shape[0]]/255\nX_test = data[df_train.shape[0]:]/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgplot = plt.imshow(X[3], cmap='gray')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = [str(_) for _ in range(10)]\n\nplt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X[i], cmap=plt.cm.binary)\n    # The CIFAR labels happen to be arrays, \n    # which is why you need the extra index\n    plt.xlabel(class_names[y[i]])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import models, layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n  layers.experimental.preprocessing.RandomRotation(0.05),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_augmented = data_augmentation(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = [str(_) for _ in range(10)]\n\nplt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X_augmented[i], cmap=plt.cm.binary)\n    # The CIFAR labels happen to be arrays, \n    # which is why you need the extra index\n    plt.xlabel(class_names[y[i]])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_all = np.concatenate((X, X_augmented), axis=0)\ny_all = np.concatenate((y, y))\ny_all = pd.get_dummies(y_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_all.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TF Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(64, activation='relu'))\n#model.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(10))\nmodel.add(layers.Softmax())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])\n\nhistory = model.fit(X_all, pd.get_dummies(y_all), validation_split=0.2, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Last score:\n- loss: 1.5545 - accuracy: 0.9580 - val_loss: 1.5464 - val_accuracy: 0.9627\n- loss: 1.4980 - accuracy: 0.9797 - val_loss: 1.5221 - val_accuracy: 0.9560\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds = pd.DataFrame({'ImageId':range(1, X_test.shape[0]+1), 'Label':np.argmax(y_test, axis=1)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds.to_csv('singleModelSubmission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Achieved 0.98492 score with single model."},{"metadata":{},"cell_type":"markdown","source":"# Multiple Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nfolds = KFold(n_splits = 10, shuffle = True, random_state = 42)\n\npredictions = np.zeros((len(X_test),10))\n\ni=1\nfor fold, (trn_idx, val_idx) in enumerate(folds.split(X_all, y_all)):\n    X_train = np.take(X_all, trn_idx, axis=0)\n    X_val = np.take(X_all, val_idx, axis=0)\n    y_train = np.take(y_all, trn_idx, axis=0)\n    y_val = np.take(y_all, val_idx, axis=0)\n    #print(X_train.shape, X_val.shape, y_train.shape, y_val.shape)\n    #break\n\n    model.compile(optimizer='adam',\n              loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])\n   \n    history = model.fit(X_train, y_train, validation_data = (X_val, y_val), epochs=5, verbose = 1)\n    \n    predictions += model.predict(X_test) / folds.n_splits \n    \n    print(\"\\n    {}/10: {} - {}\\n\".format(i, history.history['accuracy'][-1], history.history['val_accuracy'][-1]))\n    i+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.argmax(predictions, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds = pd.DataFrame({'ImageId':range(1, X_test.shape[0]+1), 'Label':np.argmax(predictions, axis=1)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds.to_csv('multipleModelSubmission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Achieved 0.99050 score."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}