{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport tensorflow.keras.layers as layers\n\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.preprocessing.image import ImageDataGenerator","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\ny_train = train.label\nX_train = train.drop('label',axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape)\nprint(test.shape)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"784 is the pixel count of the image which are all of single digits in a 28x28 pixel square. ","metadata":{}},{"cell_type":"code","source":"sns.countplot(y_train)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Numbers range from 0 to 9 and are evenly distributed which is good for when we split our data.","metadata":{}},{"cell_type":"code","source":"import matplotlib as mpl\nimport matplotlib.pyplot as plt\n\nrand_digits = X_train.iloc[[0,5,10,22]]\nfig,axes = plt.subplots(1,4)\nfor i in range(4):\n    digit=rand_digits.iloc[i,:]\n    axes[i].imshow(digit.values.reshape(28,28),cmap='gray') \n    axes[i].set_axis_off()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MNIST images are grayscaled so we only need 1 channel, more in Normalization + Reshape","metadata":{}},{"cell_type":"markdown","source":"### Null Check","metadata":{}},{"cell_type":"code","source":"X_train.isnull().any().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().any().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalization + Reshape","metadata":{}},{"cell_type":"code","source":"# /255 normalizes data to a 0 to 1 scale\n# reshape format reshape(samples, height, width, channel) channel = 3 if RGB\nX_train = (X_train.values.reshape(-1,28,28,1))/255;\ntest = (test.values.reshape(-1,28,28,1))/255;","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label Encoding","metadata":{}},{"cell_type":"code","source":"y_train = to_categorical(y_train, num_classes = 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential([\n    # Input\n    layers.Conv2D(filters = 32, kernel_size=5, activation='relu',\n                 padding='same',input_shape = (28,28,1)),\n    layers.MaxPool2D(pool_size=2), # lots of empty space in the images. need to condense them\n    layers.Dropout(rate=0.3),\n    \n    # 2\n    layers.Conv2D(filters = 64, kernel_size=3, activation = 'relu',\n                 padding='same'),\n    layers.MaxPool2D(pool_size=2),\n    layers.Dropout(rate=0.3),\n    \n    # 3\n    layers.Conv2D(filters=128, kernel_size=3, activation='relu',\n                 padding='same'),\n    layers.MaxPool2D(pool_size=2),\n    layers.Dropout(rate=0.3),\n    \n    # Classifier Head\n    layers.Flatten(),\n    layers.Dense(units=256, activation='relu'),\n    layers.Dropout(rate = 0.5),\n    layers.Dense(10, activation = 'softmax'),\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=tf.keras.optimizers.Adam(epsilon=0.01),\n    loss='categorical_crossentropy',\n    metrics=['accuracy']\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\nearly_stopping = EarlyStopping(\n    min_delta=0.001, # minimium amount of change to count as an improvement\n    patience=10, # how many epochs to wait before stopping\n    restore_best_weights=True,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n                X_train, y_train,\n                validation_data = (X_val, y_val),\n                epochs=500,\n                callbacks=[early_stopping],\n                verbose=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['accuracy', 'val_accuracy']].plot();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"results = model.predict(test)\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"results.csv\",index=False) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Epochs\n* 98.11% Accuracy with 3 epochs\n* 99.164% Accuracy with 10 epochs\n* 99.235% Accuracy ~25 epochs -> implemented early stopping with 500 epochs","metadata":{}}]}