{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"FOLDERNAME = '/kaggle/input'\ntrain_df = pd.read_csv(os.path.join(FOLDERNAME, 'digit-recognizer/train.csv'))\ntest_df = pd.read_csv(os.path.join(FOLDERNAME, 'digit-recognizer/test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_train = train_df.drop(columns=['label']).to_numpy()\ntarget_train = train_df['label'].to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_samples(input_arr, target_arr, num_classes=10):\n    assert (num_classes > 0 and num_classes <= 10), 'Number of classes must be in range [1-10]'\n    fig, axs = plt.subplots(1, num_classes, figsize=(1.5*num_classes, 2))\n    if num_classes == 1:\n        axs.imshow(input_arr[target_arr==0][0].reshape(28, 28), cmap='gray')\n        axs.axis('off')\n        axs.set_title('Digit 0')\n    else:\n        for i in range(num_classes):\n            axs[i].imshow(input_arr[target_arr==i][0].reshape(28, 28), cmap='gray')\n            axs[i].axis('off')\n            axs[i].set_title('Digit {}'.format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_samples(input_train, target_train, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(input_train, target_train, test_size=0.2, stratify=target_train)\nX_train = X_train / 255.0\nX_val = X_val / 255.0\n# X_train = X_train.reshape(-1, 28, 28)\n# X_val = X_val.reshape(-1, 28, 28)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create Dataset object\n\ntrain_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train))\nval_ds = tf.data.Dataset.from_tensor_slices((X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, optimizer, train_ds, val_ds, batch_size=32, num_epochs=1, print_every=1, is_training=False):\n    train_ds = train_ds.batch(batch_size)\n    val_ds = val_ds.batch(batch_size)\n    \n    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n    \n    train_loss = tf.keras.metrics.Mean(name='train_loss')\n    train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_acc')\n    \n    val_loss = tf.keras.metrics.Mean(name='val_loss')\n    val_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='val_acc')\n    \n    for epoch in range(num_epochs):\n        train_loss.reset_states()\n        train_accuracy.reset_states()\n        print('Epoch {}'.format(epoch+1))\n        t = 0\n        \n        for X, y in train_ds:\n            \n            with tf.GradientTape() as tape:\n                scores = model(X, training=is_training)\n                loss = loss_fn(y, scores)\n                gradients = tape.gradient(loss, model.trainable_variables)\n                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n                \n                train_loss.update_state(loss)\n                train_accuracy.update_state(y, scores)\n                t += 1\n            if t % print_every == 0:\n                print('Iter {}, train loss: {:.4f}, train acc: {:.4f}'.format(t, train_loss.result(), train_accuracy.result()))\n                      \n        val_loss.reset_states()\n        val_accuracy.reset_states()\n\n        for Xval, yval in val_ds:\n            scores_val = model(Xval, training=False)\n            t_loss = loss_fn(yval, scores_val)\n\n            val_loss.update_state(t_loss)\n            val_accuracy.update_state(yval, scores_val)\n        print('- train loss: {:.4f}, val loss: {:.4f}, train acc: {:.4f}, val acc: {:.4f}'.format(train_loss.result(), val_loss.result(),\n                                                                                                train_accuracy.result(), val_accuracy.result()))\n                \n        print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(tf.keras.Model):\n    def __init__(self, hidden_size1, hidden_size2, dropout_rate, num_classes):\n        super(Model, self).__init__()\n        self.fc1 = tf.keras.layers.Dense(hidden_size1)\n        self.bn1 = tf.keras.layers.BatchNormalization()\n        self.relu1 = tf.keras.layers.ReLU()\n        self.dropout1 = tf.keras.layers.Dropout(dropout_rate)\n        self.fc2 = tf.keras.layers.Dense(hidden_size2)\n        self.bn2 = tf.keras.layers.BatchNormalization()\n        self.relu2 = tf.keras.layers.ReLU()\n        self.dropout2 = tf.keras.layers.Dropout(dropout_rate)\n        self.fc_final = tf.keras.layers.Dense(num_classes, activation='softmax')\n        \n    def call(self, x, training=False):\n        x = self.fc1(x)\n#         x = self.bn1(x)\n        x = self.relu1(x)\n        x = self.dropout1(x)\n        x = self.fc2(x)\n#         x = self.bn2(x)\n        x = self.relu2(x)\n        x = self.dropout2(x)\n        x = self.fc_final(x)\n        return x\n    \n    \nmodel = Model(500, 300, 0.5, 10)\nlr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(0.001, decay_steps=5000, decay_rate=.95)\noptimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(model, optimizer, train_ds, val_ds, 64, 10, 200, is_training=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_test = test_df.to_numpy()\nX_test = input_test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = tf.data.Dataset.from_tensor_slices(input_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_label = np.array([])\nfor test_batch in test_ds.batch(64):\n    scores = model(test_batch, training=False)\n    test_label = np.concatenate((test_label, np.argmax(scores, axis=1)))\n    \ntest_label = test_label.astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output_df = pd.DataFrame({'ImageId': np.arange(1, len(test_label)+1), 'Label': test_label})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output_df.to_csv('output.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}