{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output. ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bc4d820457f5cd25acec8f73c05316d71685ddb"},"cell_type":"code","source":"import tensorflow as tf\nfrom keras.applications.inception_resnet_v2 import InceptionResNetV2\nfrom keras import backend as K\nimport sys\nimport keras\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, Model\nfrom keras.layers import Input,Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D, MaxPool2D\nfrom keras.optimizers import RMSprop, Adam, sgd, Adamax\nfrom keras.callbacks import TensorBoard, ModelCheckpoint, ReduceLROnPlateau\nimport skimage\nfrom skimage.transform import rescale\nfrom keras.utils import np_utils\nfrom scipy.misc import imresize\nimport matplotlib.image as mpimg\nimport numpy as np\nfrom keras.preprocessing.image import load_img\nimport shutil\nfrom glob import glob\nfrom keras import regularizers\nfrom keras.regularizers import l2\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.np_utils import to_categorical\nfrom keras.layers.advanced_activations import PReLU\nfrom keras.callbacks import CSVLogger\ndef l1_reg(weight_matrix):\n    return 0.01 * K.sum(K.abs(weight_matrix))\nimport gc\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\ntf.keras.backend.clear_session()\nfrom keras.datasets import mnist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d58c7251e140c695c4e06664636fe3eb2e1a9b8b"},"cell_type":"code","source":"# MODEL_NUMBER = 1\nNB_EPOCH = 40\nBATCH_SIZE = 32\nVERBOSE = 1\nNB_CLASSES = 10\nOPTIMIZER = Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)\n# OPTIMIZE = RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.0)\nN_HIDDEN = 128\nVALIDATION_SPLIT = 0.05\nDROPOUT = 0.3\nRESHAPED = (28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c469254b8ae9829e4cb3035034aa87fa60b14b01"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0899d27e3bbf11f1ca927165ed957cfaf2da232d"},"cell_type":"code","source":"x_train = train.drop(labels=[\"label\"], axis=1)\nx_train = x_train / 255.0\nx_train = x_train.values.reshape(-1, 28, 28, 1)\ny_train = train[\"label\"]\ny_train = np.array(to_categorical(y_train, num_classes=10))\nx_test = test / 255.0\nx_test = x_test.values.reshape(-1, 28, 28, 1)\nprint(x_train.shape, y_train.shape, x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wc -l ../input/train.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(x_train_mnist, y_train_mnist), (x_val_mnist, y_val_mnist) = mnist.load_data()\nx_train_mnist = np.concatenate((x_train_mnist, x_val_mnist))\ny_train_mnist = np.concatenate((y_train_mnist, y_val_mnist))\n\nx_train_mnist = x_train_mnist.reshape((x_train_mnist.shape[0], 28, 28, 1))\nX_train_mnist = x_train_mnist / 255\ny_train_mnist = np.array(to_categorical(y_train_mnist.reshape((y_train_mnist.shape[0], 1)),num_classes=10))\nprint(x_train_mnist.shape, y_train_mnist.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = np.concatenate((x_train, x_train_mnist))\ny_train = np.concatenate((y_train, y_train_mnist))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf88dd3fd917a4e2ee77b0a173186df4366990e0"},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        rotation_range=10,  \n        zoom_range = 0.10,  \n        width_shift_range=0.1, \n        height_shift_range=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"518773d7c6601b7e7732d1a1e69ba5ef74c069ae"},"cell_type":"code","source":"np.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_shape = (28,28,1)\nmodel = Sequential()\nmodel.add(Conv2D(filters = 96, kernel_size = (3, 3), padding='same', activation='relu', input_shape=input_shape))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = 96,kernel_size = (3, 3), padding='same', activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Conv2D(filters = 192, kernel_size = (3,3), activation='relu', padding='same'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = 256, kernel_size = (3,3), activation='relu', padding='same'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Conv2D(filters = 256, kernel_size = (3,3), activation='relu', padding='same'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = 256, kernel_size = (3,3), activation='relu', padding='same'))\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\nmodel.add(Dense(units=10, activation='softmax'))\n\nmodel.compile(optimizer=OPTIMIZER, loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c32c24e7170f55340ad4328d0537cecf87b9397"},"cell_type":"code","source":"def create_log_dir():\n    if os.path.exists('./log_dir'):\n        shutil.rmtree('./log_dir')\n    os.mkdir('./log_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"354b91921240f4cba037c1bc9a32077011d5711d"},"cell_type":"code","source":"create_log_dir()\nprint(os.listdir(\"./\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = ReduceLROnPlateau(monitor='val_acc', patience=2, verbose=1, factor=0.5, min_lr=1.0e-5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d076de910f977d173cc40cf7ba108eff019caf24"},"cell_type":"code","source":"EarlyStopping_cb = keras.callbacks.EarlyStopping(monitor='val_acc', patience=5, verbose=1, mode='auto')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"183101970c5b43c69cf35d01dc576daa753e91e8"},"cell_type":"code","source":"csv_logger = CSVLogger('model.log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def history_plot():\n    print(history.history.keys())\n    #  \"Accuracy\"\n    plt.plot(history.history['acc'])\n    plt.plot(history.history['val_acc'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'validation'], loc='upper left')\n    plt.show()\n    # \"Loss\"\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train2, x_val2, y_train2, y_val2 = train_test_split(x_train, y_train, test_size=VALIDATION_SPLIT, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x_train2.shape)\nprint(x_val2.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model():\n    model_file = './log_dir/model_file' + '.hdf5'\n    checkpointer = ModelCheckpoint(filepath=model_file, verbose=1, monitor='val_acc', save_best_only=True)\n    history = model.fit_generator(datagen.flow(x_train2, y_train2, batch_size=BATCH_SIZE),\n                              validation_data=(x_val2, y_val2),\n                              epochs = NB_EPOCH, \n                              verbose = 2, \n                              steps_per_epoch=x_train2.shape[0] // BATCH_SIZE ,\n                              callbacks=[learning_rate, checkpointer, csv_logger, EarlyStopping_cb])\n#                               steps_per_epoch=x_train2.shape[0],                                     \n#                              callbacks=[learning_rate, checkpointer, csv_logger])\n    return(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6411515c5609f8fc7ecd913d6c4064c0432492f"},"cell_type":"code","source":"history = fit_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c436735feb59ce1b673ee00b1ec849369c17584"},"cell_type":"code","source":"history_plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00a806d7843a33ffe0c6c5b2265e2a952a9201d8"},"cell_type":"code","source":"results = np.zeros((x_test.shape[0], NB_CLASSES))\nmodel_file = './log_dir/model_file' + '.hdf5'\nmodel.load_weights(model_file)\nresults = results + model.predict(x_test)\nresults = np.argmax(results, axis=1)\nresults = pd.Series(results, name=\"Label\")\n\nsubmission = pd.concat([pd.Series(range(1, 28001), name=\"ImageId\"), results], axis=1)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13c1c1706afd83f1602e2c76ea780b76f96bd07e"},"cell_type":"code","source":"!wc -l ./submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"667c4ea42bed41ccb7051001e694753d3628e2e6"},"cell_type":"code","source":"!head ./submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}