{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers as L\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import RMSprop\nfrom keras.callbacks import ReduceLROnPlateau\n\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Since we are storing big amounts of data it's best to try to reduce memory usage as much as possible**","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n\ndef import_data(file):\n    \"\"\"create a dataframe and optimize its memory usage\"\"\"\n    df = pd.read_csv(file, parse_dates=True, keep_date_col=True)\n    df = reduce_mem_usage(df)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Loading and Preprocessing data","metadata":{}},{"cell_type":"code","source":"train = import_data('../input/digit-recognizer/train.csv')\ntest = import_data('../input/digit-recognizer/test.csv')\n\ny_lab = train['label']\ny = tf.keras.utils.to_categorical(y_lab) \ntrain.drop('label', axis=1, inplace=True)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = np.array(train).reshape(-1, 28, 28, 1)\ntest_df = np.array(test).reshape(-1, 28, 28, 1)\n\ndel train\ndel test\ndel y_lab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import load_img, img_to_array, array_to_img\n\ndef change_size(image):\n    img = array_to_img(image, scale=False) \n    img = img.resize((75, 75))\n    img = img.convert(mode='RGB')\n    arr = img_to_array(img)\n    return arr.astype(np.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_array = [change_size(img) for img in train_df]\ntrain = np.array(train_array)\ndel train_array\n\ntest_array = [change_size(img) for img in test_df]\ntest = np.array(test_array)\ndel test_array","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Data Augmentation","metadata":{}},{"cell_type":"markdown","source":"**Data augmentation** is useful to improve performance and outcomes of machine learning models by forming new and different examples to train datasets. If dataset in a machine learning model is rich and sufficient, the model performs better and more accurate.\nFor example, classic image processing activities for data augmentation are\n\n* padding\n* random rotating\n* re-scaling,\n* vertical and horizontal flipping\n* translation ( image is moved along X, Y direction)\n* cropping\n* zooming\n* darkening & brightening/color modification\n* grayscaling\n* changing contrast\n* adding noise\n* random erasing","metadata":{}},{"cell_type":"code","source":"def get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255, pixel_level=False):\n    def eraser(input_img):\n        if input_img.ndim == 3:\n            img_h, img_w, img_c = input_img.shape\n        elif input_img.ndim == 2:\n            img_h, img_w = input_img.shape\n\n        p_1 = np.random.rand()\n\n        if p_1 > p:\n            return input_img\n\n        while True:\n            s = np.random.uniform(s_l, s_h) * img_h * img_w\n            r = np.random.uniform(r_1, r_2)\n            w = int(np.sqrt(s / r))\n            h = int(np.sqrt(s * r))\n            left = np.random.randint(0, img_w)\n            top = np.random.randint(0, img_h)\n\n            if left + w <= img_w and top + h <= img_h:\n                break\n\n        if pixel_level:\n            if input_img.ndim == 3:\n                c = np.random.uniform(v_l, v_h, (h, w, img_c))\n            if input_img.ndim == 2:\n                c = np.random.uniform(v_l, v_h, (h, w))\n        else:\n            c = np.random.uniform(v_l, v_h)\n\n        input_img[top:top + h, left:left + w] = c\n\n        return input_img\n\n    return eraser","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_gen = ImageDataGenerator(rescale=1./255,\n                              featurewise_center=False,\n                              preprocessing_function=get_random_eraser(v_l=0, v_h=1),\n                              samplewise_center=False,\n                              featurewise_std_normalization=False,\n                              samplewise_std_normalization=False,\n                              zca_whitening=False,\n                              zoom_range=0.1,\n                              rotation_range=10,\n                              width_shift_range=0.2,\n                              height_shift_range=0.2,\n                              shear_range=0.3,\n                              validation_split=0.2) # 80/20 train/val split\n\ntrain_generator = image_gen.flow(train, \n                                 y,\n                                batch_size=32,\n                                shuffle=True,\n                                subset='training',\n                                seed=42)\nvalid_generator = image_gen.flow(train,\n                                 y,\n                                batch_size=16,\n                                shuffle=True,\n                                subset='validation')\n\ndel train_df\ndel test_df\ndel train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Transfer Learning model","metadata":{}},{"cell_type":"markdown","source":"It is common to perform **transfer learning** with predictive modeling problems that use **image data** as input.\n\nThis may be a prediction task that takes photographs or video data as input.\n\nFor these types of problems, it is common to use a deep learning model pre-trained for a large and challenging image classification task such as the ImageNet 1000-class photograph classification competition.\n\nSo, we will use **Resnet-50** for our network architecture as it achieves state of the art result for our classification task","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(tf.keras.applications.resnet50.ResNet50(input_shape = (75, 75, 3),\n                                pooling = 'avg',\n                                include_top = False, \n                                weights = 'imagenet'))\n\nmodel.add(L.Flatten())\nmodel.add(L.Dense(128, activation='relu'))\nmodel.add(L.Dense(10, activation='softmax'))\n\nmodel.compile(optimizer=RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0), loss='categorical_crossentropy', metrics=['accuracy'])\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in model.layers[0].layers:\n    if layer.name == 'conv5_block1_0_conv':\n        break\n    layer.trainable=False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator, validation_data=valid_generator, epochs=20, \n          steps_per_epoch=train_generator.n//train_generator.batch_size,\n         validation_steps=valid_generator.n//valid_generator.batch_size, \n          callbacks=[learning_rate_reduction])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Predictions and submission","metadata":{}},{"cell_type":"code","source":"test = test/255 ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = model.predict(test[:])\noutput = pd.DataFrame({'ImageId':[ i+1 for i in range(len(res))], \n                       'Label': [ xi.argmax() for xi in res]})\noutput.to_csv('submission_grid.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    print(\"predicted: \",res[i].argmax())\n    plt.imshow(test[i],cmap=\"gray\")\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Credits:**\n* https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n* https://www.kaggle.com/elcaiseri/mnist-simple-cnn-keras-accuracy-0-99-top-1\n* https://www.kaggle.com/donatastamosauskas/using-resnet-for-mnist\n* https://www.kaggle.com/saumandas/intro-to-transfer-learning-with-mnist","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}