{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"# Attempt D6: Time-wise mean input + all possible augmentation\nI introduce one of my approach to handle the spectrum mean data.\nThis scored about val_acc 0.76, but what's important is that this looks at data from spectrum point of view only, very different from the others.\n\nMean is calculated at the first layer of model:\n```Python\n    model.add(AveragePooling2D((1, input_shape[1]), input_shape=input_shape, strides=1))\n ```\n\nI have to start with importing dependent external modules."},{"metadata":{"_cell_guid":"f2317a0c-7c02-4071-ae85-b60c061a103d","_uuid":"08fd1c609e788b57014c35054603ee2a328cf899","collapsed":true,"trusted":false},"cell_type":"code","source":"# Quoting https://raw.githubusercontent.com/bckenstler/CLR/master/clr_callback.py\nfrom keras.callbacks import *\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n        K.set_value(self.model.optimizer.lr, self.clr())\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n            \n            \n\n  ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"628b4b06-08ed-4137-bd35-eb7aa4d25745","_uuid":"f610abb1b8eaacfaaab86224f48dbf39df16ed16","collapsed":true,"trusted":false},"cell_type":"code","source":"# Quoting https://raw.githubusercontent.com/yu4u/mixup-generator/master/random_eraser.py\nimport numpy as np\n\ndef get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255):\n    def eraser(input_img):\n        img_h, img_w, _ = input_img.shape\n        p_1 = np.random.rand()\n\n        if p_1 > p:\n            return input_img\n\n        while True:\n            s = np.random.uniform(s_l, s_h) * img_h * img_w\n            r = np.random.uniform(r_1, r_2)\n            w = int(np.sqrt(s / r))\n            h = int(np.sqrt(s * r))\n            left = np.random.randint(0, img_w)\n            top = np.random.randint(0, img_h)\n\n            if left + w <= img_w and top + h <= img_h:\n                break\n\n        c = np.random.uniform(v_l, v_h)\n        input_img[top:top + h, left:left + w, :] = c\n\n        return input_img\n\n    return eraser","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4ac90915-3986-4e22-93e7-99ce8ccdf20f","_uuid":"efbc4f578c4ffdc311427763e63cdcf17a2b60bc","collapsed":true,"trusted":false},"cell_type":"code","source":"# Quoting https://raw.githubusercontent.com/yu4u/mixup-generator/master/mixup_generator.py\nimport numpy as np\n\n\nclass MixupGenerator():\n    def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):\n        self.X_train = X_train\n        self.y_train = y_train\n        self.batch_size = batch_size\n        self.alpha = alpha\n        self.shuffle = shuffle\n        self.sample_num = len(X_train)\n        self.datagen = datagen\n\n    def __call__(self):\n        while True:\n            indexes = self.__get_exploration_order()\n            itr_num = int(len(indexes) // (self.batch_size * 2))\n\n            for i in range(itr_num):\n                batch_ids = indexes[i * self.batch_size * 2:(i + 1) * self.batch_size * 2]\n                X, y = self.__data_generation(batch_ids)\n\n                yield X, y\n\n    def __get_exploration_order(self):\n        indexes = np.arange(self.sample_num)\n\n        if self.shuffle:\n            np.random.shuffle(indexes)\n\n        return indexes\n\n    def __data_generation(self, batch_ids):\n        _, h, w, c = self.X_train.shape\n        l = np.random.beta(self.alpha, self.alpha, self.batch_size)\n        X_l = l.reshape(self.batch_size, 1, 1, 1)\n        y_l = l.reshape(self.batch_size, 1)\n\n        X1 = self.X_train[batch_ids[:self.batch_size]]\n        X2 = self.X_train[batch_ids[self.batch_size:]]\n        X = X1 * X_l + X2 * (1 - X_l)\n\n        if self.datagen:\n            for i in range(self.batch_size):\n                X[i] = self.datagen.random_transform(X[i])\n                X[i] = self.datagen.standardize(X[i])\n\n        if isinstance(self.y_train, list):\n            y = []\n\n            for y_train_ in self.y_train:\n                y1 = y_train_[batch_ids[:self.batch_size]]\n                y2 = y_train_[batch_ids[self.batch_size:]]\n                y.append(y1 * y_l + y2 * (1 - y_l))\n        else:\n            y1 = self.y_train[batch_ids[:self.batch_size]]\n            y2 = self.y_train[batch_ids[self.batch_size:]]\n            y = y1 * y_l + y2 * (1 - y_l)\n\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8029424c-b050-41e5-8e41-3b73fc802f36","_uuid":"5640cb6406752d73cdceacaa98a319991ee90315"},"cell_type":"markdown","source":"Now it's ready to start. Import starndard dependencies."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":false},"cell_type":"code","source":"# Preparation\nimport os\nimport shutil\nimport pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\n# quoted above  from mixup_generator import MixupGenerator\n# quoted above  from random_eraser import get_random_eraser\n# quoted above  from cyclic_lr import CyclicLR\n\nimport keras\nimport keras.backend as K\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, GlobalMaxPooling2D, Activation, Dropout, BatchNormalization, Flatten\nfrom keras.models import Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint, Callback, TensorBoard\n\nTRY = 'D6' # Attempt Id","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"dc9bb8dc-fa87-4c2f-9e3d-eaea6eb0b284","_uuid":"39d3a307eff63a53196d02100dd675ec6c0c28ae"},"cell_type":"markdown","source":"## Load and confirm some of data."},{"metadata":{"_cell_guid":"702a77e6-7c13-4389-b38c-0646ea61c32c","_uuid":"b9c3da91187dc6ffa13e13814ecf199e6f55808a","collapsed":true,"trusted":false},"cell_type":"code","source":"# Load dataset\ndatadir = \"../input/\"\n\nX_train_org = np.load(datadir + 'X_train.npy')\nX_test = np.load(datadir + 'X_test.npy')\ny_labels_train = pd.read_csv(datadir + 'y_train.csv', sep=',')['scene_label'].tolist()\n# Make label lists\nlabels = sorted(list(set(y_labels_train)))\nlabel2int = {l:i for i, l in enumerate(labels)}\nint2label = {i:l for i, l in enumerate(labels)}\n# Map y_train to int labels\ny_train_org = keras.utils.to_categorical([label2int[l] for l in y_labels_train])\n\n# Train/Validation split to X_train/y_train\nsplitlist = pd.read_csv(datadir + 'crossvalidation_train.csv', sep=',')['set'].tolist()\nX_train = np.array([x for i, x in enumerate(X_train_org) if splitlist[i] == 'train'])\nX_valid = np.array([x for i, x in enumerate(X_train_org) if splitlist[i] == 'test'])\ny_train = np.array([y for i, y in enumerate(y_train_org) if splitlist[i] == 'train'])\ny_valid = np.array([y for i, y in enumerate(y_train_org) if splitlist[i] == 'test'])\n\n# MIXUP Augmentation\n#tmp_X, tmp_y = mixup(X_train, y_train, alpha=1)\n#X_train, y_train = np.r_[X_train, tmp_X], np.r_[y_train, tmp_y]\n\n# Normalize dataset\nvalue_max = np.max(np.vstack([X_train, X_valid, X_test]))\nX_train = X_train / value_max\nX_valid = X_valid / value_max\nX_test = X_test / value_max\n\n# [:, 40, 501] -> [:, 40, 501, 1]\nX_train = X_train[..., np.newaxis]\nX_valid = X_valid[..., np.newaxis]\nX_test = X_test[..., np.newaxis]\n\n# Confirmation\ndef plot_dataset(X, n=3):\n    for i in range(n):\n        x = X[i]\n        plt.pcolormesh(x[..., -1])\n        plt.show()\nfor X in [X_train, X_valid, X_test]:\n    plot_dataset(X)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cce70760-4f25-42e8-88f9-f8e5dd48f2a2","_uuid":"1d46eb8aa7ea3d05d2ed6bcade3376377db9073d"},"cell_type":"markdown","source":"## Create data generator with mixup & random erasing"},{"metadata":{"_cell_guid":"c08f7024-55bb-4527-8762-6c4c747f3e36","_uuid":"b182aa2fb4bcbf554d3447c278f29a205659e566","collapsed":true,"trusted":false},"cell_type":"code","source":"batch_size = 32 \nnum_classes = len(labels)\nepochs = 50 # CHANGE TO 250, 50 is just for Kaggle kernel\n\ndatagen = ImageDataGenerator(\n    rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)\n    width_shift_range=0.4,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range=0,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip=True,  # randomly flip images\n    preprocessing_function=get_random_eraser(v_l=np.min(X_train), v_h=np.max(X_train)) # RANDOM ERASER\n)\nmixupgen = MixupGenerator(X_train, y_train, alpha=1.0, batch_size=batch_size, datagen=datagen)\ntest_datagen = ImageDataGenerator(\n    featurewise_center=False,  # set input mean to 0 over the dataset\n    featurewise_std_normalization=False,  # divide inputs by std of the dataset\n)\n\ndatagen.fit(np.r_[X_train, X_valid, X_test])\ntest_datagen.mean, test_datagen.std = datagen.mean, datagen.std\n\n# Confirmation\ndef plot_dataset(X, n=3):\n    for i in range(n):\n        x = X[i]\n        plt.pcolormesh(x[..., -1])\n        plt.show()\n\nb = next(mixupgen())\nfor X in [b[0][:5]]:\n    plot_dataset(X)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"55dbd100-585a-4c11-a7e8-f9f5ef6b2425","_uuid":"40d73db8c79baa3489acea47fdf08d4274d88c3c"},"cell_type":"markdown","source":"## Define model\nThis is the model.\n\nIt takes time-wise mean first; (40, 501) -> (40, 1). Following part is a kind of AlexNet."},{"metadata":{"_cell_guid":"c0b23935-4b89-4630-8a87-47a8cc274542","_uuid":"4c34290cc460ebec1764a217f03df60e2c21970d","collapsed":true,"trusted":false},"cell_type":"code","source":"import keras\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, GlobalMaxPooling2D, AveragePooling2D, Activation, Dropout, BatchNormalization, Flatten\nfrom keras.models import Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint, Callback, TensorBoard\n\ndef model_cnn5(input_shape): # \n    model = Sequential()\n    \n    # This computes time-wise mean\n    model.add(AveragePooling2D((1, input_shape[1]), input_shape=input_shape, strides=1))\n \n    # Followings are based on half-AlexNet\n    model.add(Conv2D(48, (11, 1),  input_shape=input_shape, strides=(2,1), activation='relu', padding='same'))\n    model.add(MaxPooling2D((3, 1), strides=(1,1)))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(128, (5, 1), strides=(2,1), activation='relu', padding='same'))\n    model.add(MaxPooling2D((3, 1), strides=(2, 1)))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(256, (3, 1), strides=1, activation='relu', padding='same'))\n    model.add(Conv2D(256, (3, 1), strides=1, activation='relu', padding='same'))\n    model.add(MaxPooling2D((2, 1), strides=(1,1)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(256, (3, 1), strides=1, activation='relu', padding='same'))\n    model.add(Conv2D(128, (3, 1), strides=1, activation='relu', padding='same'))\n    model.add(MaxPooling2D((2, 1), strides=(1,1)))\n    model.add(BatchNormalization())\n\n    model.add(Flatten())\n    model.add(Dense(128, activation='relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(64, activation='relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(num_classes, activation='softmax'))\n\n    return model\n\nmodel = model_cnn5(X_train.shape[1:])\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=keras.optimizers.Adam(lr=0.0005),\n              metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f501443f-5bd6-4e28-8dfe-81bc42a0a680","_uuid":"8897c448cabe0b7695c92da0f720e33217c655af"},"cell_type":"markdown","source":"## Get ready to train"},{"metadata":{"_cell_guid":"0d297aa5-3c59-407e-923c-a81aa61c0387","_uuid":"f2a5e6b8339cae3d436a33f0ec05baeda773c93a","collapsed":true,"trusted":false},"cell_type":"code","source":"# Prepare callback\ncallbacks = [\n    # Cyclic Learning Rate\n    CyclicLR(base_lr=0.00001, max_lr=0.0001, step_size=X_train.shape[0] // batch_size, mode='triangular'),\n\n    # Save the best model\n    ModelCheckpoint('weights%s4valid.model' % TRY,\n                monitor='val_acc',\n                verbose=1,\n                save_best_only=True,\n                save_weights_only=True),\n     keras.callbacks.TensorBoard(log_dir='./%slog' % TRY, histogram_freq=0, write_graph=True, write_images=True)\n]\n\n# Fit the model on the batches generated by mixupgen().\nmodel.fit_generator(mixupgen(),\n                    steps_per_epoch=X_train.shape[0] // batch_size,\n                    epochs=epochs,\n                    validation_data=test_datagen.flow(X_valid, y_valid), callbacks=callbacks)\nmodel.load_weights('weights%s4valid.model' % TRY)\nK.set_value(model.optimizer.lr, 0.000005)\nmodel.fit_generator(mixupgen(),\n                    steps_per_epoch=X_train.shape[0] // batch_size,\n                    epochs=15,\n                    validation_data=test_datagen.flow(X_valid, y_valid), callbacks=None)\nmodel.save_weights('weights%s4valid.model' % TRY)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a541b9a4-c969-4e49-81cb-96ed20b2a1f2","_uuid":"a670e16601148c18fcf66c1a0da42a7d0aa28bcb","collapsed":true,"trusted":false},"cell_type":"code","source":"# CONFIRMATION ON INITIAL TRAINING\nmodel.load_weights('weights%s4valid.model' % TRY)\ny_valid_preds = model.predict_generator(test_datagen.flow(X_valid, y_valid, shuffle=False))\ny_valid_pred_cls = [np.argmax(pred) for pred in y_valid_preds]\ny_valid_refs = [np.argmax(y) for y in y_valid]\n\nnp.save('preds%s4valid.npy' % TRY, y_valid_preds)\nvalid_results = [result == ref for result, ref in zip(y_valid_pred_cls, y_valid_refs)]\n\n# accuracy\nprint(np.sum(valid_results)/len(valid_results))\n\n# double check the answers\nfor result, ref in zip(y_valid_pred_cls[:10], y_valid_refs[:10]):\n    print(result, '\\t', ref)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9c279df8-7e0e-4b2d-b448-86cf327739ea","_uuid":"0738b056c58fe068d4619b35614014b1c4fa4e46","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}