{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8170,"databundleVersionId":109509,"sourceType":"competition"}],"dockerImageVersionId":46,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Attempt H2: Input data as is + all possible augmentation by AlexNet based model\nHere I introduce one of my model based on half part of AlexNet, scored about val_acc 0.79.\nThis takes input almost as is, so this is most straight forward approach.\n\nI have to start with importing dependent external modules.","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"code","source":"# Quoting https://raw.githubusercontent.com/bckenstler/CLR/master/clr_callback.py\nfrom keras.callbacks import *\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n        K.set_value(self.model.optimizer.lr, self.clr())\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n            ","metadata":{"_cell_guid":"a3edc98f-6f5a-4efc-9947-23aacd7af0e1","_uuid":"eafb38db84d443706a1c23e85bd456f0a6a776ad","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:11.140303Z","iopub.execute_input":"2025-01-06T04:12:11.140647Z","iopub.status.idle":"2025-01-06T04:12:12.686397Z","shell.execute_reply.started":"2025-01-06T04:12:11.140577Z","shell.execute_reply":"2025-01-06T04:12:12.685468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Quoting https://raw.githubusercontent.com/yu4u/mixup-generator/master/random_eraser.py\nimport numpy as np\n\ndef get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255):\n    def eraser(input_img):\n        img_h, img_w, _ = input_img.shape\n        p_1 = np.random.rand()\n\n        if p_1 > p:\n            return input_img\n\n        while True:\n            s = np.random.uniform(s_l, s_h) * img_h * img_w\n            r = np.random.uniform(r_1, r_2)\n            w = int(np.sqrt(s / r))\n            h = int(np.sqrt(s * r))\n            left = np.random.randint(0, img_w)\n            top = np.random.randint(0, img_h)\n\n            if left + w <= img_w and top + h <= img_h:\n                break\n\n        c = np.random.uniform(v_l, v_h)\n        input_img[top:top + h, left:left + w, :] = c\n\n        return input_img\n\n    return eraser","metadata":{"_cell_guid":"80ef55cf-ef0b-471a-903e-70ce5a55b938","_uuid":"698045217818ad06c81fbb91ec59043d89661248","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:12.687731Z","iopub.execute_input":"2025-01-06T04:12:12.687986Z","iopub.status.idle":"2025-01-06T04:12:12.724929Z","shell.execute_reply.started":"2025-01-06T04:12:12.687938Z","shell.execute_reply":"2025-01-06T04:12:12.724156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Quoting https://raw.githubusercontent.com/yu4u/mixup-generator/master/mixup_generator.py\nimport numpy as np\n\n\nclass MixupGenerator():\n    def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):\n        self.X_train = X_train\n        self.y_train = y_train\n        self.batch_size = batch_size\n        self.alpha = alpha\n        self.shuffle = shuffle\n        self.sample_num = len(X_train)\n        self.datagen = datagen\n\n    def __call__(self):\n        while True:\n            indexes = self.__get_exploration_order()\n            itr_num = int(len(indexes) // (self.batch_size * 2))\n\n            for i in range(itr_num):\n                batch_ids = indexes[i * self.batch_size * 2:(i + 1) * self.batch_size * 2]\n                X, y = self.__data_generation(batch_ids)\n\n                yield X, y\n\n    def __get_exploration_order(self):\n        indexes = np.arange(self.sample_num)\n\n        if self.shuffle:\n            np.random.shuffle(indexes)\n\n        return indexes\n\n    def __data_generation(self, batch_ids):\n        _, h, w, c = self.X_train.shape\n        l = np.random.beta(self.alpha, self.alpha, self.batch_size)\n        X_l = l.reshape(self.batch_size, 1, 1, 1)\n        y_l = l.reshape(self.batch_size, 1)\n\n        X1 = self.X_train[batch_ids[:self.batch_size]]\n        X2 = self.X_train[batch_ids[self.batch_size:]]\n        X = X1 * X_l + X2 * (1 - X_l)\n\n        if self.datagen:\n            for i in range(self.batch_size):\n                X[i] = self.datagen.random_transform(X[i])\n                X[i] = self.datagen.standardize(X[i])\n\n        if isinstance(self.y_train, list):\n            y = []\n\n            for y_train_ in self.y_train:\n                y1 = y_train_[batch_ids[:self.batch_size]]\n                y2 = y_train_[batch_ids[self.batch_size:]]\n                y.append(y1 * y_l + y2 * (1 - y_l))\n        else:\n            y1 = self.y_train[batch_ids[:self.batch_size]]\n            y2 = self.y_train[batch_ids[self.batch_size:]]\n            y = y1 * y_l + y2 * (1 - y_l)\n\n        return X, y","metadata":{"_cell_guid":"58f5d849-2f96-45f1-9795-8709ccbe36e5","_uuid":"0fefad5847366648a6e7c2ef57b118ce6feb605c","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:12.726259Z","iopub.execute_input":"2025-01-06T04:12:12.726578Z","iopub.status.idle":"2025-01-06T04:12:12.854442Z","shell.execute_reply.started":"2025-01-06T04:12:12.726511Z","shell.execute_reply":"2025-01-06T04:12:12.853371Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now it's ready to start. Import starndard dependencies.","metadata":{"_cell_guid":"d9df19f5-da94-4e7f-86b3-ed3f10d48817","_uuid":"fa2d661e1592ece8dd89233b664644f6e571ace2"}},{"cell_type":"code","source":"# Preparation\nimport os\nimport shutil\nimport pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\n# quoted above  from mixup_generator import MixupGenerator\n# quoted above  from random_eraser import get_random_eraser\n# quoted above  from cyclic_lr import CyclicLR\n\nimport keras\nimport keras.backend as K\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, GlobalMaxPooling2D, Activation, Dropout, BatchNormalization, Flatten\nfrom keras.models import Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint, Callback, TensorBoard\n\nTRY = 'H2' # Attempt Id","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:12.855939Z","iopub.execute_input":"2025-01-06T04:12:12.856278Z","iopub.status.idle":"2025-01-06T04:12:12.888470Z","shell.execute_reply.started":"2025-01-06T04:12:12.856225Z","shell.execute_reply":"2025-01-06T04:12:12.887640Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load and confirm some of data.","metadata":{"_cell_guid":"99e64289-db33-44a0-bac5-6476af605784","_uuid":"2307cde7dfb058313ef48f21beb47537d26e7dfe"}},{"cell_type":"code","source":"# Load dataset\ndatadir = \"../input/\"\n\nX_train_org = np.load(datadir + 'X_train.npy')\nX_test = np.load(datadir + 'X_test.npy')\ny_labels_train = pd.read_csv(datadir + 'y_train.csv', sep=',')['scene_label'].tolist()\n# Make label lists\nlabels = sorted(list(set(y_labels_train)))\nlabel2int = {l:i for i, l in enumerate(labels)}\nint2label = {i:l for i, l in enumerate(labels)}\n# Map y_train to int labels\ny_train_org = keras.utils.to_categorical([label2int[l] for l in y_labels_train])\n\n# Train/Validation split to X_train/y_train\nsplitlist = pd.read_csv(datadir + 'crossvalidation_train.csv', sep=',')['set'].tolist()\nX_train = np.array([x for i, x in enumerate(X_train_org) if splitlist[i] == 'train'])\nX_valid = np.array([x for i, x in enumerate(X_train_org) if splitlist[i] == 'test'])\ny_train = np.array([y for i, y in enumerate(y_train_org) if splitlist[i] == 'train'])\ny_valid = np.array([y for i, y in enumerate(y_train_org) if splitlist[i] == 'test'])\n\n# MIXUP Augmentation\n#tmp_X, tmp_y = mixup(X_train, y_train, alpha=1)\n#X_train, y_train = np.r_[X_train, tmp_X], np.r_[y_train, tmp_y]\n\n# Normalize dataset\nvalue_max = np.max(np.vstack([X_train, X_valid, X_test]))\nX_train = X_train / value_max\nX_valid = X_valid / value_max\nX_test = X_test / value_max\n\n# [:, 40, 501] -> [:, 40, 501, 1]\nX_train = X_train[..., np.newaxis]\nX_valid = X_valid[..., np.newaxis]\nX_test = X_test[..., np.newaxis]\n\n# Confirmation\ndef plot_dataset(X, n=3):\n    for i in range(n):\n        x = X[i]\n        plt.pcolormesh(x[..., -1])\n        plt.show()\nfor X in [X_train, X_valid, X_test]:\n    plot_dataset(X)","metadata":{"_cell_guid":"9ba6028d-3fe0-4bcd-a034-7fbb31f792e4","_uuid":"5a1b6ce3a9a8e65d768350c490cf70e2c0447708","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:12.890033Z","iopub.execute_input":"2025-01-06T04:12:12.890410Z","iopub.status.idle":"2025-01-06T04:12:17.562651Z","shell.execute_reply.started":"2025-01-06T04:12:12.890345Z","shell.execute_reply":"2025-01-06T04:12:17.561889Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"_cell_guid":"170a6249-4d84-4efb-b339-ba9c4cf7c714","_uuid":"d78b6f49e54d6f7c2d39e477a0c5c42cd4865223"}},{"cell_type":"markdown","source":"## Create data generator with mixup & random erasing","metadata":{"_cell_guid":"fed00952-dabb-49e1-9515-116cacff3ff3","_uuid":"8366ea7eb9f5af42c74f9afe1b8b29ef893616c8"}},{"cell_type":"code","source":"batch_size = 32 \nnum_classes = len(labels)\nepochs = 250 # 250 - use this to achive good score, 50 is for kernel submission purpose only.\n\ndatagen = ImageDataGenerator(\n    featurewise_center=True,  # set input mean to 0 over the dataset\n    featurewise_std_normalization=True,  # divide inputs by std of the dataset\n    rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)\n    width_shift_range=0.6,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range=0,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip=True,  # randomly flip images\n    preprocessing_function=get_random_eraser(v_l=np.min(X_train), v_h=np.max(X_train)) # RANDOM ERASER\n)\nmixupgen = MixupGenerator(X_train, y_train, alpha=1.0, batch_size=batch_size, datagen=datagen)\ntest_datagen = ImageDataGenerator(\n    featurewise_center=True,  # set input mean to 0 over the dataset\n    featurewise_std_normalization=True,  # divide inputs by std of the dataset\n)\n\ndatagen.fit(np.r_[X_train, X_valid, X_test])\ntest_datagen.mean, test_datagen.std = datagen.mean, datagen.std\n\n# Visualize some of them.\ntmp = next(mixupgen())\nfor X in [tmp[0][:5]]:\n    plot_dataset(X)","metadata":{"_cell_guid":"ebf92f50-0ef8-42e5-a3f6-f898986e5b74","_uuid":"85042b5152526a7ab79eec908a3573c4bb0b402c","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:17.563837Z","iopub.execute_input":"2025-01-06T04:12:17.564085Z","iopub.status.idle":"2025-01-06T04:12:20.813598Z","shell.execute_reply.started":"2025-01-06T04:12:17.564044Z","shell.execute_reply":"2025-01-06T04:12:20.812823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define model\nThis is based on AlexNet","metadata":{"_cell_guid":"752e2250-48d6-4485-af34-e4f4392a9fcc","_uuid":"2baccc0798cee6644a9cff83c4fd478addc2fe92"}},{"cell_type":"code","source":"def model_cnn_alexnet(input_shape): # Half part of AlexNet based, matched to input shape (40, 501)\n    model = Sequential()\n \n    model.add(Conv2D(48, 11,  input_shape=input_shape, strides=(2,3), activation='relu', padding='same'))\n    model.add(MaxPooling2D(3, strides=(1,2)))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(128, 5, strides=(2,3), activation='relu', padding='same'))\n    model.add(MaxPooling2D(3, strides=2))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(192, 3, strides=1, activation='relu', padding='same'))\n    model.add(Conv2D(192, 3, strides=1, activation='relu', padding='same'))\n    model.add(Conv2D(128, 3, strides=1, activation='relu', padding='same'))\n    model.add(MaxPooling2D(3, strides=(1,2)))\n    model.add(BatchNormalization())\n\n    model.add(Flatten())\n    model.add(Dense(256, activation='relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(256, activation='relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(num_classes, activation='softmax'))\n    return model\n\nmodel = model_cnn_alexnet(X_train.shape[1:])\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=keras.optimizers.Adam(lr=0.0001),\n              metrics=['accuracy'])\nmodel.summary()","metadata":{"_cell_guid":"20574916-f977-4075-b285-b95d4ff13b58","_uuid":"66bd6e6076145e707881765eb065426e1e10ade0","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:20.814847Z","iopub.execute_input":"2025-01-06T04:12:20.815219Z","iopub.status.idle":"2025-01-06T04:12:21.351275Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Get ready to train","metadata":{"_cell_guid":"95c3d59f-bf8e-4faf-848d-5666f92f5771","_uuid":"9e6d0a6fdd76a619874adabce4fb7d09955450bd"}},{"cell_type":"code","source":"# Prepare callback\ncallbacks = [\n    # Cyclic Learning Rate\n    CyclicLR(base_lr=0.0001, max_lr=0.001, step_size=X_train.shape[0] // batch_size, mode='triangular'),\n    # Save the best model\n    ModelCheckpoint('weights%s4valid.model' % TRY,\n                monitor='val_acc',\n                verbose=1,\n                save_best_only=True,\n                save_weights_only=True),\n     keras.callbacks.TensorBoard(log_dir='./%slog' % TRY, histogram_freq=0, write_graph=True, write_images=True)\n]\n\n# Fit the model on the batches generated by mixupgen().\nmodel.fit_generator(mixupgen(),\n                    steps_per_epoch=X_train.shape[0] // batch_size,\n                    epochs=epochs,\n                    validation_data=test_datagen.flow(X_valid, y_valid), callbacks=callbacks)\n# Fine tune\nmodel.load_weights('weights%s4valid.model' % TRY)\nK.set_value(model.optimizer.lr, 0.00001)\nmodel.fit_generator(mixupgen(),\n                    steps_per_epoch=X_train.shape[0] // batch_size,\n                    epochs=10,\n                    validation_data=test_datagen.flow(X_valid, y_valid), callbacks=None)\nmodel.save_weights('weights%s4valid.model' % TRY)","metadata":{"_cell_guid":"b1bb2a48-20ac-4c89-8c7c-1653b1c69db0","_uuid":"e23ec4a60a7fa952312faed535227f2a88770515","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T04:12:21.352589Z","iopub.execute_input":"2025-01-06T04:12:21.352840Z","iopub.status.idle":"2025-01-06T10:11:26.232778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CONFIRMATION ON INITIAL TRAINING\nmodel.load_weights('weights%s4valid.model' % TRY)\ny_valid_preds = model.predict_generator(test_datagen.flow(X_valid, y_valid, shuffle=False))\ny_valid_pred_cls = [np.argmax(pred) for pred in y_valid_preds]\ny_valid_refs = [np.argmax(y) for y in y_valid]\n\nnp.save('preds%s4valid.npy' % TRY, y_valid_preds)\nvalid_results = [result == ref for result, ref in zip(y_valid_pred_cls, y_valid_refs)]\n\n# accuracy\nprint(np.sum(valid_results)/len(valid_results))\n\n# double check the answers\nfor result, ref in zip(y_valid_pred_cls[:10], y_valid_refs[:10]):\n    print(result, '\\t', ref)","metadata":{"_cell_guid":"758ddcf3-d6f5-4d79-891b-52968d5f51cf","_uuid":"3d6c8d51edaf638e7d42cb889fed1435048a5e3a","trusted":true,"execution":{"iopub.status.busy":"2025-01-06T10:11:26.235828Z","iopub.execute_input":"2025-01-06T10:11:26.236127Z","iopub.status.idle":"2025-01-06T10:11:34.594123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_cell_guid":"17548b1b-efdc-48f7-ba7c-282d400a08cb","_uuid":"e18620a54e4f681106c1f1e23c94b061e793d167","trusted":true},"outputs":[],"execution_count":null}]}