{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Pytorch modules\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.utils.data import DataLoader\n\n#Import metcirs\nfrom sklearn.metrics import confusion_matrix, accuracy_score\n\n#Import math libs\nimport numpy as np\nfrom scipy import signal\n\n#Plotting\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import make_axes_locatable\nfrom mlxtend.plotting import plot_confusion_matrix\n\n#Status bar\nfrom tqdm.notebook import tqdm\n\n# Load files\nimport json\nimport os\nimport pandas as pd\n\nfrom collections import namedtuple","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:05.793799Z","iopub.execute_input":"2022-09-05T10:56:05.794444Z","iopub.status.idle":"2022-09-05T10:56:07.573767Z","shell.execute_reply.started":"2022-09-05T10:56:05.794261Z","shell.execute_reply":"2022-09-05T10:56:07.572719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"def rand_diap(num, diap):\n        low = np.log(diap[0])\n        high = np.log(diap[1])\n        tmp = np.random.rand(num)\n        tmp *= high - low\n        tmp += low\n        return np.exp(tmp)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.575753Z","iopub.execute_input":"2022-09-05T10:56:07.576670Z","iopub.status.idle":"2022-09-05T10:56:07.583441Z","shell.execute_reply.started":"2022-09-05T10:56:07.576632Z","shell.execute_reply":"2022-09-05T10:56:07.581279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constants","metadata":{}},{"cell_type":"code","source":"ROOT_DIR = '../input/classification-of-defects-in-a-rotary-machine'\nWORK_DIR = './'\n\nSENSORS_LIST = np.array([0, 1, 2, 3, 8, 9])\nSENSORS_NUM = len(SENSORS_LIST)\nSENSORS_NAMES =  np.array(['X_left_bearing',\n                            'Y_left_bearing',\n                            'X_right_bearing',\n                            'Y_right_bearing',\n                            'X_left_bearing_smoosed',\n                            'Y_left_bearing_smoosed',\n                            'X_right_bearing_smoosed',\n                            'Y_right_bearing_smoosed',\n                            'Temperature_left',\n                            'Temperature_right'])[SENSORS_LIST]\n\nCLASSES_NUM = 4\nEPOCHS_NUM = 500\nMETRIC = accuracy_score\nMETRIC_NAME = 'accuracy'\nCRITERION = nn.CrossEntropyLoss()\nHYPS = []\n\nfor hidden_layers in [1, 2, 4, 8, 16]:\n    for frame_length in [50, 100, 200, 400]:\n        for reg_coef in [0, 5e-4]:\n            for learning_rate in rand_diap(10, (1e-3, 1e-5)):\n                for hidden_layer_size in [1, 2, 4, 8, 16]:\n                    HYPS.append({'frame_length': frame_length,\n                                'hidden_layers': hidden_layers,\n                                'reg_coef': reg_coef,\n                                'learning_rate': learning_rate,\n                                'hidden_layer_size': hidden_layer_size})\n                    \nMEAN = np.array([5.506143093109131, \n                6.501108169555664, \n                6.133607864379883, \n                7.200148582458496, \n                5.506143093109131, \n                6.501108646392822, \n                6.133608341217041, \n                7.200153827667236, \n                1109.86279296875, \n                1108.7576904296875])[SENSORS_LIST]\n\nSTD = np.array([0.21205657720565796,\n                0.14815978705883026,\n                0.4135267734527588,\n                0.1607103794813156,\n                0.20784716308116913,\n                0.13870318233966827,\n                0.411641925573349,\n                0.15539060533046722,\n                6.792581558227539,\n                6.066262245178223])[SENSORS_LIST]","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.585142Z","iopub.execute_input":"2022-09-05T10:56:07.586247Z","iopub.status.idle":"2022-09-05T10:56:07.602081Z","shell.execute_reply.started":"2022-09-05T10:56:07.586217Z","shell.execute_reply":"2022-09-05T10:56:07.601069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create datasets","metadata":{}},{"cell_type":"code","source":"class DefectsDataset(torch.utils.data.Dataset):\n    def __init__(self, exp_nums=[1, 2, 3, 4], n=1000, sensors_list=SENSORS_LIST, mode='train'):\n        self.inputs = []\n        self.targets = []\n        self.n = n\n        self.sensors_list = sensors_list\n        \n        files = os.listdir(ROOT_DIR)\n        for file in files:\n            if mode == 'train' and 'data' in file and int(file.split('_')[1]) in exp_nums:\n                class_index = int(file.split('_')[2][:-4]) - 1\n                self.load_csv(class_index, os.path.join(ROOT_DIR, file))\n\n            elif mode == 'submit' and 'Series' in file and int(file.split('_')[1][:-4]) in exp_nums:\n                class_index = 0\n                self.load_csv(class_index, os.path.join(ROOT_DIR, file))\n        \n        self.inputs = torch.from_numpy(np.array(self.inputs).astype(np.float32))\n        self.targets = torch.from_numpy(np.array(self.targets).astype(np.int64))\n        \n        for i in range(len(MEAN)):\n            self.inputs[:, i] = (self.inputs[:, i] - MEAN[i]) / STD[i]\n    \n    def load_csv(self, class_index, path):\n        df = pd.read_csv(path)\n        df = df.to_numpy().astype(np.float32).T\n        df[8, :] = signal.savgol_filter(self.smoothing(df[8, :]), 1501, 1)\n        df[9, :] = signal.savgol_filter(self.smoothing(df[9, :]), 1501, 1)\n        for i in range(df.shape[1] // self.n):\n            self.inputs.append(df[self.sensors_list, i * self.n:(i + 1) * self.n])\n            self.targets.append(class_index)\n    \n    @staticmethod\n    def smoothing(array):\n        for i in range(len(array) // 1000):\n            try:\n                array[i * 1000:(i + 1) * 1000] = np.linspace(array[i * 1000], array[(i + 1) * 1000], 1000)\n            except IndexError:\n                array[i * 1000:(i + 1) * 1000] = np.linspace(array[i * 1000], array[((i + 1) * 1000) - 1], 1000)\n        return array\n                \n    def __getitem__(self, index):\n        return self.inputs[index], self.targets[index]\n    \n    def __len__(self):\n        return len(self.targets)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.605344Z","iopub.execute_input":"2022-09-05T10:56:07.606838Z","iopub.status.idle":"2022-09-05T10:56:07.622048Z","shell.execute_reply.started":"2022-09-05T10:56:07.606800Z","shell.execute_reply":"2022-09-05T10:56:07.621021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create callback for train process","metadata":{}},{"cell_type":"code","source":"class EarlyStop():\n    \"\"\"Callback for early stop train process.\n    \n    Args:\n        monitor (str): value for monitoring.\n        patience (int): Number of epochs with no improvement after which training will be stopped.\n        mode (str): One of {\"min\", \"max\"}. In min mode, training will stop when the quantity monitored has stopped decreasing.\n            In \"max\" mode it will stop when the quantity monitored has stopped increasing.\n    \n    Attributes:\n        history (dict): Dict of lists with train history. Key \"monitor\" contains list of monitoring values. \n        steps (int): Number of passed epoches. \n        best_step (int): Number of best epoch. \n        best_monitor (float): Best of monitoring value.\n        model (Model): Training model\n    \"\"\"\n    \n    def __init__(self, monitor, patience, mode):\n        self.monitor = monitor\n        self.patience = patience\n        self.mode = mode\n        self.history = None\n        self.steps = -1\n        self.best_step = -1\n        self.best_monitor = None\n            \n    def start(self, history, model):\n        \"\"\"Start and init callback.\n        \n        Args:\n            history (dict): Dict of lists with train history. Key \"monitor\" contains list of monitoring values. \n            model (Model): Training model.\n        \"\"\"\n        \n        self.history = history\n        self.model = model\n        \n    def step(self):\n        \"\"\"Make a step of callback.\n        \n        Returns:\n            tuple: (event, stop):\n                event (str): Decription of event. If event not did not happen then event = ''.\n                stop (bool): Flag of stopping train process.\n        \"\"\"\n        \n        self.steps += 1\n        \n        if self.mode == 'max':\n            if self.best_monitor is None or self.history[self.monitor][-1] > self.best_monitor:\n                self.best_monitor = self.history[self.monitor][-1]\n                self.best_step = self.steps\n        elif self.mode == 'min':\n            if self.best_monitor is None or self.history[self.monitor][-1] < self.best_monitor:\n                self.best_monitor = self.history[self.monitor][-1]\n                self.best_step = self.steps\n        \n        if self.steps - self.best_step > self.patience:\n            return 'Early stop with {}: {:.4f}'.format(self.monitor, self.history[self.monitor][self.best_step]), True\n        return None, False\n    \n    def stop(self):\n        \"\"\"Delete model from callback.\"\"\"\n        \n        self.model = None\n        torch.cuda.empty_cache()\n\n\nclass SaveBest():\n    \"\"\"Callback for save model if there is an improvement.\n    \n    Args:\n        monitor (str): value for monitoring.\n        model_path (str): Path for saving model.\n        mode (str): One of {\"min\", \"max\"}. In min mode, training will stop when the quantity monitored has stopped decreasing.\n            In \"max\" mode it will stop when the quantity monitored has stopped increasing.\n    \n    Attributes:\n        history (dict): Dict of lists with train history. Key \"monitor\" contains list of monitoring values. \n        steps (int): Number of passed epoches. \n        best_step (int): Number of best epoch. \n        best_monitor (float): Best of monitoring value.\n        model (Model): Training model\n    \"\"\"\n    \n    def __init__(self, monitor, model_path, mode):\n        self.monitor = monitor\n        self.model_path = model_path\n        self.mode = mode\n        self.history = None\n        self.steps = -1\n        self.best_step = -1\n        self.best_monitor = None\n    \n    def start(self, history, model):\n        \"\"\"Start and init callback. Save first version of model.\n        \n        Args:\n            history (dict): Dict of lists with train history. Key \"monitor\" contains list of monitoring values. \n            model (Model): Training model\n        \"\"\"\n        \n        self.history = history\n        self.model = model\n        torch.save(self.model.state_dict(), self.model_path)\n    \n    def step(self):\n        \"\"\"Make a step of callback.\n        \n        Returns:\n            tuple: (event, stop):\n                event (str): Decription of event. If event not did not happen then event = ''.\n                stop (bool): Flag of stopping train process.\n        \"\"\"\n        \n        self.steps += 1\n        \n        if self.mode == 'max':\n            if self.best_monitor is None or self.history[self.monitor][-1] > self.best_monitor:\n                self.best_monitor = self.history[self.monitor][-1]\n                self.best_step = self.steps\n        elif self.mode == 'min':\n            if self.best_monitor is None or self.history[self.monitor][-1] < self.best_monitor:\n                self.best_monitor = self.history[self.monitor][-1]\n                self.best_step = self.steps\n        \n        if self.steps == self.best_step:\n            torch.save(self.model.state_dict(), self.model_path)\n            return 'Save model with {}: {:.4f}'.format(self.monitor, self.history[self.monitor][self.best_step]), False\n        return None, False\n    \n    def stop(self):\n        \"\"\"Delete model from callback.\"\"\"\n        \n        self.model = None\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.623955Z","iopub.execute_input":"2022-09-05T10:56:07.624401Z","iopub.status.idle":"2022-09-05T10:56:07.646650Z","shell.execute_reply.started":"2022-09-05T10:56:07.624350Z","shell.execute_reply":"2022-09-05T10:56:07.645431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions for work with model","metadata":{}},{"cell_type":"code","source":"def predict(loader, model):\n    \"\"\"Make model prediction on loader.\n\n    Args:\n        model (Model): Model for training.\n        loader (DataLoader): loader with data.\n\n    Returns:\n        tuple: (act_classes, pred_classes):\n            outputs_all (torch.Tensor): Tensor of all outputs.\n            labels_all (torch.Tensor): Tensor of actual labels.\n    \"\"\"\n    model.eval()\n\n    labels_all = None\n    outputs_all = None\n\n    for inputs, labels in iter(loader):\n        if torch.cuda.is_available():\n            inputs, labels = inputs.to('cuda'), labels.to('cuda')\n\n        outputs = model.forward(inputs)\n        if labels_all is None:\n            labels_all = labels.cpu()\n            outputs_all = outputs.cpu()\n\n        else:\n            labels_all = torch.cat((labels_all, labels.cpu()))\n            outputs_all = torch.cat((outputs_all, outputs.cpu()), 0)\n\n    model = None\n    inputs = None\n    labels = None\n    outputs = None\n    sm = None\n\n    torch.cuda.empty_cache()\n\n    return outputs_all, labels_all\n\n\ndef evaluate(outputs, labels):\n    \"\"\"Make model evaluation on outputs and labels.\n\n    Args:\n        outputs (torch.Tensor): Tensor of outputs.\n        labels (torch.Tensor): Tensor of actual labels.\n\n    Returns:\n        tuple: (criterion, metric):\n            criterion (float): Calculated criterion.\n            metric (float): Calculated metric.\n    \"\"\"\n    sm = torch.nn.Softmax(dim=1)\n    predictions = sm(outputs).argmax(1)\n    sm = None\n    return CRITERION(outputs, labels), METRIC(labels, predictions)\n\n\ndef train(model, train_loader, val_loader, optimizer, criterion, metric_fn, metric_name,\n                     epochs=20, print_every=10, callbacks=None, lr_scheduler=None):\n    \"\"\"Make train process.\n\n    Args:\n        model (Model): Model for training.\n        train_loader (DataLoader): loader with train data.\n        val_loader (DataLoader): loader with validation data.\n        optimizer (Optimizer): Optimizer.\n        criterion (callable): Function for loss calculation.\n        metric_fn (callable): Function for metric calculation.\n        metric_name (str): Name of metric.\n        epochs (int): Number of epoches.\n        print_every (int): Number of iteration for update statusbar.\n        callbacks (list): List of callbacks\n\n    Returns:\n        history (dict): Dict of lists with train history.\n    \"\"\"\n\n    history = {'Train loss': [], 'Val loss': [], 'Train {}'.format(metric_name): [], 'Val {}'.format(metric_name): []}\n\n    if callbacks:\n        for i in callbacks:\n            i.start(history, model)\n    \n    sm = torch.nn.Softmax(dim=1)\n\n    for e in range(epochs):\n        model.train()\n\n        running_loss = 0\n\n        stop = False\n\n        steps = 0\n        \n        running_metric = 0\n\n        train_print = ''\n        bar = tqdm(range(len(train_loader)), desc=\"Epoch {}/{}\".format(e + 1, epochs), postfix=train_print)\n\n        for inputs, labels in iter(train_loader):\n            steps += 1\n\n            if torch.cuda.is_available():\n                inputs, labels = inputs.to('cuda'), labels.to('cuda')\n\n            optimizer.zero_grad()\n            output = model.forward(inputs)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n\n            with torch.no_grad():\n                running_loss += loss.item()\n                probabilities = sm(output).argmax(dim=1).cpu()\n\n                running_metric += metric_fn(labels.cpu(), probabilities)\n\n            if steps % print_every == 0:\n                train_print = \"Train loss: {:.4f}, Train {}: {:.4f}\".format(running_loss / steps, metric_name,\n                                                                            running_metric / steps)\n                bar.postfix = train_print\n                model.train()\n\n            if steps != len(train_loader):\n                bar.update()\n\n        if lr_scheduler:\n            lr_scheduler.step()\n\n        with torch.no_grad():\n            val_loss, val_metric = evaluate(*predict(val_loader, model))\n\n\n        train_print = \"Train loss: {:.4f}, Val loss: {:.4f}, Train {}: {:.4f}, Val {}: {:.4f}\".format(\n            running_loss / steps,\n            val_loss,\n            metric_name,\n            running_metric / steps,\n            metric_name,\n            val_metric)\n\n        history['Train loss'].append(running_loss / steps)\n        history['Train {}'.format(metric_name)].append(running_metric / steps)\n        history['Val loss'].append(val_loss.item())\n        history['Val {}'.format(metric_name)].append(val_metric)\n\n        if callbacks:\n            for i in callbacks:\n                state_text, state = i.step()\n                if state_text:\n                    train_print += ', ' + state_text\n                if state:\n                    stop = True\n\n        bar.postfix = train_print\n        bar.update()\n        bar.close()\n\n        if stop:\n            if callbacks:\n                for i in callbacks:\n                    i.stop()\n            model = None\n            inputs = None\n            labels = None\n            outputs = None\n            loss = None\n            sm = None\n\n            torch.cuda.empty_cache()\n\n            break\n\n        inputs = None\n        labels = None\n        outputs = None\n        loss = None\n\n        torch.cuda.empty_cache()\n\n    if callbacks:\n        for i in callbacks:\n            i.stop()\n\n    model = None\n\n    torch.cuda.empty_cache()\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.648504Z","iopub.execute_input":"2022-09-05T10:56:07.648896Z","iopub.status.idle":"2022-09-05T10:56:07.674323Z","shell.execute_reply.started":"2022-09-05T10:56:07.648854Z","shell.execute_reply":"2022-09-05T10:56:07.673329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_samples(dataset, title):\n    \"\"\"Show samples of dataset with label.\n\n    Args:\n        dataset (ImageFolder): ImageFolder dataset.\n        title (str): Title of images.\n    \"\"\"\n    fig, ax = plt.subplots(nrows=SENSORS_NUM, ncols=CLASSES_NUM, figsize=(20, 25))\n    indices = torch.randperm(len(dataset))\n    colors = ['brown', 'r', 'orange', 'y', 'g', 'b', 'darkblue', 'magenta', 'grey', 'black']\n    for i in range(CLASSES_NUM):\n        for ind in indices:\n            if dataset[ind][1] == i:\n                break\n        for sensor_ind in range(SENSORS_NUM):\n            ax[sensor_ind, i].plot(dataset[ind][0][sensor_ind], c=colors[sensor_ind])\n            ax[sensor_ind, i].set_title(title +\n                                        '\\nClass: ' + str(int(dataset[ind][1])) +\n                                        '\\nSensor: ' + str(SENSORS_NAMES[sensor_ind]))\n\n    plt.subplots_adjust(hspace=.4)\n    plt.show()\n    \ndef show_predictions(model, dataset, title):\n    \"\"\"Show samples of dataset with label and predictions.\n    \n    Args:\n        dataset (ImageFolder): ImageFolder dataset.\n        title (str): Title of images.\n    \"\"\"\n    fig, ax = plt.subplots(nrows=SENSORS_NUM, ncols=CLASSES_NUM, figsize=(20, 25))\n    sm = torch.nn.Softmax(dim=1)\n    indices = torch.randperm(len(dataset))\n    colors = ['brown', 'r', 'orange', 'y', 'g', 'b', 'darkblue', 'magenta', 'grey', 'black']\n    for i in range(CLASSES_NUM):\n        for ind in indices:\n            if dataset[ind][1] == i:\n                break\n        with torch.no_grad():\n            example = dataset[ind][0].unsqueeze(0)\n            if torch.cuda.is_available():\n                example = example.to('cuda')\n            pred = sm(model(example)).argmax(1)\n            \n        for sensor_ind in range(SENSORS_NUM):\n            ax[sensor_ind, i].plot(dataset[ind][0][sensor_ind], c=colors[sensor_ind])\n            ax[sensor_ind, i].set_title(title +\n                                        '\\nClass: ' + str(int(dataset[ind][1])) +\n                                        '\\nPrediction: ' + str(pred.item()) +\n                                        '\\nSensor: ' + str(SENSORS_NAMES[sensor_ind]))\n\n    plt.subplots_adjust(hspace=.4)\n    plt.show()\n\ndef plot_train_history(history, metric_name):\n    \"\"\"Plot train history.\n\n    Args:\n        history (dict): Dict of lists with train history..\n    \"\"\"\n\n    fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(17, 10))\n\n    ax[0].plot(history['Train loss'], c='r')\n    ax[0].plot(history['Val loss'], c='g')\n    ax[0].set_title('Loss')\n    ax[0].set_xlabel('Epochs')\n    ax[0].set_ylabel('Loss')\n    ax[0].set_yscale('log')\n    ax[0].legend(['Train', 'Val'])\n\n    ax[1].plot(history['Train {}'.format(metric_name)], c='r')\n    ax[1].plot(history['Val {}'.format(metric_name)], c='g')\n    ax[1].set_title(metric_name)\n    ax[1].set_xlabel('Epochs')\n    ax[1].set_ylabel(metric_name)\n    ax[1].legend(['Train', 'Val'])\n\n    plt.show()\n    \ndef plot_cm(outputs, labels, title):\n    \"\"\"Show confusion matrix for outputs and labels.\n\n    Args:\n        outputs (torch.Tensor): Tensor of outputs.\n        labels (torch.Tensor): Tensor of actual labels.\n    \"\"\"\n\n    sm = torch.nn.Softmax(dim=1)\n    predictions = sm(outputs).argmax(1)\n\n    cm = confusion_matrix(labels, predictions)\n    plot_confusion_matrix(cm, figsize=(5, 5), cmap=plt.cm.Blues)\n    plt.xticks(range(CLASSES_NUM), [i for i in range(CLASSES_NUM)])\n    plt.yticks(range(CLASSES_NUM), [i for i in range(CLASSES_NUM)])\n    plt.xlabel('Predicted Label')\n    plt.ylabel('True Label')\n    plt.title(title + \" CM\")\n    plt.show()\n\n    \ndef dict2str(dict1):\n    out = str(dict1).replace(\"}\", \"\")\n    out = str(out).replace(\"{\", \"\")\n    out = str(out).replace(\"\\\"\", \"\")\n    out = str(out).replace(\"\\'\", \"\")\n    out = str(out).replace(\":\", \"\")\n    return out","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.675888Z","iopub.execute_input":"2022-09-05T10:56:07.676402Z","iopub.status.idle":"2022-09-05T10:56:07.703656Z","shell.execute_reply.started":"2022-09-05T10:56:07.676350Z","shell.execute_reply":"2022-09-05T10:56:07.702308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class MLP(nn.Module):\n    def __init__(self, hidden_layer_size, hidden_layers, num_classes, frame_length):\n        super(MLP, self).__init__()\n        layers = []\n        for i in range(hidden_layers):\n            if not i:\n                layers += [nn.Linear(frame_length * SENSORS_NUM, hidden_layer_size, bias=False),\n                            nn.BatchNorm1d(hidden_layer_size),\n                            nn.ReLU(inplace=True)]\n            else:\n                layers += [nn.Linear(hidden_layer_size, hidden_layer_size, bias=False),\n                            nn.BatchNorm1d(hidden_layer_size),\n                            nn.ReLU(inplace=True)]\n            \n        layers.append(nn.Linear(hidden_layer_size, num_classes))\n                      \n        self.classifer = nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = torch.flatten(x, 1)\n        x = self.classifer(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.705110Z","iopub.execute_input":"2022-09-05T10:56:07.706165Z","iopub.status.idle":"2022-09-05T10:56:07.717674Z","shell.execute_reply.started":"2022-09-05T10:56:07.706128Z","shell.execute_reply":"2022-09-05T10:56:07.716632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"os.makedirs(WORK_DIR + 'models', exist_ok=True)\nos.makedirs(WORK_DIR + 'history', exist_ok=True)\n\nrun_record = {}\n\nframe_length_old = 0\n\nfor hyp in HYPS[:2]:\n\n    if frame_length_old != hyp['frame_length']:\n        train_dataset = DefectsDataset([1, 3, 5, 7, 9], hyp['frame_length'])\n        val_dataset = DefectsDataset([2, 4, 6], hyp['frame_length'])\n        test_dataset = DefectsDataset([8, 10], hyp['frame_length'])\n        \n        train_loader = DataLoader(train_dataset, batch_size = 64, shuffle = True)\n        val_loader = DataLoader(val_dataset, batch_size = 32)\n        test_loader = DataLoader(val_dataset, batch_size = 32)\n        \n    frame_length_old = hyp['frame_length']\n    \n    if hyp is HYPS[0]:\n        show_samples(train_dataset, 'Train')\n        show_samples(val_dataset, 'Val')\n        \n    print((dict2str(hyp)))\n\n    model_path = (f'{WORK_DIR}models/MLP_{dict2str(hyp)}.pth')\n\n    model = MLP(hyp['hidden_layer_size'], hyp['hidden_layers'], CLASSES_NUM, hyp['frame_length'])\n          \n    if torch.cuda.is_available():\n        model = model.to('cuda')\n        \n    callbacks = [EarlyStop(f'Val {METRIC_NAME}', 5, 'max'), SaveBest(f'Val {METRIC_NAME}', model_path, 'max')]\n    \n    optimizer = optim.Adam(model.parameters(), lr=hyp['learning_rate'], weight_decay=hyp['reg_coef'])\n\n    history = train(model, train_loader, val_loader, optimizer, CRITERION,\n                    metric_fn = METRIC, metric_name = METRIC_NAME,\n                    epochs = EPOCHS_NUM, callbacks = callbacks)\n\n    run_record[model_path] = {'hyperparams':hyp,\n                              'history': history,\n                              'final_val_metric': callbacks[1].best_monitor}\n    \n    with open(f'{WORK_DIR}history/MLP.json', 'w') as fp:\n        json.dump(run_record, fp)\n\n    model = None\n    optimizer = None\n\n    torch.cuda.empty_cache()\n\n\nbest_val_metric = None\nbest_hyperparams = None\nbest_run = None\nbest_model_path = None\n\nfor key, train_info in run_record.items():\n    if best_val_metric is None or best_val_metric < train_info['final_val_metric']:\n        best_val_metric = train_info['final_val_metric']\n        best_hyperparams = train_info['hyperparams']\n        best_run = train_info\n        best_model_path = key\n                              \ntrain_dataset = DefectsDataset([1, 3, 5, 7, 9], best_hyperparams['frame_length'])\nval_dataset = DefectsDataset([2, 4, 6], best_hyperparams['frame_length'])\ntest_dataset = DefectsDataset([8, 10], best_hyperparams['frame_length'])\n\ntrain_loader = DataLoader(train_dataset, batch_size = 32)\nval_loader = DataLoader(val_dataset, batch_size = 32)\ntest_loader = DataLoader(test_dataset, batch_size = 32)\n\nwith open(f'{WORK_DIR}history/MLP.json', 'w') as fp:\n    json.dump(run_record, fp)\n        \nprint(\"Best val metric: %4.4f, best hyperparams: %s\" % (best_val_metric, dict2str(best_hyperparams)))\n\nmodel = MLP(best_hyperparams[\"hidden_layer_size\"], best_hyperparams[\"hidden_layers\"],\n            CLASSES_NUM, best_hyperparams['frame_length'])\n\nmodel.load_state_dict(torch.load(best_model_path))\ntorch.save(model.state_dict(), f'{WORK_DIR}models/MLP.pth')\nif torch.cuda.is_available():\n    model = model.to('cuda')","metadata":{"execution":{"iopub.status.busy":"2022-09-05T10:56:07.719606Z","iopub.execute_input":"2022-09-05T10:56:07.720022Z","iopub.status.idle":"2022-09-05T11:06:02.602985Z","shell.execute_reply.started":"2022-09-05T10:56:07.719987Z","shell.execute_reply":"2022-09-05T11:06:02.600891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Print model","metadata":{}},{"cell_type":"code","source":"print(model)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T11:06:02.606967Z","iopub.execute_input":"2022-09-05T11:06:02.607278Z","iopub.status.idle":"2022-09-05T11:06:02.613069Z","shell.execute_reply.started":"2022-09-05T11:06:02.607250Z","shell.execute_reply":"2022-09-05T11:06:02.612008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate model","metadata":{}},{"cell_type":"code","source":"with torch.no_grad():\n    preds_train = predict(train_loader, model)\n    train_loss, train_metric = evaluate(*preds_train)\n    preds_val = predict(val_loader, model)\n    val_loss, val_metric = evaluate(*preds_val)\n    preds_test = predict(test_loader, model)\n    test_loss, test_metric = evaluate(*preds_test)\n    \nprint(\"Train loss: {:.4f}\".format(train_loss))  \nprint(\"Val loss: {:.4f}\".format(val_loss))  \nprint(\"Test loss: {:.4f}\".format(test_loss))  \nprint(\"Train {}: {:.4f}\".format(METRIC_NAME, train_metric))\nprint(\"Val {}: {:.4f}\".format(METRIC_NAME, val_metric))\nprint(\"Test {}: {:.4f}\".format(METRIC_NAME, test_metric))","metadata":{"execution":{"iopub.status.busy":"2022-09-05T11:06:02.614757Z","iopub.execute_input":"2022-09-05T11:06:02.615568Z","iopub.status.idle":"2022-09-05T11:06:10.257324Z","shell.execute_reply.started":"2022-09-05T11:06:02.615527Z","shell.execute_reply":"2022-09-05T11:06:10.256011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot train history","metadata":{}},{"cell_type":"code","source":"plot_train_history(best_run['history'], \"accuracy\")","metadata":{"execution":{"iopub.status.busy":"2022-09-05T11:06:10.258917Z","iopub.execute_input":"2022-09-05T11:06:10.259315Z","iopub.status.idle":"2022-09-05T11:06:10.877299Z","shell.execute_reply.started":"2022-09-05T11:06:10.259277Z","shell.execute_reply":"2022-09-05T11:06:10.876369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show predictions","metadata":{}},{"cell_type":"code","source":"show_predictions(model, train_dataset, 'Train')\nshow_predictions(model, val_dataset, 'Val')\nshow_predictions(model, test_dataset, 'Test')","metadata":{"execution":{"iopub.status.busy":"2022-09-05T11:06:10.879152Z","iopub.execute_input":"2022-09-05T11:06:10.879560Z","iopub.status.idle":"2022-09-05T11:06:21.843214Z","shell.execute_reply.started":"2022-09-05T11:06:10.879516Z","shell.execute_reply":"2022-09-05T11:06:21.842281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot confusion matrix","metadata":{}},{"cell_type":"code","source":"plot_cm(*preds_train, 'Train')\nplot_cm(*preds_val, 'Val')\nplot_cm(*preds_test, 'Test')","metadata":{"execution":{"iopub.status.busy":"2022-09-05T11:06:21.844684Z","iopub.execute_input":"2022-09-05T11:06:21.845608Z","iopub.status.idle":"2022-09-05T11:06:22.474296Z","shell.execute_reply.started":"2022-09-05T11:06:21.845570Z","shell.execute_reply":"2022-09-05T11:06:22.472998Z"},"trusted":true},"execution_count":null,"outputs":[]}]}