{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":77476,"databundleVersionId":8462666,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. introduction\n- data : fashion mnist\n- model : vision transformer\n- tag : PCA, viT, CNN\n","metadata":{}},{"cell_type":"code","source":"import random\nimport torch\nimport numpy as np\nimport os\n\nrandom_seed= 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:34:22.087416Z","iopub.execute_input":"2024-05-21T05:34:22.087792Z","iopub.status.idle":"2024-05-21T05:34:22.099344Z","shell.execute_reply.started":"2024-05-21T05:34:22.087761Z","shell.execute_reply":"2024-05-21T05:34:22.098422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:34:24.166450Z","iopub.execute_input":"2024-05-21T05:34:24.167229Z","iopub.status.idle":"2024-05-21T05:34:24.171052Z","shell.execute_reply.started":"2024-05-21T05:34:24.167197Z","shell.execute_reply":"2024-05-21T05:34:24.170107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/2024-smarcle-ks-3-fashion-mnist2/train.csv')\ntest_df = pd.read_csv('/kaggle/input/2024-smarcle-ks-3-fashion-mnist2/test.csv')\nsubmit = pd.read_csv('/kaggle/input/2024-smarcle-ks-3-fashion-mnist2/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:14.752674Z","iopub.execute_input":"2024-05-21T05:33:14.753693Z","iopub.status.idle":"2024-05-21T05:33:22.450022Z","shell.execute_reply.started":"2024-05-21T05:33:14.753655Z","shell.execute_reply":"2024-05-21T05:33:22.448973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:22.451130Z","iopub.execute_input":"2024-05-21T05:33:22.451409Z","iopub.status.idle":"2024-05-21T05:33:22.484188Z","shell.execute_reply.started":"2024-05-21T05:33:22.451385Z","shell.execute_reply":"2024-05-21T05:33:22.483230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape, test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:22.486194Z","iopub.execute_input":"2024-05-21T05:33:22.486725Z","iopub.status.idle":"2024-05-21T05:33:22.491407Z","shell.execute_reply.started":"2024-05-21T05:33:22.486699Z","shell.execute_reply":"2024-05-21T05:33:22.490393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to numpy\nx_train = train_df.drop(['label'], axis=1).values\ny_train = train_df['label'].values\nx_test = test_df.values","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:22.492758Z","iopub.execute_input":"2024-05-21T05:33:22.493084Z","iopub.status.idle":"2024-05-21T05:33:22.619697Z","shell.execute_reply.started":"2024-05-21T05:33:22.493053Z","shell.execute_reply":"2024-05-21T05:33:22.618754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to img format\nx_train = x_train.reshape(-1, 28, 28)\nx_test = x_test.reshape(-1, 28, 28)\nprint(x_train.shape, x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:22.620877Z","iopub.execute_input":"2024-05-21T05:33:22.621127Z","iopub.status.idle":"2024-05-21T05:33:22.626497Z","shell.execute_reply.started":"2024-05-21T05:33:22.621104Z","shell.execute_reply":"2024-05-21T05:33:22.625657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. visualization","metadata":{}},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:22.627536Z","iopub.execute_input":"2024-05-21T05:33:22.627793Z","iopub.status.idle":"2024-05-21T05:33:23.111736Z","shell.execute_reply.started":"2024-05-21T05:33:22.627770Z","shell.execute_reply":"2024-05-21T05:33:23.110946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = x_train[0]\nplt.imshow(img, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:23.112751Z","iopub.execute_input":"2024-05-21T05:33:23.112996Z","iopub.status.idle":"2024-05-21T05:33:23.419923Z","shell.execute_reply.started":"2024-05-21T05:33:23.112975Z","shell.execute_reply":"2024-05-21T05:33:23.419088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker','Bag','Ankle boot']\nsample_size = 9\nrandom_idx = np.random.randint(60000, size=sample_size)\n\nplt.figure(figsize=(5,5))\nfor i, idx in enumerate(random_idx):\n  plt.subplot(3,3,i+1)\n  plt.xticks([])\n  plt.yticks([])\n  plt.imshow(x_train[idx], cmap='gray')\n  plt.xlabel(class_names[y_train[idx]])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:33:23.421285Z","iopub.execute_input":"2024-05-21T05:33:23.421534Z","iopub.status.idle":"2024-05-21T05:33:23.887560Z","shell.execute_reply.started":"2024-05-21T05:33:23.421511Z","shell.execute_reply":"2024-05-21T05:33:23.886187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import FloatTensor, LongTensor\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\n\n# validation data split\nX_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train, random_state=random_seed, stratify=y_train, test_size=0.3)\nprint(x_train.dtype, y_train.dtype, x_test.dtype)\nprint(X_train.dtype, Y_train.dtype, X_valid.dtype)\n\n# numpy to tensor\nX_train = FloatTensor(X_train)\nY_train = LongTensor(Y_train)\nX_valid = FloatTensor(X_valid)\nY_valid = LongTensor(Y_valid)\n\nprint(Y_train.dtype)\nx_train = FloatTensor(x_train)\ny_train = LongTensor(y_train)\nx_test = FloatTensor(x_test)\n\n# tensor to Dataset\ntrain_dataset_small = TensorDataset(X_train, Y_train)\nvalid_dataset = TensorDataset(X_valid, Y_valid)\ntrain_dataset = TensorDataset(x_train, y_train)\ntest_dataset = TensorDataset(x_test)\n\n# Dataset to DataLoader\ntrain_dataloader_small = DataLoader(train_dataset_small, batch_size=128)\nvalid_dataloader = DataLoader(valid_dataset, batch_size=128)\ntrain_dataloader = DataLoader(train_dataset, batch_size=128)\ntest_dataloader = DataLoader(test_dataset, batch_size=128, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:36:44.507931Z","iopub.execute_input":"2024-05-21T05:36:44.508442Z","iopub.status.idle":"2024-05-21T05:36:46.059008Z","shell.execute_reply.started":"2024-05-21T05:36:44.508409Z","shell.execute_reply":"2024-05-21T05:36:46.058111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Model","metadata":{}},{"cell_type":"code","source":"import torch\n\nclass DeepNeuralNework(torch.nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(DeepNeuralNework, self).__init__()\n        self.input_layer = torch.nn.Linear(in_features=input_dim, out_features=hidden_dim, bias=True)\n        self.hidden_layer1 = torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim, bias=True)\n        self.hidden_layer2 = torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim, bias=True)\n        self.hidden_layer3 = torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim, bias=True)\n        self.output_layer = torch.nn.Linear(in_features=hidden_dim, out_features=output_dim, bias=True)\n        self.activation = torch.nn.Tanh()\n    def forward(self, x):\n        x = self.input_layer(x)\n        x = self.activation(x)\n        x = self.hidden_layer1(x)\n        x = self.activation(x)\n        x = self.hidden_layer2(x)\n        x = self.activation(x)\n        x = self.hidden_layer3(x)\n        x = self.activation(x)\n        x = self.output_layer(x)\n        x = self.activation(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:37:32.867180Z","iopub.execute_input":"2024-05-21T05:37:32.867594Z","iopub.status.idle":"2024-05-21T05:37:32.877433Z","shell.execute_reply.started":"2024-05-21T05:37:32.867564Z","shell.execute_reply":"2024-05-21T05:37:32.876524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 최소 제곱 오차 함수\ndef loss_mse(pred, y):\n    y_onehot = torch.nn.functional.one_hot(y, num_classes=10).cuda()\n    loss = torch.mean((y_onehot-pred)**2)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:37:33.198681Z","iopub.execute_input":"2024-05-21T05:37:33.199414Z","iopub.status.idle":"2024-05-21T05:37:33.204203Z","shell.execute_reply.started":"2024-05-21T05:37:33.199384Z","shell.execute_reply":"2024-05-21T05:37:33.203298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def makeHistory():# 평가 지표\n    history = dict()\n    history['train_acc'] = list()\n    history['valid_acc'] = list()\n    history['train_loss'] = list()\n    history['valid_loss'] = list()\n    return history","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:37:50.756806Z","iopub.execute_input":"2024-05-21T05:37:50.757519Z","iopub.status.idle":"2024-05-21T05:37:50.762665Z","shell.execute_reply.started":"2024-05-21T05:37:50.757484Z","shell.execute_reply":"2024-05-21T05:37:50.761547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(dataloader, optimizer, model, loss_fn, log_dict):\n    acc = 0\n    loss = 0\n    model.train()\n    for X, y in dataloader:\n        X = X.view(-1, 784).cuda()\n        y = y.cuda()\n        \n        pred = model(X)\n        cost = loss_fn(pred, y)\n        \n        optimizer.zero_grad()\n        cost.backward()\n        optimizer.step()\n        \n        acc += torch.sum(torch.argmax(pred, dim=1)==y).item()\n        loss += cost.item()\n    \n    log_dict['train_acc'].append(acc / len(dataloader.dataset))\n    log_dict['train_loss'].append(loss)\n    \n    return model \n\ndef valid_epoch(dataloader, model, loss_fn, log_dict):\n    acc = 0\n    loss = 0\n    model.train()\n    for X, y in dataloader:\n        X = X.view(-1, 784).cuda()\n        y = y.cuda()\n        \n        pred = model(X)\n        cost = loss_fn(pred, y)\n        \n        acc += torch.sum(torch.argmax(pred, dim=1)==y).item()\n        loss += cost.item()\n    \n    log_dict['valid_acc'].append(acc / len(dataloader.dataset))\n    log_dict['valid_loss'].append(loss)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:38:10.771777Z","iopub.execute_input":"2024-05-21T05:38:10.772607Z","iopub.status.idle":"2024-05-21T05:38:10.782288Z","shell.execute_reply.started":"2024-05-21T05:38:10.772576Z","shell.execute_reply":"2024-05-21T05:38:10.781316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_model(model, dataloader):\n    y_test_pred = []\n    model.eval()\n    for X in dataloader:\n        X = X[0].view(-1, 784).cuda()\n        pred = model(X)\n        pred = torch.argmax(pred, dim=1).cpu().numpy()\n        print(pred)\n        y_test_pred.extend(pred)\n    return y_test_pred","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:39:36.440597Z","iopub.execute_input":"2024-05-21T05:39:36.441201Z","iopub.status.idle":"2024-05-21T05:39:36.447320Z","shell.execute_reply.started":"2024-05-21T05:39:36.441167Z","shell.execute_reply":"2024-05-21T05:39:36.446373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\nepoches = 30\n\nDNN = DeepNeuralNework(input_dim=784, hidden_dim=1024, output_dim=10)\nDNN = DNN.cuda()\nhistory = makeHistory()\n\n# optimizer\nAdam = torch.optim.Adam(DNN.parameters(), lr=0.001)\nSGD = torch.optim.SGD(DNN.parameters(), lr=0.01)\nAdagrad = torch.optim.Adagrad(DNN.parameters(), lr=0.001)\nAdadelta = torch.optim.Adadelta(DNN.parameters(), lr=0.001)\nRMSprop = torch.optim.RMSprop(DNN.parameters(), lr=0.001)\n\nfor iter in tqdm(range(epoches)):\n    DNN = train_epoch(train_dataloader_small, SGD, DNN, loss_mse, history)\n    with torch.no_grad():\n        DNN = valid_epoch(valid_dataloader, DNN, loss_mse, history)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:38:52.887628Z","iopub.execute_input":"2024-05-21T05:38:52.887973Z","iopub.status.idle":"2024-05-21T05:39:36.438885Z","shell.execute_reply.started":"2024-05-21T05:38:52.887946Z","shell.execute_reply":"2024-05-21T05:39:36.438002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib.gridspec import GridSpec\ndef print_log(loss_fn_name, history):\n    x = range(1, len(history['train_acc']) + 1)\n    gs = GridSpec(nrows=2, ncols=2)\n    fig = plt.figure(figsize=(10, 10))\n    ax1 = fig.add_subplot(gs[0, 0])\n    ax2 = fig.add_subplot(gs[0, 1])\n\n    # acc\n    ax1.set_title(\"accuracy\")\n    ax1.plot(x, history['train_acc'], 'r')\n    ax1.plot(x, history['valid_acc'], 'b')\n    ax1.legend(['train_acc', 'valid_acc'])\n    ax1.set_xlabel(\"epoches\")\n    ax1.set_ylabel(\"accuracy\")\n    \n    # loss\n    ax2.set_title(str(loss_fn_name) +\"_loss\")\n    ax2.plot(x, history['train_loss'], 'r--')\n    ax2.plot(x, history['valid_loss'], 'b--')\n    ax2.legend(['train_loss', 'valid_loss'])\n    ax2.set_xlabel(\"epoches\")\n    ax2.set_ylabel(\"loss\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:39:48.867103Z","iopub.execute_input":"2024-05-21T05:39:48.867442Z","iopub.status.idle":"2024-05-21T05:39:48.876483Z","shell.execute_reply.started":"2024-05-21T05:39:48.867417Z","shell.execute_reply":"2024-05-21T05:39:48.875482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_log(\"DNN MSE\", history)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:39:49.607026Z","iopub.execute_input":"2024-05-21T05:39:49.607468Z","iopub.status.idle":"2024-05-21T05:39:50.219312Z","shell.execute_reply.started":"2024-05-21T05:39:49.607434Z","shell.execute_reply":"2024-05-21T05:39:50.218360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    y_test_pred = test_model(DNN, test_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:39:53.387707Z","iopub.execute_input":"2024-05-21T05:39:53.388387Z","iopub.status.idle":"2024-05-21T05:39:53.593102Z","shell.execute_reply.started":"2024-05-21T05:39:53.388355Z","shell.execute_reply":"2024-05-21T05:39:53.592161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['label'] = y_test_pred\nsubmit.to_csv(\"submit.csv\", header=True, index=None)\nsubmit.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T05:40:01.063371Z","iopub.execute_input":"2024-05-21T05:40:01.063756Z","iopub.status.idle":"2024-05-21T05:40:01.123897Z","shell.execute_reply.started":"2024-05-21T05:40:01.063726Z","shell.execute_reply":"2024-05-21T05:40:01.122943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}