{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# VGG Nets tutorial with Transfer learning","metadata":{}},{"cell_type":"markdown","source":"This notebooks is using **pretrained VGG Nets (vgg19_bn, vgg11_bn,...)**with Pytorch.","metadata":{}},{"cell_type":"markdown","source":"Ver2: Change model (vgg19_bn → vgg11_bn) gets higher score.(0.9675 → **0.97846**)","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport numpy as np\nimport copy\nimport re \nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nimport random\nimport os\nimport torch\nimport torchvision\nimport torchvision.transforms as transform\n#from efficientnet_pytorch import EfficientNet\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport cv2 \nfrom sklearn.metrics import classification_report\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nimport datetime","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:54:56.878381Z","iopub.execute_input":"2022-02-27T08:54:56.878733Z","iopub.status.idle":"2022-02-27T08:54:59.386876Z","shell.execute_reply.started":"2022-02-27T08:54:56.878660Z","shell.execute_reply":"2022-02-27T08:54:59.386154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:54:59.388460Z","iopub.execute_input":"2022-02-27T08:54:59.388711Z","iopub.status.idle":"2022-02-27T08:55:04.262331Z","shell.execute_reply.started":"2022-02-27T08:54:59.388677Z","shell.execute_reply":"2022-02-27T08:55:04.261584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.263598Z","iopub.execute_input":"2022-02-27T08:55:04.263831Z","iopub.status.idle":"2022-02-27T08:55:04.268600Z","shell.execute_reply.started":"2022-02-27T08:55:04.263799Z","shell.execute_reply":"2022-02-27T08:55:04.267874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_df.iloc[:, 1:]\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.270734Z","iopub.execute_input":"2022-02-27T08:55:04.271165Z","iopub.status.idle":"2022-02-27T08:55:04.295851Z","shell.execute_reply.started":"2022-02-27T08:55:04.271103Z","shell.execute_reply":"2022-02-27T08:55:04.295109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train_df.iloc[:, 0]\ny_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.296829Z","iopub.execute_input":"2022-02-27T08:55:04.297005Z","iopub.status.idle":"2022-02-27T08:55:04.304805Z","shell.execute_reply.started":"2022-02-27T08:55:04.296982Z","shell.execute_reply":"2022-02-27T08:55:04.303590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check to see if there is any evident data imbalance.","metadata":{}},{"cell_type":"code","source":"n_class = len(set(y_train))\n\nplt.rcParams['figure.figsize'] = (8, 5)\nplt.bar(y_train.value_counts().index, y_train.value_counts())\nplt.xticks(np.arange(n_class))\nplt.xlabel('Class', fontsize=8)\nplt.ylabel('Count', fontsize=8)\nplt.grid('on', axis='y')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.306092Z","iopub.execute_input":"2022-02-27T08:55:04.306345Z","iopub.status.idle":"2022-02-27T08:55:04.534781Z","shell.execute_reply.started":"2022-02-27T08:55:04.306314Z","shell.execute_reply":"2022-02-27T08:55:04.534034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df\nX_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.535895Z","iopub.execute_input":"2022-02-27T08:55:04.536306Z","iopub.status.idle":"2022-02-27T08:55:04.553674Z","shell.execute_reply.started":"2022-02-27T08:55:04.536267Z","shell.execute_reply":"2022-02-27T08:55:04.553054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, train_size=0.8, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.554680Z","iopub.execute_input":"2022-02-27T08:55:04.554992Z","iopub.status.idle":"2022-02-27T08:55:04.782115Z","shell.execute_reply.started":"2022-02-27T08:55:04.554954Z","shell.execute_reply":"2022-02-27T08:55:04.781372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape)\nprint(X_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.783270Z","iopub.execute_input":"2022-02-27T08:55:04.783922Z","iopub.status.idle":"2022-02-27T08:55:04.788617Z","shell.execute_reply.started":"2022-02-27T08:55:04.783880Z","shell.execute_reply":"2022-02-27T08:55:04.787948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{}},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, features, labels, transforms):\n        self.x = features\n        self.y = labels\n        self.transform = transforms\n        \n    def __len__(self):\n        return len(self.x)\n    \n    def __getitem__(self, idx):\n        return self.transform(self.x[idx]), self.y[idx]","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.791344Z","iopub.execute_input":"2022-02-27T08:55:04.791950Z","iopub.status.idle":"2022-02-27T08:55:04.798615Z","shell.execute_reply.started":"2022-02-27T08:55:04.791913Z","shell.execute_reply":"2022-02-27T08:55:04.797920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GetData(data, label=False):\n    data = data.to_numpy()\n    if label == True:\n        data = data.astype('uint8')\n        data = torch.from_numpy(data).type(torch.LongTensor)\n    else:\n        data = data.reshape(-1, 1, 28, 28)\n        data = data.astype('float32')\n        data = torch.from_numpy(data)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.799696Z","iopub.execute_input":"2022-02-27T08:55:04.800381Z","iopub.status.idle":"2022-02-27T08:55:04.807592Z","shell.execute_reply.started":"2022-02-27T08:55:04.800338Z","shell.execute_reply":"2022-02-27T08:55:04.806845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = GetData(X_train)\nX_val = GetData(X_val)\nX_test = GetData(X_test)\n\ny_train = GetData(y_train, label=True)\ny_val = GetData(y_val, label=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.808940Z","iopub.execute_input":"2022-02-27T08:55:04.809741Z","iopub.status.idle":"2022-02-27T08:55:04.926141Z","shell.execute_reply.started":"2022-02-27T08:55:04.809705Z","shell.execute_reply":"2022-02-27T08:55:04.925391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(3, 3, figsize=(12, 12))\naxies = axs.flat\nimages, labels = X_train, y_train\nplt.suptitle('number features')\nfor i, ax in enumerate(axs.flat): #axs.flatとは何か\n    image, label = images[i], labels[i]\n        \n    ax.imshow(image.view(28, 28), cmap='gray') #ax.imshowの引数\n    ax.set(xlabel='pixel number', title=f'{label}') ","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:04.927357Z","iopub.execute_input":"2022-02-27T08:55:04.927613Z","iopub.status.idle":"2022-02-27T08:55:05.919902Z","shell.execute_reply.started":"2022-02-27T08:55:04.927566Z","shell.execute_reply":"2022-02-27T08:55:05.919232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transforms","metadata":{}},{"cell_type":"code","source":"transforms = {   'train' :             transform.Compose([ transform.ToPILImage(),\n                                       transform.Resize((224, 224)),\n                                       transform.Grayscale(num_output_channels=3),\n                                       transform.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5),\n                                       transform.RandomAutocontrast(p=0.5),\n                                       transform.RandomRotation(degrees=3),\n                                       transform.ToTensor(),\n                                       transform.Normalize(\n                                       [0.13141274452209473, 0.13141274452209473, 0.13141274452209473],\n                                       [0.30904173851013184, 0.30904173851013184, 0.30904173851013184]),\n]),\n                   \n                    'val'   :           transform.Compose([ transform.ToPILImage(),\n                                       transform.Resize((224, 224)),\n                                       transform.Grayscale(num_output_channels=3),\n                                       transform.ToTensor(),\n                                       transform.Normalize(\n                                       [0.13141274452209473, 0.13141274452209473, 0.13141274452209473],\n                                       [0.30904173851013184, 0.30904173851013184, 0.30904173851013184]),\n])}","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:05.921299Z","iopub.execute_input":"2022-02-27T08:55:05.921528Z","iopub.status.idle":"2022-02-27T08:55:05.929345Z","shell.execute_reply.started":"2022-02-27T08:55:05.921495Z","shell.execute_reply":"2022-02-27T08:55:05.928400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = MyDataset(X_train, y_train, transforms=transforms['train'])\nval_dataset   = MyDataset(X_val, y_val, transforms=transforms['val'])","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:05.930857Z","iopub.execute_input":"2022-02-27T08:55:05.931108Z","iopub.status.idle":"2022-02-27T08:55:05.939051Z","shell.execute_reply.started":"2022-02-27T08:55:05.931074Z","shell.execute_reply":"2022-02-27T08:55:05.938410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    MODEL        = 'vgg11'\n    NUM_EPOCHS   = 15\n    IMG_SIZE     = 224\n    BATCH_SIZE   = 32\n    OPTIMIZER    = 'Adam'\n    SCHEDULER    = 'StepLR'\n    LR           = 0.001\n    BETAS        =  (0.9, 0.999)\n    EPS          = 1e-08\n    WEIGHT_DECAY = 0\n    STEP_SIZE    = 2\n    GAMMA        = 0.1\n    SHUFFLE      = True\n    SEED         = 42\n    EXP_NAME     = 'Vgg11-224×224-aug'","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:05.940325Z","iopub.execute_input":"2022-02-27T08:55:05.940584Z","iopub.status.idle":"2022-02-27T08:55:05.951039Z","shell.execute_reply.started":"2022-02-27T08:55:05.940540Z","shell.execute_reply":"2022-02-27T08:55:05.950377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    print('> SEEDING DONE')\n    \nset_seed(CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:05.952618Z","iopub.execute_input":"2022-02-27T08:55:05.952837Z","iopub.status.idle":"2022-02-27T08:55:05.964332Z","shell.execute_reply.started":"2022-02-27T08:55:05.952813Z","shell.execute_reply":"2022-02-27T08:55:05.963470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DataLoader","metadata":{}},{"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_size=CFG.BATCH_SIZE, shuffle=CFG.SHUFFLE)\nval_loader = DataLoader(val_dataset, batch_size=CFG.BATCH_SIZE, shuffle=CFG.SHUFFLE)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:05.965696Z","iopub.execute_input":"2022-02-27T08:55:05.966025Z","iopub.status.idle":"2022-02-27T08:55:05.971534Z","shell.execute_reply.started":"2022-02-27T08:55:05.965986Z","shell.execute_reply":"2022-02-27T08:55:05.970757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:05.973025Z","iopub.execute_input":"2022-02-27T08:55:05.973455Z","iopub.status.idle":"2022-02-27T08:55:06.663324Z","shell.execute_reply.started":"2022-02-27T08:55:05.973419Z","shell.execute_reply":"2022-02-27T08:55:06.662566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load model(VGG11)","metadata":{}},{"cell_type":"code","source":"vgg11 = torchvision.models.vgg11(pretrained=True)\nfor param in vgg11.parameters():\n    param.grad_requires = False\n\nvgg11.classifier[6] = nn.Linear(4096, 10, bias=True)\nprint(vgg11)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:50.046010Z","iopub.execute_input":"2022-02-27T08:55:50.046426Z","iopub.status.idle":"2022-02-27T08:55:51.540789Z","shell.execute_reply.started":"2022-02-27T08:55:50.046384Z","shell.execute_reply":"2022-02-27T08:55:51.539940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = vgg11","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:55.191057Z","iopub.execute_input":"2022-02-27T08:55:55.191590Z","iopub.status.idle":"2022-02-27T08:55:55.195165Z","shell.execute_reply.started":"2022-02-27T08:55:55.191550Z","shell.execute_reply":"2022-02-27T08:55:55.194256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.classifier.parameters(), lr=CFG.LR, betas=CFG.BETAS, eps=CFG.EPS, weight_decay=CFG.WEIGHT_DECAY)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=CFG.STEP_SIZE, gamma=CFG.GAMMA)\n\n\n#scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.7, patience=3, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:56.746913Z","iopub.execute_input":"2022-02-27T08:55:56.747467Z","iopub.status.idle":"2022-02-27T08:55:56.753317Z","shell.execute_reply.started":"2022-02-27T08:55:56.747425Z","shell.execute_reply":"2022-02-27T08:55:56.752405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Functions","metadata":{}},{"cell_type":"code","source":"def train_epoch(loader):\n    total_loss = 0\n    training_loss = []\n    avg_loss = 0\n    total_num = 0\n    data = tqdm(loader)\n    model.train()\n    for inputs, label in data:\n        inputs, label = inputs.to(device), label.to(device)\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):\n            output = model(inputs)\n            loss = criterion(output, label)\n        label = label.to('cpu').detach().numpy()\n        total_loss += loss.item()\n        total_num += len(label)\n        loss.backward()\n        optimizer.step()\n        \n    avg_loss = total_loss / total_num\n    training_loss = training_loss.append(avg_loss)\n    print(f'Training_loss: {avg_loss}')\n    scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:55:58.972827Z","iopub.execute_input":"2022-02-27T08:55:58.973105Z","iopub.status.idle":"2022-02-27T08:55:58.979992Z","shell.execute_reply.started":"2022-02-27T08:55:58.973072Z","shell.execute_reply":"2022-02-27T08:55:58.979351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cal_val_accuracy(epoch, loader):\n    total_counts = 0\n    correct = 0\n    data = tqdm(loader)\n    if epoch == 0:\n        best_accuracy= 0\n    \n    model.eval()\n    for inputs, label in data:\n        with torch.no_grad():\n            inputs, label = inputs.to(device), label.to(device)\n            output = model(inputs)\n            _, pred = torch.max(output, 1)\n            total_counts += label.size(0)\n            correct += (pred == label).sum().item()\n    val_accuracy = 100 * (correct / total_counts)\n    print(f'val_accuracy : {val_accuracy}')\n    return val_accuracy","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:56:00.432163Z","iopub.execute_input":"2022-02-27T08:56:00.432761Z","iopub.status.idle":"2022-02-27T08:56:00.438889Z","shell.execute_reply.started":"2022-02-27T08:56:00.432722Z","shell.execute_reply":"2022-02-27T08:56:00.438242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"num_epochs = CFG.NUM_EPOCHS\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nmodel.to(device)\nbest_accuracy = 0\nfor epoch in range(num_epochs):\n    train_epoch(train_loader)\n    val_accuracy = cal_val_accuracy(epoch=epoch, loader=val_loader)\n    if val_accuracy >= best_accuracy:\n        print(f'val_accuracy Improved ! {best_accuracy} ---> {val_accuracy}')\n        best_accuracy = val_accuracy\n        best_epoch = epoch\n        PATH = f\"best_epoch-{epoch+1}.pth\"\n        best_model = copy.deepcopy(model.state_dict())","metadata":{"execution":{"iopub.status.busy":"2022-02-27T08:56:02.981794Z","iopub.execute_input":"2022-02-27T08:56:02.982055Z","iopub.status.idle":"2022-02-27T09:01:26.880652Z","shell.execute_reply.started":"2022-02-27T08:56:02.982021Z","shell.execute_reply":"2022-02-27T09:01:26.879282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Data","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, features, transforms):\n        self.x = features\n        self.transform = transforms\n    \n    def __len__(self):\n        return len(self.x)\n    \n    def __getitem__(self, idx):\n        return self.transform(self.x[idx])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(X_test, transforms=transforms['val'])\ntest_loader = DataLoader(test_dataset, batch_size=CFG.BATCH_SIZE, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmit_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(best_model) #Load best model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict","metadata":{}},{"cell_type":"code","source":"prediction = []\nmodel.eval()\ndata = tqdm(test_loader)\nfor inputs in data:\n    with torch.no_grad():\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        _, pred = torch.max(outputs, 1)\n        pred = pred.to('cpu').detach().numpy()\n        prediction.extend(pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pred = prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df.Label = final_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}