{"cells":[{"metadata":{"trusted":false,"_uuid":"356dc99ae65d2380d79d30fc1071c47e26f3508b"},"cell_type":"code","source":"from __future__ import print_function","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5e21eaf36771ab69af3ead5e41fc8db0070375aa"},"cell_type":"code","source":"%matplotlib inline\n%reload_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ec2c721c862a0106fdc37ec2260370c2e240c9d"},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.backends.cudnn as cudnn\nimport matplotlib.pyplot as plt\nimport torchvision\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom os.path import join, exists, expanduser\nfrom keras.layers import Flatten, Dense, Dropout, Reshape, Permute, Activation, \\\n    Input, merge, Lambda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ac19fbc4d4ebab114bfa490dae57852e6f828c0"},"cell_type":"code","source":"use_cuda = torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"957b12b9fc2caf165479d89895fb6c55037ac682"},"cell_type":"code","source":"!ls ../input/dogsdata/data/data","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b125cce7e5a6e391dca4ac49e9e992c1cca021d5"},"cell_type":"code","source":"%%writefile AlexNet.py\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\ndef conv_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        #nn.init.xavier_uniform(m.weight, gain=np.sqrt(2))\n        nn.init.normal_(m.weight, mean=0, std=1)\n        nn.init.constant(m.bias, 0)\n\nclass AlexNet(nn.Module):\n\n    def __init__(self, num_classes, inputs=3):\n        super(AlexNet, self).__init__()\n        self.features = nn.Sequential(\n            nn.Conv2d(inputs, 64, kernel_size=11, stride=4, padding=5),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n        )\n        self.classifier = nn.Linear(256, num_classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6a966eceb7fa8423db13c659493274885e46d6e"},"cell_type":"code","source":"%load AlexNet.py\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\ndef conv_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        #nn.init.xavier_uniform(m.weight, gain=np.sqrt(2))\n        nn.init.normal_(m.weight, mean=0, std=1)\n        nn.init.constant(m.bias, 0)\n\nclass AlexNet(nn.Module):\n\n    def __init__(self, num_classes, inputs=3):\n        super(AlexNet, self).__init__()\n        self.features = nn.Sequential(\n            nn.Conv2d(inputs, 64, kernel_size=11, stride=4, padding=5),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n        )\n        self.classifier = nn.Linear(256, num_classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d3ce7b0d5049037f5d27bc071292f2e0f251edc"},"cell_type":"code","source":"%%writefile train_test.py\nimport torch\nimport torch.optim as optim\nfrom torch.autograd import Variable\nimport config as cf\nimport time\nimport numpy as np\n\nuse_cuda = torch.cuda.is_available()\n\nbest_acc = 0\n\ndef train(epoch, net, trainloader, criterion):\n    net.train()\n    train_loss = 0\n    correct = 0\n    total = 0\n    optimizer = optim.Adam(net.parameters(), lr=cf.lr, weight_decay=5e-4)\n    train_loss_stacked = np.array([0])\n\n    print('\\n=> Training Epoch #%d, LR=%.4f' %(epoch, cf.lr))\n    for batch_idx, (inputs_value, targets) in enumerate(trainloader):\n        if use_cuda:\n            inputs_value, targets = inputs_value.cuda(), targets.cuda() # GPU settings\n        optimizer.zero_grad()\n        inputs_value, targets = Variable(inputs_value), Variable(targets)\n        outputs = net(inputs_value)               # Forward Propagation\n        loss = criterion(outputs, targets)  # Loss\n        loss.backward()  # Backward Propagation\n        optimizer.step() # Optimizer update\n\n        train_loss += loss.data[0]\n        _, predicted = torch.max(outputs.data, 1)\n        total += targets.size(0)\n        correct += predicted.eq(targets.data).cpu().sum()\n        train_loss_stacked = np.append(train_loss_stacked, loss.data[0].cpu().numpy())\n    print ('| Epoch [%3d/%3d] \\t\\tLoss: %.4f Acc@1: %.3f%%'\n                %(epoch, cf.num_epochs, loss.data[0], 100.*correct/total))\n\n    return train_loss_stacked\n\n\ndef test(epoch, net, testloader, criterion):\n    global best_acc\n    net.eval()\n    test_loss = 0\n    correct = 0\n    total = 0\n    test_loss_stacked = np.array([0])\n    for batch_idx, (inputs_value, targets) in enumerate(testloader):\n        if use_cuda:\n            inputs_value, targets = inputs_value.cuda(), targets.cuda()\n        with torch.no_grad():\n            inputs_value, targets = Variable(inputs_value), Variable(targets)\n        outputs = net(inputs_value)\n        loss = criterion(outputs, targets)\n\n        test_loss += loss.data[0]\n        _, predicted = torch.max(outputs.data, 1)\n        total += targets.size(0)\n        correct += predicted.eq(targets.data).cpu().sum()\n        test_loss_stacked = np.append(test_loss_stacked, loss.data[0].cpu().numpy())\n\n\n    # Save checkpoint when best model\n    acc = 100. * correct / total\n    print(\"\\n| Validation Epoch #%d\\t\\t\\tLoss: %.4f Acc@1: %.2f%%\" % (epoch, loss.data[0], acc))\n\n\n\n    if acc > best_acc:\n        best_acc = acc\n    print('* Test results : Acc@1 = %.2f%%' % (best_acc))\n\n    return test_loss_stacked\n\ndef start_train_test(net,trainloader, testloader, criterion):\n    elapsed_time = 0\n\n    for epoch in range(cf.start_epoch, cf.start_epoch + cf.num_epochs):\n        start_time = time.time()\n\n        train_loss = train(epoch, net, trainloader, criterion)\n        test_loss = test(epoch, net, testloader, criterion)\n\n        epoch_time = time.time() - start_time\n        elapsed_time += epoch_time\n        print('| Elapsed time : %d:%02d:%02d' % (get_hms(elapsed_time)))\n\n    return train_loss.tolist(), test_loss.tolist()\n\ndef get_hms(seconds):\n    m, s = divmod(seconds, 60)\n    h, m = divmod(m, 60)\n\n    return h, m, s\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8c27675aa1cb30346ccd41b777eb0e83ca82648"},"cell_type":"code","source":"%load train_test.py\nimport torch\nimport torch.optim as optim\nfrom torch.autograd import Variable\nimport config as cf\nimport time\nimport numpy as np\n\nuse_cuda = torch.cuda.is_available()\n\nbest_acc = 0\n\ndef train(epoch, net, trainloader, criterion):\n    net.train()\n    train_loss = 0\n    correct = 0\n    total = 0\n    optimizer = optim.Adam(net.parameters(), lr=cf.lr, weight_decay=5e-4)\n    train_loss_stacked = np.array([0])\n\n    print('\\n=> Training Epoch #%d, LR=%.4f' %(epoch, cf.lr))\n    for batch_idx, (inputs_value, targets) in enumerate(trainloader):\n        if use_cuda:\n            inputs_value, targets = inputs_value.cuda(), targets.cuda() # GPU settings\n        optimizer.zero_grad()\n        inputs_value, targets = Variable(inputs_value), Variable(targets)\n        outputs = net(inputs_value)               # Forward Propagation\n        loss = criterion(outputs, targets)  # Loss\n        loss.backward()  # Backward Propagation\n        optimizer.step() # Optimizer update\n\n        train_loss += loss.data[0]\n        _, predicted = torch.max(outputs.data, 1)\n        total += targets.size(0)\n        correct += predicted.eq(targets.data).cpu().sum()\n        train_loss_stacked = np.append(train_loss_stacked, loss.data[0].cpu().numpy())\n    print ('| Epoch [%3d/%3d] \\t\\tLoss: %.4f Acc@1: %.3f%%'\n                %(epoch, cf.num_epochs, loss.data[0], 100.*correct/total))\n\n    return train_loss_stacked\n\n\ndef test(epoch, net, testloader, criterion):\n    global best_acc\n    net.eval()\n    test_loss = 0\n    correct = 0\n    total = 0\n    test_loss_stacked = np.array([0])\n    for batch_idx, (inputs_value, targets) in enumerate(testloader):\n        if use_cuda:\n            inputs_value, targets = inputs_value.cuda(), targets.cuda()\n        with torch.no_grad():\n            inputs_value, targets = Variable(inputs_value), Variable(targets)\n        outputs = net(inputs_value)\n        loss = criterion(outputs, targets)\n\n        test_loss += loss.data[0]\n        _, predicted = torch.max(outputs.data, 1)\n        total += targets.size(0)\n        correct += predicted.eq(targets.data).cpu().sum()\n        test_loss_stacked = np.append(test_loss_stacked, loss.data[0].cpu().numpy())\n\n\n    # Save checkpoint when best model\n    acc = 100. * correct / total\n    print(\"\\n| Validation Epoch #%d\\t\\t\\tLoss: %.4f Acc@1: %.2f%%\" % (epoch, loss.data[0], acc))\n\n\n\n    if acc > best_acc:\n        best_acc = acc\n    print('* Test results : Acc@1 = %.2f%%' % (best_acc))\n\n    return test_loss_stacked\n\ndef start_train_test(net,trainloader, testloader, criterion):\n    elapsed_time = 0\n\n    for epoch in range(cf.start_epoch, cf.start_epoch + cf.num_epochs):\n        start_time = time.time()\n\n        train_loss = train(epoch, net, trainloader, criterion)\n        test_loss = test(epoch, net, testloader, criterion)\n\n        epoch_time = time.time() - start_time\n        elapsed_time += epoch_time\n        print('| Elapsed time : %d:%02d:%02d' % (get_hms(elapsed_time)))\n\n    return train_loss.tolist(), test_loss.tolist()\n\ndef get_hms(seconds):\n    m, s = divmod(seconds, 60)\n    h, m = divmod(m, 60)\n\n    return h, m, s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e794712c61bf4469007b3e323ee25696521c182f"},"cell_type":"code","source":"from train_test import start_train_test\nfrom AlexNet import AlexNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d86fed54a9077c987dc299ddf088464555f4d91"},"cell_type":"code","source":"%%writefile config.py\nstart_epoch = 1\nnum_epochs = 10\nbatch_size = 256\noptim_type = 'Adam'\nresize=32\nlr=0.001\n\nmean = {\n    'cifar10': (0.4914, 0.4822, 0.4465),\n    'cifar100': (0.5071, 0.4867, 0.4408),\n    'mnist': (0.1307,),\n    'stl10': (0.485, 0.456, 0.406),\n}\n\nstd = {\n    'cifar10': (0.2023, 0.1994, 0.2010),\n    'cifar100': (0.2675, 0.2565, 0.2761),\n    'mnist': (0.3081,),\n    'stl10': (0.229, 0.224, 0.225),\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31c23e3c79c6af186e67a0b610478093d4631134"},"cell_type":"code","source":"%load config.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d80312e65ede7227efebb761c07d9160215275e"},"cell_type":"code","source":"%%writefile transform.py\nimport torchvision.transforms as transforms\n\nimport config as cf\n\ndef transform_training():\n\n    transform_train = transforms.Compose([\n        transforms.Resize((cf.resize, cf.resize)),\n        transforms.RandomCrop(32, padding=4),\n        # transforms.RandomHorizontalFlip(),\n        # CIFAR10Policy(),\n        transforms.ToTensor(),\n    ])  # meanstd transformation\n\n    return transform_train\n\ndef transform_testing():\n\n    transform_test = transforms.Compose([\n        transforms.Resize((cf.resize, cf.resize)),\n        transforms.RandomCrop(32, padding=4),\n        # transforms.RandomHorizontalFlip(),\n        # CIFAR10Policy(),\n        transforms.ToTensor(),\n    ])\n\n    return transform_test\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb176ba9c185f7ea5ccec95e627a5838f6ef7284"},"cell_type":"code","source":"%load transform.py\nimport torchvision.transforms as transforms\n\nimport config as cf\n\ndef transform_training():\n\n    transform_train = transforms.Compose([\n        transforms.Resize((cf.resize, cf.resize)),\n        transforms.RandomCrop(32, padding=4),\n        # transforms.RandomHorizontalFlip(),\n        # CIFAR10Policy(),\n        transforms.ToTensor(),\n    ])  # meanstd transformation\n\n    return transform_train\n\ndef transform_testing():\n\n    transform_test = transforms.Compose([\n        transforms.Resize((cf.resize, cf.resize)),\n        transforms.RandomCrop(32, padding=4),\n        # transforms.RandomHorizontalFlip(),\n        # CIFAR10Policy(),\n        transforms.ToTensor(),\n    ])\n\n    return transform_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52942ef67e4b82afad01149fbad7bb7d3b9d931f"},"cell_type":"code","source":"import torch\nimport torchvision\nimport sys\nfrom transform import transform_training, transform_testing\nimport torchvision.datasets as datasets\nimport config as cf\nimport os\ndata_path = '../input/dogsdata/data/data'\nimport torchvision.transforms as transforms\ndef dataset(dataset_name):\n    if(dataset_name  == 'dog-breed'):\n         print(\"| Preparing dog-breed dataset...\")\n         trainset = datasets.ImageFolder(os.path.join(data_path, 'train'),transform_training())\n         testset = datasets.ImageFolder(os.path.join(data_path, 'test'),transform=transform_testing())                  \n         outputs = 16\n         inputs = 3\n    trainloader = torch.utils.data.DataLoader(trainset, batch_size=cf.batch_size, shuffle=True, num_workers=4)\n    testloader = torch.utils.data.DataLoader(testset, batch_size=cf.batch_size, shuffle=False, num_workers=4)\n\n    return trainloader, testloader, outputs, inputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"656b0c90628f5084d8b2117771ba32e30368683c"},"cell_type":"code","source":"data_dir = '../input/dog-breed-identification'\nINPUT_SIZE = 224\nNUM_CLASSES = 16\nSEED = 1987\nlabels = pd.read_csv(join(data_dir, 'labels.csv'))\nsample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d03894e90ed9e6b3625d03b8ada19e48c3acf12"},"cell_type":"code","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\ngroup = labels.groupby(by='breed', as_index=False).agg({'id': pd.Series.nunique})\n\ngroup = group.sort_values('id',ascending=False)\n\nprint(group)\n\nlabels['rank'] = group['breed']\n\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\nnp.random.seed(seed=SEED)\nrnd = np.random.random(len(labels))\ntrain_idx = rnd < 0.8\nvalid_idx = rnd >= 0.8\ny_train = labels_pivot[selected_breed_list].values\nytr = y_train[train_idx]\nyv = y_train[valid_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f7c253f0a9e20ac2bc2998b0bf7d377a3d060b2"},"cell_type":"code","source":"trainloader, testloader, outputs, inputs = dataset('dog-breed')\nprint ('Output classes: {}\\nInput channels: {}'.format(outputs, inputs))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca4967de6810d35619495e7faa44c608107fb484"},"cell_type":"code","source":"def imshow(inp, title=None):\n    \"\"\"Imshow for Tensor.\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    inp = np.clip(inp, 0, 1)\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001)  # pause a bit so that plots are updated\n\n\n# Get a batch of training data\ninputdata, classes = next(iter(trainloader))\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputdata)\n\nimshow(out)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"722bedd8cf484cac5d5d383234fc22b573e3f529"},"cell_type":"code","source":"net = AlexNet(num_classes = outputs, inputs=inputs)\nfile_name = 'alexnet-'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"732bf506a68204fde8c4f5fbb88956867e69e7f6"},"cell_type":"code","source":"if use_cuda:\n    net.cuda()\n    net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))\n    cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43f102e1349761b9f8b63a0b48eb1767bfdb84c4"},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1f6fafb542c964dd4a9227af54720f648c3b2e3"},"cell_type":"code","source":"train_loss, test_loss = start_train_test(net, trainloader, testloader, criterion)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12fb3854e559b864aa0500939e3468bde30ec70c"},"cell_type":"code","source":"plt.plot(train_loss)\nplt.ylabel('Train Loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e5e536c74d6b1ff94ba496a64075bcd89195702"},"cell_type":"code","source":"plt.plot(test_loss)\nplt.ylabel('Test Loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5eb548d2f9e0e20b74ab2710c9d18992934cdad6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}