{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip install timm\nfrom IPython.display import Image as ImageIpython\nfrom IPython.core.display import HTML ","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:45.893478Z","iopub.status.busy":"2021-06-10T06:46:45.892842Z","iopub.status.idle":"2021-06-10T06:46:54.43031Z","shell.execute_reply":"2021-06-10T06:46:54.429739Z","shell.execute_reply.started":"2021-06-10T06:46:16.700297Z"},"papermill":{"duration":8.573929,"end_time":"2021-06-10T06:46:54.430471","exception":false,"start_time":"2021-06-10T06:46:45.856542","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Table of contents\n* [Initial Setup](#section-one)\n* [LR Schedulers & Question](#section-two)\n* [Mixup & Question](#section-three)\n* [Architectures & Question](#section-four)\n* [Mixed Precision & Question](#section-five)\n* [Label Smoothing & Question](#section-six)","metadata":{"papermill":{"duration":0.024071,"end_time":"2021-06-10T06:46:54.479207","exception":false,"start_time":"2021-06-10T06:46:54.455136","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"Imagenet improvement on Resnet-50 using tricks\")\nImageIpython(url= \"https://miro.medium.com/max/890/1*4y2Rdl0i-kvNh2SCMZZBuw.png\")","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:54.534381Z","iopub.status.busy":"2021-06-10T06:46:54.533587Z","iopub.status.idle":"2021-06-10T06:46:54.541254Z","shell.execute_reply":"2021-06-10T06:46:54.540796Z","shell.execute_reply.started":"2021-06-10T06:46:22.220886Z"},"papermill":{"duration":0.037856,"end_time":"2021-06-10T06:46:54.541399","exception":false,"start_time":"2021-06-10T06:46:54.503543","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-one\"></a>\n## Initial Set up - just run through these steps","metadata":{"papermill":{"duration":0.024545,"end_time":"2021-06-10T06:46:54.59111","exception":false,"start_time":"2021-06-10T06:46:54.566565","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import time\nfrom tqdm import tqdm_notebook as tqdm\n#import tqdm.notebook import tqdm\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport torch\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim import lr_scheduler\nimport timm\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:54.646194Z","iopub.status.busy":"2021-06-10T06:46:54.645464Z","iopub.status.idle":"2021-06-10T06:46:58.267474Z","shell.execute_reply":"2021-06-10T06:46:58.267891Z","shell.execute_reply.started":"2021-06-10T06:46:22.231694Z"},"papermill":{"duration":3.651911,"end_time":"2021-06-10T06:46:58.268054","exception":false,"start_time":"2021-06-10T06:46:54.616143","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l ../input/bird-watch-spring-2022","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:58.322538Z","iopub.status.busy":"2021-06-10T06:46:58.321822Z","iopub.status.idle":"2021-06-10T06:46:58.963901Z","shell.execute_reply":"2021-06-10T06:46:58.963466Z","shell.execute_reply.started":"2021-06-10T06:46:22.241365Z"},"papermill":{"duration":0.670727,"end_time":"2021-06-10T06:46:58.964021","exception":false,"start_time":"2021-06-10T06:46:58.293294","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class args:\n    lr = 0.0001\n    epochs = 5\n    batch_size = 32\n    num_workers = 8\n    folds = 5","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:59.020934Z","iopub.status.busy":"2021-06-10T06:46:59.019751Z","iopub.status.idle":"2021-06-10T06:46:59.022035Z","shell.execute_reply":"2021-06-10T06:46:59.022418Z","shell.execute_reply.started":"2021-06-10T06:46:22.91213Z"},"papermill":{"duration":0.032335,"end_time":"2021-06-10T06:46:59.022541","exception":false,"start_time":"2021-06-10T06:46:58.990206","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alldf = pd.read_csv('../input/bird-watch-spring-2022/train.csv')\n# Split the training dataset into a training and a validation\nvaldf = alldf[::args.folds]\ntrndf = alldf[~alldf.filename.isin(valdf.filename)]\n# Load our test data\ntstdf = pd.read_csv('../input/bird-watch-spring-2022/test.csv')\nmetadf = pd.read_csv('../input/bird-watch-spring-2022/metadata.csv')\nmetadf = metadf.set_index('label')\nprint(f'File shapes -- train : {trndf.shape}, valid : {valdf.shape}, test : {tstdf.shape}')\ntrndf.head()","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:59.078616Z","iopub.status.busy":"2021-06-10T06:46:59.078074Z","iopub.status.idle":"2021-06-10T06:46:59.187913Z","shell.execute_reply":"2021-06-10T06:46:59.188498Z","shell.execute_reply.started":"2021-06-10T06:46:22.918604Z"},"papermill":{"duration":0.14109,"end_time":"2021-06-10T06:46:59.188677","exception":false,"start_time":"2021-06-10T06:46:59.047587","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgnetmeans = [0.22363983, 0.18190407, 0.2523437 ]\nimgnetstds = [0.32451536, 0.2956294,  0.31335256]\n# Using albumentations, check some examples here : https://albumentations.readthedocs.io/en/latest/examples.html \ndef trntransforms():\n    return A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.Transpose(p=0.5),\n        ToTensorV2(),\n        ])\n\ndef tsttransforms():\n    return A.Compose([\n        ToTensorV2(),\n    ])\n\nclass BirdDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        self.data = df\n        self.img_dir = '../input/bird-watch-spring-2022/'\n        self.transform = transform\n        self.mode = mode\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        \n        fname = self.data.iloc[idx]['filename']\n        image = cv2.imread(f'{self.img_dir}/{fname}')\n        if self.transform is not None:\n            image = self.transform(image = image)['image']\n        image = image.float() / 255.\n        label = -1 if self.mode=='test' else self.data.iloc[idx]['label']\n        \n        return image, label","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:59.249457Z","iopub.status.busy":"2021-06-10T06:46:59.248769Z","iopub.status.idle":"2021-06-10T06:46:59.251179Z","shell.execute_reply":"2021-06-10T06:46:59.25161Z","shell.execute_reply.started":"2021-06-10T06:46:22.979533Z"},"papermill":{"duration":0.036573,"end_time":"2021-06-10T06:46:59.251732","exception":false,"start_time":"2021-06-10T06:46:59.215159","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define our dataset\ntrndataset = BirdDataset(trndf, 'train', trntransforms())\nvaldataset = BirdDataset(valdf, 'valid', tsttransforms())\ntstdataset = BirdDataset(tstdf, 'test', tsttransforms())\nloaderargs = {'num_workers' : args.num_workers, 'batch_size':args.batch_size, 'pin_memory': False, 'drop_last': False}\ntrnloader = DataLoader(trndataset, shuffle = True, **loaderargs)\nvalloader = DataLoader(valdataset, shuffle = False, **loaderargs)\ntstloader = DataLoader(tstdataset, shuffle = False, **loaderargs)","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:59.309699Z","iopub.status.busy":"2021-06-10T06:46:59.309025Z","iopub.status.idle":"2021-06-10T06:46:59.311727Z","shell.execute_reply":"2021-06-10T06:46:59.311342Z","shell.execute_reply.started":"2021-06-10T06:46:22.99164Z"},"papermill":{"duration":0.033664,"end_time":"2021-06-10T06:46:59.311841","exception":false,"start_time":"2021-06-10T06:46:59.278177","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creates efficientnet-b0 architecture\ndevice = torch.device(\"cuda:0\")\nmodel = timm.create_model('efficientnet_b2', pretrained = True)\nmodel = model.to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=args.lr)\ncriterion = torch.nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:46:59.370192Z","iopub.status.busy":"2021-06-10T06:46:59.369688Z","iopub.status.idle":"2021-06-10T06:47:07.00041Z","shell.execute_reply":"2021-06-10T06:47:06.999935Z","shell.execute_reply.started":"2021-06-10T06:46:23.00455Z"},"papermill":{"duration":7.662926,"end_time":"2021-06-10T06:47:07.000547","exception":false,"start_time":"2021-06-10T06:46:59.337621","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-two\"></a>\n# LR Schedulers","metadata":{"papermill":{"duration":0.026465,"end_time":"2021-06-10T06:47:07.053592","exception":false,"start_time":"2021-06-10T06:47:07.027127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Create a scheduler which will warmup and cooldown over 20 epochs.\nfrom timm.scheduler.cosine_lr import CosineLRScheduler\n\nn_epochs = 20\nn_warmup_epochs = 2\nn_steps = len(trnloader)\n\nscheduler = CosineLRScheduler(\n            optimizer,\n            t_initial= n_steps * n_epochs + 1,\n            lr_min=0.00001,\n            warmup_lr_init=0.00001,\n            warmup_t= n_steps * n_warmup_epochs + 1)","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:07.111913Z","iopub.status.busy":"2021-06-10T06:47:07.111081Z","iopub.status.idle":"2021-06-10T06:47:07.116105Z","shell.execute_reply":"2021-06-10T06:47:07.115697Z","shell.execute_reply.started":"2021-06-10T06:46:23.295273Z"},"papermill":{"duration":0.035868,"end_time":"2021-06-10T06:47:07.116218","exception":false,"start_time":"2021-06-10T06:47:07.08035","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's visualise how this changes the LR\nlrls = []\nglobal_step = 0\nfor epoch in range(n_epochs):\n    for step in range(len(trnloader)):\n        #train_step(...)\n        scheduler.step(global_step)\n        global_step+=1\n        lrls.append(optimizer.param_groups[0]['lr']) \n    #validate_epoch(...)\n# Plot\nax = pd.Series(lrls).plot(logy=True, figsize = (10, 4))\nfor i in range(0,n_epochs*n_steps+1,n_steps) : ax.axvline(i, linewidth=0.2, color='r', linestyle='--')\nax.set_xlabel(\"steps\")\nax.set_ylabel(\"LR (log scale)\")","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:07.192167Z","iopub.status.busy":"2021-06-10T06:47:07.181897Z","iopub.status.idle":"2021-06-10T06:47:07.910456Z","shell.execute_reply":"2021-06-10T06:47:07.909993Z","shell.execute_reply.started":"2021-06-10T06:46:23.303357Z"},"papermill":{"duration":0.767567,"end_time":"2021-06-10T06:47:07.910587","exception":false,"start_time":"2021-06-10T06:47:07.14302","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Cosine Annealing is a type of learning rate schedule that has the effect of starting with a large learning rate that is relatively rapidly decreased to a minimum value before being increased rapidly again. The resetting of the learning rate acts like a simulated restart of the learning process and the re-use of good weights as the starting point of the restart is referred to as a \"warm restart\" in contrast to a \"cold restart\" where a new set of small random numbers may be used as a starting point.","metadata":{"papermill":{"duration":0.026928,"end_time":"2021-06-10T06:47:07.964983","exception":false,"start_time":"2021-06-10T06:47:07.938055","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lrls = []\nglobal_step = 0\nfor epoch in range(n_epochs * 6):\n    for step in range(len(trnloader)):\n        #train_step(...)\n        scheduler.step(global_step)\n        global_step+=1\n        lrls.append(optimizer.param_groups[0]['lr']) \n    #validate_epoch(...)\n# Plot\nax = pd.Series(lrls).plot(logy=True, figsize = (10, 4))\nfor i in range(0,n_epochs * 6 *n_steps+1,n_steps) : ax.axvline(i, linewidth=0.2, color='r', linestyle='--')\nax.set_xlabel(\"steps\")\nax.set_ylabel(\"LR (log scale)\")","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:08.05386Z","iopub.status.busy":"2021-06-10T06:47:08.033438Z","iopub.status.idle":"2021-06-10T06:47:09.030221Z","shell.execute_reply":"2021-06-10T06:47:09.029507Z","shell.execute_reply.started":"2021-06-10T06:46:23.761639Z"},"papermill":{"duration":1.037807,"end_time":"2021-06-10T06:47:09.030364","exception":false,"start_time":"2021-06-10T06:47:07.992557","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"#subsection-two-one\"></a>\n### Question : Can you implement a cosine learning rate schedule which has 4 epochs warmup and reaches a minimum at 15 epochs, using the n_steps from `trnloader`. ","metadata":{"papermill":{"duration":0.028924,"end_time":"2021-06-10T06:47:09.088675","exception":false,"start_time":"2021-06-10T06:47:09.059751","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.028364,"end_time":"2021-06-10T06:47:09.146238","exception":false,"start_time":"2021-06-10T06:47:09.117874","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.028164,"end_time":"2021-06-10T06:47:09.202807","exception":false,"start_time":"2021-06-10T06:47:09.174643","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-three\"></a>\n## Mixup","metadata":{"papermill":{"duration":0.028065,"end_time":"2021-06-10T06:47:09.259359","exception":false,"start_time":"2021-06-10T06:47:09.231294","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ImageIpython(url= \"https://forums.fast.ai/uploads/default/original/3X/4/b/4b00023c65aa58fbe58b02271de08949e53c64b9.png\")","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:09.32021Z","iopub.status.busy":"2021-06-10T06:47:09.319696Z","iopub.status.idle":"2021-06-10T06:47:09.325601Z","shell.execute_reply":"2021-06-10T06:47:09.325118Z","shell.execute_reply.started":"2021-06-10T06:46:24.77706Z"},"papermill":{"duration":0.038142,"end_time":"2021-06-10T06:47:09.325716","exception":false,"start_time":"2021-06-10T06:47:09.287574","status":"completed"},"tags":[],"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets try this very simply. \nimg1 = cv2.imread(f'../input/bird-watch-spring-2022/{trndf.iloc[0].filename}')\nimg2 = cv2.imread(f'../input/bird-watch-spring-2022/{trndf.iloc[1].filename}')\nmixup_alpha = 0.6\nimg_mixed = (img1 * mixup_alpha + img2 * (1-mixup_alpha)).astype(np.uint8)\nImage.fromarray(np.concatenate([img1,img2,img_mixed], 1))","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:09.387676Z","iopub.status.busy":"2021-06-10T06:47:09.387066Z","iopub.status.idle":"2021-06-10T06:47:09.498059Z","shell.execute_reply":"2021-06-10T06:47:09.498483Z","shell.execute_reply.started":"2021-06-10T06:46:24.784711Z"},"papermill":{"duration":0.144212,"end_time":"2021-06-10T06:47:09.498618","exception":false,"start_time":"2021-06-10T06:47:09.354406","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beta = 1.0\n\nfor step, batch in enumerate(trnloader):\n    if step > 20: \n        break\n    inputs = batch[0].to(device, dtype=torch.float)\n    labels = batch[1].to(device).long()\n    \n    # Get out a random value form a distribution    \n    lam = np.random.beta(beta, beta)\n    rand_index = torch.randperm(inputs.size()[0]).to(device) # make an index which reorders the batch\n    \n    # Reorder the labels\n    labels_a = labels\n    labels_b = labels[rand_index]\n    \n    # Partially mixup up the batch\n    inputs_mixed = lam * inputs + (1 - lam) * inputs[rand_index]\n    \n    optimizer.zero_grad()\n    output = model(inputs_mixed)\n    \n    # Partial loss against original labels, partial loss against mixed up labels\n    loss = criterion(output, labels_a) * lam + criterion(output, labels_b) * (1. - lam)\n    loss.backward()\n    optimizer.step()\n    \n    # Note, do not mixup your validation or test data !! Just train, make the model sweat....\n\n    ","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:09.577175Z","iopub.status.busy":"2021-06-10T06:47:09.576591Z","iopub.status.idle":"2021-06-10T06:47:16.5035Z","shell.execute_reply":"2021-06-10T06:47:16.503005Z"},"papermill":{"duration":6.969839,"end_time":"2021-06-10T06:47:16.50364","exception":false,"start_time":"2021-06-10T06:47:09.533801","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Questions \nCan you plot the different distributions of `lam`. (Something like ... `pd.Series([np.random.beta(beta, beta) for i in range(1000)]).hist(bins=100)`).   \nWhy would we us larger or smaller values of `lam` ?   \nCan you understand how the loss is calculated above, and why ?   \nCan you guess how you would implement cutmix in the example above ?","metadata":{"papermill":{"duration":0.03535,"end_time":"2021-06-10T06:47:16.575127","exception":false,"start_time":"2021-06-10T06:47:16.539777","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.034701,"end_time":"2021-06-10T06:47:16.644784","exception":false,"start_time":"2021-06-10T06:47:16.610083","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.035081,"end_time":"2021-06-10T06:47:16.715179","exception":false,"start_time":"2021-06-10T06:47:16.680098","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-four\"></a>\n# Different Architectures","metadata":{"papermill":{"duration":0.03481,"end_time":"2021-06-10T06:47:16.784982","exception":false,"start_time":"2021-06-10T06:47:16.750172","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"#### Above we chose to work with efficientnet-b0, can you initialise a pretrained `mixnet-xl` model ? Tip, check the results table [here](https://github.com/rwightman/pytorch-image-models/blob/master/results/results-imagenet.csv)","metadata":{"papermill":{"duration":0.034719,"end_time":"2021-06-10T06:47:16.854821","exception":false,"start_time":"2021-06-10T06:47:16.820102","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model = timm.create_model('efficientnet_b2', pretrained = True)","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:16.931706Z","iopub.status.busy":"2021-06-10T06:47:16.930854Z","iopub.status.idle":"2021-06-10T06:47:17.182637Z","shell.execute_reply":"2021-06-10T06:47:17.182157Z"},"papermill":{"duration":0.292812,"end_time":"2021-06-10T06:47:17.182768","exception":false,"start_time":"2021-06-10T06:47:16.889956","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Our model still has 1000 classes outputted. How would you initialise it with 10 classes in one line of code. Tip, try checking https://fastai.github.io/timmdocs/","metadata":{"papermill":{"duration":0.035121,"end_time":"2021-06-10T06:47:17.253611","exception":false,"start_time":"2021-06-10T06:47:17.21849","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.034903,"end_time":"2021-06-10T06:47:17.32384","exception":false,"start_time":"2021-06-10T06:47:17.288937","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.035104,"end_time":"2021-06-10T06:47:17.394176","exception":false,"start_time":"2021-06-10T06:47:17.359072","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.034819,"end_time":"2021-06-10T06:47:17.463993","exception":false,"start_time":"2021-06-10T06:47:17.429174","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-five\"></a>\n# Mixed Precision Training","metadata":{"papermill":{"duration":0.035298,"end_time":"2021-06-10T06:47:17.534574","exception":false,"start_time":"2021-06-10T06:47:17.499276","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ImageIpython(url= \"https://developer-blogs.nvidia.com/wp-content/uploads/2019/10/Screen-Shot-2019-10-18-at-7.31.09-AM-624x328.png\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2021-06-10T06:47:17.609665Z","iopub.status.busy":"2021-06-10T06:47:17.609044Z","iopub.status.idle":"2021-06-10T06:47:17.61409Z","shell.execute_reply":"2021-06-10T06:47:17.614477Z"},"papermill":{"duration":0.04462,"end_time":"2021-06-10T06:47:17.614607","exception":false,"start_time":"2021-06-10T06:47:17.569987","status":"completed"},"tags":[],"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Can you implement mixed precision in the below training loop. Tip : Check [this](https://pytorch.org/docs/stable/notes/amp_examples.html#typical-mixed-precision-training) example. You will need to comment out some of the below lines, its not too tough.\n### See how large you can make your batchsize with mixed precision, and without mixed precision (you may need ot restart the kernels a few times.)","metadata":{"papermill":{"duration":0.041204,"end_time":"2021-06-10T06:47:17.693796","exception":false,"start_time":"2021-06-10T06:47:17.652592","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from torch.cuda.amp import autocast, GradScaler\n\nmodel.to(device)\nmodel.train()\n\nfor step, batch in enumerate(trnloader):\n    if step > 20:\n        break\n    inputs = batch[0].to(device, dtype=torch.float)\n    labels = batch[1].to(device).long()\n    optimizer.zero_grad()\n    outputs = model(inputs)\n    loss = criterion(outputs, labels)\n    loss.backward()\n    optimizer.step()","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:17.770793Z","iopub.status.busy":"2021-06-10T06:47:17.77024Z","iopub.status.idle":"2021-06-10T06:47:24.43953Z","shell.execute_reply":"2021-06-10T06:47:24.439014Z"},"papermill":{"duration":6.710393,"end_time":"2021-06-10T06:47:24.439675","exception":false,"start_time":"2021-06-10T06:47:17.729282","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.035787,"end_time":"2021-06-10T06:47:24.511963","exception":false,"start_time":"2021-06-10T06:47:24.476176","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.03565,"end_time":"2021-06-10T06:47:24.583243","exception":false,"start_time":"2021-06-10T06:47:24.547593","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-six\"></a>\n# Label Smoothing\nThe generalization and learning speed of a multi-class neural network can often\nbe significantly improved by using soft targets that are a weighted average of the\nhard targets and the uniform distribution over labels. Smoothing the labels in this\nway prevents the network from becoming over-confident and label smoothing has\nbeen used in many state-of-the-art models,","metadata":{"papermill":{"duration":0.035708,"end_time":"2021-06-10T06:47:24.6548","exception":false,"start_time":"2021-06-10T06:47:24.619092","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ImageIpython(url= \"https://paperswithcode.com/media/methods/image3_1_oTiwmLN.png\")","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:24.733736Z","iopub.status.busy":"2021-06-10T06:47:24.733096Z","iopub.status.idle":"2021-06-10T06:47:24.736202Z","shell.execute_reply":"2021-06-10T06:47:24.736608Z"},"papermill":{"duration":0.045263,"end_time":"2021-06-10T06:47:24.736733","exception":false,"start_time":"2021-06-10T06:47:24.69147","status":"completed"},"tags":[],"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Can you implement label smoothing in the below training loop. Tip : Check [this](https://github.com/pytorch/pytorch/issues/7455#issuecomment-513062631) or [this](https://github.com/pytorch/pytorch/issues/7455#issuecomment-759175034) example. ","metadata":{"papermill":{"duration":0.035804,"end_time":"2021-06-10T06:47:24.808677","exception":false,"start_time":"2021-06-10T06:47:24.772873","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from torch.cuda.amp import autocast, GradScaler\n\nmodel.to(device)\nmodel.train()\ncriterion = torch.nn.CrossEntropyLoss()\n\nfor step, batch in enumerate(trnloader):\n    if step > 20:\n        break\n    inputs = batch[0].to(device, dtype=torch.float)\n    labels = batch[1].to(device).long()\n    optimizer.zero_grad()\n    outputs = model(inputs)\n    loss = criterion(outputs, labels)\n    loss.backward()\n    optimizer.step()","metadata":{"execution":{"iopub.execute_input":"2021-06-10T06:47:24.887328Z","iopub.status.busy":"2021-06-10T06:47:24.88654Z","iopub.status.idle":"2021-06-10T06:47:30.913862Z","shell.execute_reply":"2021-06-10T06:47:30.914415Z"},"papermill":{"duration":6.069627,"end_time":"2021-06-10T06:47:30.91459","exception":false,"start_time":"2021-06-10T06:47:24.844963","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.035871,"end_time":"2021-06-10T06:47:30.987196","exception":false,"start_time":"2021-06-10T06:47:30.951325","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.035771,"end_time":"2021-06-10T06:47:31.059178","exception":false,"start_time":"2021-06-10T06:47:31.023407","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}