{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Dogs breeds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n!git clone --quiet https://github.com/yabhi0807/libml1.git /kaggle/tmp/fastai # This is my repo with all the fastai(updated) libraries \nsys.path.append('/kaggle/tmp/fastai')\n!mkdir /kaggle/tmp/data/\n!ln -s /kaggle/tmp/* /kaggle/working/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Download the weights","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# This will take a minute to run\n!wget --quiet -O /kaggle/tmp/fastai/weights.tgz http://files.fast.ai/models/weights.tgz\n!tar xvfz /kaggle/tmp/fastai/weights.tgz -C /kaggle/tmp/fastai\n!rm -rf /kaggle/tmp/fastai/weights.tgz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir /kaggle/tmp/data/dogbreed\n!cp -r ../input/dog-breed-identification/* /kaggle/tmp/data/dogbreed\n!ls /kaggle/tmp/data/dogbreed/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.torch_imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import *\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.cuda.set_device(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/tmp/data/dogbreed/\"\nsz = 224\narch = resnext101_64\nbs = 40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_csv = f'{PATH}labels.csv'\nn = len(list(open(label_csv))) - 1 # header is not counted (-1)\nval_idxs = get_cv_idxs(n) # random 20% data for validation set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(val_idxs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initial exploration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls {PATH}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_df = pd.read_csv(label_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(label_df['breed'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_df.pivot_table(index=\"breed\", aggfunc=len).sort_values('id', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\ndata = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test', # we need to specify where the test set is if you want to submit to Kaggle competitions\n                                   val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fn = PATH + data.trn_ds.fnames[0]; fn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = PIL.Image.open(fn); img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img.size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"size_d = {k: PIL.Image.open(PATH + k).size for k in data.trn_ds.fnames}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_sz, col_sz = list(zip(*size_d.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_sz = np.array(row_sz); col_sz = np.array(col_sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_sz[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(row_sz);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(row_sz[row_sz < 1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(col_sz);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(col_sz[col_sz < 1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data.trn_ds), len(data.test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data.classes), data.classes[:6]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initial model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(sz, bs): # sz: image size, bs: batch size\n    tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\n    data = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test',\n                                       val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)\n    \n    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/13\n    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/37\n    return data if sz > 300 else data.resize(340, 'tmp') # Reading the jpgs and resizing is slow for big images, so resizing them all to 340 first saves time\n\n#Source:   \n#    def resize(self, targ, new_path):\n#        new_ds = []\n#        dls = [self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl]\n#        if self.test_dl: dls += [self.test_dl, self.test_aug_dl]\n#        else: dls += [None,None]\n#        t = tqdm_notebook(dls)\n#        for dl in t: new_ds.append(self.resized(dl, targ, new_path))\n#        t.close()\n#        return self.__class__(new_ds[0].path, new_ds, self.bs, self.num_workers, self.classes)\n#File:      fastai/dataset.py","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Precompute","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = get_data(sz, bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/tmp/fastai/weights/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit(1e-2, 5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augment","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = get_data(sz, bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True, ps=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit(1e-2, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.precompute = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit(1e-2, 5, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('224_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('224_pre')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Increase size","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Starting training on small images for a few epochs, then switching to bigger images, and continuing training is an amazingly effective way to avoid overfitting.\n\n# http://forums.fast.ai/t/planet-classification-challenge/7824/96\n# set_data doesn’t change the model at all. It just gives it new data to train with.\nlearn.set_data(get_data(299, bs)) \nlearn.freeze()\n\n#Source:   \n#    def set_data(self, data, precompute=False):\n#        super().set_data(data)\n#        if precompute:\n#            self.unfreeze()\n#            self.save_fc1()\n#            self.freeze()\n#            self.precompute = True\n#        else:\n#            self.freeze()\n#File:      fastai/conv_learner.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.summary() # uncommnet to see the complete model summary of all layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit(1e-2, 1, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Validation loss is much lower than training loss. This is a sign of underfitting. Cycle_len=1 may be too short. Let's set cycle_mult=2 to find better parameter.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# When you are under fitting, it means cycle_len=1 is too short (learning rate is getting reset before it had the chance to zoom in properly).\n# learn.fit(1e-2, 3, cycle_len=1, cycle_mult=2) # 1+2+4 = 7 epochs (Uncommnet for better accuracy)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training loss and validation loss are getting closer and smaller. We are on right track.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"log_preds, y = learn.TTA() # (5, 2044, 120), (2044,)\nprobs = np.mean(np.exp(log_preds),0)\naccuracy_np(probs, y), metrics.log_loss(y, probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data.val_ds.y), data.val_ds.y[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.fit(1e-2, 1, cycle_len=2) # 1+1 = 2 epochs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.save('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# log_preds, y = learn.TTA()\n# probs = np.mean(np.exp(log_preds),0)\n# accuracy_np(probs, y), metrics.log_loss(y, probs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This dataset is so similar to ImageNet dataset. Training convolution layers doesn't help much. We are not going to unfreeze.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Create submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data.test_ds.fnames","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log_preds, y = learn.TTA(is_test=True) # use test dataset rather than validation dataset\nprobs = np.mean(np.exp(log_preds),0)\n#accuracy_np(probs, y), metrcs.log_loss(y, probs) # This does not make sense since test dataset has no labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probs.shape # (n_images, n_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(probs)\ndf.columns = data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.insert(0, 'id', [o[5:-4] for o in data.test_ds.fnames])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUBM = f'{PATH}/subm/'\nos.makedirs(SUBM, exist_ok=True)\ndf.to_csv(f'{SUBM}subm.gz', compression='gzip', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(f'{SUBM}subm.gz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp data/dogbreed/subm/* ./","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Individual prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fn = data.val_ds.fnames[0]\nfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Image.open(PATH + fn).resize((150, 150))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Method 1.\ntrn_tfms, val_tfms = tfms_from_model(arch, sz)\nds = FilesIndexArrayDataset([fn], np.array([0]), val_tfms, PATH)\ndl = DataLoader(ds)\npreds = learn.predict_dl(dl)\nnp.argmax(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data.classes[np.argmax(preds)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Method 2.\ntrn_tfms, val_tfms = tfms_from_model(arch, sz)\nim = val_tfms(open_image(PATH + fn)) # open_image() returns numpy.ndarray\npreds = learn.predict_array(im[None]) # we can only predict minbatch\nnp.argmax(preds)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}