{"cells":[{"metadata":{"_uuid":"6f305cfeb9a9eba9c0785f0209e0414fc0e986ff"},"cell_type":"markdown","source":"# Dogs breeds\n\nhttps://youtu.be/JNxcznsrRb8?t=1h31m8s"},{"metadata":{"trusted":true,"_uuid":"1ceba0897a3ae54684756eaa93bb1b2b9b33e5ff"},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"860c5169715c77da3d7788c56682b9991418c2bb"},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.torch_imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import *\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a08928c8e431908fbe38f8c58e6323375cf9a29"},"cell_type":"code","source":"torch.cuda.set_device(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91ee367c9a76da08d0ff60d71e735842b57a596f"},"cell_type":"markdown","source":"Kaggle Dog Breed Identification. Get data from https://www.kaggle.com/c/dog-breed-identification"},{"metadata":{"trusted":true,"_uuid":"3136b8b5ad7b567f1e6896d8aa7af32f5915dd27"},"cell_type":"code","source":"print(os.listdir(\"../input/\"))\nPATH = \"../input/\"\nsz = 224\narch = resnext101_64\nbs = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"273455060d419638849c82689911dfe2de21b2a6"},"cell_type":"code","source":"label_csv = f'{PATH}labels.csv'\nn = len(list(open(label_csv))) - 1 # header is not counted (-1)\nval_idxs = get_cv_idxs(n) # random 20% data for validation set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50b5afd2f3cf705b8928404937df1381b30f2e7e"},"cell_type":"code","source":"n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1158157db3fa6f15653e6a2735663d23d5a4aa9"},"cell_type":"code","source":"len(val_idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd1a31ef75b782cbe667f66035740e1932c66e5"},"cell_type":"code","source":"# If you haven't downloaded weights.tgz yet, download the file.\n#     http://forums.fast.ai/t/error-when-trying-to-use-resnext50/7555\n#     http://forums.fast.ai/t/lesson-2-in-class-discussion/7452/222\n!wget -O weights.tgz http://files.fast.ai/models/weights.tgz\n!tar xvfz weights.tgz -C /opt/conda/lib/python3.6/site-packages/fastai/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cc5e9910a2eb54e9a789c5e153948ec45b02766"},"cell_type":"markdown","source":"## Initial exploration"},{"metadata":{"trusted":true,"_uuid":"92476ac1720aa393b8e8e51410c4467bb963736b"},"cell_type":"code","source":"!ls {PATH}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39d84d221b90fe7adbef8dddcd9f9f92749f4cd6"},"cell_type":"code","source":"label_df = pd.read_csv(label_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"715ac118f78ee6af14bbbb2ee4bf3c8994155c64"},"cell_type":"code","source":"label_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3dbd67ae5e6e3adf79bd29ea8540be0c1f0559c"},"cell_type":"code","source":"label_df.pivot_table(index=\"breed\", aggfunc=len).sort_values('id', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f499c710c0a8704762c9e293f085a882611aff20"},"cell_type":"code","source":"tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\ndata = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test', # we need to specify where the test set is if you want to submit to Kaggle competitions\n                                   val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ede8e0be4b44f78aa0c0d21927738ab171f51431"},"cell_type":"code","source":"fn = PATH + data.trn_ds.fnames[0]; fn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8227b2cb61335db014be8ac565ca2c23266c102"},"cell_type":"code","source":"img = PIL.Image.open(fn); img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62220fca45969db07ec04477173548d186e99ec2"},"cell_type":"code","source":"img.size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58b69f45d6abc9e2f6b9bd6c82bdfe270df34747"},"cell_type":"code","source":"size_d = {k: PIL.Image.open(PATH + k).size for k in data.trn_ds.fnames}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9e64a49a380eaef82de852158bbd1111cbe7164"},"cell_type":"code","source":"row_sz, col_sz = list(zip(*size_d.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"294678fa15de6b053950acbb20ea8f937807ed8d"},"cell_type":"code","source":"row_sz = np.array(row_sz); col_sz = np.array(col_sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70269af168b500372cdcdeccbbdd35a31db986d4"},"cell_type":"code","source":"row_sz[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa3e3128267f9ae1af9cee195cc589b3e24aa672"},"cell_type":"code","source":"plt.hist(row_sz);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1faa93832189bcb3328b83350fd2ac390cfd60c"},"cell_type":"code","source":"plt.hist(row_sz[row_sz < 1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dd81fff0399ec8144fbb9a6c04dfb939ed1d5f5"},"cell_type":"code","source":"plt.hist(col_sz);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57294f44a5c5cf44b535d09ae99568c215ec416f"},"cell_type":"code","source":"plt.hist(col_sz[col_sz < 1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61a617ef1bb485cc5fcd588adfbfbc78a06ae3a0"},"cell_type":"code","source":"len(data.trn_ds), len(data.test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71367cadbb48950aef883c2e196882ae3dfb1261"},"cell_type":"code","source":"len(data.classes), data.classes[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3728e724597ebce7b368114f04d3816201204ae2"},"cell_type":"markdown","source":"## Initial model"},{"metadata":{"trusted":true,"_uuid":"b1c12fd08198698c5684f927860def0e4f675c9c"},"cell_type":"code","source":"TMP_PATH = \"/tmp/tmp\"\nMODEL_PATH = \"/tmp/model\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8af68d4e51c7ff81e91c3b6d4313c98a5cabd6e7"},"cell_type":"code","source":"def get_data(sz, bs): # sz: image size, bs: batch size\n    tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\n    data = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test',\n                                       val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)\n    \n    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/13\n    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/37\n    return data if sz > 300 else data.resize(340, TMP_PATH) # Reading the jpgs and resizing is slow for big images, so resizing them all to 340 first saves time\n\n#Source:   \n#    def resize(self, targ, new_path):\n#        new_ds = []\n#        dls = [self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl]\n#        if self.test_dl: dls += [self.test_dl, self.test_aug_dl]\n#        else: dls += [None,None]\n#        t = tqdm_notebook(dls)\n#        for dl in t: new_ds.append(self.resized(dl, targ, new_path))\n#        t.close()\n#        return self.__class__(new_ds[0].path, new_ds, self.bs, self.num_workers, self.classes)\n#File:      ~/fastai/courses/dl1/fastai/dataset.py","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a70f48fc962de80c61da32553530c4dd87ef9748"},"cell_type":"markdown","source":"### Precompute"},{"metadata":{"trusted":true,"_uuid":"0988cb4fb21da68b12a9fe6afdd5ae6848322ec2"},"cell_type":"code","source":"data = get_data(sz, bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc3cc6b8eab4e6fd0d9dad23baf857901915dd8c"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68ca1d0f2593a4dc076f5f3377f1c5c5b315869a"},"cell_type":"code","source":"learn.fit(1e-2, 5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73c78c9948797ab453514dbe7dbc55f8e66ec8a0"},"cell_type":"markdown","source":"### Augment"},{"metadata":{"trusted":true,"_uuid":"dc1a6c5fb56fa4014fb60e95dc593f9f7eba7b6c"},"cell_type":"code","source":"from sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4b555c83515ef60cabf2a35ca76f837207e1a29"},"cell_type":"code","source":"data = get_data(sz, bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d3b8506b935c78c9a30e57f3fa567ff2d90a70d"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True, ps=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"062e75a23e6f9731c4fd0140425d458075ad7916"},"cell_type":"code","source":"learn.fit(1e-2, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b296bceb33a4b9d107eff7d6400a4a91dcd7ae78"},"cell_type":"code","source":"learn.precompute = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec5c97eb615c6c073231ed845efbff60654e21a5"},"cell_type":"code","source":"learn.fit(1e-2, 5, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4462f4c5cd2822d4608313c4f7df8462ceece8c"},"cell_type":"code","source":"learn.save('224_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd14a39cc03df77230d75df47ebef4e8adcfbc70"},"cell_type":"code","source":"learn.load('224_pre')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c2dc9ae8eff917efb0545828ad839b3c587eedc"},"cell_type":"markdown","source":"## Increase size"},{"metadata":{"trusted":true,"_uuid":"8bf0ffe2bf22bd27f81588f76aad5f4d1a9cd4af"},"cell_type":"code","source":"# Starting training on small images for a few epochs, then switching to bigger images, and continuing training is an amazingly effective way to avoid overfitting.\n\n# http://forums.fast.ai/t/planet-classification-challenge/7824/96\n# set_data doesn’t change the model at all. It just gives it new data to train with.\nlearn.set_data(get_data(299, bs)) \nlearn.freeze()\n\n#Source:   \n#    def set_data(self, data, precompute=False):\n#        super().set_data(data)\n#        if precompute:\n#            self.unfreeze()\n#            self.save_fc1()\n#            self.freeze()\n#            self.precompute = True\n#        else:\n#            self.freeze()\n#File:      ~/fastai/courses/dl1/fastai/conv_learner.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"229da7345108f08b87ff2888f1ba509c1f2892d2"},"cell_type":"code","source":"learn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d223b1392cf3980c105a459040e04378da3cc99"},"cell_type":"code","source":"learn.fit(1e-2, 3, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a5c4644ce0a85881cea02566d3388cd025b8496"},"cell_type":"markdown","source":"Validation loss is much lower than training loss. This is a sign of underfitting. Cycle_len=1 may be too short. Let's set cycle_mult=2 to find better parameter."},{"metadata":{"trusted":true,"_uuid":"dd8b9d8b1f308f28d2906e8037ed1c25fa5afb26"},"cell_type":"code","source":"# When you are under fitting, it means cycle_len=1 is too short (learning rate is getting reset before it had the chance to zoom in properly).\nlearn.fit(1e-2, 3, cycle_len=1, cycle_mult=2) # 1+2+4 = 7 epochs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b381d535431ff67f7f3213390ca6806e2c47a1d6"},"cell_type":"markdown","source":"Training loss and validation loss are getting closer and smaller. We are on right track."},{"metadata":{"trusted":true,"_uuid":"7d7a203c143253b446ad97217677faa3d5e6fdd9"},"cell_type":"code","source":"log_preds, y = learn.TTA() # (5, 2044, 120), (2044,)\nprobs = np.mean(np.exp(log_preds),0)\naccuracy_np(probs, y), metrics.log_loss(y, probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f43c62bcdc507a62bf69713b84b1639d1b1e5a15"},"cell_type":"code","source":"len(data.val_ds.y), data.val_ds.y[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d347899fb29d6346b855be3d433dbff48eebf9fd"},"cell_type":"code","source":"learn.save('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34afdb6f8896bad6d8f4ca21556846c845039389"},"cell_type":"code","source":"learn.load('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f46c47573557fdd2e1a85244b5fb10beedcd4911"},"cell_type":"code","source":"learn.fit(1e-2, 1, cycle_len=2) # 1+1 = 2 epochs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"883f3fbced058997d0cbb1df51ea3990036aed93"},"cell_type":"code","source":"learn.save('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2d1f6111dbd04afb4e1f18fbbaf0d990664a201"},"cell_type":"code","source":"log_preds, y = learn.TTA()\nprobs = np.mean(np.exp(log_preds),0)\naccuracy_np(probs, y), metrics.log_loss(y, probs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"969ea5cb365883bf145adc447a649e360fa43e99"},"cell_type":"markdown","source":"This dataset is so similar to ImageNet dataset. Training convolution layers doesn't help much. We are not going to unfreeze."},{"metadata":{"_uuid":"3fbc0bfa18781f5ea6d53035ed0c12fa6a3f148d"},"cell_type":"markdown","source":"## Create submission\n\nhttps://youtu.be/9C06ZPF8Uuc?t=1905"},{"metadata":{"trusted":true,"_uuid":"7031ebe71d33f3ad464a86e8c4ceb3742f10f9b9"},"cell_type":"code","source":"data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95b75488915944c33b0f3c29680e595f17ec1a02"},"cell_type":"code","source":"data.test_ds.fnames","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6fc00272b6fbdce0bd2d9df4bea9ccd007f0372"},"cell_type":"code","source":"log_preds, y = learn.TTA(is_test=True) # use test dataset rather than validation dataset\nprobs = np.mean(np.exp(log_preds),0)\n#accuracy_np(probs, y), metrcs.log_loss(y, probs) # This does not make sense since test dataset has no labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3bd90a381b4f7ac05d08cb7da4c4a3edcd49034"},"cell_type":"code","source":"probs.shape # (n_images, n_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7a6f22501469d9dada054b2e88fce217ef6e34c"},"cell_type":"code","source":"df = pd.DataFrame(probs)\ndf.columns = data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5488d919e3cc247ddbfddc3c3a1201f71e0339c7"},"cell_type":"code","source":"df.insert(0, 'id', [o[5:-4] for o in data.test_ds.fnames])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71603cfdcee1ae182f1c6fb604af3eda24338047"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2caa0b7a60819e1b93f1fe98baccbfe8fab9e53e"},"cell_type":"code","source":"SUBM = f'/tmp/subm/'\nos.makedirs(SUBM, exist_ok=True)\ndf.to_csv(f'{SUBM}subm.gz', compression='gzip', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e03d08792f70a96de21af679e3b685d5c8f80a70"},"cell_type":"code","source":"FileLink(f'{SUBM}subm.gz')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0aba8d00e247a603aec983c70ef693d95f688bbf"},"cell_type":"markdown","source":"## Individual prediction"},{"metadata":{"trusted":true,"_uuid":"1ebd446fd7b7da373808cfeff2cc19fcb8004ab7"},"cell_type":"code","source":"fn = data.val_ds.fnames[0]\nfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5e280acf85c798f683d12058a7b98866bb0400e"},"cell_type":"code","source":"Image.open(PATH + fn).resize((150, 150))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cb748a59e017d532a248934df401dd20e534b63"},"cell_type":"code","source":"# Method 1.\ntrn_tfms, val_tfms = tfms_from_model(arch, sz)\nds = FilesIndexArrayDataset([fn], np.array([0]), val_tfms, PATH)\ndl = DataLoader(ds)\npreds = learn.predict_dl(dl)\nnp.argmax(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"265517bf8191d475065faa1ef31f5c20e1cd0b99"},"cell_type":"code","source":"learn.data.classes[np.argmax(preds)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"563c233ef50d1749e32bce880a20d43ad34e6a14"},"cell_type":"code","source":"# Method 2.\ntrn_tfms, val_tfms = tfms_from_model(arch, sz)\nim = val_tfms(open_image(PATH + fn)) # open_image() returns numpy.ndarray\npreds = learn.predict_array(im[None])\nnp.argmax(preds)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}