{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install fastai==0.7.0 --no-deps\n!pip install torch==0.4.1 torchvision==0.2.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b29a9bb7a8d954a0431bdad7528d52f29a1f136"},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.torch_imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import *\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1775ccbfc093b7b368506fdebbf04955bf6b9a3c"},"cell_type":"code","source":"# verify GPU\nprint(torch.cuda.is_available(), torch.backends.cudnn.enabled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"577eeaaf5e415c2b9c536e43fe4b584e437d1b8f"},"cell_type":"code","source":"PATH = \"../input/\"\n#models = \"../input/pytorch-pretrained-models/\"\nsz = 224\narch = resnext101_64\nbs = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5728e5c962fd670cdcd98a67c61fd7d093139c73"},"cell_type":"code","source":"# If you haven't downloaded weights.tgz yet, download the file.\n#     http://forums.fast.ai/t/error-when-trying-to-use-resnext50/7555\n#     http://forums.fast.ai/t/lesson-2-in-class-discussion/7452/222\n!wget -O weights.tgz http://files.fast.ai/models/weights.tgz\n!tar xvfz weights.tgz -C /opt/conda/lib/python3.6/site-packages/fastai/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f01275073802e68aa18dc86d95294c0120ffa37d"},"cell_type":"code","source":"label_csv = f'{PATH}labels.csv'\nn = len(list(open(label_csv))) - 1 # header is not counted (-1)\nval_idxs = get_cv_idxs(n) # random 20% data for validation set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a4ae2ee6bb8f80b9776edb29c845a5e2fc6a06a"},"cell_type":"code","source":"n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f1eb566ff3ff3563cf38d2ba90637fad5c371a9"},"cell_type":"code","source":"len(val_idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efeb24d7b37baa22692591edb8fd54ea45bfb836"},"cell_type":"code","source":"label_df = pd.read_csv(label_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cb8b0d1b405ba39859fb7a2018dd1362c583d50"},"cell_type":"code","source":"label_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae1d5ca085802b03b34a6c6fd86f8adc027eeec5"},"cell_type":"code","source":"label_df.pivot_table(index=\"breed\", aggfunc=len).sort_values('id', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a8bf4dc7daaa6ab0e59729d5b7579a05e2c1e29"},"cell_type":"code","source":"tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\ndata = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test', # we need to specify where the test set is if you want to submit to Kaggle competitions\n                                   val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6f5cda1448fe01f3297a139e9e2ad80cc315da9"},"cell_type":"code","source":"fn = PATH + data.trn_ds.fnames[0]; fn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88ece1d99c3ea439bab4b32e2b428f0e4da20893"},"cell_type":"code","source":"img = PIL.Image.open(fn); img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67685b66fb82032702fd51cce1a37ddc6bcab1b4"},"cell_type":"code","source":"img.size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3828129dd93124f9003fccd25557a496debc47fc"},"cell_type":"code","source":"size_d = {k: PIL.Image.open(PATH + k).size for k in data.trn_ds.fnames}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39fb2e53563487cc0411e636dc679ceb59bd7b88"},"cell_type":"code","source":"row_sz, col_sz = list(zip(*size_d.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0062950598fb5488d56c456753507601d4bfa7a"},"cell_type":"code","source":"row_sz = np.array(row_sz); col_sz = np.array(col_sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"692210fe22c87ecc5b23d39493e9ada9e5293337"},"cell_type":"code","source":"row_sz[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21b43170a6c5aaa94f2bd918fcfc9ad3138f6c52"},"cell_type":"code","source":"plt.hist(row_sz);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32634f4438c38b23ee453806b848c5cf39207def"},"cell_type":"code","source":"plt.hist(row_sz[row_sz < 1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f63fc67fb954cee63d06f96d03ff06b157ace519"},"cell_type":"code","source":"plt.hist(col_sz);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82787048fe78bb161b1500b913cec3d9ad338840"},"cell_type":"code","source":"plt.hist(col_sz[col_sz < 1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19461b5c40afde6091dad0fa727a3a4041c04312"},"cell_type":"code","source":"len(data.trn_ds), len(data.test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7413c16ff2e27d808fc44c1f888f400219531d56"},"cell_type":"code","source":"len(data.classes), data.classes[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e2a0cd343ebebf1fc5d125f6124c5c57900d42e"},"cell_type":"code","source":"TMP_PATH = \"/tmp/tmp\"\nMODEL_PATH = \"/tmp/model\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2904040f44c99105a82d2f3ae63f984346cc3bf"},"cell_type":"code","source":"def get_data(sz, bs): # sz: image size, bs: batch size\n    tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\n    data = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test',\n                                       val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)\n    \n    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/13\n    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/37\n    return data if sz > 300 else data.resize(340, TMP_PATH) # Reading the jpgs and resizing is slow for big images, so resizing them all to 340 first saves time\n\n#Source:   \n#    def resize(self, targ, new_path):\n#        new_ds = []\n#        dls = [self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl]\n#        if self.test_dl: dls += [self.test_dl, self.test_aug_dl]\n#        else: dls += [None,None]\n#        t = tqdm_notebook(dls)\n#        for dl in t: new_ds.append(self.resized(dl, targ, new_path))\n#        t.close()\n#        return self.__class__(new_ds[0].path, new_ds, self.bs, self.num_workers, self.classes)\n#File:      ~/fastai/courses/dl1/fastai/dataset.py","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"729d1a194b6c718ba11d378dfabab58df6600078"},"cell_type":"markdown","source":"Precompute"},{"metadata":{"trusted":true,"_uuid":"18b0a82bef171ecd4518dc88cf42200b1f52faa2"},"cell_type":"code","source":"data = get_data(sz, bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"061275667c208d9158589c81fe60df8b38611ad7"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6161a07e74f8ff59c8b01b730be02ab8251a8488"},"cell_type":"code","source":"learn.fit(1e-2, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3812c339b58f8cc9131ae08f180194e61b0448b8"},"cell_type":"code","source":"from sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe76646b63c5c1840f200134b25d8b5b526dc04c"},"cell_type":"code","source":"data = get_data(sz, bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c636e8d772aa2c7ead15a32f736c94072355295c"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True, ps=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a81f198962b2dd72f328e3336838fb4c82b06872"},"cell_type":"code","source":"learn.fit(1e-2, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2942660e31ea8e33344ebf12bf688f8b8594deb7"},"cell_type":"code","source":"learn.precompute = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39712f28d03949ecf1b5e07fa2d60feb62089387"},"cell_type":"code","source":"learn.fit(1e-2, 5, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bca919c5d1cd5a11ea1715c1e67f0ad8486930d0"},"cell_type":"code","source":"learn.save('224_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449149c10331754aa1576131ee9c2a1dbf148ecc"},"cell_type":"code","source":"learn.load('224_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec28c20d0239a3dc25d0df89bf6e43762ad6e523"},"cell_type":"code","source":"# Starting training on small images for a few epochs, then switching to bigger images, and continuing training is an amazingly effective way to avoid overfitting.\n\n# http://forums.fast.ai/t/planet-classification-challenge/7824/96\n# set_data doesn’t change the model at all. It just gives it new data to train with.\nlearn.set_data(get_data(299, bs)) \nlearn.freeze()\n\n#Source:   \n#    def set_data(self, data, precompute=False):\n#        super().set_data(data)\n#        if precompute:\n#            self.unfreeze()\n#            self.save_fc1()\n#            self.freeze()\n#            self.precompute = True\n#        else:\n#            self.freeze()\n#File:      ~/fastai/courses/dl1/fastai/conv_learner.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a07fd771d51c71c05ae971965ceea52e1720b69"},"cell_type":"code","source":"#learn.summary()\nlearn.fit(1e-2, 3, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e433c71d704e5f5ceaf8fe188136ed67f21c3ab9"},"cell_type":"code","source":"# When you are under fitting, it means cycle_len=1 is too short (learning rate is getting reset before it had the chance to zoom in properly).\nlearn.fit(1e-2, 3, cycle_len=1, cycle_mult=2) # 1+2+4 = 7 epochs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba2e1cceed7daaa7a37bc52a572ef060e7f6e72b"},"cell_type":"code","source":"\nlog_preds, y = learn.TTA() # (5, 2044, 120), (2044,)\nprobs = np.mean(np.exp(log_preds),0)\naccuracy_np(probs, y), metrics.log_loss(y, probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd4b4b2bd483a8ea4984df56f9cd33a61212aab"},"cell_type":"code","source":"len(data.val_ds.y), data.val_ds.y[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab9f84280ecd332e91ae3eabbc8a996b74f4715c"},"cell_type":"code","source":"learn.save('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3794d98cf3074579f3c471654a2d05ebf1089895"},"cell_type":"code","source":"learn.load('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"145341461fa2dc47f9433ca1f10566cfd2a4f3ac"},"cell_type":"code","source":"learn.fit(1e-2, 1, cycle_len=2) # 1+1 = 2 epochs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"267257f5d48cb875600e9bf581348001bb05ab1c"},"cell_type":"code","source":"learn.save('299_pre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f7cb95f12dd0d0f5b34c431b4bbdcadc3729c78"},"cell_type":"code","source":"log_preds, y = learn.TTA()\nprobs = np.mean(np.exp(log_preds),0)\naccuracy_np(probs, y), metrics.log_loss(y, probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaf795436def6c9ea60280fb0e03b3d4c103440e"},"cell_type":"code","source":"data.classes\n#data.test_ds.fnames","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19fe72a4be3b36e07cbc628a6ecc3dbd50d0207f"},"cell_type":"code","source":"log_preds, y = learn.TTA(is_test=True) # use test dataset rather than validation dataset\nprobs = np.mean(np.exp(log_preds),0)\n#accuracy_np(probs, y), metrcs.log_loss(y, probs) # This does not make sense since test dataset has no labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6283d7c7789409252f2e96087796b6ff5668bee4"},"cell_type":"code","source":"probs.shape # (n_images, n_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e74e2ae0a7219b8474158fd4de1d83b5315dcae4"},"cell_type":"code","source":"df = pd.DataFrame(probs)\ndf.columns = data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"863189c639cc1164d5154c909f0238e5c40e025e"},"cell_type":"code","source":"df.insert(0, 'id', [o[5:-4] for o in data.test_ds.fnames])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0caf6cea0610a32bfea54ae4c30ab8e391240bfa"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f9b0da04531fee9cce33000288117daefc2059e"},"cell_type":"code","source":"SUBM = f'subm/'\nos.makedirs(SUBM, exist_ok=True)\ndf.to_csv(f'{SUBM}subm.gz', compression='gzip', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18c991580166ba1d08b6b098e7499876e7f13687"},"cell_type":"code","source":"FileLink(f'{SUBM}subm.gz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd5c8ea744b5bec029d18e0e00c7bb3f422610c6"},"cell_type":"code","source":"fn = data.val_ds.fnames[0]\nfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51c48736efb6ac9427c889a08c22d51884339d6e"},"cell_type":"code","source":"Image.open(PATH + fn).resize((150, 150))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af6d510588e3fe65aaee3b030cb551a69423131b"},"cell_type":"code","source":"# Method 1.\ntrn_tfms, val_tfms = tfms_from_model(arch, sz)\nds = FilesIndexArrayDataset([fn], np.array([0]), val_tfms, PATH)\ndl = DataLoader(ds)\npreds = learn.predict_dl(dl)\nnp.argmax(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2017307199609e7550cbdf04f85f306089dd1a62"},"cell_type":"code","source":"learn.data.classes[np.argmax(preds)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3769b8ad512d476143c0242565307ea9c6a85dbc"},"cell_type":"code","source":"# Method 2.\ntrn_tfms, val_tfms = tfms_from_model(arch, sz)\nim = val_tfms(open_image(PATH + fn)) # open_image() returns numpy.ndarray\npreds = learn.predict_array(im[None])\nnp.argmax(preds)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}