{"cells":[{"metadata":{"_uuid":"6da2d665c290723d528a37eba8b9ee2a035054da"},"cell_type":"markdown","source":"## Dog Breed Identification with CNN"},{"metadata":{"_uuid":"cc7dcfacce9227ceddf92933788b0953e707d881"},"cell_type":"markdown","source":"**Kaggle kernels have contain common deep learning toolkits, like tensorflow, pytorch, keras, even fastai **"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dcb4a39595d56df3bcef28c9ac54c128dc4c4416"},"cell_type":"code","source":"# enable IE to call matlplotlib\n%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b620fd90022cb8d314778838cc0e4741e6281701"},"cell_type":"markdown","source":"**Here we import the libraries we need in the following notebook.**"},{"metadata":{"trusted":true,"_uuid":"99b1ea7d493adc0da44baa778fcd849f8e8f5e73","collapsed":true},"cell_type":"code","source":"#??fastai.imports\n# This file contains all the main external libs we'll use\n# Here we import other libraries except fastai (or other deep learning tools)\nfrom fastai.imports import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"from fastai.transforms import *\nfrom fastai.conv_learner import *\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true,"_uuid":"41a45adeac4a1d7df9df472d111ea428b1e56be2","collapsed":true},"cell_type":"code","source":"PATH = '../input/'\nsz = 224\narch = resnet34\nbs = 128\n!ls {PATH}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"728f6f07dcdf8775870b1a3aa5cffd43df0a62de","collapsed":true},"cell_type":"code","source":"print('CUDA is available?', torch.cuda.is_available())\nprint('pytorch version:', torch.__version__)\ntorch.backends.cudnn.enabled","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc9c51ccc5056e0dcddf18b8858313e61bca2f6a"},"cell_type":"markdown","source":"## Look at images"},{"metadata":{"trusted":true,"_uuid":"0112a91da44e9627213e5cc2ee7ed6f25eca46c1","collapsed":true},"cell_type":"code","source":"files = os.listdir(f'{PATH}train')[:5]\nfiles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67307ebf224024d9cf72f44814266756103eb8ca","collapsed":true},"cell_type":"code","source":"img = plt.imread(f'{PATH}train/{files[0]}')\nplt.imshow(img);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e80f49024f9a35a3acaacda479d1963dccd96458","collapsed":true},"cell_type":"code","source":"img.shape\nimg[:4,:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d85f0eeff2db318a98d5a9bd1444f69166b2c516","collapsed":true},"cell_type":"code","source":"label_csv = f'{PATH}labels.csv'\nn = len(list(open(label_csv))) - 1\nval_idxs = get_cv_idxs(n)\nval_idxs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3fa5de807bdd2154fd7eae7038befa75425173c0","collapsed":true},"cell_type":"code","source":"# make soft link for dataset only if using Kaggle Kernel\n!ln -s {PATH}train \n!ln -s {PATH}test\n!ls","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18021535a7be9cda4f86da7b9062ca7b3bcbcfa5"},"cell_type":"markdown","source":"## Our first model: quick start"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8ef8e0a1f3038c84fb4e02fa45834fa36ec64a5f"},"cell_type":"code","source":"PATH = '.'\ntfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)\ndata = ImageClassifierData.from_csv(PATH, 'train', label_csv, bs=bs, tfms=tfms,  \n                            suffix='.jpg', val_idxs=val_idxs, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a3e7c4ff5a56273e6fe1fb23efaf656de10a1bb","collapsed":true},"cell_type":"code","source":"#??ConvLearner.pretrained\nlearn = ConvLearner.pretrained(arch, data, precompute=True, ps=0.5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffbfa5fc1b8f5e1cd97261c30860c5ee33bf4a91"},"cell_type":"markdown","source":"**Choose a learning rate**"},{"metadata":{"trusted":true,"_uuid":"60af9eb7260e57c13190ada613ed8d1a996044a7","collapsed":true},"cell_type":"code","source":"lrf = learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"febe8243b708ec9b75914528ec12f9cbf5f1a008","collapsed":true},"cell_type":"code","source":"learn.sched.plot()\n# learning rate corresponding to the lowset point location, then div 10 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c227d1f4e5daab37ff14c4499e4636820b93d39","collapsed":true},"cell_type":"code","source":"#learn.fit(0.5, 3)\n# different batch size corresponding to different best learning rate","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41dabe5e7309a4e86047f2f54472168ee1ae82c9"},"cell_type":"markdown","source":"## Improving our model:\n**Data augmentation**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"45e9499d346cacc6b4f6da8f946f3dba820afb26"},"cell_type":"code","source":"tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"911b1fc54a8b2a009b38ffabc673a55660df83ad"},"cell_type":"code","source":"def get_augs():\n    data = ImageClassifierData.from_csv(PATH, 'train', label_csv, bs=bs, tfms=tfms,\n                    val_idxs=val_idxs, suffix='.jpg', num_workers=4)\n    x, _ = next(iter(data.aug_dl))\n    return data.trn_ds.denorm(x)[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a50e1adf179e30707678bdb0ddcc36f0e09b2be3","collapsed":true},"cell_type":"code","source":"ims = np.stack([get_augs() for i in range(6)])\nplots(ims, rows=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d80abfd2584c2a86350f3fdb9f291635237a5dbf"},"cell_type":"code","source":"# create new data object including this augmentation\ndata = ImageClassifierData.from_csv(PATH, 'train', label_csv, bs=bs, tfms=tfms, \n                        val_idxs=val_idxs, suffix='.jpg', num_workers=4)\nlearn = ConvLearner.pretrained(arch, data, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15019e2a9070e0318fba807a988abfef0d3cd954","collapsed":true},"cell_type":"code","source":"learn.fit(0.5, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a32dab592eb177fada2ed5ac0be47598f7bb3683","collapsed":true},"cell_type":"code","source":"# As we want to use data augmentation, should find a new best learning rate\nlrf = learn.lr_find()\nlearn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9938d2be45bb3d09ace35e756d57434ce008a3e","collapsed":true},"cell_type":"code","source":"learn.precompute = False\nlearn.fit(0.1, 3, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91c4afaecb9a61ab25c7270dc91a74ed6c994d19","collapsed":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb79ffd29ba9a0410d1e5b7ce1fe403d73744341","collapsed":true},"cell_type":"code","source":"learn.save('224_lastlayer')\nlearn.load('224_lastlayer')\n!ls 'models'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"185715d27fc6db88413c30a2f56928b3bcfde74f"},"cell_type":"markdown","source":"## Fine-tuning and differential learning rate annealing\nNow we have a good final layer trained, we can try fine-tuneing the other layers. Just call unfreeze()"},{"metadata":{"trusted":true,"_uuid":"60c7e5420c72cd39fb1eaf010495ae082f6cc0d1","collapsed":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7002f36200d4dabf0a8dd4326414671d37e0200c","collapsed":true},"cell_type":"code","source":"# After unfreeze the former layers, we should find learning rate again\nlearn.lr_find()\nlearn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63580ce80bb73911a7edb30650c1ea272f3c4a2a","collapsed":true},"cell_type":"code","source":"#lr = np.array([5e-5, 5e-4, 5e-3])\nlr = np.array([1e-4, 1e-3, 1e-2])\nlearn.fit(lr, 3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3868be8207c513b8a52a5364fbe749814a9debef","collapsed":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8301d24664ff011a3bc62ebe65069495bc034bee","collapsed":true},"cell_type":"code","source":"learn.save('224_all')\nlearn.load('224_all')\n# Using FileLink can generate Link to download files, like, trianed model\n#FileLink('./models/224_all.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aee5c9b6a4e8476d41a0be2d8ef106f790030dff"},"cell_type":"markdown","source":"## Analyzing results"},{"metadata":{"trusted":true,"_uuid":"a937444cb082d68fcaa1369c34dffe4af2846145","collapsed":true},"cell_type":"code","source":"log_preds, y = learn.TTA()\nprobs = np.mean(np.exp(log_preds), 0)\naccuracy_np(probs, y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bee8e8f7a27b8da44fb84f5c14edc898f715be51"},"cell_type":"markdown","source":"**Confusion matrix**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e5d31088589c4e867dd79fa4b45db78d2ce1869b"},"cell_type":"code","source":"preds = np.argmax(probs, axis=1)\nprobs = probs[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c791699ea745bb330f3d1050de191982d8f0fead","collapsed":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(y, preds)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0cd9f7a916d6c9af4f27d5aa5a510549b29971a","collapsed":true},"cell_type":"code","source":"plot_confusion_matrix(cm, data.classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3143f7d3233c3ccde5c6a03207e35c21f011b7c","collapsed":true},"cell_type":"code","source":"!rm -rf train\n!rm -rf test\n!rm -rf tmp/\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7a66a99b56145795e25d2da5778ee54cd2788730"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}