{"cells":[{"metadata":{"_uuid":"103c0cb8f8e25764d2c3ab7a95366fd812bb907a"},"cell_type":"markdown","source":"You can find more about fast.ai [here](http://course.fast.ai)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"#put these at the top of every notebook, to get automatic reloading and inline plotting\n%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import confusion_matrix\nimport os, random\nfrom shutil import copy, copytree #important for creating new working directory\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b357db59c825781d4d4f6542a34997108fa64594"},"cell_type":"code","source":"#fast.ai \nfrom fastai.imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import*\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac4b6c76f58032fb68234d606c1ec64eef56d86d"},"cell_type":"markdown","source":"**Prepare Images**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9b18829c97d278cdfa3907cb2144508ce26dd3a9"},"cell_type":"code","source":"#define train and test paths\ntrain_path = '../input/train/'\ntest_path = '../input/test/'\n#define numeber of train and valid images\ntrain_samples = 10000\nvalid_samples = 2500","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8b81a41d3042d40d57414647f2ebbb20c80e3531"},"cell_type":"code","source":"#create a list of all dog images in the train folder\ntrain_dog = [train_path+i for i in os.listdir(train_path) if 'dog' in i]\n#create a list of all cat images in the train folder\ntrain_cat = [train_path+i for i in os.listdir(train_path) if 'cat' in i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a02be20823689de41bfd39c4dfe62a25bc7811cd"},"cell_type":"code","source":"#shuffle images of dogs and cats\nrandom.shuffle(train_dog)\nrandom.shuffle(train_cat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9f9cc2aa68c433840801a2cb3bb15c2095c5a87c"},"cell_type":"code","source":"#only considering a number of images for both the training and valid set\ntrain_dog_images = train_dog[:train_samples]\ntrain_cat_images = train_cat[:train_samples]\nvalid_dog_images = train_dog[- valid_samples:]\nvalid_cat_images = train_cat[- valid_samples:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"57bc5554b199d631a44343eab1ea8281b52ecfd4"},"cell_type":"code","source":"#create new working directory \nos.makedirs('../working/dogcats/valid/cat/')\nos.makedirs('../working/dogcats/valid/dog/')\nos.makedirs('../working/dogcats/train/cat/')\nos.makedirs('../working/dogcats/train/dog/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e5db0f9643b30ba400ee2dccceb31d2b78a8311b"},"cell_type":"code","source":"#copy train images from input directory to working directory\nfor i in range(0,train_samples):\n    shutil.copy(train_dog_images[i], '../working/dogcats/train/dog/')\n    shutil.copy(train_cat_images[i], '../working/dogcats/train/cat/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"370e214772ffab5bf6feb6c0f978fa1f9615904a"},"cell_type":"code","source":"#copy valid images from input directory to working directory\nfor i in range(0,valid_samples):\n    shutil.copy(valid_dog_images[i], '../working/dogcats/valid/dog/')\n    shutil.copy(valid_cat_images[i], '../working/dogcats/valid/cat/') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac60bf796e0352e5ca9e4c477586f024487d1dff","collapsed":true},"cell_type":"code","source":"#create directory then copy test images to the new directory\nshutil.copytree(test_path, '../working/dogcats/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a8a850d9e432b5a45122ff9d2630df6a0b692f1","collapsed":true},"cell_type":"code","source":"#view folders in the new directory \nos.listdir('../working/dogcats/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cffcea0f7807c0341d2cb3da95defae4103b7b9","collapsed":true},"cell_type":"code","source":"len(os.listdir('../working/dogcats/test/'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"bdba175c3937a4b51e2b17c4d09462def5bd0ae1"},"cell_type":"code","source":"path = '../working/dogcats/'\nimage_size = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8874566b0aab363860397e6643963c3458c5d5b9","collapsed":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4039b22e2e1af242a0f9449120a98edf475868b7","collapsed":true},"cell_type":"code","source":"#activate an NVidia special accelerated functions for deep learning in a package called CuDNN\ntorch.backends.cudnn.enabled ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"01628b2c6c5cda06deb096f36f1b3734f065eab0"},"cell_type":"markdown","source":"**Create and train a model**"},{"metadata":{"trusted":true,"_uuid":"547422e8fb3254a21cc126145da055870f746852","collapsed":true},"cell_type":"code","source":"#create a model based on resnet34\narch = resnet34\ntfms = tfms_from_model(arch, image_size, aug_tfms=transforms_side_on, max_zoom=1.1)\ndata = ImageClassifierData.from_paths(path, tfms=tfms, test_name='test')\nlearn = ConvLearner.pretrained(arch, data, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdd498c65435b9494b8decbbf2403896ea947d9d","collapsed":true},"cell_type":"code","source":"#find the best learning rate\nlrf = learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d459bd47d6923550810c7434f061634662ad88fa","collapsed":true},"cell_type":"code","source":"#plot learning rate against loss to determine the best learning rate\nlearn.sched.plot() \n#best alpha is 0.01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0eb04d07502ccaaecff518ef27b2a4f7a6eefb9","collapsed":true},"cell_type":"code","source":"#fit our model\nlearn.fit(0.01, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0f4e9afa959a509627703b99fe23f15d7999561","collapsed":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"682cb81106d2dea46a30ac070518332de7bf17d2"},"cell_type":"markdown","source":"**Fine Tuning **"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d479f07a1577767fb6cfdc74b708c35d27b89645"},"cell_type":"code","source":"#precompute  True >> means we are using the output of the pretrained model and passing it to the last layer\n# which is a way of saving time\nlearn.precompute = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32ca10656f1b9bf09560f1ee9fcd6feafef9a1d1","collapsed":true},"cell_type":"code","source":"#n_cycle is the number of times of resetting the learning rate back to 0.01\n#cycle_len is the number of times of resetting the learning rate per an epoch\nlearn.fit(0.01, n_cycle=3, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6d52eefd6ae8f123385e4ef097259ca0d90a6f3","collapsed":true},"cell_type":"code","source":"#plot learning rate\nlearn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d18e47f1f1e39b584ae2b09cbbf9fdcb6d5cdb7","collapsed":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"97362979aa1e47b61dd16a187a7356f320042267"},"cell_type":"code","source":"#unfreeze all layers\nlearn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e2604a95baf727aa5858cce3a3b411c613bca69","collapsed":true},"cell_type":"code","source":"#set a differential learning rate\nlr=np.array([1e-4,1e-3,1e-2])\nlearn.fit(lr, 3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f36404785042462969f5ff9e82eade9eeefc84c6","collapsed":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48c52dacae8fb2b78ea4f5c7abdb6d735f650b38","collapsed":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8024bfc65d5eeba599b5c8e9e24ae451e67dcb75"},"cell_type":"markdown","source":"**Analyze Results**"},{"metadata":{"trusted":true,"_uuid":"af0d5725bba42061c2856fe3edc93da84d9a3eed","collapsed":true},"cell_type":"code","source":"#TTA makes predictions not just on the images in your validation set, \n#but also makes predictions on a number of randomly augmented versions\nvalid_log_predictions,y = learn.TTA()\nvalid_prob_predictions = np.mean(np.exp(valid_log_predictions),0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a255f0072516d7fea02eff26dcdde5c98cb334bd","collapsed":true},"cell_type":"code","source":"#calculate accuracy of valid predictions\naccuracy_np(valid_prob_predictions, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3d947a50e433708eeb88d93f2431055441ab44dc"},"cell_type":"code","source":"#return a list of (1 if dog or 0 if cat)\nvalid_predictions = np.argmax(valid_prob_predictions, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eedd969cacbd6b2f0825ca31dec20849430c16c5","collapsed":true},"cell_type":"code","source":"#plot a confussion matrix\ncm = confusion_matrix(y, valid_predictions)\nplot_confusion_matrix(cm, data.classes)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9d622744c05906457919d00970d60f02b210448"},"cell_type":"markdown","source":"**Predictions**"},{"metadata":{"trusted":true,"_uuid":"4036cd1400068320cb0d5cd931142c1ad42abd8a","collapsed":true},"cell_type":"code","source":"#predict\n#log_predictions = learn.predict(is_test=True)\nlog_predictions,_ = learn.TTA(is_test=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9442dfc5c834adec55a51cd4235dce380acd728e"},"cell_type":"code","source":"#prob_predictions = np.exp(log_predictions[:,1])\nprob_predictions = np.mean(np.exp(log_predictions),0)\nprob_predictions = prob_predictions[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d127088d565bb7ef389268bf9d09d3d4d3044325","collapsed":true},"cell_type":"code","source":"submission = pd.DataFrame({'id':os.listdir(f'{path}test'), 'label':prob_predictions})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b192a678b8af49e01ac1c44649e75ca5dba5c328"},"cell_type":"code","source":"! rm -rf ../working/dogcats/ #remove \"working\" directory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c08f4cff68699c306a77060ee5da8a729f2e0c9e"},"cell_type":"code","source":"submission['id'] = submission['id'].map(lambda x: x.split('.')[0])\nsubmission['id'] = submission['id'].astype(int)\nsubmission = submission.sort_values('id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9459eb16bbabeb4beb6e7d708f042c2c3372149f"},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}