{"cells":[{"metadata":{"_uuid":"04499cc33864de76e3a500d4d72167325ab28898"},"cell_type":"markdown","source":"# Important\n\n**I have used only 16000 of the available 25000 images of the training data due to kaggle kernel limitations of Disk space. Please make the necessary changes in the code if you want to train on the complete dataset.**\n\n**I realize this is not a well explained kernel, but I primarily wrote this to verify if it was possible to come up with an end-to-end solution using fastai inside kaggle kernels**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport glob\nimport shutil\nimport os\nimport gc\nimport pathlib\nprint(os.listdir(\"../input/\"))\n%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5696bd0a2b1445e45dee01f5a4a660e901fcf05e"},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import *\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d5add01b7f01cd49594ef2270264c46af16920f","scrolled":true,"collapsed":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13745f426ac6359808e159c77e5aaafffe1b31d3","collapsed":true},"cell_type":"code","source":"torch.backends.cudnn.enabled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5801fd45da67b39bca0d63088741027fb79d39e8"},"cell_type":"code","source":"dog_indexes = []\ncat_indexes = []\ncount = 0\ncat_count = 1\ndog_count = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d9f922ef68f215e0901759c84e6f918cf08c595"},"cell_type":"markdown","source":"**Indexes for Validation**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fc4f29f553a8b93708d43390a4afda854b92dd59"},"cell_type":"code","source":"for name in os.listdir('../input/dogs-vs-cats-redux-kernels-edition/train/'):\n    if 'cat' in name and cat_count <= 8000:\n        cat_indexes.append(name.split('.')[1])\n        cat_count += 1\n    if 'dog' in name and dog_count <= 8000:\n        dog_indexes.append(name.split('.')[1])\n        dog_count += 1\n    if dog_count > 8000 and cat_count > 8000:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaa5260a6cbc77408d46e0c387ee907d2a589ed5","collapsed":true},"cell_type":"code","source":"print ('Dog!\\n',len(dog_indexes), '\\nCat!\\n', len(cat_indexes))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7c3858cbcbad087e08d763aabf1f998020964df"},"cell_type":"markdown","source":"**Randomly sampling indexes to add images to Validation set. Make changes to validation set if required.**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"aa995f8e4f9259d50de779e2129a2a8e318afbab"},"cell_type":"code","source":"cat_val_list = random.sample(cat_indexes, 1200)\ndog_val_list = random.sample(dog_indexes, 1200)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10a0c14f4d49268c792055282b2028c6ad04c983"},"cell_type":"markdown","source":"**Folders to store the images in format that will allow to run fastai algorithms**"},{"metadata":{"trusted":true,"_uuid":"f25b5671f3a52bd6464eef5792c77621b2d4c8ab","collapsed":true},"cell_type":"code","source":"os.makedirs('../working/dogcats/valid/cats/')\nos.makedirs('../working/dogcats/valid/dogs/')\nos.makedirs('../working/dogcats/train/cats/')\nos.makedirs('../working/dogcats/train/dogs/')\nos.makedirs('../working/dogcats/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f8d0cab0b4c51a077f3ba128ad8214ca356868ba"},"cell_type":"code","source":"train_dir = \"../input/dogs-vs-cats-redux-kernels-edition/train/\"\ntest_dir = \"../input/dogs-vs-cats-redux-kernels-edition/test/\"\ncat_train_dir = \"../working/dogcats/train/cats/\"\ncat_valid_dir = \"../working/dogcats/valid/cats/\"\ndog_train_dir = \"../working/dogcats/train/dogs/\"\ndog_valid_dir = \"../working/dogcats/valid/dogs/\"\ndogcats_test = \"../working/dogcats/test/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7dea8cfd2dc0075ac14194bac80a6962927ce4e4"},"cell_type":"code","source":"PATH = \"../working/dogcats/\"\nsz=224","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e1a89fa0fa24d68fe43cad6a264c4ebf400bf42"},"cell_type":"markdown","source":"**Only moving 8000 images of each class due to size constraints. Make changes if required.**"},{"metadata":{"trusted":true,"_uuid":"cba01281147d187b0e03004c36ed095786e7e829","collapsed":true},"cell_type":"code","source":"for jpgfile in iglob(os.path.join(train_dir, \"cat*.jpg\")):\n    if count >= 8000:\n        break\n    count += 1\n    if jpgfile.split('.')[3] in cat_val_list:\n        shutil.copy(jpgfile, cat_valid_dir)\n    else:\n        shutil.copy(jpgfile, cat_train_dir)\n\ncount = 0\n\nfor jpgfile in iglob(os.path.join(train_dir, \"dog*.jpg\")):\n    if count >= 8000:\n        break\n    count += 1\n    if jpgfile.split('.')[3] in dog_val_list:\n        shutil.copy(jpgfile, dog_valid_dir)\n    else:\n        shutil.copy(jpgfile, dog_train_dir)\n        ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed5ece5536f4af094ff22f9e01f32b77e8cfdc3c","trusted":true,"collapsed":true},"cell_type":"code","source":"for jpgfile in iglob(os.path.join(test_dir, \"*.jpg\")):\n    shutil.copy(jpgfile, dogcats_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87faa7ec1610797651b6be8a3ee143c851f77e2d","collapsed":true},"cell_type":"code","source":"cache_dir = os.path.expanduser(os.path.join('~', '.torch'))\nif not os.path.exists(cache_dir):\n    os.makedirs(cache_dir)\nmodels_dir = os.path.join(cache_dir, 'models')\nif not os.path.exists(models_dir):\n    os.makedirs(models_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ccc62b31532f467ce09575a59c562bc30b0b7f5","collapsed":true},"cell_type":"code","source":"!cp ../input/resnet34/resnet34.pth /tmp/.torch/models/resnet34-333f7ec4.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce105b515d5ee91046f386352ad637f7b3c5076b","collapsed":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26b350932bb21eef3e1b442ae3a64cc9cd9bec89"},"cell_type":"markdown","source":"**Data Augmentation**"},{"metadata":{"_uuid":"92ced348c3717f4345a1e6b6f70d0b9dfab35492","trusted":true,"collapsed":true},"cell_type":"code","source":"tfms = tfms_from_model(resnet34, sz, aug_tfms=transforms_side_on, max_zoom=1.1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26d5a7b115678e4c9116f3b31436cf5e65b7c630"},"cell_type":"markdown","source":"**Transfer Learning**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"503f69f42c495cf9f1110925e4e660705a3864ea"},"cell_type":"code","source":"arch=resnet34","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6649ba2afea7e7d747b3f8035075c0848270c143","collapsed":true},"cell_type":"code","source":"data = ImageClassifierData.from_paths(PATH, tfms=tfms, test_name='test')\nlearn = ConvLearner.pretrained(arch, data, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"448997448fc3c25f9a0c47c12e60c66ba33a81e4"},"cell_type":"markdown","source":"**Begin Training**"},{"metadata":{"trusted":true,"_uuid":"39afb922c0e1bc4b865db267770e869f644a54bc","collapsed":true},"cell_type":"code","source":"learn.fit(1e-2, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f2bce170aa3941ba69d2e43e99a498e580485c92"},"cell_type":"code","source":"learn.precompute=False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a860047421d8d03a8b2db49c55d014c1bc1a5b4","collapsed":true},"cell_type":"code","source":"learn.fit(1e-2, 3, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e714cdb3151d9baa11efd40ccfab857d1ba8f8e9","collapsed":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a854b58f03a350f7fdaad239ed4d87755e5dc419","collapsed":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e633bc0f464dabe913b906919c29cac734d4ed09"},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5168f5c86f15a9c7eeb95a7155824abb209be34c"},"cell_type":"code","source":"lr=np.array([1e-4,1e-3,1e-2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14e82624f532ffe6c0c051dd9dfff09a2f9eb8bf","collapsed":true},"cell_type":"code","source":"learn.fit(lr, 3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81d42231d3647172826f45f92919acb77ecb8704","collapsed":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52039b96995560f737f936c336f9de0d08e59485","collapsed":true},"cell_type":"code","source":"log_preds,y = learn.TTA()\nprobs = np.mean(np.exp(log_preds),0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3211579fa76fbd19bd9e8c1b285dd122900033ea","collapsed":true},"cell_type":"code","source":"accuracy_np(probs, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6e6899df6a8fd3fa3dbbbab229a2b4b730102d68"},"cell_type":"code","source":"preds = np.argmax(probs, axis=1)\nprobs = probs[:,1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fd61736a682382c812454ec3519515366f6d501"},"cell_type":"markdown","source":"**Prediction on Test Data**"},{"metadata":{"trusted":true,"_uuid":"c5b4cf220f42498d9a23739d23f0dc8d6753613a","collapsed":true},"cell_type":"code","source":"temp = learn.predict(is_test=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eefa632016b9b3d33bc4a31de1e0e4bc16a29141","collapsed":true},"cell_type":"code","source":"temp.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dfa7642eb10c1d7372a2911d6f4097db94cb8845"},"cell_type":"code","source":"pred_test = np.argmax(temp, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c68466ba95c8f22815bd22913a76439d1893a3c5","collapsed":true},"cell_type":"code","source":"pred_test[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30b92f0a45acf8cb46e85618200fe192919ef102","collapsed":true},"cell_type":"code","source":"probs = np.exp(temp[:,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45457793316c8f5cc27f80ed5e7d88479b335afc","collapsed":true},"cell_type":"code","source":"probs[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eb134bf08de59cde7b40f95e43c8ffef4760117","collapsed":true},"cell_type":"code","source":"os.listdir(f'{PATH}test')[:4]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f8d4e1adad22ccce555f8146f7a2544783a4501"},"cell_type":"markdown","source":"**Creation of submission CSV**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c8ed36396dd10bea2566cff0d3f4a1f4e3669275"},"cell_type":"code","source":"submission = pd.DataFrame({'id':os.listdir(f'{PATH}test'), 'label':probs})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c8251c6d9b1974821fb36065b23f73d93b191c4","collapsed":true},"cell_type":"code","source":"! rm -rf ../working/dogcats/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c621e1bd788aa52a92c86d2c7019cf6f57228caa","collapsed":true},"cell_type":"code","source":"submission['id'] = submission['id'].map(lambda x: x.split('.')[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f400f90552adb066e31504dac9e9cfb1d69f042","collapsed":true},"cell_type":"code","source":"submission['id'] = submission['id'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83063963f944667c5801537890fa921c70052eed","collapsed":true},"cell_type":"code","source":"submission = submission.sort_values('id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"87ba9d5418da2669f740abe8310f47ac03430055"},"cell_type":"code","source":"submission.to_csv('../working/output.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}