{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport fastai\nfrom fastai import vision\n\nimport pretrainedmodels as pm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create training and validation data\n\nCode in the cell below is used to split the raw training data set in `../input/train/train` into interim training and \nvalidation data sets. The interim training and validation data sets are stored in `./data/interim/train` and \n`./data/interim/valid`, respectively.\n\nThe raw training data set is divided into 80% interim training data and 20% interim validation data using startified \nsampling on the image labels so that the distribution of labels in to raw training data is preserved in the interim \ntraining and validation data sets."},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport os\nimport shutil\nfrom typing import List\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn import model_selection\n\nPREFIX = \"./data\"\nSEED = 42\nTEST_SIZE = 0.2\n\ndef _filepaths_to_dataframe(paths: List[str]) -> pd.DataFrame:\n    \"\"\"Converts filepaths to a Pandas DataFrame.\"\"\"\n    results = {\"label\": [], \"filename\": []}\n    for path in paths:\n        _, _, _, _, _label, _ = path.split('/')\n        results[\"label\"].append(_label)\n        results[\"filename\"].append(path)\n    df = (pd.DataFrame\n            .from_dict(results))\n    return df\n\n\ndef _make_interim_training_data(prefix: str, df: pd.DataFrame) -> None:\n    if not os.path.isdir(f\"{prefix}/interim/train\"):\n        os.makedirs(f\"{prefix}/interim/train\")\n\n    for _, row in df.iterrows():\n        label, path = row\n        filename = (os.path\n                      .basename(path))\n        if not os.path.isdir(f\"{prefix}/interim/train/{label}\"):\n            os.mkdir(f\"{prefix}/interim/train/{label}\")\n        shutil.copy(path, f\"{prefix}/interim/train/{label}/{filename}\")\n\n        \ndef _make_interim_validation_data(prefix: str, df: pd.DataFrame) -> None:\n    if not os.path.isdir(f\"{prefix}/interim/valid\"):\n        os.makedirs(f\"{prefix}/interim/valid\")\n\n    for _, row in df.iterrows():\n        label, path = row\n        filename = (os.path\n                      .basename(path))\n        if not os.path.isdir(f\"{prefix}/interim/valid/{label}\"):\n            os.mkdir(f\"{prefix}/interim/valid/{label}\")\n        shutil.copy(path, f\"{prefix}/interim/valid/{label}/{filename}\")\n\n        \nfilepaths = glob.glob(f\"../input/train/train/*/*.jpg\", recursive=True)\ndf = _filepaths_to_dataframe(filepaths)\nprng = np.random.RandomState(SEED)\n\ntraining_df, validation_df = model_selection.train_test_split(df,\n                                                              test_size=TEST_SIZE,\n                                                              random_state=prng,\n                                                              stratify=df[\"label\"])\n    \nif not os.path.isdir(PREFIX):\n    os.mkdir(PREFIX)\n_make_interim_training_data(PREFIX, training_df)\n_make_interim_validation_data(PREFIX, validation_df)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(15, 8))\n\n_ = (df.loc[:, \"label\"]\n       .value_counts()\n       .plot\n       .bar(ax=axes[0], title=\"Raw\"))\n\n_ = (training_df.loc[:, \"label\"]\n                .value_counts()\n                .plot\n                .bar(ax=axes[1], title=\"Training\"))\n\n_ = (validation_df.loc[:, \"label\"]\n                  .value_counts()\n                  .plot\n                  .bar(ax=axes[2], title=\"Validation\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n\nset_seed(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating a `ImageDataBunch`\n\nTo create the training data set we use standard data augmentation techniques. All parameters defining the transformations used for data augmentation are left at their default values (unless otherwise specified)."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"_transform_kwargs = {\"do_flip\": True,\n                     \"flip_vert\": True,  # default is False\n                     \"max_rotate\": 180,  # default is 10\n                     \"max_zoom\": 1.2,    # default is 1.1\n                     \"max_lighting\": 0.2,\n                     \"max_warp\": 0.2,\n                     \"p_affine\": 0.75,\n                     \"p_lighting\": 0.7,\n                    }\n        \n_transforms = vision.get_transforms(**_transform_kwargs)\n\n_data_bunch_kwargs = {\"path\": \"./data/interim\",\n                      \"train\": \"train\",\n                      \"valid\": \"valid\",\n                      \"bs\": 16,\n                      \"size\": 448,\n                      \"ds_tfms\": _transforms,\n                      \"test\": \"../../../input/test/test\",  ## hack to access the test data without copying to ./data\n                     }\n\nimage_data_bunch = (vision.ImageDataBunch\n                          .from_folder(**_data_bunch_kwargs)\n                          .normalize())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_data_bunch.train_ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_data_bunch.valid_ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_data_bunch.test_ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring the data\n\nAlways important to understand what the images that are being fed into your model actually look like.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_data_bunch.show_batch(figsize=(20,20))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitting the model"},{"metadata":{},"cell_type":"markdown","source":"## Transfer Learning\n\nFor computer vision applications always start by trying transfer learning with a standard architecture: [SE-ResNeXt-101](https://arxiv.org/pdf/1803.09820.pdf)."},{"metadata":{"trusted":true},"cell_type":"code","source":"_base_arch = lambda arg: pm.se_resnext101_32x4d(num_classes=1000, pretrained=\"imagenet\")\nlearner = vision.cnn_learner(image_data_bunch,\n                             base_arch=_base_arch,\n                             pretrained=True,\n                             metrics=vision.error_rate,\n                             model_dir=\"/kaggle/working/models/se-resnext101-32x4d\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(learner.recorder\n        .plot())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_optimal_lr(recorder):\n    \"\"\"Extract the optimal learning rate from recorder data.\"\"\"\n    optimal_lr = 0\n    minimum_loss = float(\"inf\")\n    for loss, lr in zip(recorder.losses, recorder.lrs):\n        if loss < minimum_loss:\n            optimal_lr = lr\n            minimum_loss = loss\n    return optimal_lr, minimum_loss\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define a callback that stores state of \"best\" model.\n# N.B. best model is re-loaded when training completes\n_save_model_kwargs = {\"every\": \"improvement\",\n                      \"monitor\": \"valid_loss\",\n                      \"name\": \"best-model-stage-1\"}\n_save_model = (fastai.callbacks\n                     .SaveModelCallback(learner, **_save_model_kwargs))\n\n# if validation loss < training loss either learning rate too low or not enough training epoch\nlearner.fit_one_cycle(15, callbacks=[_save_model])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring the model's predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_interp = (vision.ClassificationInterpretation\n                    .from_learner(learner))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_interp.plot_top_losses(16, figsize=(20,20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_interp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_interp.most_confused()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Unfreezing, fine-tuning, and learning rates"},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(learner.recorder\n        .plot())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_save_model_kwargs = {\"every\": \"improvement\",\n                      \"monitor\": \"valid_loss\",\n                      \"name\": \"best-model-stage-2\"}\n_save_model = (fastai.callbacks\n                     .SaveModelCallback(learner, **_save_model_kwargs))\nlearner.fit_one_cycle(15, max_lr=slice(1e-6, 1e-4), callbacks=[_save_model])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploring results of the fine-tuned model"},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_interp = (vision.ClassificationInterpretation\n                    .from_learner(learner))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_interp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test-Time Augmentation (TTA)"},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_class_probabilities, _ = learner.TTA(ds_type=fastai.basic_data.DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Creating a submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"_predicted_classes = (predicted_class_probabilities.argmax(dim=1)\n                                                   .numpy())\n_class_labels = np.array(['cbb','cbsd','cgm','cmd','healthy'])\n_predicted_class_labels = _class_labels[_predicted_classes]\n\n_filenames = np.array([item.name for item in image_data_bunch.test_ds.items])\n\nsubmission = (pd.DataFrame\n                .from_dict({'Category': _predicted_class_labels,'Id': _filenames}))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree(PREFIX)  # necessary not to overwhlem Kaggle with unused output files","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}