{"cells":[{"metadata":{},"cell_type":"markdown","source":"#  MNIST with FastAI ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"> Thanks to this notebook \"MNIST Resnet_bigger+Mish_in_shortcut+transforms\" that was used partially arrange the dataset\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 1. Imports  and file check","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this import is to avoid warnings","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings # remove Warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch.nn.functional\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"the one below is to reload the latest packages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"FastAI imports","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\nfrom fastai.metrics import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Standard imports for ML","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Quick exam of the dataset provided","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/digit-recognizer/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is the path were you find the datasets in Kaggle","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"quick look of train and test dataset, the only difference is the label column on the train dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Formatting the data for ML","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Class created in the [\"MNIST Resnet_bigger+Mish_in_shortcut+transforms\"](https://www.kaggle.com/rincewind007/mnist-resnet-bigger-mish-in-shortcut-transforms/comments) to format the data for FastAI models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomImageList(ImageList):\n    def open(self, fn):\n        if(fn.size == 785):\n            fn = fn[1:]\n        img = fn.reshape(28,28)\n        img = np.stack((img,)*1, axis=-1)\n        return Image(pil2tensor(img, dtype=np.float32))\n    \n    @classmethod\n    def from_csv_custom(cls, path:PathOrStr, csv_name:str, imgIdx:int=1, header:str='infer', **kwargs)->'ItemList': \n        df = pd.read_csv(Path(path)/csv_name, header=header)\n        res = super().from_df(df, path=path, cols=0, **kwargs)\n        \n        res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1).values\n        \n        return res\n    \n    @classmethod\n    def from_csv_custom_test(cls, path:PathOrStr, csv_name:str, imgIdx:int=1, header:str='infer', **kwargs)->'ItemList': \n        df = pd.read_csv(Path(path)/csv_name, header=header)\n        res = super().from_df(df, path=path, cols=0, **kwargs)\n        \n        res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1).values\n        print(res)\n        return res\n    \n    \n    \n    @classmethod\n    def from_df_custom(cls, path:PathOrStr, df:DataFrame, imgIdx:int=1, header:str='', **kwargs)->'ItemList': \n        res = super().from_df(df, path=path, cols=0, **kwargs)\n        \n        res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1).values\n        \n        return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = CustomImageList.from_csv_custom_test(path=path, csv_name='test.csv', imgIdx=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The test dataset contains 28000 items 1,28,28","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (CustomImageList.from_csv_custom(path=path, csv_name='train.csv', imgIdx=1)\n                .split_by_rand_pct(.02)\n                .label_from_df(cols='label') #cols='label'\n                .add_test(test, label=0)\n                .transform(get_transforms(do_flip=False,max_rotate=15,max_warp=0.4))\n                .databunch(bs=128, num_workers=0)\n                .normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We created our data for the training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.train_ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"the train dataset contains 41160 images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.valid_ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The validation dataset contains the 2% of the training data, in total 840 images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We now created the databunch of 128 images to use in our model and we quickly visually examine the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(rows=5, figsize=(7,7))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Create the model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We create our learner now using the data that we just formatted and chosing the model.\nHere FastAi is very flexible, I personally experimented with:\n* resnet18 \n* resnet34\n* resnet50\n* resnet101\n* resnet152\n\nYou can find more info about the models [here](https://docs.fast.ai/vision.models.html#Computer-Vision-models-zoo)\n\nI encourage you to try and change this values in the model selection and analyze the results, you will learn a lot","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"In order to keep it simple I started with the one who should give the best results","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_selected = models.resnet152","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"create the learner","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data, model_selected, \n                    metrics=error_rate,callback_fns=ShowGraph,model_dir='/kaggle/working').mixup()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"classic FastAI [learning rate finder](https://docs.fast.ai/basic_train.html#Model-fitting-methods)\nthis is the link to the [scientific paper](https://arxiv.org/abs/1506.01186) if you want to go deeper","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"plot of the learning rate with [suggested best lr](https://docs.fast.ai/callbacks.lr_finder.html#Suggested-LR) to adopt","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot(suggestion=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"set the lr suggested as lr for training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"min_grad_lr = learn.recorder.min_grad_lr\nmin_grad_lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Fit the model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Fit the model with the [one cycle policy](https://docs.fast.ai/callbacks.one_cycle.html#Training-with-the-1cycle-policy)\n\nhere is the scientific [paper link](https://arxiv.org/abs/1803.09820) if you want to go deeper","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I suggest to experiment with the epoch number, for me 30 epochs works pretty well","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(30,min_grad_lr)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"remember to save your job ... of course I learned the hard way","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('r152-fit1')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit with the full dataset","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Now we are going to train with the full dataset removing the split, so we are going to use all the 42000 ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (CustomImageList.from_csv_custom(path=path, csv_name='train.csv', imgIdx=1)\n                #.split_by_rand_pct(.02) --- this split has been removed\n                .split_none()# this line replaces the previous split\n                .label_from_df(cols='label') #cols='label'\n                .add_test(test, label=0)\n                .transform(get_transforms(do_flip=False,max_rotate=15,max_warp=0.4))\n                .databunch(bs=128, num_workers=0)\n                .normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we replace the data in our learner with the new data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.train_ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now the data contains all the 42000 images of the training set","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"we replace the data inside the model to retrain with those extra 840 samples","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data = data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fit for extra 10 epochs with the full dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(10,min_grad_lr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"and save again","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('r152-fit2')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Unfreeze","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"[Unfreeze the whole model](https://docs.fast.ai/basic_train.html#Learner.unfreeze) in order to train all the layers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Follow what we did before","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot(suggestion=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"min_grad_lr = learn.recorder.min_grad_lr\nmin_grad_lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fit for an extra 10 epochs.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(10,min_grad_lr)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Predictions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"> In order to have a correct interpretation of out data we should compare it towards our validation dataset, BUT we also trained the validation dataset this time in order to achieve the maximum score on Kaggle, so we are forced to look at our model performance through something that we trained.\n> > For this reason we recreate the dataset with the command line below.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (CustomImageList.from_csv_custom(path=path, csv_name='train.csv', imgIdx=1)\n                .split_by_rand_pct(.02)\n                .label_from_df(cols='label') #cols='label'\n                .add_test(test, label=0)\n                .transform(get_transforms(do_flip=False,max_rotate=15,max_warp=0.4))\n                .databunch(bs=128, num_workers=0)\n                .normalize(imagenet_stats))\nlearn.data = data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We create the predictions and the output file","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the predictions\npredictions, *_ = learn.get_preds(DatasetType.Test)\nlabels = np.argmax(predictions, 1)\n# output to a file\nsubmission_df = pd.DataFrame({'ImageId': list(range(1,len(labels)+1)), 'label': labels})\nsubmission_df.to_csv(f'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"quick look at the submission format","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create the [classification interpretation](https://docs.fast.ai/vision.learner.html#ClassificationInterpretation) from FastAI","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"and explore the [confusion matrix](https://docs.fast.ai/train.html#ClassificationInterpretation.plot_confusion_matrix)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_confusion_matrix()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"for vision the [plot top losses](https://docs.fast.ai/vision.learner.html#_cl_int_plot_top_losses) is anyway my favourite","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_top_losses(9,cmap='gray')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> This resnet18 scored 0.99014, wich is not bad.\n\n> The best score was achieved with resnet152 with a score of 0.99600.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"If you were so patient to read up to here I hope you found some valuable information ...","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"... and if you liked it ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# ***... please upvote ... Thank you!!! :)))***","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}