{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"#The Following cell of code is used everytime FASTAI library is used.\n#They tell the notebook to reload any changes made to any libraries used.\n#They also ensure that any graphs are plotted are shown in this notebook\n%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:25.469958Z","iopub.execute_input":"2022-04-27T01:09:25.470405Z","iopub.status.idle":"2022-04-27T01:09:25.548186Z","shell.execute_reply.started":"2022-04-27T01:09:25.470368Z","shell.execute_reply":"2022-04-27T01:09:25.547477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:25.549553Z","iopub.execute_input":"2022-04-27T01:09:25.549830Z","iopub.status.idle":"2022-04-27T01:09:25.619667Z","shell.execute_reply.started":"2022-04-27T01:09:25.549794Z","shell.execute_reply":"2022-04-27T01:09:25.618654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.metrics import *\nfrom fastai.vision.all import *\nfrom fastai.callback.fp16 import *\nfrom torchvision.utils import save_image\nfrom PIL import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-27T01:09:25.621208Z","iopub.execute_input":"2022-04-27T01:09:25.621471Z","iopub.status.idle":"2022-04-27T01:09:25.708936Z","shell.execute_reply.started":"2022-04-27T01:09:25.621436Z","shell.execute_reply":"2022-04-27T01:09:25.708032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the train and test datasets","metadata":{}},{"cell_type":"code","source":"#import train and test data\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:25.710906Z","iopub.execute_input":"2022-04-27T01:09:25.711433Z","iopub.status.idle":"2022-04-27T01:09:31.201156Z","shell.execute_reply.started":"2022-04-27T01:09:25.711391Z","shell.execute_reply":"2022-04-27T01:09:31.200310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:31.203575Z","iopub.execute_input":"2022-04-27T01:09:31.203900Z","iopub.status.idle":"2022-04-27T01:09:31.314024Z","shell.execute_reply.started":"2022-04-27T01:09:31.203861Z","shell.execute_reply":"2022-04-27T01:09:31.313351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:31.316188Z","iopub.execute_input":"2022-04-27T01:09:31.317488Z","iopub.status.idle":"2022-04-27T01:09:31.412616Z","shell.execute_reply.started":"2022-04-27T01:09:31.317439Z","shell.execute_reply":"2022-04-27T01:09:31.411945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PATH ","metadata":{}},{"cell_type":"markdown","source":"**We have to try and get the dataset into a folder format, from the existing format, which will make it easier to use fastai's functions.**","metadata":{}},{"cell_type":"code","source":"path = Path('/kaggle/input/digit-recognizer')\nPath.BASE_PATH = path","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:31.415438Z","iopub.execute_input":"2022-04-27T01:09:31.415642Z","iopub.status.idle":"2022-04-27T01:09:31.490249Z","shell.execute_reply.started":"2022-04-27T01:09:31.415616Z","shell.execute_reply":"2022-04-27T01:09:31.489399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path.ls()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:31.491997Z","iopub.execute_input":"2022-04-27T01:09:31.492533Z","iopub.status.idle":"2022-04-27T01:09:31.563927Z","shell.execute_reply.started":"2022-04-27T01:09:31.492492Z","shell.execute_reply":"2022-04-27T01:09:31.563058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Data","metadata":{}},{"cell_type":"markdown","source":"**Currently, it is not even an image, just a 0s and 1s, as seen from the training set. Using the functions below, we can convert them into images:**","metadata":{}},{"cell_type":"markdown","source":"**normalization is very important to make sure all values lie between 0 and 1.**","metadata":{}},{"cell_type":"code","source":"#Normalize\ntrain.iloc[:,1:] = train.iloc[:,1:]/255\ntest = test/255","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:38.498378Z","iopub.execute_input":"2022-04-27T01:09:38.498651Z","iopub.status.idle":"2022-04-27T01:09:39.816209Z","shell.execute_reply.started":"2022-04-27T01:09:38.498621Z","shell.execute_reply":"2022-04-27T01:09:39.815414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Function to convert a vector representation of an image to tensor\ndef vetcor_to_image(vector, shape):\n    image_matrix = tensor(vector).reshape(shape)\n    return(image_matrix)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:40.343771Z","iopub.execute_input":"2022-04-27T01:09:40.344499Z","iopub.status.idle":"2022-04-27T01:09:40.428564Z","shell.execute_reply.started":"2022-04-27T01:09:40.344441Z","shell.execute_reply":"2022-04-27T01:09:40.427628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert training data to images\npath = Path('train')\nif not path.exists():\n    path.mkdir()\nfor i in range(train.shape[0]):\n    img = vetcor_to_image(train.iloc[i,1:], shape=(28,28))\n    dest = (path/train['label'][i].astype(str))\n    dest.mkdir(exist_ok=True)\n    save_image(img,f'{dest}/{i}.png')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:09:41.020246Z","iopub.execute_input":"2022-04-27T01:09:41.020516Z","iopub.status.idle":"2022-04-27T01:11:21.609079Z","shell.execute_reply.started":"2022-04-27T01:09:41.020486Z","shell.execute_reply":"2022-04-27T01:11:21.608348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert test data to images\npath = Path('test')\nif not path.exists():\n    path.mkdir()\nfor i in range(test.shape[0]):\n    img = vetcor_to_image(test.iloc[i,:], shape=(28,28))\n    save_image(img,f'{path}/{i}.png')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:11:21.610630Z","iopub.execute_input":"2022-04-27T01:11:21.610884Z","iopub.status.idle":"2022-04-27T01:12:47.957127Z","shell.execute_reply.started":"2022-04-27T01:11:21.610851Z","shell.execute_reply":"2022-04-27T01:12:47.956376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display images","metadata":{}},{"cell_type":"code","source":" def displayRandomImagesFromEveryFolder(directory=Path('train'), samplesPerDigit=5):\n\n    fig = plt.figure(figsize=(5,10))\n    \n    for rowIndex in range(1, 10):\n        subdirectory = str(rowIndex)\n        path = directory/subdirectory\n        images = os.listdir(path)\n        for sampleIndex in range(1,samplesPerDigit+1):\n            randomNumber = random.randint(0, len(images)-1)\n            image = Image.open(path/images[randomNumber])\n            ax = fig.add_subplot(10, 5, samplesPerDigit*rowIndex + sampleIndex)\n            ax.axis(\"off\")\n            \n            plt.imshow(image, cmap='gray')\n            \n    \n    plt.show()\n    \ndisplayRandomImagesFromEveryFolder()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:12:47.958546Z","iopub.execute_input":"2022-04-27T01:12:47.958810Z","iopub.status.idle":"2022-04-27T01:12:49.356745Z","shell.execute_reply.started":"2022-04-27T01:12:47.958774Z","shell.execute_reply":"2022-04-27T01:12:49.355993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The dataset has been converted into images!**\n\n**We can move on to getting the data from folders, and seperating them into training and validation sets.**","metadata":{}},{"cell_type":"code","source":"#create datablock & dataloaders\npath = Path('train')\ndblock = DataBlock(blocks=(ImageBlock, CategoryBlock),\n                  get_items=get_image_files,\n                  get_y=parent_label,\n                  splitter=RandomSplitter(seed=777),\n                  batch_tfms=Normalize(imagenet_stats))\ndls = dblock.dataloaders(path,bs=256)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:20:11.306816Z","iopub.execute_input":"2022-04-27T01:20:11.307085Z","iopub.status.idle":"2022-04-27T01:20:14.850139Z","shell.execute_reply.started":"2022-04-27T01:20:11.307057Z","shell.execute_reply":"2022-04-27T01:20:14.849438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build a simple Model","metadata":{}},{"cell_type":"code","source":"learn = cnn_learner(dls, resnet34,\n                    loss_func = F.cross_entropy,\n                    metrics = accuracy)\n# let's get an idea of how accurate the model is.\nlearn.fine_tune(3,base_lr=1e-3)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:20:16.769166Z","iopub.execute_input":"2022-04-27T01:20:16.769462Z","iopub.status.idle":"2022-04-27T01:22:55.945319Z","shell.execute_reply.started":"2022-04-27T01:20:16.769425Z","shell.execute_reply":"2022-04-27T01:22:55.944543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find a good learning rate","metadata":{}},{"cell_type":"code","source":"lr_min,lr_steep = learn.lr_find(suggest_funcs=(minimum, steep))","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:22:55.947376Z","iopub.execute_input":"2022-04-27T01:22:55.947649Z","iopub.status.idle":"2022-04-27T01:23:14.239381Z","shell.execute_reply.started":"2022-04-27T01:22:55.947613Z","shell.execute_reply":"2022-04-27T01:23:14.238590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Minimum/10: {lr_min:.2e}, steepest point: {lr_steep:.2e}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:23:14.240924Z","iopub.execute_input":"2022-04-27T01:23:14.241166Z","iopub.status.idle":"2022-04-27T01:23:14.317618Z","shell.execute_reply.started":"2022-04-27T01:23:14.241133Z","shell.execute_reply":"2022-04-27T01:23:14.316955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Add mixed-precision training\n\nOne of the downside of deeper architectures is that they take quite a bit longer to train. One technique that can speed things up a lot is mixed-precision training. This refers to using less-precise numbers (half-precision floating point, also called fp16) where possible during training. nearly all current NVIDIA GPUs support a special feature called tensor cores that can dramatically speed up neural network training, by 2-3x. They also require a lot less GPU memory. To enable this feature in fastai, just add to_fp16() after your Learner creation (you also need to import the module). It often even improve the accuracy of the model, because it adds a tiny bit of randomness to it! (not always though)","metadata":{}},{"cell_type":"markdown","source":"**train fit_one_cycle for 3 cycles get an idea of how accurate the model would be.**","metadata":{}},{"cell_type":"code","source":"learn = cnn_learner(dls,resnet34,loss_func = F.cross_entropy, metrics = accuracy).to_fp16()\nlearn.fit_one_cycle(3,1e-3) ","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:23:32.001825Z","iopub.execute_input":"2022-04-27T01:23:32.002369Z","iopub.status.idle":"2022-04-27T01:25:23.894045Z","shell.execute_reply.started":"2022-04-27T01:23:32.002332Z","shell.execute_reply":"2022-04-27T01:25:23.893266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Unfreezing and Transfer Learning","metadata":{}},{"cell_type":"code","source":"learn.unfreeze()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:25:23.895996Z","iopub.execute_input":"2022-04-27T01:25:23.896485Z","iopub.status.idle":"2022-04-27T01:25:23.975656Z","shell.execute_reply.started":"2022-04-27T01:25:23.896438Z","shell.execute_reply":"2022-04-27T01:25:23.974658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_min,lr_steep = learn.lr_find(suggest_funcs=(minimum, steep))","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:25:23.977130Z","iopub.execute_input":"2022-04-27T01:25:23.977399Z","iopub.status.idle":"2022-04-27T01:25:44.250392Z","shell.execute_reply.started":"2022-04-27T01:25:23.977363Z","shell.execute_reply":"2022-04-27T01:25:44.249574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Minimum/10: {lr_min:.2e}, steepest point: {lr_steep:.2e}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:25:44.252554Z","iopub.execute_input":"2022-04-27T01:25:44.252823Z","iopub.status.idle":"2022-04-27T01:25:44.328048Z","shell.execute_reply.started":"2022-04-27T01:25:44.252787Z","shell.execute_reply":"2022-04-27T01:25:44.326977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(30 ,lr_max=slice(1e-4, 1e-2))","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:26:23.780236Z","iopub.execute_input":"2022-04-27T01:26:23.780512Z","iopub.status.idle":"2022-04-27T01:45:40.670389Z","shell.execute_reply.started":"2022-04-27T01:26:23.780478Z","shell.execute_reply":"2022-04-27T01:45:40.669546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.recorder.plot_loss()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:45:40.672217Z","iopub.execute_input":"2022-04-27T01:45:40.672643Z","iopub.status.idle":"2022-04-27T01:45:41.028339Z","shell.execute_reply.started":"2022-04-27T01:45:40.672604Z","shell.execute_reply":"2022-04-27T01:45:41.027582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.save('model1')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:58:33.506410Z","iopub.execute_input":"2022-04-27T01:58:33.506674Z","iopub.status.idle":"2022-04-27T01:58:33.998077Z","shell.execute_reply.started":"2022-04-27T01:58:33.506644Z","shell.execute_reply":"2022-04-27T01:58:33.997093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"#Prediction on test set\npath = Path('test')\ntest_files = get_image_files(path)\ntest_dl = dls.test_dl(test_files)\ntest_dl.show_batch()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T01:57:52.610245Z","iopub.execute_input":"2022-04-27T01:57:52.610498Z","iopub.status.idle":"2022-04-27T01:57:53.583217Z","shell.execute_reply.started":"2022-04-27T01:57:52.610463Z","shell.execute_reply":"2022-04-27T01:57:53.582533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_score, _, pred_classes = learn.get_preds(dl=test_dl, with_decoded=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:04:13.512285Z","iopub.execute_input":"2022-04-27T02:04:13.512547Z","iopub.status.idle":"2022-04-27T02:04:29.365637Z","shell.execute_reply.started":"2022-04-27T02:04:13.512516Z","shell.execute_reply":"2022-04-27T02:04:29.364867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities = class_score[0].tolist()\n[f\"{index}: {probabilities[index]}\" for index in range(len(probabilities))]","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:04:29.367584Z","iopub.execute_input":"2022-04-27T02:04:29.368065Z","iopub.status.idle":"2022-04-27T02:04:29.445816Z","shell.execute_reply.started":"2022-04-27T02:04:29.368022Z","shell.execute_reply":"2022-04-27T02:04:29.445062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_score = np.argmax(class_score, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:04:33.175377Z","iopub.execute_input":"2022-04-27T02:04:33.175775Z","iopub.status.idle":"2022-04-27T02:04:33.248700Z","shell.execute_reply.started":"2022-04-27T02:04:33.175739Z","shell.execute_reply":"2022-04-27T02:04:33.247997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_score[1].item()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:04:34.008111Z","iopub.execute_input":"2022-04-27T02:04:34.008401Z","iopub.status.idle":"2022-04-27T02:04:34.088962Z","shell.execute_reply.started":"2022-04-27T02:04:34.008368Z","shell.execute_reply":"2022-04-27T02:04:34.087877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"path = Path(\"../input/digit-recognizer\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:05:36.564973Z","iopub.execute_input":"2022-04-27T02:05:36.565225Z","iopub.status.idle":"2022-04-27T02:05:36.636973Z","shell.execute_reply.started":"2022-04-27T02:05:36.565194Z","shell.execute_reply":"2022-04-27T02:05:36.636211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission =  pd.read_csv(path/\"sample_submission.csv\")\ndisplay(sample_submission.head(2))\ndisplay(sample_submission.tail(2))","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:05:43.702203Z","iopub.execute_input":"2022-04-27T02:05:43.702462Z","iopub.status.idle":"2022-04-27T02:05:43.801893Z","shell.execute_reply.started":"2022-04-27T02:05:43.702433Z","shell.execute_reply":"2022-04-27T02:05:43.801205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove file extension from filename\nImageId = [os.path.splitext(path)[0] for path in os.listdir(Path('test'))]\n# typecast to int so that file can be sorted by ImageId\nImageId = [int(path) for path in ImageId]\n# +1 because index starts at 1 in the submission file\nImageId = [ID+1 for ID in ImageId]","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:06:33.212553Z","iopub.execute_input":"2022-04-27T02:06:33.213236Z","iopub.status.idle":"2022-04-27T02:06:33.354349Z","shell.execute_reply.started":"2022-04-27T02:06:33.213196Z","shell.execute_reply":"2022-04-27T02:06:33.353621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission  = pd.DataFrame({\n    \"ImageId\": ImageId,\n    \"Label\": class_score\n})\n# submission.sort_values(by=[\"ImageId\"], inplace = True)\nsubmission.to_csv(\"submission.csv\", index=False)\ndisplay(submission.head(3))\ndisplay(submission.tail(3))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-04-27T02:06:34.410291Z","iopub.execute_input":"2022-04-27T02:06:34.411030Z","iopub.status.idle":"2022-04-27T02:06:34.542724Z","shell.execute_reply.started":"2022-04-27T02:06:34.410978Z","shell.execute_reply":"2022-04-27T02:06:34.542059Z"},"trusted":true},"execution_count":null,"outputs":[]}]}