{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U fastai","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:18.618454Z","iopub.execute_input":"2022-11-18T17:04:18.619013Z","iopub.status.idle":"2022-11-18T17:04:37.205632Z","shell.execute_reply.started":"2022-11-18T17:04:18.618881Z","shell.execute_reply":"2022-11-18T17:04:37.204302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-18T17:04:37.211400Z","iopub.execute_input":"2022-11-18T17:04:37.214139Z","iopub.status.idle":"2022-11-18T17:04:37.226552Z","shell.execute_reply.started":"2022-11-18T17:04:37.214092Z","shell.execute_reply":"2022-11-18T17:04:37.225307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic EDA","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\n\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:37.231177Z","iopub.execute_input":"2022-11-18T17:04:37.233594Z","iopub.status.idle":"2022-11-18T17:04:41.779383Z","shell.execute_reply.started":"2022-11-18T17:04:37.231949Z","shell.execute_reply":"2022-11-18T17:04:41.778418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:41.785076Z","iopub.execute_input":"2022-11-18T17:04:41.787497Z","iopub.status.idle":"2022-11-18T17:04:41.799858Z","shell.execute_reply.started":"2022-11-18T17:04:41.787458Z","shell.execute_reply":"2022-11-18T17:04:41.798944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Convert Pixel Columns to single NP Array Image Column","metadata":{}},{"cell_type":"code","source":"columns = df.columns\npixel_columns = columns[1:]\npixel_columns","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:41.804565Z","iopub.execute_input":"2022-11-18T17:04:41.807334Z","iopub.status.idle":"2022-11-18T17:04:41.817712Z","shell.execute_reply.started":"2022-11-18T17:04:41.807293Z","shell.execute_reply":"2022-11-18T17:04:41.816878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df.copy()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:41.822403Z","iopub.execute_input":"2022-11-18T17:04:41.823818Z","iopub.status.idle":"2022-11-18T17:04:41.938229Z","shell.execute_reply.started":"2022-11-18T17:04:41.823782Z","shell.execute_reply":"2022-11-18T17:04:41.937086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"image\"] = df_train[pixel_columns].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:41.940026Z","iopub.execute_input":"2022-11-18T17:04:41.940724Z","iopub.status.idle":"2022-11-18T17:04:43.653742Z","shell.execute_reply.started":"2022-11-18T17:04:41.940686Z","shell.execute_reply":"2022-11-18T17:04:43.652467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[[\"label\", \"image\"]]\ndf_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:43.658765Z","iopub.execute_input":"2022-11-18T17:04:43.661309Z","iopub.status.idle":"2022-11-18T17:04:43.687521Z","shell.execute_reply.started":"2022-11-18T17:04:43.661265Z","shell.execute_reply":"2022-11-18T17:04:43.686536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import Test Data","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\ndf_test[\"image\"] = df_test[pixel_columns].values.tolist()\ndf_test = df_test[[\"image\"]]\ndf_test.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:43.691755Z","iopub.execute_input":"2022-11-18T17:04:43.694098Z","iopub.status.idle":"2022-11-18T17:04:47.423058Z","shell.execute_reply.started":"2022-11-18T17:04:43.694058Z","shell.execute_reply":"2022-11-18T17:04:47.421952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create and Clean File Directories","metadata":{}},{"cell_type":"code","source":"import shutil\nimport os\n\nfrom pathlib import Path\n\ntrain_data = Path(\"/tmp/train\")\ntest_data = Path(\"/tmp/test\")\n\ndef _create_subdirectories(path):\n    for label in range(10):\n        os.mkdir(path/str(label))\n\ndef create_directories():\n    os.mkdir(train_data)\n    _create_subdirectories(train_data)\n    os.mkdir(test_data)\n    \ndef clean_directories():\n    shutil.rmtree(train_data, ignore_errors=True)\n    shutil.rmtree(test_data, ignore_errors=True)\n\nclean_directories()\ncreate_directories()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:47.433684Z","iopub.execute_input":"2022-11-18T17:04:47.436560Z","iopub.status.idle":"2022-11-18T17:04:47.449253Z","shell.execute_reply.started":"2022-11-18T17:04:47.436516Z","shell.execute_reply":"2022-11-18T17:04:47.448112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Write Pixel Data as Files for FastAI\n\nBecause for some reason it MUST be files on disk. I hate it.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nfrom pathlib import Path\n\n\nfor index, row in df_train.iterrows():\n    image_name = index\n    label, image = str(row[0]), np.array(row[1])\n    file_path = train_data/f\"{label}\"/f\"{image_name}.jpg\"\n    # 784 pixels is a 28x28 image\n    image = image.reshape(28, 28)\n    # Our values are 0 -> 2^8 greyscale\n    image = image.astype(np.uint8)\n    Image.fromarray(image).save(file_path)\n    \n    \nfor index, row in df_test.iterrows():\n    image_name = index\n    image = np.array(row[0])\n    file_path = test_data/f\"{image_name}.jpg\"\n    # 784 pixels is a 28x28 image\n    image = image.reshape(28, 28)\n    # Our values are 0 -> 2^8 greyscale\n    image = image.astype(np.uint8)\n    Image.fromarray(image).save(file_path)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:04:47.453749Z","iopub.execute_input":"2022-11-18T17:04:47.456523Z","iopub.status.idle":"2022-11-18T17:05:28.739033Z","shell.execute_reply.started":"2022-11-18T17:04:47.456480Z","shell.execute_reply":"2022-11-18T17:05:28.737875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -la /tmp/test | head -n 10","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:28.744607Z","iopub.execute_input":"2022-11-18T17:05:28.746991Z","iopub.status.idle":"2022-11-18T17:05:30.411890Z","shell.execute_reply.started":"2022-11-18T17:05:28.746949Z","shell.execute_reply":"2022-11-18T17:05:30.410396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -la /tmp/train/1 | head -n 10","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:30.417756Z","iopub.execute_input":"2022-11-18T17:05:30.418197Z","iopub.status.idle":"2022-11-18T17:05:31.658744Z","shell.execute_reply.started":"2022-11-18T17:05:30.418152Z","shell.execute_reply":"2022-11-18T17:05:31.657390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setup FastAI","metadata":{}},{"cell_type":"code","source":"from fastai.data.all import *\nfrom fastai.vision.all import *","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:31.660724Z","iopub.execute_input":"2022-11-18T17:05:31.665290Z","iopub.status.idle":"2022-11-18T17:05:35.964739Z","shell.execute_reply.started":"2022-11-18T17:05:31.665243Z","shell.execute_reply":"2022-11-18T17:05:35.963427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Setup Data Loader\n\n--------------\n\n* Are we setting test and train data here? Or just train.  CUrrent setup just points to training path\n* WE probably also want a Stratified Splitter, but not that big of a deal because all of our classes are equal","metadata":{}},{"cell_type":"code","source":"digits = DataBlock(\n    blocks=(ImageBlock, CategoryBlock), \n    get_items=get_image_files, \n    splitter=RandomSplitter(valid_pct=0.2, seed=42),\n    get_y=parent_label,\n    item_tfms=Resize(128))\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:35.971360Z","iopub.execute_input":"2022-11-18T17:05:35.975141Z","iopub.status.idle":"2022-11-18T17:05:35.987564Z","shell.execute_reply.started":"2022-11-18T17:05:35.975090Z","shell.execute_reply":"2022-11-18T17:05:35.986392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = digits.dataloaders(train_data)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:35.994535Z","iopub.execute_input":"2022-11-18T17:05:35.997686Z","iopub.status.idle":"2022-11-18T17:05:45.763795Z","shell.execute_reply.started":"2022-11-18T17:05:35.997643Z","shell.execute_reply":"2022-11-18T17:05:45.762715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.valid.show_batch(max_n=4, nrows=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:45.768884Z","iopub.execute_input":"2022-11-18T17:05:45.771360Z","iopub.status.idle":"2022-11-18T17:05:46.397566Z","shell.execute_reply.started":"2022-11-18T17:05:45.771316Z","shell.execute_reply":"2022-11-18T17:05:46.396246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Augment Data with resizing","metadata":{}},{"cell_type":"code","source":"digits.new(item_tfms=RandomResizedCrop(128, min_scale=0.3)).dataloaders(train_data).train.show_batch(max_n=4, nrows=1, unique=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:46.399593Z","iopub.execute_input":"2022-11-18T17:05:46.400393Z","iopub.status.idle":"2022-11-18T17:05:48.337231Z","shell.execute_reply.started":"2022-11-18T17:05:46.400351Z","shell.execute_reply":"2022-11-18T17:05:48.336164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add Data Augmentation, DONT FLIP","metadata":{}},{"cell_type":"code","source":"digits.new(\n    item_tfms=RandomResizedCrop(224, min_scale=0.5), \n    batch_tfms=aug_transforms(mult=2, do_flip=False)).dataloaders(train_data).train.show_batch(max_n=8, nrows=2, unique=True)\n# dls = digits.dataloaders(path)\n# dls.train.show_batch(max_n=8, nrows=2, unique=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:48.342153Z","iopub.execute_input":"2022-11-18T17:05:48.344805Z","iopub.status.idle":"2022-11-18T17:05:54.788278Z","shell.execute_reply.started":"2022-11-18T17:05:48.344759Z","shell.execute_reply":"2022-11-18T17:05:54.787061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"digits = digits.new(\n    item_tfms=RandomResizedCrop(224, min_scale=0.5),\n    batch_tfms=aug_transforms(mult=2, do_flip=False))\ndls = digits.dataloaders(train_data)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:54.789685Z","iopub.execute_input":"2022-11-18T17:05:54.790867Z","iopub.status.idle":"2022-11-18T17:05:56.012678Z","shell.execute_reply.started":"2022-11-18T17:05:54.790817Z","shell.execute_reply":"2022-11-18T17:05:56.011453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls, resnet18, metrics=error_rate)\nlearn.fine_tune(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:05:56.014549Z","iopub.execute_input":"2022-11-18T17:05:56.015298Z","iopub.status.idle":"2022-11-18T17:27:27.814104Z","shell.execute_reply.started":"2022-11-18T17:05:56.015258Z","shell.execute_reply":"2022-11-18T17:27:27.813006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:27:27.815686Z","iopub.execute_input":"2022-11-18T17:27:27.816078Z","iopub.status.idle":"2022-11-18T17:28:10.505064Z","shell.execute_reply.started":"2022-11-18T17:27:27.816044Z","shell.execute_reply":"2022-11-18T17:28:10.503958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp.plot_top_losses(5, nrows=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:28:10.506668Z","iopub.execute_input":"2022-11-18T17:28:10.507058Z","iopub.status.idle":"2022-11-18T17:28:11.125961Z","shell.execute_reply.started":"2022-11-18T17:28:10.507024Z","shell.execute_reply":"2022-11-18T17:28:11.124924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Clean Dataset\n\n-----------\n\n* There are definitely some poorly labelled digits we should clean\n* Also there is just some ambiguity in digits\n* Finally, we could potentially hunt down more training data and do a stratified Kfold vs a randomized sample\n","metadata":{}},{"cell_type":"markdown","source":"### Predictions\n\n----------\n\n* Figure out how to attach test data to our data loader","metadata":{}},{"cell_type":"markdown","source":"### Save Model\n","metadata":{}},{"cell_type":"code","source":"learn.export(\"no_flip_10_fine_tune.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:28:11.128241Z","iopub.execute_input":"2022-11-18T17:28:11.128560Z","iopub.status.idle":"2022-11-18T17:28:12.054714Z","shell.execute_reply.started":"2022-11-18T17:28:11.128528Z","shell.execute_reply":"2022-11-18T17:28:12.053746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Test Data into DLS\n\n---------\n\n* There must be a better way to do this. The training part takes just a path and finds the layout, we shouldnt need to explicitly pass all of the file names.\n* I hate that everything requires specific on disk file layouts. I already had the images as an NP Array! So wasteful.\n* THe ordering here is important, we need to keep the order of the file index from the data frame, if we arent careful we will sort by strings vs integers or submit in the wrong order\n","metadata":{}},{"cell_type":"code","source":"def file_sort(path):\n    return int(path.name.split(\".\")[0])\n\n\ntest_files = [fn for fn in sorted((test_data).glob('**/*'), key=file_sort) if fn.is_file()]\ntest_dl = learn.dls.test_dl(test_files)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:28:12.056431Z","iopub.execute_input":"2022-11-18T17:28:12.056801Z","iopub.status.idle":"2022-11-18T17:28:12.426638Z","shell.execute_reply.started":"2022-11-18T17:28:12.056759Z","shell.execute_reply":"2022-11-18T17:28:12.425550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds, _ = learn.get_preds(dl=test_dl)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:28:12.428426Z","iopub.execute_input":"2022-11-18T17:28:12.428837Z","iopub.status.idle":"2022-11-18T17:29:10.084397Z","shell.execute_reply.started":"2022-11-18T17:28:12.428797Z","shell.execute_reply":"2022-11-18T17:29:10.083168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds, _, decoded = learn.get_preds(dl=test_dl, with_decoded=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:29:10.086759Z","iopub.execute_input":"2022-11-18T17:29:10.087378Z","iopub.status.idle":"2022-11-18T17:30:09.658763Z","shell.execute_reply.started":"2022-11-18T17:29:10.087341Z","shell.execute_reply":"2022-11-18T17:30:09.657595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head /kaggle/input/digit-recognizer/sample_submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:33:03.913540Z","iopub.execute_input":"2022-11-18T17:33:03.913899Z","iopub.status.idle":"2022-11-18T17:33:04.943883Z","shell.execute_reply.started":"2022-11-18T17:33:03.913867Z","shell.execute_reply":"2022-11-18T17:33:04.942599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Organize Submission\n\n-----------\n\n* There has to be a better way to do this than zipping together results.\n* Need to validate that our results are in the right order (hopefully sorting above did the trick)\n","metadata":{}},{"cell_type":"code","source":"df_submit = pd.DataFrame({'Label': decoded.numpy()})\ndf_submit['ImageId'] = df_submit.index + 1\n\ndf_submit = df_submit[[\"ImageId\", \"Label\"]]","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:33:14.029400Z","iopub.execute_input":"2022-11-18T17:33:14.029898Z","iopub.status.idle":"2022-11-18T17:33:14.044811Z","shell.execute_reply.started":"2022-11-18T17:33:14.029851Z","shell.execute_reply":"2022-11-18T17:33:14.043573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit.to_csv(\"/kaggle/working/version_4.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:33:18.811286Z","iopub.execute_input":"2022-11-18T17:33:18.811656Z","iopub.status.idle":"2022-11-18T17:33:18.846386Z","shell.execute_reply.started":"2022-11-18T17:33:18.811624Z","shell.execute_reply":"2022-11-18T17:33:18.845436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head /kaggle/working/version_4.csv","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:33:22.434456Z","iopub.execute_input":"2022-11-18T17:33:22.435423Z","iopub.status.idle":"2022-11-18T17:33:23.439809Z","shell.execute_reply.started":"2022-11-18T17:33:22.435371Z","shell.execute_reply":"2022-11-18T17:33:23.438571Z"},"trusted":true},"execution_count":null,"outputs":[]}]}