{"cells":[{"metadata":{"trusted":true,"_uuid":"308d6ffb1d4534e567cb3befce8c73a08976d9a2"},"cell_type":"code","source":"#This kernel applies the fastAI libraries to the well-known MNIST dataset.  \n#There are a gazillion other descriptions of what MNIST is so I will not spend time giving an explanation here","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#standard imports\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"../input\")) #look for the files we'll be using\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#we'll be using fastAI for this project\nfrom fastai import *\nfrom fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"91de8d1d6f1089cf3bf682e911358425b7f2c4c8"},"cell_type":"code","source":"#load the training data\ndf_train = pd.read_csv('../input/train.csv')\ndf_train['fn'] = df_train.index\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2263c2d462f2b9e83beb9067ea69f8209824236d","scrolled":true},"cell_type":"code","source":"#df_train.head() just shows 0's for the pixel values  so let's find a spot where they're not zero\ndf_train.iloc[0,185:193]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3e246538a5cef2ae352eaf4f968ca50bbd3bfc1"},"cell_type":"markdown","source":"Borrowed this next class from https://www.kaggle.com/heye0507/fastai-1-0-with-customized-itemlist.  Thank you @Dreamdragon.  He has a good description of how it works there, so check that out."},{"metadata":{"trusted":true,"_uuid":"d8135a60976e39401835b0e8c6645797a1a04324"},"cell_type":"code","source":"class PixelImageItemList(ImageList):\n    def open(self,fn):\n        regex = re.compile(r'\\d+')\n        fn = re.findall(regex,fn)\n        df = self.inner_df[self.inner_df.fn.values == int(fn[0])]\n        df_fn = df[df.fn.values == int(fn[0])]\n        img_pixel = df_fn.drop(labels=['label','fn'],axis=1).values\n        img_pixel = img_pixel.reshape(28,28)\n        img_pixel = np.stack((img_pixel,)*3,axis=-1)\n        return vision.Image(pil2tensor(img_pixel,np.float32).div_(255))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68d3735b3ff925c815715c559c2905719d20e272"},"cell_type":"code","source":"#reshape each 784 line of ints into 28x28xRGB images using PixelImageItemList.  \n#Random_split will take 20% of the data to use for validation\n#label contains the *value* that the image is supposed to represent\nsrc = (PixelImageItemList.from_df(df_train,'.',cols='fn')\n      .split_by_rand_pct()\n      .label_from_df(cols='label'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"8753118620ad14b31524924c66f4b24a96f1a19d"},"cell_type":"code","source":"#make sure our src looks like it's of the right shape and that we have both training and validation sets.\nsrc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54d81f48fd833febc545c7496066eec7a9a76985"},"cell_type":"code","source":"#Add transforms to the data since people will write numbers larger vs. smaller and at slight differnt angles of rotation\n#do_flip=false means that people *don't* write their numbers backwards!\ntfms = get_transforms(max_rotate=15., max_zoom=1.2, max_lighting=0, max_warp=0, do_flip=False)\ndata = ImageDataBunch.create_from_ll(src, ds_tfms=tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20f9a80753a5cb213f4d5c7f84980b8f14c536ae","scrolled":true},"cell_type":"code","source":"#hey, now we have images that look like real numbers!\ndata.show_batch(rows=3, figsize=(6,6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fb5d1c8bb368b15b8de7591b9f73548c7d58cb5"},"cell_type":"code","source":"#standard resnet model.  Initially we'll leave the model frozen and just train the last couple of layers\nlearn = cnn_learner(data,models.resnet50,metrics=accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90ba394a49265f9070af1440507a724983e065bb"},"cell_type":"code","source":"#find the best learning rate (look for a nice steep downards slope)\nlearn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"714ed547da66dc5105da793b474fc78498f7074d"},"cell_type":"code","source":"#Now let's do some learning!\n#Weight decay helps avoid overfitting by taking into account the sum of the activation parameters within the \n#loss function, and therefore helps minimize the overall model complexity\nlearn.fit_one_cycle(6,slice(1e-2), wd=.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f4a181c50095585f86ea54abc9cf31a649eca35"},"cell_type":"code","source":"learn.save('50-stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13d84282b0f89afb42682bee4ec67f73fd3c62e0"},"cell_type":"code","source":"#learn.load('50-stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94da08faec74ef040e2ebffe45dced305bb75d8a"},"cell_type":"code","source":"#OK, now we'll unfreeze and train the whole model\nlearn.unfreeze()\nlearn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"9fd556d74267088c189d7b93d9c30c4a81edd632"},"cell_type":"code","source":"learn.fit_one_cycle(9,slice(2e-3,2e-5), wd=.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3902e3ae91f2592d34e9f94e10ba4b3bd5e73058"},"cell_type":"code","source":"#99.6% accuracy is a pretty good result so let's save results and move on.\nlearn.save('50-stage-2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e478a31c12fe1148af3bcf6b02c9149aa91abf9"},"cell_type":"code","source":"#Out of interest, let's see where our model messed up...\ninterp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_top_losses(9,figsize=(6,6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b01b975ca9e55ef83b5984559b3afc61244f895"},"cell_type":"code","source":"#OK, we're done building our model.  Let's run agains the Kaggle test set and see how we do.\n#Load in the test data\ndf_test = pd.read_csv('../input/test.csv')\ndf_test['label'] = 0\ndf_test['fn'] = df_test.index\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab06a83efc01551208538a7841103883e0aa0669"},"cell_type":"code","source":"#Run our predictions\nlearn.data.add_test(PixelImageItemList.from_df(df_test, path='.', cols='fn'))\ntest_pred, test_y, test_loss = learn.get_preds(ds_type=DatasetType.Test, with_loss=True)\ntest_result = torch.argmax(test_pred,dim=1)\nresult = test_result.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccddd450e6be0339cb055b59af7bfa00478866a3"},"cell_type":"code","source":"#Check that the predictions is of the right length and width (e.g. number of posible values)\ntest_pred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3592938629b0754345ca187e37d4995eca7ac1b9"},"cell_type":"code","source":"#Values look OK (e.g. range from 0-0)\nresult","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c77435156fe684eca7884fee57e6146850b5873c"},"cell_type":"code","source":"#create a CSV file to submit\nfinal = pd.Series(result,name='Label')\nsubmission = pd.concat([pd.Series(range(1,28001),name='ImageId'),final],axis=1)\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ea7c39bfe38aee63a3ddd735b73091e462fc188"},"cell_type":"code","source":"#Cross my fingers and submit1\n!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dae2e05cfd0f6241680310bc83e365a6d427bb4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}