{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv\nprint(os.listdir(\"../input\"))\n%matplotlib inline\nfrom fastai.basics import *\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d15265bde5383390baa8b136aa1b6bff7220dacd"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2e1fb865e22f21ffcff5eec71998242070d3ba0"},"cell_type":"code","source":"x = np.array(train.drop('label', axis = 1),  dtype = 'float_'); y = np.array(train.label)\nX_train, X_test, y_train, y_test = train_test_split(x,  y, train_size = 0.8)\n#X_train= np.array(X_train); X_test = np.array(X_test)\n#y_train= np.array(y_train); y_test = np.array(y_test)\nprint('x train: ', X_train.shape, '| x test: ', X_test.shape, '| y train', y_train.shape, ' | y test: ', y_test.shape)\nprint('Type of X: ', type(X_train))\nprint('Type of y: ', type(y_train))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b44c63e3f0e8f8054a9b5301cbbcfd4e4c8e771"},"cell_type":"code","source":"print('Type of X: ', type(X_train))\nplt.imshow(X_train[4].reshape((28,28)), cmap = 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78706b6c16cb2f899344857a41b37f09554f1fab"},"cell_type":"code","source":"x_train, y_train, x_valid, y_valid = map(torch.tensor, (X_train, y_train, X_test, y_test))\nx_train = x_train.float(); x_valid = x_valid.float()\nn,c = x_train.shape\nx_train.shape, y_train.min(), y_train.max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"578da03a7e2422ff096f7df947f41a8512327d53"},"cell_type":"markdown","source":"* The TorchDataset is a pytorch function that index zips up data and labels into a pytorch dataset, allows you to index into a dataset and get an x value and y value (like a list of x,y tuples)\n* Then DataBunch - fastai wrapper for pytorch Data Loader which combines a dataset and a sampler, and provides a single- or multi-process iteratros over the dataset\n    * Allows you to ask for the next thing, since the databunch is an itterator\n    * Makes sure training is shuffled and all the data is put on GPU, etc"},{"metadata":{"trusted":true,"_uuid":"bf271b277e34b11ccc1d33a37a4da4f02b1ef621"},"cell_type":"code","source":"bs = 64\ntrain_ds = TensorDataset(x_train, y_train)\nvalid_ds = TensorDataset(x_valid, y_valid)\ndata = DataBunch.create(train_ds, valid_ds, bs = bs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0024c14c1f48852e5ed18895f585defa43752452"},"cell_type":"markdown","source":"Show how this plays out in practice, taking the first batch  by calling iter on the train data loader"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e0ea649b553d739c202612f936114646eadd4b69"},"cell_type":"code","source":"x,y = next(iter(data.train_dl))\nx.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d49e44c90f6c4e706593ac6d348e4bfb2266a557"},"cell_type":"markdown","source":"Going to create a nn.Module with a simple linear layer (basically a NN with a single layer that follows the form y_hat = x@a + bias\n* the foreward method passes throuh xb (the single batch of x data)\n* every nn.Module acts as a function that you can call\n\n "},{"metadata":{"trusted":true,"_uuid":"74185c650811b5641865d1ce64dab758fda5bf3f"},"cell_type":"code","source":"class Mnist_Logistic(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.lin = nn.Linear(784, 10, bias = True)\n        \n    def forward(self, xb): return self.lin(xb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ecc761afbbab95a08d76f594dd0150570dcafe1"},"cell_type":"code","source":"#put the function/model on the GPU\nmodel = Mnist_Logistic().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"256d3a982e1a158b8946210ce49ea3b31caa3bce"},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c4f9be9cfd3de7b09c2d261e36bb71f86623f6"},"cell_type":"code","source":"model.lin","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"517bff7d932b264bbd365804f2c6e632bf6ca72d"},"cell_type":"markdown","source":"Model automatically stores all the parameters that are contained in the models"},{"metadata":{"trusted":true,"_uuid":"368822695072e0e5c6e69172a3d91b728b3a0257"},"cell_type":"code","source":"[p.shape for p in model.parameters()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"177d8cf29337f84b61176f1aacc54d07938fc54d"},"cell_type":"code","source":"lr = 1e-2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5dac6b8a284301d03ad137258acf3c99cfd01001"},"cell_type":"code","source":"loss_func = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa626acebb8d67de8853ce0becefb3efd5bbf1eb"},"cell_type":"code","source":"def update(x,y,lr):\n    wd = 1e-6\n    y_hat = model(x)\n    w2 = 0.\n    for p in model.parameters(): w2 += (p**2).sum()\n    #add tpo regular loss\n    loss = loss_func(y_hat, y) + w2*wd\n    loss.backward()\n    with torch.no_grad():\n        for p in model.parameters():\n            p.sub_(lr*p.grad)\n            p.grad.zero_()\n    return loss.item()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"450368e53bbee5bd61c2c91af45a360923d1e7eb"},"cell_type":"code","source":"losses = [update(x,y,1e-2) for x,y in data.train_dl]\nplt.plot(losses)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"203f286afaa11495302e893bf883247faecaa0d1"},"cell_type":"markdown","source":"Since we 'solved' the linear function in pytorch/fasltai using weight decay and back propigation, building onto that to get a neural network."},{"metadata":{"trusted":true,"_uuid":"9a7b0cfdc54a3a3a06734efaa145f9053f580021"},"cell_type":"code","source":"class Mnist_NN(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.lin1 = nn.Linear(784, 50, bias = True)\n        self.lin2 = nn.Linear(50, 10, bias = True)\n        \n    def forward(self, xb): \n        x = self.lin1(xb)\n        x = F.relu(x)\n        return self.lin2(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41a29f722f778b20e792c98a7e8c1a4113320f3b"},"cell_type":"code","source":"model = Mnist_NN().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c935cd1c05f1d81db8b4f5a1d12d196b0bd5002"},"cell_type":"code","source":"losses = [update(x,y,1e-2) for x,y in data.train_dl]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d401c31adb729b420b6a7b9627be7c05820e55f4"},"cell_type":"code","source":"x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a3af7c3a5a4007c71fd1e5d10e313848e9848ea"},"cell_type":"code","source":"plt.plot(losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7763e180529e6c213849b77421621909e1874d86"},"cell_type":"code","source":"def update(x,y,lr, wd):\n    opt = optim.Adam(model.parameters(), lr, weight_decay = wd)\n    #This line replaces the weight decay step and allows the use of .step to go one \n    #loop through the gradient descent\n    y_hat = model(x)\n    loss = loss_func(y_hat, y)\n    loss.backward()\n    opt.step()\n    opt.zero_grad()\n    return loss.item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d85be8ef47f36dca0dda192a58e0f37d7d41a3fd"},"cell_type":"code","source":"losses=[update(x,y,5e-3, 0.0001) for x,y in data.train_dl]\nplt.plot(losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6041f27d483e603828ad636a261760645c3e00c"},"cell_type":"code","source":"#FastAI just wraps all the update function we did above to make it into a neat-er package \n#and to all the functionality with FASTAI with the updater \n??Learner","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6621113e47f3abccf9d327dea6471816f285255"},"cell_type":"code","source":"learn = Learner(data, Mnist_NN(), loss_func = nn.CrossEntropyLoss(), metrics = accuracy)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d291cee07395b68fdbeb9982b28a6a63a21578bd"},"cell_type":"markdown","source":"Now we can use the lr_find method on our Mnist_NN model to find an optimal learning rate\n\n* Note that nn.CrossEntropyLoss includes the softmax function\n* the direct output of our current model does NOT include softmax"},{"metadata":{"trusted":true,"_uuid":"646b4ed2cdf3525749b4cbe07f0c6de222d96f8d"},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f554d677f0a25304fefda8f8c8f3fa36be6a16a0"},"cell_type":"code","source":"learn.fit_one_cycle(1, 5e-5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"25cd70e8deb8fe1bff91669b815e64e41c97c654"},"cell_type":"markdown","source":"Recall that fit_one_cycle does a loop of varying learning rate annealing"},{"metadata":{"trusted":true,"_uuid":"e8fa4e6474474855d3cf346a510f49e042a77ad9"},"cell_type":"code","source":"learn.recorder.plot_lr(show_moms = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5a3e837a3e15268158b3188c14bfc0527adf36e"},"cell_type":"markdown","source":"Tested a few different learning rates to quickly visualize where the best loss function occurs relative to the lr_find graph\n\nIt looks like the point with the highest (stable) downward slope gives the best loss results\n* This, I think, intuitively makes sense since the fit_one_cycle slides up and down the lr curve, so mayve a good stable region helps prevent divergence"},{"metadata":{"trusted":true,"_uuid":"8b635e2c549ee2e312d426342e6e19aef395c0f4"},"cell_type":"code","source":"#lr = 1e-5 #valid_loss = 1.76\n#lr = 1e-4 #valid_loss = 0.37\n#lr = 5e-4 #valid_loss = 0.18\nlr = 1e-3 #valid_loss = 0.16\nepochs = 5\nlearn = Learner(data, Mnist_NN(), loss_func = nn.CrossEntropyLoss(), metrics = accuracy)\nlearn.fit_one_cycle(epochs,lr)\nlearn.recorder.plot_losses()\nlearn.recorder.plot_lr(show_moms = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f6e2d23e41d91ac8fff30496621de12c00545a0b"},"cell_type":"markdown","source":"FastAI calculates the exponentially weighted moving average of the losses (which is why the losses model is smoth)"},{"metadata":{"_uuid":"48b0717ad2364ebf51caa48a1f187ad257bf91c3"},"cell_type":"markdown","source":"Softmax as a last layer in classification always adds up to a total of 1 for all the predictions\n* Single lable multi label classification always uses SoftMax and Cross-Entorpy Loss"},{"metadata":{"trusted":true,"_uuid":"50500859a590fd3eb323f62ae6576cb80edebb66"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}