{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\nWorking with tensors\n ","metadata":{}},{"cell_type":"code","source":"import torch\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.161504Z","iopub.execute_input":"2023-02-17T04:01:28.161947Z","iopub.status.idle":"2023-02-17T04:01:28.167009Z","shell.execute_reply.started":"2023-02-17T04:01:28.161912Z","shell.execute_reply":"2023-02-17T04:01:28.165762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1D tensor\nx = torch.empty(3)\nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.172059Z","iopub.execute_input":"2023-02-17T04:01:28.172476Z","iopub.status.idle":"2023-02-17T04:01:28.181115Z","shell.execute_reply.started":"2023-02-17T04:01:28.172439Z","shell.execute_reply":"2023-02-17T04:01:28.179879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2D Tensor\nx = torch.empty(2, 3)\nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.182851Z","iopub.execute_input":"2023-02-17T04:01:28.183655Z","iopub.status.idle":"2023-02-17T04:01:28.190279Z","shell.execute_reply.started":"2023-02-17T04:01:28.183614Z","shell.execute_reply":"2023-02-17T04:01:28.189198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating Tensor with Random Values**","metadata":{}},{"cell_type":"code","source":"x = torch.rand(2, 3)\nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.194982Z","iopub.execute_input":"2023-02-17T04:01:28.196026Z","iopub.status.idle":"2023-02-17T04:01:28.205015Z","shell.execute_reply.started":"2023-02-17T04:01:28.195979Z","shell.execute_reply":"2023-02-17T04:01:28.203666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tensor with all ones**","metadata":{}},{"cell_type":"code","source":"x = torch.ones(4, 4, dtype= torch.int) # We can also specify dtype\nprint(x)\nprint(x.size()) # Size of tensor","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.209617Z","iopub.execute_input":"2023-02-17T04:01:28.210016Z","iopub.status.idle":"2023-02-17T04:01:28.218078Z","shell.execute_reply.started":"2023-02-17T04:01:28.209983Z","shell.execute_reply":"2023-02-17T04:01:28.217036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Converting List into Tensor**","metadata":{}},{"cell_type":"code","source":"x = torch.tensor([2.5, 0.1])\nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.231877Z","iopub.execute_input":"2023-02-17T04:01:28.232612Z","iopub.status.idle":"2023-02-17T04:01:28.238514Z","shell.execute_reply.started":"2023-02-17T04:01:28.232550Z","shell.execute_reply":"2023-02-17T04:01:28.237470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Converting NumPy array into Tensor**","metadata":{}},{"cell_type":"code","source":"a = np.ones(5)\nb = torch.from_numpy(a)\nprint(b)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.268888Z","iopub.execute_input":"2023-02-17T04:01:28.269560Z","iopub.status.idle":"2023-02-17T04:01:28.275198Z","shell.execute_reply.started":"2023-02-17T04:01:28.269515Z","shell.execute_reply":"2023-02-17T04:01:28.274418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a += 1\nprint(b) # Tensor gets modiefied too","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.309204Z","iopub.execute_input":"2023-02-17T04:01:28.309835Z","iopub.status.idle":"2023-02-17T04:01:28.316829Z","shell.execute_reply.started":"2023-02-17T04:01:28.309791Z","shell.execute_reply":"2023-02-17T04:01:28.315503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Basic Operations on Tensors**","metadata":{}},{"cell_type":"code","source":"x = torch.rand(2, 2)\ny = torch.rand(2, 2)\nprint(x) \nprint(y)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.321291Z","iopub.execute_input":"2023-02-17T04:01:28.322059Z","iopub.status.idle":"2023-02-17T04:01:28.330869Z","shell.execute_reply.started":"2023-02-17T04:01:28.322024Z","shell.execute_reply":"2023-02-17T04:01:28.329653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z = x + y # We can also do z = torch.add(x, y)\nprint(z)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.332462Z","iopub.execute_input":"2023-02-17T04:01:28.333041Z","iopub.status.idle":"2023-02-17T04:01:28.339810Z","shell.execute_reply.started":"2023-02-17T04:01:28.332991Z","shell.execute_reply":"2023-02-17T04:01:28.338884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Every function which have a underscore in the end will perform an inplace operation in PyTorch","metadata":{}},{"cell_type":"code","source":"y.add_(x) # Adds x to y","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.346889Z","iopub.execute_input":"2023-02-17T04:01:28.347665Z","iopub.status.idle":"2023-02-17T04:01:28.354258Z","shell.execute_reply.started":"2023-02-17T04:01:28.347607Z","shell.execute_reply":"2023-02-17T04:01:28.353429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Subtraction\nz = torch.sub(x, y)\n# Multiplication\nz = torch.mul(x, y)\n# Division\nz = torch.div(x, y)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.359207Z","iopub.execute_input":"2023-02-17T04:01:28.359548Z","iopub.status.idle":"2023-02-17T04:01:28.365244Z","shell.execute_reply.started":"2023-02-17T04:01:28.359517Z","shell.execute_reply":"2023-02-17T04:01:28.364421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Slicing**","metadata":{}},{"cell_type":"code","source":"x = torch.rand(5, 3)\nprint(x)\nprint(x[3, :]) # Similar to NumPy","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.367718Z","iopub.execute_input":"2023-02-17T04:01:28.368722Z","iopub.status.idle":"2023-02-17T04:01:28.378296Z","shell.execute_reply.started":"2023-02-17T04:01:28.368689Z","shell.execute_reply":"2023-02-17T04:01:28.377297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x[1, 1].item()) # Use this if we only have one element in tensor","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.379925Z","iopub.execute_input":"2023-02-17T04:01:28.381009Z","iopub.status.idle":"2023-02-17T04:01:28.387095Z","shell.execute_reply.started":"2023-02-17T04:01:28.380976Z","shell.execute_reply":"2023-02-17T04:01:28.386273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reshaping**","metadata":{}},{"cell_type":"code","source":"x = torch.rand(4, 4)\ny = x.view(16) # Converted to 1D\nprint(y.size())\ny = x.view(-1, 2)\nprint(y.size())","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.392353Z","iopub.execute_input":"2023-02-17T04:01:28.392911Z","iopub.status.idle":"2023-02-17T04:01:28.398945Z","shell.execute_reply.started":"2023-02-17T04:01:28.392879Z","shell.execute_reply":"2023-02-17T04:01:28.398047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nAutograd\n ","metadata":{}},{"cell_type":"code","source":"x = torch.rand(3, requires_grad= True) # This additional argument must be specified if we want to calculate gradients later\nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.403211Z","iopub.execute_input":"2023-02-17T04:01:28.403986Z","iopub.status.idle":"2023-02-17T04:01:28.416032Z","shell.execute_reply.started":"2023-02-17T04:01:28.403952Z","shell.execute_reply":"2023-02-17T04:01:28.414956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = x + 2\nz = y * y\nz = z.mean()\nprint(y)\nprint(z)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.417778Z","iopub.execute_input":"2023-02-17T04:01:28.418320Z","iopub.status.idle":"2023-02-17T04:01:28.426309Z","shell.execute_reply.started":"2023-02-17T04:01:28.418289Z","shell.execute_reply":"2023-02-17T04:01:28.425371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z.backward() # Calculates the gradient (dz/dx)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.428012Z","iopub.execute_input":"2023-02-17T04:01:28.428618Z","iopub.status.idle":"2023-02-17T04:01:28.434692Z","shell.execute_reply.started":"2023-02-17T04:01:28.428586Z","shell.execute_reply":"2023-02-17T04:01:28.433462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x.grad) # Prints the gradients","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.436835Z","iopub.execute_input":"2023-02-17T04:01:28.437212Z","iopub.status.idle":"2023-02-17T04:01:28.446023Z","shell.execute_reply.started":"2023-02-17T04:01:28.437182Z","shell.execute_reply":"2023-02-17T04:01:28.444842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = x.detach() # detach method turns the requires_grad parameter to False\nprint(y)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.448327Z","iopub.execute_input":"2023-02-17T04:01:28.449138Z","iopub.status.idle":"2023-02-17T04:01:28.458025Z","shell.execute_reply.started":"2023-02-17T04:01:28.449106Z","shell.execute_reply":"2023-02-17T04:01:28.456752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Calculating Gradients**","metadata":{}},{"cell_type":"code","source":"random_weights = torch.ones(3, requires_grad= True)\n\nfor epoch in range(3):\n output = (random_weights * 2).sum()\n # Backward pass(gradients)\n output.backward()\n print(random_weights.grad)\n\n random_weights.grad.zero_() # Setting the gradients back to zero again","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.459775Z","iopub.execute_input":"2023-02-17T04:01:28.460815Z","iopub.status.idle":"2023-02-17T04:01:28.468904Z","shell.execute_reply.started":"2023-02-17T04:01:28.460780Z","shell.execute_reply":"2023-02-17T04:01:28.467961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Backpropagation**","metadata":{}},{"cell_type":"code","source":"x = torch.tensor(1.0) # Input\ny = torch.tensor(2.0) # Output\nw = torch.tensor(1.0, requires_grad= True) # Initializing weights\n\n# Forward pass\ny_hat = w * x # Predicted value\nloss = (y_hat - y) ** 2 # Loss function for Regression tasks (MSE)\nprint(loss)\n\n# Backward pass\nloss.backward() \nprint(w.grad) # 2*(w*x - y) = 2*(1 - 2)\n\n# Update weights and do the same thing for more number of epochs","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.473396Z","iopub.execute_input":"2023-02-17T04:01:28.473746Z","iopub.status.idle":"2023-02-17T04:01:28.482195Z","shell.execute_reply.started":"2023-02-17T04:01:28.473715Z","shell.execute_reply":"2023-02-17T04:01:28.481460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nGradient Descent\n ","metadata":{}},{"cell_type":"code","source":"x = torch.tensor([1, 2, 3, 4], dtype= torch.float32)\ny = torch.tensor([2, 4, 6, 8], dtype= torch.float32) # y = 2x\nw = torch.tensor(0.0, dtype= torch.float32, requires_grad= True)\n\ndef forward_pass(x):\n return w * x\n\n\ndef loss(y, y_pred):\n return ((y_pred - y)**2).mean()\n\nprint(f\"Prediction before training: f(5): {forward_pass(5)}\")\n\n\n# Training\nlearning_rate = 0.01\nepochs = 30\n\nfor epoch in range(epochs):\n\n #Prediction\n y_pred = forward_pass(x)\n\n # Loss\n l = loss(y, y_pred)\n\n # Backward pass\n l.backward() # dl/dw\n\n # Update weights manually\n with torch.no_grad():\n w -= learning_rate * w.grad\n \n # Setting the gradients back to zero\n w.grad.zero_()\n\n if epoch % 5 == 0:\n print(f\"Epoch: {epoch + 1}: w = {w:3f} : loss = {l:.8f}\")\n\nprint(f\"Prediction after training: f(5): {forward_pass(5):3f}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.484537Z","iopub.execute_input":"2023-02-17T04:01:28.484874Z","iopub.status.idle":"2023-02-17T04:01:28.499337Z","shell.execute_reply.started":"2023-02-17T04:01:28.484844Z","shell.execute_reply":"2023-02-17T04:01:28.498189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Optimizer and Loss function in PyTorch**","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\n\nx = torch.tensor([1, 2, 3, 4], dtype= torch.float32)\ny = torch.tensor([2, 4, 6, 8], dtype= torch.float32)\nw = torch.tensor(0.0, dtype= torch.float32, requires_grad= True)\n\nprint(f\"Prediction before training: f(5): {forward_pass(5)}\")\n\n# Training\nlearning_rate = 0.01\nepochs = 30\n\nloss = nn.MSELoss()\noptimizer = torch.optim.SGD([w], lr= learning_rate)\n\nfor epoch in range(epochs):\n\n #Prediction\n y_pred = forward_pass(x)\n\n # Loss\n l = loss(y, y_pred)\n\n # Backward pass\n l.backward() # dl/dw\n\n # Update weights\n optimizer.step()\n\n # Rest the gradients\n optimizer.zero_grad()\n\n if epoch % 5 == 0:\n print(f\"Epoch: {epoch + 1}: w = {w:3f} : loss = {l:.8f}\")\n\nprint(f\"Prediction after training: f(5): {forward_pass(5):3f}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.501253Z","iopub.execute_input":"2023-02-17T04:01:28.502078Z","iopub.status.idle":"2023-02-17T04:01:28.518987Z","shell.execute_reply.started":"2023-02-17T04:01:28.502045Z","shell.execute_reply":"2023-02-17T04:01:28.518168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nLinear Regression\n ","metadata":{}},{"cell_type":"code","source":"from sklearn import datasets\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.520542Z","iopub.execute_input":"2023-02-17T04:01:28.521537Z","iopub.status.idle":"2023-02-17T04:01:28.527039Z","shell.execute_reply.started":"2023-02-17T04:01:28.521503Z","shell.execute_reply":"2023-02-17T04:01:28.525829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data\nX_numpy, y_numpy = datasets.make_regression(n_samples= 100, n_features= 1, noise = 1, random_state= 1)\nplt.figure(figsize= (10, 6))\nplt.scatter(X_numpy, y_numpy)\nplt.xlabel(\"X\")\nplt.ylabel(\"Y\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.552017Z","iopub.execute_input":"2023-02-17T04:01:28.552927Z","iopub.status.idle":"2023-02-17T04:01:28.790311Z","shell.execute_reply.started":"2023-02-17T04:01:28.552893Z","shell.execute_reply":"2023-02-17T04:01:28.789348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nX = torch.from_numpy(X_numpy.astype(np.float32))\ny = torch.from_numpy(X_numpy.astype(np.float32))\ny = y.view(y.shape[0], 1) # Reshaping y to make it 2D\n\nn_samples, n_features = X.shape\n\n\n# Building the model\nclass LinearRegression(nn.Module):\n def __init__(self,input_size,output_size):\n # super function inherits from nn.Module and we can access everything in nn.Module\n super(LinearRegression,self).__init__()\n self.linear = nn.Linear(input_size, output_size)\n\n def forward(self,x):\n return self.linear(x)\n\n\ninput_size = n_features\noutput_size = 1\nmodel = LinearRegression(input_size,output_size)\n\n\n# Loss and optimizer\nmse = nn.MSELoss()\noptimizer = torch.optim.SGD(model.parameters(), lr= 0.01)\ncost = []\n\n# Training\nfor epoch in range(100):\n\n # Forward pass\n y_pred = model(X)\n loss = mse(y_pred, y)\n\n # Backward pass\n loss.backward()\n\n # Update\n optimizer.step()\n cost.append(loss.data)\n optimizer.zero_grad()\n\n\n if (epoch + 1) % 10 == 0:\n print(f\"Epoch: {epoch + 1}, Loss = {loss.item():.4f}\")\n\nplt.plot(range(100), cost)\nplt.xlabel(\"Number of Iterations\")\nplt.ylabel(\"Loss\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:28.791911Z","iopub.execute_input":"2023-02-17T04:01:28.792398Z","iopub.status.idle":"2023-02-17T04:01:29.089817Z","shell.execute_reply.started":"2023-02-17T04:01:28.792367Z","shell.execute_reply":"2023-02-17T04:01:29.088582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted = model(X).detach().numpy()\nplt.figure(figsize= (10, 6))\nplt.scatter(X, y, label = 'original', color = 'red')\nplt.scatter(X, predicted, label = 'predicted', color = 'blue')\nplt.ylabel(\"Y\")\nplt.xlabel(\"X\")\nplt.title(\"Original vs Predicted values\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:29.091074Z","iopub.execute_input":"2023-02-17T04:01:29.091422Z","iopub.status.idle":"2023-02-17T04:01:29.405830Z","shell.execute_reply.started":"2023-02-17T04:01:29.091390Z","shell.execute_reply":"2023-02-17T04:01:29.404710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nLogistic Regression\n ","metadata":{}},{"cell_type":"code","source":"from sklearn import datasets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:29.408865Z","iopub.execute_input":"2023-02-17T04:01:29.409197Z","iopub.status.idle":"2023-02-17T04:01:29.414268Z","shell.execute_reply.started":"2023-02-17T04:01:29.409166Z","shell.execute_reply":"2023-02-17T04:01:29.413117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = datasets.load_breast_cancer()\nX, y = data.data, data.target\n\nn_samples, n_features = X.shape # We have 569 samples and 30 different features\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state= 42)\n\n# Scaling the data\nsc = StandardScaler()\nX_train = sc.fit_transform(X_train)\nX_test = sc.transform(X_test)\n\n# Converting the training and test set into tensors\nX_train = torch.from_numpy(X_train.astype(np.float32))\nX_test = torch.from_numpy(X_test.astype(np.float32))\ny_train = torch.from_numpy(y_train.astype(np.float32))\ny_test = torch.from_numpy(y_test.astype(np.float32))\n\n# Converting y_train and y_test into 2D\ny_train = y_train.view(y_train.shape[0], 1)\ny_test = y_test.view(y_test.shape[0], 1)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:29.416186Z","iopub.execute_input":"2023-02-17T04:01:29.416649Z","iopub.status.idle":"2023-02-17T04:01:29.439535Z","shell.execute_reply.started":"2023-02-17T04:01:29.416606Z","shell.execute_reply":"2023-02-17T04:01:29.438393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building the Model\nclass Model(nn.Module):\n def __init__(self, n_input_features):\n super(Model, self).__init__()\n self.linear = nn.Linear(n_input_features, 1) # 30 input features and 1 output feature\n\n def forward(self, x):\n y_pred = torch.sigmoid(self.linear(x)) # y_pred = sigmoid(output)\n return y_pred\n\nmodel = Model(n_features)\n\n\n# Loss and optimizer\ncriterion = nn.BCELoss() # binary cross entropy\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01)\ncost = []\n\n# Training loop\nfor epoch in range(100):\n\n # Forward pass\n y_pred = model(X_train)\n loss = criterion(y_pred, y_train)\n\n # Backward pass\n loss.backward()\n\n optimizer.step()\n cost.append(loss.data)\n optimizer.zero_grad()\n\n if (epoch+1) % 10 == 0:\n print(f'epoch: {epoch+1}, loss = {loss.item():.4f}')\n\n\nwith torch.no_grad():\n y_predicted = model(X_test)\n y_predicted_cls = y_predicted.round() # Either predicted value will be 0 or 1\n acc = y_predicted_cls.eq(y_test).sum() / float(y_test.shape[0])\n plt.plot(range(100), cost)\n plt.xlabel(\"Number of Iterations\")\n plt.ylabel(\"Loss\")\n plt.show()\n print(f'accuracy: {acc.item():.4f}')","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:29.441366Z","iopub.execute_input":"2023-02-17T04:01:29.442026Z","iopub.status.idle":"2023-02-17T04:01:29.757088Z","shell.execute_reply.started":"2023-02-17T04:01:29.441980Z","shell.execute_reply":"2023-02-17T04:01:29.756325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Softmax Function**","metadata":{}},{"cell_type":"code","source":"x = torch.tensor([2.0, 1.0, 3.0])\noutputs = torch.softmax(x, dim= 0)\nprint(outputs, outputs.sum())","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:29.758219Z","iopub.execute_input":"2023-02-17T04:01:29.758843Z","iopub.status.idle":"2023-02-17T04:01:29.766928Z","shell.execute_reply.started":"2023-02-17T04:01:29.758807Z","shell.execute_reply":"2023-02-17T04:01:29.765664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **CrossEntropyLoss**","metadata":{}},{"cell_type":"markdown","source":"> `nn.CrossEntropyLoss()` function in PyTorch applies softmax internally, therefore don't use softmax whenever you use this function","metadata":{}},{"cell_type":"code","source":"Y = torch.tensor([0])\nY_pred_good = torch.tensor([[2.0, 1.0, 0.1]])\nY_pred_bad = torch.tensor([[0.5, 2.0, 0.3]])\n\n\nloss = nn.CrossEntropyLoss()\nl1 = loss(Y_pred_good, Y)\nl2 = loss(Y_pred_bad, Y)\n\nprint(f'PyTorch Loss1: {l1.item():.4f}')\nprint(f'PyTorch Loss2: {l2.item():.4f}') # loss is more in this case as predictions are not so good\n","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:29.768788Z","iopub.execute_input":"2023-02-17T04:01:29.769384Z","iopub.status.idle":"2023-02-17T04:01:29.781029Z","shell.execute_reply.started":"2023-02-17T04:01:29.769348Z","shell.execute_reply":"2023-02-17T04:01:29.779806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nANN\n ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom torch.utils.data import DataLoader\n\n# Loading data\ntrain = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\",dtype = np.float32)\n\n# Splitting data into features and labels\ny_numpy = train.label.values\nX_numpy = train.loc[:,train.columns != \"label\"].values\nX_numpy_normalized = X_numpy / 255.\n\n# train test split. \nX_train, X_test, y_train, y_test = train_test_split(X_numpy_normalized, y_numpy, test_size = 0.2, random_state = 42)\n\n# Converting the data into tensors\nX_train = torch.from_numpy(X_train)\ny_train = torch.from_numpy(y_train).type(torch.LongTensor) # data type is long\nX_test = torch.from_numpy(X_test)\ny_test = torch.from_numpy(y_test).type(torch.LongTensor) # data type is long\n\n# batch_size, epoch and iteration\nbatch_size = 100\nn_iters = 10000\nnum_epochs = n_iters / (len(X_train) / batch_size)\nnum_epochs = int(num_epochs)\n\n# Pytorch train and test sets\ntrain = torch.utils.data.TensorDataset(X_train, y_train)\ntest = torch.utils.data.TensorDataset(X_test,y_test)\n\n# data loader\ntrain_loader = DataLoader(train, batch_size = batch_size, shuffle = False)\ntest_loader = DataLoader(test, batch_size = batch_size, shuffle = False)\n\n# visualizing one of the image in the training set\nplt.imshow(X_numpy[20].reshape(28,28))\nplt.axis(\"off\")\nplt.title(str(y_numpy[20]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:06:06.196259Z","iopub.execute_input":"2023-02-17T04:06:06.196711Z","iopub.status.idle":"2023-02-17T04:06:09.451474Z","shell.execute_reply.started":"2023-02-17T04:06:06.196672Z","shell.execute_reply":"2023-02-17T04:06:09.450364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArtificialNeuralNetwork(nn.Module):\n \n def __init__(self, input_dim, hidden_dim, output_dim):\n super(ArtificialNeuralNetwork, self).__init__()\n \n # Layer1: 784 --> 150\n self.l1 = nn.Linear(input_dim, hidden_dim) \n self.relu1 = nn.ReLU()\n \n # Layer2: 150 --> 150\n self.l2 = nn.Linear(hidden_dim, hidden_dim)\n self.tanh2 = nn.Tanh()\n \n # Layer3: 150 --> 150\n self.l3 = nn.Linear(hidden_dim, hidden_dim)\n self.elu3 = nn.ELU()\n \n # Layer4 (readout): 150 --> 10\n self.l4 = nn.Linear(hidden_dim, output_dim) \n \n def forward(self, x):\n # Layer 1\n out = self.l1(x)\n out = self.relu1(out)\n \n # Layer 2\n out = self.l2(out)\n out = self.tanh2(out)\n \n # Layer 3\n out = self.l3(out)\n out = self.elu3(out)\n \n # Layer 4 (readout)\n out = self.l4(out)\n return out\n \n \n\ninput_dim = 28*28\nhidden_dim = 150 # Hyperparameter (hidden layer size)\noutput_dim = 10\n\nmodel = ArtificialNeuralNetwork(input_dim, hidden_dim, output_dim)\n\n# Cross Entropy Loss \nerror = nn.CrossEntropyLoss()\n\n# SGD Optimizer\nlearning_rate = 0.02\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:33.189160Z","iopub.execute_input":"2023-02-17T04:01:33.189637Z","iopub.status.idle":"2023-02-17T04:01:33.203901Z","shell.execute_reply.started":"2023-02-17T04:01:33.189605Z","shell.execute_reply":"2023-02-17T04:01:33.202814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training\ncount = 0\ncost = []\niteration_list = []\naccuracy_list = []\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n\n images = images.reshape(-1, 28*28)\n \n # Clear gradients\n optimizer.zero_grad()\n \n # Forward propagation\n outputs = model(images)\n \n # Calculate softmax and cross entropy loss\n loss = error(outputs, labels)\n \n # Calculating gradients\n loss.backward()\n \n # Update parameters\n optimizer.step()\n \n count += 1\n \n if count % 50 == 0:\n # Calculate Accuracy \n correct = 0\n total = 0\n # Predict test dataset\n for images, labels in test_loader:\n\n test = images.reshape(-1, 28*28)\n \n # Forward propagation\n outputs = model(test)\n \n # Get predictions from the maximum value\n predicted = torch.max(outputs.data, 1)[1]\n \n # Total number of labels\n total += len(labels)\n\n # Total correct predictions\n correct += (predicted == labels).sum()\n \n accuracy = 100 * correct / float(total)\n \n cost.append(loss.data)\n iteration_list.append(count)\n accuracy_list.append(accuracy)\n \n if count % 500 == 0:\n print('Iteration: {} Loss: {} Accuracy: {} %'.format(count, loss.data, accuracy))","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:01:33.205123Z","iopub.execute_input":"2023-02-17T04:01:33.205589Z","iopub.status.idle":"2023-02-17T04:02:29.274631Z","shell.execute_reply.started":"2023-02-17T04:01:33.205526Z","shell.execute_reply":"2023-02-17T04:02:29.273198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize = (20, 5))\n\n# Plotting loss\nax1 = sns.lineplot(x = iteration_list, y = np.array(cost), ax= ax[0])\nax1.set(title = 'Loss', xlabel = 'Epochs')\n\n# Plotting accuracy\nax2 = sns.lineplot(x = iteration_list, y = np.array(accuracy_list), ax= ax[1])\nax2.set(title = 'Accuracy', xlabel = 'Epochs');","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:02:29.276532Z","iopub.execute_input":"2023-02-17T04:02:29.277227Z","iopub.status.idle":"2023-02-17T04:02:30.044686Z","shell.execute_reply.started":"2023-02-17T04:02:29.277192Z","shell.execute_reply":"2023-02-17T04:02:30.043554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nCNN\n ","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n \n def __init__(self):\n super(CNN, self).__init__()\n \n # Convolution 1\n self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n # We will get 16 feature maps as the output of the first convolutional layer\n self.relu1 = nn.ReLU()\n \n # Max pool 1\n self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n \n # Convolution 2\n self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n self.relu2 = nn.ReLU()\n \n # Max pool 2\n self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n \n # Fully connected 1\n self.fc1 = nn.Linear(32 * 4 * 4, 10) \n \n def forward(self, x):\n # Convolution 1\n out = self.cnn1(x)\n out = self.relu1(out)\n \n # Max pool 1\n out = self.maxpool1(out)\n \n # Convolution 2 \n out = self.cnn2(out)\n out = self.relu2(out)\n \n # Max pool 2 \n out = self.maxpool2(out)\n \n # flatten\n out = out.view(out.size(0), -1)\n\n # Linear function (readout)\n out = self.fc1(out)\n \n return out\n\nbatch_size = 100\nn_iters = 2500\nnum_epochs = n_iters / (len(X_train) / batch_size)\nnum_epochs = int(num_epochs)\n \nmodel = CNN()\nerror = nn.CrossEntropyLoss()\nlearning_rate = 0.02\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:02:30.046492Z","iopub.execute_input":"2023-02-17T04:02:30.047284Z","iopub.status.idle":"2023-02-17T04:02:30.062335Z","shell.execute_reply.started":"2023-02-17T04:02:30.047235Z","shell.execute_reply":"2023-02-17T04:02:30.061230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training\ncount = 0\ncost = []\niteration_list = []\naccuracy_list = []\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n\n images = images.reshape(100,1, 28, 28)\n \n # Clear gradients\n optimizer.zero_grad()\n \n # Forward propagation\n outputs = model(images)\n \n # Calculate softmax and cross entropy loss\n loss = error(outputs, labels)\n \n # Calculating gradients\n loss.backward()\n \n # Update parameters\n optimizer.step()\n \n count += 1\n \n if count % 50 == 0:\n # Calculate Accuracy \n correct = 0\n total = 0\n # Predict test dataset\n for images, labels in test_loader:\n\n test = images.reshape(100, 1, 28, 28)\n \n # Forward propagation\n outputs = model(test)\n \n # Get predictions from the maximum value\n predicted = torch.max(outputs.data, 1)[1]\n \n # Total number of labels\n total += len(labels)\n\n # Total correct predictions\n correct += (predicted == labels).sum()\n \n accuracy = 100 * correct / float(total)\n \n cost.append(loss.data)\n iteration_list.append(count)\n accuracy_list.append(accuracy)\n \n if count % 500 == 0:\n print('Iteration: {} Loss: {} Accuracy: {} %'.format(count, loss.data, accuracy))","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:02:30.063921Z","iopub.execute_input":"2023-02-17T04:02:30.064378Z","iopub.status.idle":"2023-02-17T04:03:46.536914Z","shell.execute_reply.started":"2023-02-17T04:02:30.064331Z","shell.execute_reply":"2023-02-17T04:03:46.535504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize = (20, 5))\n\n# Plotting loss\nax1 = sns.lineplot(x = iteration_list, y = np.array(cost), ax= ax[0])\nax1.set(title = 'Loss', xlabel = 'Epochs')\n\n# Plotting accuracy\nax2 = sns.lineplot(x = iteration_list, y = np.array(accuracy_list), ax= ax[1])\nax2.set(title = 'Accuracy', xlabel = 'Epochs');","metadata":{"execution":{"iopub.status.busy":"2023-02-17T04:03:46.538780Z","iopub.execute_input":"2023-02-17T04:03:46.541936Z","iopub.status.idle":"2023-02-17T04:03:47.101659Z","shell.execute_reply.started":"2023-02-17T04:03:46.541884Z","shell.execute_reply":"2023-02-17T04:03:47.100103Z"},"trusted":true},"execution_count":null,"outputs":[]}]}