{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-05T20:55:23.167809Z","iopub.execute_input":"2023-07-05T20:55:23.168169Z","iopub.status.idle":"2023-07-05T20:55:23.172951Z","shell.execute_reply.started":"2023-07-05T20:55:23.168140Z","shell.execute_reply":"2023-07-05T20:55:23.172047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:23.177936Z","iopub.execute_input":"2023-07-05T20:55:23.178766Z","iopub.status.idle":"2023-07-05T20:55:27.901408Z","shell.execute_reply.started":"2023-07-05T20:55:23.178730Z","shell.execute_reply":"2023-07-05T20:55:27.900417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:27.903443Z","iopub.execute_input":"2023-07-05T20:55:27.903794Z","iopub.status.idle":"2023-07-05T20:55:27.920057Z","shell.execute_reply.started":"2023-07-05T20:55:27.903762Z","shell.execute_reply":"2023-07-05T20:55:27.919134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train.isnull().sum().sort_values(ascending = False))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:27.921656Z","iopub.execute_input":"2023-07-05T20:55:27.922257Z","iopub.status.idle":"2023-07-05T20:55:27.966822Z","shell.execute_reply.started":"2023-07-05T20:55:27.922226Z","shell.execute_reply":"2023-07-05T20:55:27.965754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(test.isnull().sum().sort_values(ascending = False))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:27.969918Z","iopub.execute_input":"2023-07-05T20:55:27.970270Z","iopub.status.idle":"2023-07-05T20:55:27.996682Z","shell.execute_reply.started":"2023-07-05T20:55:27.970238Z","shell.execute_reply":"2023-07-05T20:55:27.995672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Shape of Training Set : {train.shape}\")\nprint(f\"Shape of Test Set : {test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:27.998500Z","iopub.execute_input":"2023-07-05T20:55:27.998864Z","iopub.status.idle":"2023-07-05T20:55:28.003961Z","shell.execute_reply.started":"2023-07-05T20:55:27.998832Z","shell.execute_reply":"2023-07-05T20:55:28.002984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.iloc[:,1:]\nY = train.label","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:28.005460Z","iopub.execute_input":"2023-07-05T20:55:28.006118Z","iopub.status.idle":"2023-07-05T20:55:28.014434Z","shell.execute_reply.started":"2023-07-05T20:55:28.006087Z","shell.execute_reply":"2023-07-05T20:55:28.013372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Second Example Before Scaling**","metadata":{}},{"cell_type":"code","source":"image = X.iloc[1]\nplt.imshow(image.to_numpy().reshape(28,28), cmap='gray')\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:28.016020Z","iopub.execute_input":"2023-07-05T20:55:28.016369Z","iopub.status.idle":"2023-07-05T20:55:28.093891Z","shell.execute_reply.started":"2023-07-05T20:55:28.016336Z","shell.execute_reply":"2023-07-05T20:55:28.092833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scaler(Features):\n\n  mu = np.mean(Features , axis = 0)\n\n  sigma = np.std(Features , axis = 0)\n\n  sigma[sigma == 0] = 1e-9  \n\n  Features_norm = (Features - mu) / sigma\n  \n  return Features_norm","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:28.095426Z","iopub.execute_input":"2023-07-05T20:55:28.095737Z","iopub.status.idle":"2023-07-05T20:55:28.103750Z","shell.execute_reply.started":"2023-07-05T20:55:28.095708Z","shell.execute_reply":"2023-07-05T20:55:28.102328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_norm = scaler(X)\nx_test_norm = scaler(test)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:28.105015Z","iopub.execute_input":"2023-07-05T20:55:28.105408Z","iopub.status.idle":"2023-07-05T20:55:28.863457Z","shell.execute_reply.started":"2023-07-05T20:55:28.105367Z","shell.execute_reply":"2023-07-05T20:55:28.862503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Second Example After Scaling**\n","metadata":{}},{"cell_type":"code","source":"image = x_train_norm.iloc[1]\nplt.imshow(image.to_numpy().reshape(28,28), cmap='gray')\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:28.867858Z","iopub.execute_input":"2023-07-05T20:55:28.868173Z","iopub.status.idle":"2023-07-05T20:55:28.943700Z","shell.execute_reply.started":"2023-07-05T20:55:28.868136Z","shell.execute_reply":"2023-07-05T20:55:28.942669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train , x_cv , y_train , y_cv = train_test_split(x_train_norm,Y,test_size = 0.2 , random_state = 0)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:28.945297Z","iopub.execute_input":"2023-07-05T20:55:28.945954Z","iopub.status.idle":"2023-07-05T20:55:29.192498Z","shell.execute_reply.started":"2023-07-05T20:55:28.945921Z","shell.execute_reply":"2023-07-05T20:55:29.191546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_hot_encoder(y):\n    \n    unique_values = list(set(y))\n    num_unique = len(unique_values)\n    y_encoded = []\n    \n    for value in y:\n        encoded_vector = [0] * num_unique\n        index = unique_values.index(value)\n        encoded_vector[index] = 1\n        y_encoded.append(encoded_vector)\n        \n    return y_encoded\n        \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.194074Z","iopub.execute_input":"2023-07-05T20:55:29.194418Z","iopub.status.idle":"2023-07-05T20:55:29.202033Z","shell.execute_reply.started":"2023-07-05T20:55:29.194387Z","shell.execute_reply":"2023-07-05T20:55:29.201003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_encoded = one_hot_encoder(y_train)\ny_encoded = pd.DataFrame(y_encoded,columns = ['y0','y1','y2','y3','y4','y5','y6','y7','y8','y9'])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.203629Z","iopub.execute_input":"2023-07-05T20:55:29.203962Z","iopub.status.idle":"2023-07-05T20:55:29.385962Z","shell.execute_reply.started":"2023-07-05T20:55:29.203929Z","shell.execute_reply":"2023-07-05T20:55:29.385153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.to_numpy()\nx_cv = x_cv.to_numpy()\nx_test = x_test_norm.to_numpy()\ny_encoded = y_encoded.to_numpy()\ny_cv = y_cv.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.387386Z","iopub.execute_input":"2023-07-05T20:55:29.387779Z","iopub.status.idle":"2023-07-05T20:55:29.393180Z","shell.execute_reply.started":"2023-07-05T20:55:29.387748Z","shell.execute_reply":"2023-07-05T20:55:29.392335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sigmoid_fun(z):\n  \n  return 1/(1+np.exp(-z))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.394471Z","iopub.execute_input":"2023-07-05T20:55:29.395028Z","iopub.status.idle":"2023-07-05T20:55:29.403471Z","shell.execute_reply.started":"2023-07-05T20:55:29.394996Z","shell.execute_reply":"2023-07-05T20:55:29.402664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cost_fun(x,y,w,b,lambda_):\n\n  loss = reg_cost = 0.0\n\n  for i in range(x.shape[0]):\n    loss += ( y[i] * np.log(sigmoid_fun(np.dot(x[i],w)+b)) ) + ( (1-y[i]) * np.log(1-sigmoid_fun(np.dot(x[i],w)+b)) ) \n  \n  for j in range(x.shape[1]):\n    reg_cost += w[j] ** 2\n \n  total_cost = (loss / -x.shape[0]) + ((lambda_ / (2 * x.shape[0])) * reg_cost)\n    \n  return total_cost","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.404600Z","iopub.execute_input":"2023-07-05T20:55:29.405170Z","iopub.status.idle":"2023-07-05T20:55:29.415095Z","shell.execute_reply.started":"2023-07-05T20:55:29.405136Z","shell.execute_reply":"2023-07-05T20:55:29.414183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gradients_fun(x,y,w,b,lambda_):\n\n  dj_dw = np.zeros((x.shape[1],))     \n  dj_db = 0.\n\n  for i in range(x.shape[0]):      \n    error = sigmoid_fun(np.dot(x[i],w)+b) - y[i]    \n    for j in range(x.shape[1]):\n      dj_dw[j] += error * x[i,j]\n    dj_db += error\n\n  dj_dw = dj_dw/x.shape[0]                                 \n  dj_db = dj_db/x.shape[0]\n  \n  for j in range(x.shape[1]):\n    dj_dw[j] +=  (lambda_ / x.shape[0]) * w[j]\n                            \n  return dj_dw, dj_db","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.416255Z","iopub.execute_input":"2023-07-05T20:55:29.416521Z","iopub.status.idle":"2023-07-05T20:55:29.426912Z","shell.execute_reply.started":"2023-07-05T20:55:29.416498Z","shell.execute_reply":"2023-07-05T20:55:29.426106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gradient_descent(x,y,alpha,num_iterations,lambda_):\n\n  w = np.zeros((x.shape[1],))\n  b = 0\n  cost_values = []\n\n  for i in range(num_iterations):\n    dj_dw, dj_db = gradients_fun(x,y,w,b,lambda_)\n    w = w - alpha * dj_dw\n    b = b - alpha * dj_db\n    if i<100000:\n      cost_values.append(cost_fun(x, y, w , b, lambda_))\n    if i% math.ceil(num_iterations/10) == 0 or i == num_iterations - 1:\n      print(f\"Iteration {i} -> Cost = {cost_values[-1]}\")\n\n  return w, b","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.428396Z","iopub.execute_input":"2023-07-05T20:55:29.429052Z","iopub.status.idle":"2023-07-05T20:55:29.443189Z","shell.execute_reply.started":"2023-07-05T20:55:29.429021Z","shell.execute_reply":"2023-07-05T20:55:29.442414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def accuracy_score(y_pred,y):\n  return np.mean(y_pred == y) * 100","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.444502Z","iopub.execute_input":"2023-07-05T20:55:29.444996Z","iopub.status.idle":"2023-07-05T20:55:29.455346Z","shell.execute_reply.started":"2023-07-05T20:55:29.444957Z","shell.execute_reply":"2023-07-05T20:55:29.454376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nW = []\nB = []\nfor i in range(len(Y.unique())):\n    \n    print(\"-------------------------------------------\")\n    print(f\"Model_{i+1} : \")\n    \n    learning_rate = 0.1\n    num_iters = 50\n    lambda_ = 0.01\n    \n    y_train = y_encoded[:,i]\n    w, b = gradient_descent(x_train,y_train,learning_rate,num_iters,lambda_)\n\n    W.append(w)\n    B.append(b)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T20:55:29.456852Z","iopub.execute_input":"2023-07-05T20:55:29.457492Z","iopub.status.idle":"2023-07-06T00:15:25.515920Z","shell.execute_reply.started":"2023-07-05T20:55:29.457462Z","shell.execute_reply":"2023-07-06T00:15:25.514786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(z):\n    z_exp = np.exp(z)\n    z_exp_sum = np.sum(z_exp,axis = 1,keepdims=True)\n    P = z_exp / z_exp_sum\n    return P","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:15:25.517454Z","iopub.execute_input":"2023-07-06T00:15:25.517849Z","iopub.status.idle":"2023-07-06T00:15:25.523489Z","shell.execute_reply.started":"2023-07-06T00:15:25.517811Z","shell.execute_reply":"2023-07-06T00:15:25.522512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W = np.array(W)\nB = np.array(B)\nZ = np.dot(x_cv,W.T) + b\nprobabilites_predictions = softmax(Z)\npredicted_labels = np.argmax(probabilites_predictions,axis = 1)\nprint(f\"Accuracy on cv set : {accuracy_score(predicted_labels,y_cv)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:15:25.524896Z","iopub.execute_input":"2023-07-06T00:15:25.525511Z","iopub.status.idle":"2023-07-06T00:15:25.571732Z","shell.execute_reply.started":"2023-07-06T00:15:25.525477Z","shell.execute_reply":"2023-07-06T00:15:25.570671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Z = np.dot(x_test,W.T) + b\nprobabilites_predictions = softmax(Z)\npredicted_labels = np.argmax(probabilites_predictions,axis = 1)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:15:25.573375Z","iopub.execute_input":"2023-07-06T00:15:25.573964Z","iopub.status.idle":"2023-07-06T00:15:25.663802Z","shell.execute_reply.started":"2023-07-06T00:15:25.573918Z","shell.execute_reply":"2023-07-06T00:15:25.662344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df = pd.DataFrame({'ImageId': range(1, len(predicted_labels)+1),\n                           'Label': predicted_labels})\nresults_df","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:15:25.665434Z","iopub.execute_input":"2023-07-06T00:15:25.666095Z","iopub.status.idle":"2023-07-06T00:15:25.682832Z","shell.execute_reply.started":"2023-07-06T00:15:25.666060Z","shell.execute_reply":"2023-07-06T00:15:25.681628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nresults_df.to_csv('predictions.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:15:25.684675Z","iopub.execute_input":"2023-07-06T00:15:25.685365Z","iopub.status.idle":"2023-07-06T00:15:25.815156Z","shell.execute_reply.started":"2023-07-06T00:15:25.685330Z","shell.execute_reply":"2023-07-06T00:15:25.814243Z"},"trusted":true},"execution_count":null,"outputs":[]}]}