{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport umap\nfrom collections import Counter\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-09T19:09:34.780181Z","iopub.execute_input":"2023-01-09T19:09:34.781531Z","iopub.status.idle":"2023-01-09T19:09:57.786239Z","shell.execute_reply.started":"2023-01-09T19:09:34.781481Z","shell.execute_reply":"2023-01-09T19:09:57.784521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ndf_test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:09:57.789467Z","iopub.execute_input":"2023-01-09T19:09:57.790379Z","iopub.status.idle":"2023-01-09T19:10:04.002969Z","shell.execute_reply.started":"2023-01-09T19:09:57.790337Z","shell.execute_reply":"2023-01-09T19:10:04.001989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:04.004318Z","iopub.execute_input":"2023-01-09T19:10:04.005002Z","iopub.status.idle":"2023-01-09T19:10:04.029054Z","shell.execute_reply.started":"2023-01-09T19:10:04.004960Z","shell.execute_reply":"2023-01-09T19:10:04.027324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.dropna(axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:04.032401Z","iopub.execute_input":"2023-01-09T19:10:04.032863Z","iopub.status.idle":"2023-01-09T19:10:04.296832Z","shell.execute_reply.started":"2023-01-09T19:10:04.032809Z","shell.execute_reply":"2023-01-09T19:10:04.295351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_data = df_train.groupby(['label'])['label'].count()\ngroup_data","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:04.298612Z","iopub.execute_input":"2023-01-09T19:10:04.298959Z","iopub.status.idle":"2023-01-09T19:10:04.324424Z","shell.execute_reply.started":"2023-01-09T19:10:04.298930Z","shell.execute_reply":"2023-01-09T19:10:04.323213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_labels = df_train['label'].copy()\ndf_train_data = df_train.drop(['label'], axis=1).copy()\ndf_test_id = df_test['id'].copy()\ndf_test_data = df_test.drop(['id'], axis=1).copy()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:04.325942Z","iopub.execute_input":"2023-01-09T19:10:04.326322Z","iopub.status.idle":"2023-01-09T19:10:04.738702Z","shell.execute_reply.started":"2023-01-09T19:10:04.326290Z","shell.execute_reply":"2023-01-09T19:10:04.737523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_labels.shape)\nprint(df_train_data.shape)\nprint(df_test_id.shape)\nprint(df_test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:04.740321Z","iopub.execute_input":"2023-01-09T19:10:04.740682Z","iopub.status.idle":"2023-01-09T19:10:04.747442Z","shell.execute_reply.started":"2023-01-09T19:10:04.740650Z","shell.execute_reply":"2023-01-09T19:10:04.746133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# The original images of the handwritten digits can be viewed using plt and reshaping each row \n# from the dataframe into a 2D image with a size of 28 by 28 pixels.\n# This for loop will display the first 10 images from the training data, along with their labels:\nfor i in range(10):\n    ax= plt.subplot(1,10 ,i+1)\n    im=ax.imshow(df_train_data.iloc[i].values.reshape(28,28))\n    plt.xlabel=''\n    ax.axes.get_xaxis().set_visible(False)\n    ax.axes.get_yaxis().set_visible(False)\n    plt.title(df_train_labels.iloc[i])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:04.749801Z","iopub.execute_input":"2023-01-09T19:10:04.750385Z","iopub.status.idle":"2023-01-09T19:10:05.219880Z","shell.execute_reply.started":"2023-01-09T19:10:04.750337Z","shell.execute_reply":"2023-01-09T19:10:05.218328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reducer = umap.UMAP(random_state=42)\nembedding = reducer.fit_transform(df_train_data)\nembedding.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:10:05.222232Z","iopub.execute_input":"2023-01-09T19:10:05.222751Z","iopub.status.idle":"2023-01-09T19:11:35.428136Z","shell.execute_reply.started":"2023-01-09T19:10:05.222704Z","shell.execute_reply":"2023-01-09T19:11:35.426389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(reducer.embedding_[:, 0], reducer.embedding_[:, 1], s= 5, c=df_train_labels, cmap='Spectral')\nplt.gca().set_aspect('equal', 'datalim')\nplt.colorbar(boundaries=np.arange(11)-0.5).set_ticks(np.arange(10))\nplt.title('Visualizing Kannada MNIST with UMAP', fontsize=24);","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:11:35.433535Z","iopub.execute_input":"2023-01-09T19:11:35.433929Z","iopub.status.idle":"2023-01-09T19:11:37.279927Z","shell.execute_reply.started":"2023-01-09T19:11:35.433895Z","shell.execute_reply":"2023-01-09T19:11:37.278574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init(x,y):\n    layer=np.random.uniform(-1.,1.,size=(x,y))/np.sqrt(x*y)\n    return layer.astype(np.float32)\n\nnp.random.seed(42)\nl1=init(28*28,128)\nl2=init(128,10)\nprint(l1.shape, l2.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:11:37.281909Z","iopub.execute_input":"2023-01-09T19:11:37.282424Z","iopub.status.idle":"2023-01-09T19:11:37.293927Z","shell.execute_reply.started":"2023-01-09T19:11:37.282378Z","shell.execute_reply":"2023-01-09T19:11:37.292193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sigmoid funstion\ndef sigmoid(x):\n    return 1/(np.exp(-x)+1)    \n\n#derivative of sigmoid\ndef d_sigmoid(x):\n    return (np.exp(- x)) / ((np.exp(- x) + 1) ** 2)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:11:37.295986Z","iopub.execute_input":"2023-01-09T19:11:37.297044Z","iopub.status.idle":"2023-01-09T19:11:37.305296Z","shell.execute_reply.started":"2023-01-09T19:11:37.296957Z","shell.execute_reply":"2023-01-09T19:11:37.303799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Softmax\ndef softmax(x):\n    exp_element=np.exp(x-x.max())\n    return exp_element/np.sum(exp_element,axis=0)\n\n#derivative of softmax\ndef d_softmax(x):\n    exp_element=np.exp(x-x.max())\n    return exp_element/np.sum(exp_element,axis=0)*(1-exp_element/np.sum(exp_element,axis=0))","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:11:37.307524Z","iopub.execute_input":"2023-01-09T19:11:37.308093Z","iopub.status.idle":"2023-01-09T19:11:37.318320Z","shell.execute_reply.started":"2023-01-09T19:11:37.308037Z","shell.execute_reply":"2023-01-09T19:11:37.316852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#forward and backward pass\ndef forward_backward_pass(x,y):\n    \"\"\" Perform forward and back propagation to caculate ouput, update_l1, update_l2 \n\n    Args:\n        x (ndarray (m,n)): train data \n        y (ndarray (m,1)): train label\n\n    Returns:\n        out (ndarray (m,10)): output\n        update_l1 (ndarray (n,k)): weight parameters\n        update_l2 (ndarray (k,10)): weight parameters\n    \"\"\"\n\n    # create one-hot vector for output\n    targets = np.zeros((len(y),10), np.float32)\n    targets[range(targets.shape[0]),y] = 1\n \n    # perform forward propagation\n    x_l1=x.dot(l1)\n    x_sigmoid=sigmoid(x_l1)\n    x_l2=x_sigmoid.dot(l2)\n    out=softmax(x_l2)\n    \n    # perform back propagation\n    error=2*(out-targets)/out.shape[0]*d_softmax(x_l2)\n    update_l2=x_sigmoid.T@error\n    \n    error=((l2).dot(error.T)).T*d_sigmoid(x_l1)\n    update_l1=x.T@error\n\n    return out,update_l1,update_l2 ","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:11:37.320222Z","iopub.execute_input":"2023-01-09T19:11:37.320743Z","iopub.status.idle":"2023-01-09T19:11:37.334372Z","shell.execute_reply.started":"2023-01-09T19:11:37.320695Z","shell.execute_reply":"2023-01-09T19:11:37.332743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=10000\nlr=0.001\nbatch=128\nlosses,accuracies,val_accuracies=[],[],[]\nX_train = df_train_data.to_numpy()\nY_train = df_train_labels.to_numpy()\nfor i in range(epochs):\n    sample=np.random.randint(0,X_train.shape[0],size=(batch))\n    x=X_train[sample].reshape((-1,28*28))\n    y=Y_train[sample]\n \n\n    out,update_l1,update_l2=forward_backward_pass(x,y)\n  \n    category=np.argmax(out,axis=1)\n    accuracy=(category==y).mean()\n    accuracies.append(accuracy)\n    \n    loss=((category-y)**2).mean()\n    losses.append(loss.item())\n    \n    l1=l1-lr*update_l1\n    l2=l2-lr*update_l2\n    \n    # if(i%20==0):    \n    #     X_val=X_val.reshape((-1,28*28))\n    #     val_out=np.argmax(softmax(sigmoid(X_val.dot(l1)).dot(l2)),axis=1)\n    #     val_acc=(val_out==Y_val).mean()\n    #     val_accuracies.append(val_acc.item())\n    if(i%500==0): print(f'For {i}th epoch: train accuracy: {accuracy:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:20:59.617893Z","iopub.execute_input":"2023-01-09T19:20:59.618458Z","iopub.status.idle":"2023-01-09T19:22:24.658922Z","shell.execute_reply.started":"2023-01-09T19:20:59.618415Z","shell.execute_reply":"2023-01-09T19:22:24.657168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(X_test_data, X_test_id, l1, l2):\n    x_l1=X_test_data.dot(l1)\n    x_sigmoid=sigmoid(x_l1)\n    x_l2=x_sigmoid.dot(l2)\n    output=softmax(x_l2)\n\n    labels = pd.DataFrame(columns=['id','label'])\n    for i in range(len(output)):\n        label = np.argmax(output.iloc[i])\n        point = X_test_id[i]\n        labels = pd.concat([labels, pd.Series({'id': point, 'label': label}).to_frame().T],ignore_index=True)\n    return labels","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:25:33.924579Z","iopub.execute_input":"2023-01-09T19:25:33.925034Z","iopub.status.idle":"2023-01-09T19:25:33.934811Z","shell.execute_reply.started":"2023-01-09T19:25:33.924983Z","shell.execute_reply":"2023-01-09T19:25:33.932978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = predict(df_test_data, df_test_id, l1, l2)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:26:52.335520Z","iopub.execute_input":"2023-01-09T19:26:52.335956Z","iopub.status.idle":"2023-01-09T19:26:59.050343Z","shell.execute_reply.started":"2023-01-09T19:26:52.335923Z","shell.execute_reply":"2023-01-09T19:26:59.048868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outfile = 'submission.csv'\ny_pred.to_csv(outfile, index=False)\nprint('Finished creating subsmission file {}'.format(outfile))","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:36:51.990887Z","iopub.execute_input":"2023-01-09T19:36:51.991375Z","iopub.status.idle":"2023-01-09T19:36:52.011073Z","shell.execute_reply.started":"2023-01-09T19:36:51.991338Z","shell.execute_reply":"2023-01-09T19:36:52.009622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}