{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:47:46.177162Z","iopub.execute_input":"2025-07-31T13:47:46.177496Z","iopub.status.idle":"2025-07-31T13:47:46.506978Z","shell.execute_reply.started":"2025-07-31T13:47:46.177469Z","shell.execute_reply":"2025-07-31T13:47:46.506088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#importing library\n\nimport matplotlib.pyplot as plt \n\nfrom sklearn.model_selection import train_test_split\n\n#impot torch\nimport torch\nimport torch.nn as nn  #model used to build neural network\nfrom torchvision import transforms, models #preprocessing images and access pretrained models \nfrom torch.autograd import Variable #for automatic gradient tracking \nfrom torch import optim #to access optimizers\nimport torch.nn.functional as F\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:47:46.508415Z","iopub.execute_input":"2025-07-31T13:47:46.508984Z","iopub.status.idle":"2025-07-31T13:47:55.901633Z","shell.execute_reply.started":"2025-07-31T13:47:46.508959Z","shell.execute_reply":"2025-07-31T13:47:55.900619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# load Data","metadata":{}},{"cell_type":"code","source":"train_df=pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\",dtype=np.float32)\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\", dtype=np.float32)\nsample_sub = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\ntrain_df.label.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:47:55.902468Z","iopub.execute_input":"2025-07-31T13:47:55.902833Z","iopub.status.idle":"2025-07-31T13:48:01.655239Z","shell.execute_reply.started":"2025-07-31T13:47:55.902814Z","shell.execute_reply":"2025-07-31T13:48:01.654277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:01.657411Z","iopub.execute_input":"2025-07-31T13:48:01.657785Z","iopub.status.idle":"2025-07-31T13:48:01.705135Z","shell.execute_reply.started":"2025-07-31T13:48:01.657751Z","shell.execute_reply":"2025-07-31T13:48:01.704147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:01.705944Z","iopub.execute_input":"2025-07-31T13:48:01.706202Z","iopub.status.idle":"2025-07-31T13:48:01.744533Z","shell.execute_reply.started":"2025-07-31T13:48:01.706182Z","shell.execute_reply":"2025-07-31T13:48:01.743709Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# Do normalization by dividing all pixels features by 255 this makes scale between 0 and 1 (this called min-max normalization or scaling)\n\n#separate Features and labels \ntarget=train_df.label.values\n\nfeatures=train_df.drop('label',axis=1).values/255\n\n#splitting into training and test set \nfeatures_train,features_dev,target_train,target_dev=train_test_split(features,target,test_size=0.2, random_state=42)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:01.745613Z","iopub.execute_input":"2025-07-31T13:48:01.745951Z","iopub.status.idle":"2025-07-31T13:48:02.198040Z","shell.execute_reply.started":"2025-07-31T13:48:01.745921Z","shell.execute_reply":"2025-07-31T13:48:02.197249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Conert Numpy arrays to pytorch tensors\nfeaturesTrain=torch.tensor(features_train,dtype=torch.float32)\ntargetTrain=torch.tensor(target_train,dtype=torch.long)\n\nfeaturesDev=torch.tensor(features_dev, dtype=torch.float32)\ntargetDev=torch.tensor(target_dev, dtype=torch.long)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:02.198892Z","iopub.execute_input":"2025-07-31T13:48:02.199118Z","iopub.status.idle":"2025-07-31T13:48:02.269473Z","shell.execute_reply.started":"2025-07-31T13:48:02.199101Z","shell.execute_reply":"2025-07-31T13:48:02.268540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#set batch size \n\nbatch_size=256\n\n#Combine faetures and taget tensors into dataset that can be loaded by DataLoader\ntrain=torch.utils.data.TensorDataset(featuresTrain,targetTrain)\ndev=torch.utils.data.TensorDataset(featuresDev,targetDev)\n\n\n\n#dataloader\n\ntrain_loader=torch.utils.data.DataLoader(train,batch_size=batch_size,shuffle=True)\ndev_loader=torch.utils.data.DataLoader(dev,batch_size=batch_size,shuffle=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:02.270389Z","iopub.execute_input":"2025-07-31T13:48:02.270677Z","iopub.status.idle":"2025-07-31T13:48:02.277311Z","shell.execute_reply.started":"2025-07-31T13:48:02.270649Z","shell.execute_reply":"2025-07-31T13:48:02.276314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize one of the images in data set\n\ndef visualize_image(data,index,pred=False,val=0):\n    #convert flatten image to 28*28\n    plt.imshow(data[index].reshape(28,28))\n    #remove the label from the drawn image\n    plt.axis('off')\n    plt.title(\"Handwritten Digit Image\")\n    plt.show()\n\n\nvisualize_image(features,12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:02.278263Z","iopub.execute_input":"2025-07-31T13:48:02.278579Z","iopub.status.idle":"2025-07-31T13:48:02.657735Z","shell.execute_reply.started":"2025-07-31T13:48:02.278558Z","shell.execute_reply":"2025-07-31T13:48:02.656756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_train.shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:02.660362Z","iopub.execute_input":"2025-07-31T13:48:02.660764Z","iopub.status.idle":"2025-07-31T13:48:02.666565Z","shell.execute_reply.started":"2025-07-31T13:48:02.660741Z","shell.execute_reply":"2025-07-31T13:48:02.665563Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This means we have 784 input features , this means that input layer will have 784 neurons","metadata":{}},{"cell_type":"code","source":"np.unique(target)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:02.667467Z","iopub.execute_input":"2025-07-31T13:48:02.667710Z","iopub.status.idle":"2025-07-31T13:48:02.686460Z","shell.execute_reply.started":"2025-07-31T13:48:02.667692Z","shell.execute_reply":"2025-07-31T13:48:02.685342Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This means we have 10 Classes, so output layer will have 10 neurons \n","metadata":{}},{"cell_type":"markdown","source":"# Network  Architecture","metadata":{}},{"cell_type":"code","source":"# for the first time i don't know how to setup teh archeticture so i will do try differnt archecticture automaticalaly \n#hyperparameter tunning / archeticture search\n\n#loop over different hidden layers sizes,build model with it, train and evaluate per each \n\n#hidden layer if multiple better to go in pyramid shape (larger then smaller) as earlier layer learn basic low level features , smaller next learn higher level combinations from previous layeers \n\nhidden_sizes=[(128,),(256,),(256,128),(512,256,128)]\n\ndef build_model(hidden_layers, activation_fn=nn.ReLU):\n    #empty list to store layers in the model \n    layers=[]\n    input_size=784 #this is the size of the input layer\n    for hidden_size in hidden_layers:\n        #fully connected (linaer) layer from the input to the hidden layer\n        layers.append(nn.Linear(input_size,hidden_size))\n        #add activation function after the linear layer to introduce non linearity  \n        layers.append(activation_fn())\n        input_size=hidden_size\n    layers.append(nn.Linear(input_size,10))\n    #unpack the list of layers into sequential model\n    return nn.Sequential(*layers)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:48:02.687744Z","iopub.execute_input":"2025-07-31T13:48:02.687993Z","iopub.status.idle":"2025-07-31T13:48:02.701631Z","shell.execute_reply.started":"2025-07-31T13:48:02.687975Z","shell.execute_reply":"2025-07-31T13:48:02.700469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Training loop for each model \n\ndef train_model(model,train_loader, dev_loader, epochs=5,learning_rate=0.001):\n    #store loss values per epoch\n    train_losses=[]\n    dev_losses=[]\n    \n    #use cross entropy as a loss function (used for multiclass classification)\n    criterion=nn.CrossEntropyLoss()\n    #define optimizer algo for weight updates lr=0.001 learning rate  \n    optimizer=optim.Adam(model.parameters(),lr=learning_rate)\n\n    #loop over training process epochs times\n    for epoch in range(epochs):\n        model.train()\n        running_train_loss=0.0\n        train_batches=0\n        #loop over batches in training data and their labels\n        for X_batch,y_batch in train_loader:\n            #reset previous gradients before backpropagation to prevent accumilation from previous batches\n            optimizer.zero_grad()\n            #performs forward pass through the model \n            outputs=model(X_batch)\n            #computing loss by comparing predicted outputs abd actual labels \n            loss=criterion(outputs,y_batch)\n            #performs backpropagation (calculate gradients of loss with respect to weights)\n            loss.backward()\n            # update model weights using the calculated gradients\n            optimizer.step()\n            #evaluation phase after each epoch \n            running_train_loss+=loss.item()\n            train_batches+=1\n        avg_train_loss=running_train_loss /train_batches\n        train_losses.append(avg_train_loss)\n        model.eval()\n        #intiaize counters to com pute validation accuracy\n        running_dev_loss=0.0\n        dev_batches=0\n        correct=0\n        total=0\n         #disable gradient tracking to speed and save memory \n        with torch.no_grad():\n                #loop over development data in batches\n                for X_batch,y_batch in dev_loader:\n                    outputs=model(X_batch)\n                    loss=criterion(outputs,y_batch)\n                    running_dev_loss+=loss.item()\n                    dev_batches+=1\n                    #get the predicted class (torch.max(outputs,1 ) > return the max value of the choosen predicted class and its index), we care of the index(class of the highest score)\n                    _,predicted=torch.max(outputs,1)\n                    # compare predictions to actual lables by counting how many of them were correct\n                    total+=y_batch.size(0) #return the number of samples in the current batch\n                    correct +=(predicted==y_batch).sum().item() #return how many prediction is true \n                avg_dev_loss=running_dev_loss/dev_batches\n                dev_losses.append(avg_dev_loss)\n                #compute and print accuracy\n                acc=100*correct/total\n                #display accuracy for each epoch\n                print(f\"Epoch{epoch+1},Dev Accuracy: {acc:.2f}%\")\n            \n    return train_losses,dev_losses\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:50:33.284266Z","iopub.execute_input":"2025-07-31T13:50:33.284700Z","iopub.status.idle":"2025-07-31T13:50:33.294230Z","shell.execute_reply.started":"2025-07-31T13:50:33.284670Z","shell.execute_reply":"2025-07-31T13:50:33.293169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##loop over differnt architecture\n\nfor hidden_config in hidden_sizes:\n    for lr in [0.01,0.001,0.0001]:\n        print(f\"\\n trying architecture : {hidden_config} with learning rate: {lr}\")\n        model=build_model(hidden_config,nn.ReLU)\n        train_model(model,train_loader,dev_loader,learning_rate=lr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:50:35.800239Z","iopub.execute_input":"2025-07-31T13:50:35.800911Z","iopub.status.idle":"2025-07-31T13:51:53.131540Z","shell.execute_reply.started":"2025-07-31T13:50:35.800882Z","shell.execute_reply":"2025-07-31T13:51:53.130642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"So, simple best architecture is (256,) [784 > 256 >10] with accuracy architecture 97.20% and learning rate 0.01","metadata":{}},{"cell_type":"markdown","source":"# Visualizing Model Performance ","metadata":{}},{"cell_type":"code","source":"#choose the best model to draw:\nmodel=build_model((256,))\ntrain_losses, dev_losses = train_model(model, train_loader, dev_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:52:54.562149Z","iopub.execute_input":"2025-07-31T13:52:54.563006Z","iopub.status.idle":"2025-07-31T13:53:00.054934Z","shell.execute_reply.started":"2025-07-31T13:52:54.562974Z","shell.execute_reply":"2025-07-31T13:53:00.053846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nplt.plot(train_losses, label='Training Loss')\nplt.plot(dev_losses, label='Dev Loss')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Loss Curve\")\nplt.legend()\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T13:53:26.608007Z","iopub.execute_input":"2025-07-31T13:53:26.608448Z","iopub.status.idle":"2025-07-31T13:53:26.808616Z","shell.execute_reply.started":"2025-07-31T13:53:26.608419Z","shell.execute_reply":"2025-07-31T13:53:26.807731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}