{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Introduction**\n\nThis notebook is a based on Efficientnet without augmentations and should be easy to understand for anyone. In the Session options -> Accelerator ->  select GPU","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-05T11:08:18.426395Z","iopub.execute_input":"2025-11-05T11:08:18.427077Z","iopub.status.idle":"2025-11-05T11:08:18.433083Z","shell.execute_reply.started":"2025-11-05T11:08:18.427052Z","shell.execute_reply":"2025-11-05T11:08:18.432476Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Import packages**","metadata":{}},{"cell_type":"code","source":"import torch \nimport torch.nn as nn \nimport matplotlib.pyplot as plt # visualization\nfrom torch.utils.data import DataLoader, TensorDataset # load the data\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.optim import SGD, Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR # Learning rate scheduler\nfrom tqdm import tqdm # track the progress \nimport timm #importing efficientnet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T11:08:24.569331Z","iopub.execute_input":"2025-11-05T11:08:24.570149Z","iopub.status.idle":"2025-11-05T11:08:24.573993Z","shell.execute_reply.started":"2025-11-05T11:08:24.570120Z","shell.execute_reply":"2025-11-05T11:08:24.573266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Import training data from Kaggle**","metadata":{}},{"cell_type":"code","source":"train_data=pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T11:08:27.585029Z","iopub.execute_input":"2025-11-05T11:08:27.585588Z","iopub.status.idle":"2025-11-05T11:08:30.431524Z","shell.execute_reply.started":"2025-11-05T11:08:27.585565Z","shell.execute_reply":"2025-11-05T11:08:30.430673Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's look at the first 5 rows:","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T11:08:33.477990Z","iopub.execute_input":"2025-11-05T11:08:33.478669Z","iopub.status.idle":"2025-11-05T11:08:33.504340Z","shell.execute_reply.started":"2025-11-05T11:08:33.478641Z","shell.execute_reply":"2025-11-05T11:08:33.503535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The dataset contains the label (0 to 9) in the first column and then either a value between 0 and 255 for each of the 784 pixels. The higher the number, the darker the color. We will have a look at the first picture:","metadata":{}},{"cell_type":"code","source":"# Take first row [0], leave out the label column [1:]\nx1=train_data.iloc[0,1:].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:19:55.566499Z","iopub.execute_input":"2025-11-05T12:19:55.566880Z","iopub.status.idle":"2025-11-05T12:19:55.571179Z","shell.execute_reply.started":"2025-11-05T12:19:55.566856Z","shell.execute_reply":"2025-11-05T12:19:55.570511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For visualization of the image, reshape the first into a 28 x 28 shape:\nx1re=x1.reshape(28,28)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:19:57.772954Z","iopub.execute_input":"2025-11-05T12:19:57.773629Z","iopub.status.idle":"2025-11-05T12:19:57.776927Z","shell.execute_reply.started":"2025-11-05T12:19:57.773604Z","shell.execute_reply":"2025-11-05T12:19:57.776270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the reshaped image\nplt.imshow(x1re)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:47:21.577849Z","iopub.execute_input":"2025-11-03T11:47:21.578105Z","iopub.status.idle":"2025-11-03T11:47:21.713932Z","shell.execute_reply.started":"2025-11-03T11:47:21.578082Z","shell.execute_reply":"2025-11-03T11:47:21.713173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Split the training data into training and validation set**","metadata":{}},{"cell_type":"code","source":"# Generate a config class:\nclass CFG:\n    batch_size = 64 # Specifies how many images are in one batch\n    lr = 1e-3 # Learning rate, how much the model adapts the parameters in each step\n    num_epochs = 10 # Number of epochs, how long the model is training\n    n_folds = 5 # Number of folds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:01.406534Z","iopub.execute_input":"2025-11-05T12:20:01.406836Z","iopub.status.idle":"2025-11-05T12:20:01.410836Z","shell.execute_reply.started":"2025-11-05T12:20:01.406814Z","shell.execute_reply":"2025-11-05T12:20:01.410049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = train_data\ny = train_data['label']\ncv = StratifiedKFold(n_splits=CFG.n_folds) # Stratified k fold preserves number of occurences in dataset\nsplit = list(cv.split(x,y)) # generates a list of the train and validation split for each fold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:03.430943Z","iopub.execute_input":"2025-11-05T12:20:03.431227Z","iopub.status.idle":"2025-11-05T12:20:03.448675Z","shell.execute_reply.started":"2025-11-05T12:20:03.431205Z","shell.execute_reply":"2025-11-05T12:20:03.447653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add a fold column into the train dataset\ntrain_data['fold'] = -1 #dummy which gets overwriten in next 2 lines\nfor fold,(tr_idx, val_idx) in enumerate(split):\n    train_data.iloc[val_idx,-1] = fold \n# loops over the just generated split list with training and validation index and adds this into the fold column","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:05.233316Z","iopub.execute_input":"2025-11-05T12:20:05.233603Z","iopub.status.idle":"2025-11-05T12:20:05.240504Z","shell.execute_reply.started":"2025-11-05T12:20:05.233581Z","shell.execute_reply":"2025-11-05T12:20:05.239620Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Here us what the training data looks like now:","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:47:21.747398Z","iopub.execute_input":"2025-11-03T11:47:21.747651Z","iopub.status.idle":"2025-11-03T11:47:21.765881Z","shell.execute_reply.started":"2025-11-03T11:47:21.747625Z","shell.execute_reply":"2025-11-03T11:47:21.765174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Are the folds the same size?\ntrain_data['fold'].value_counts(normalize=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:47:21.766497Z","iopub.execute_input":"2025-11-03T11:47:21.766668Z","iopub.status.idle":"2025-11-03T11:47:21.777911Z","shell.execute_reply.started":"2025-11-03T11:47:21.766655Z","shell.execute_reply":"2025-11-03T11:47:21.777314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Yes, the data is evenly split into 5 folds, each containing 20% of the data.","metadata":{}},{"cell_type":"code","source":"# How is the distribution of the lables in the dataset?\n#train_data['label'].value_counts(normalize=True).sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:47:21.778479Z","iopub.execute_input":"2025-11-03T11:47:21.778645Z","iopub.status.idle":"2025-11-03T11:47:21.788745Z","shell.execute_reply.started":"2025-11-03T11:47:21.778632Z","shell.execute_reply":"2025-11-03T11:47:21.788124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#plt.hist(train_data['label'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:47:21.790705Z","iopub.execute_input":"2025-11-03T11:47:21.790865Z","iopub.status.idle":"2025-11-03T11:47:21.796487Z","shell.execute_reply.started":"2025-11-03T11:47:21.790852Z","shell.execute_reply":"2025-11-03T11:47:21.795805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Similar counts for all digits.","metadata":{}},{"cell_type":"markdown","source":"**Prepare the dataloader of training and validation set**","metadata":{}},{"cell_type":"code","source":"def prepare_loaders(fold, batch_size):\n    train_df = train_data[train_data[\"fold\"]!=fold].reset_index(drop=True)\n    valid_df = train_data[train_data[\"fold\"]==fold].reset_index(drop=True)\n\n    X = train_df.iloc[:,1:-1].values / 255 # norm picture to values between 0-1\n    X = X.reshape(X.shape[0],1,28,28)\n    y = train_df.iloc[:,0].values\n    train_dataset = TensorDataset(torch.from_numpy(X), torch.from_numpy(y))\n\n    X_val = valid_df.iloc[:,1:-1].values / 255 # norm picture to values between 0-1\n    X_val = X_val.reshape(X_val.shape[0],1,28,28)\n    y_val = valid_df.iloc[:,0].values\n    valid_dataset = TensorDataset(torch.from_numpy(X_val), torch.from_numpy(y_val))\n\n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, pin_memory=True, drop_last=False)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:12.411076Z","iopub.execute_input":"2025-11-05T12:20:12.411339Z","iopub.status.idle":"2025-11-05T12:20:12.417461Z","shell.execute_reply.started":"2025-11-05T12:20:12.411320Z","shell.execute_reply":"2025-11-05T12:20:12.416707Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Load the training and validation loader for fold 0.","metadata":{}},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(fold=0, batch_size=CFG.batch_size) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:15.685563Z","iopub.execute_input":"2025-11-05T12:20:15.686062Z","iopub.status.idle":"2025-11-05T12:20:16.173283Z","shell.execute_reply.started":"2025-11-05T12:20:15.686037Z","shell.execute_reply":"2025-11-05T12:20:16.172637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = timm.create_model('tf_efficientnetv2_s.in21k_ft_in1k', pretrained=True, in_chans=1,\n            num_classes=10) \nmodel.cuda() # Push the model to GPU\noptimizer = Adam(model.parameters(), lr=CFG.lr) # Stochastic gradient descent\nscheduler = CosineAnnealingLR(optimizer, T_max=10, eta_min=0) # Learning rate scheduler, cosine annealing\ncriterion = nn.CrossEntropyLoss() #loss function","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:18.753607Z","iopub.execute_input":"2025-11-05T12:20:18.754306Z","iopub.status.idle":"2025-11-05T12:20:21.511931Z","shell.execute_reply.started":"2025-11-05T12:20:18.754283Z","shell.execute_reply":"2025-11-05T12:20:21.511080Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Generate the NN model\nInitiate a very simple NN with 4 linear layers.","metadata":{}},{"cell_type":"markdown","source":"Train the model and calculate validation loss (Cross Entropy Loss) and accuracy. \nSave the model parameter (later loaded in inference notebook). ","metadata":{}},{"cell_type":"code","source":"train_loss = []\nv_loss2 = []\nacc2 = []\nfor i in range(CFG.num_epochs):\n    model.train()\n    for data in tqdm(train_loader):\n        # Set the gradients to zero\n        optimizer.zero_grad()\n        # Run a forward pass\n        feature, target = data\n        prediction = model(feature.float().cuda())\n        prediction = prediction.cpu()\n        # Compute the loss\n        loss = criterion(prediction, target)\n        # Compute the gradients\n        loss.backward()\n        # Update the model's parameters\n        optimizer.step()\n        train_loss += [loss.item()]\n    val_loss = []\n    accuracy = []\n    model.eval()\n    with torch.no_grad():\n        for data in tqdm(valid_loader):\n            feature, target = data\n            prediction = model(feature.float().cuda())\n            prediction = prediction.cpu()\n            loss = criterion(prediction, target)\n            val_loss += [loss.item()]\n            accuracy += [(prediction.softmax(1).argmax(1) == target).float().mean()]\n    scheduler.step() # update learning rate\n    v_loss = np.array(val_loss).mean()\n    print(f'Epoch: {i}')\n    print(f'Val_loss: {v_loss}')\n    acc = np.array(accuracy).mean()\n    print(f'Accuracy: {acc}')\n    v_loss2 += [v_loss]\n    acc2 += [acc]\n    PATH = f\"last_epoch-{fold:02d}.bin\"\n    torch.save(model.state_dict(), PATH) # Saves the model weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T12:20:28.713233Z","iopub.execute_input":"2025-11-05T12:20:28.713514Z","execution_failed":"2025-11-05T15:58:30.912Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Plot the training and validation loss**","metadata":{}},{"cell_type":"code","source":"all_loss = np.array(train_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:33:20.71954Z","iopub.execute_input":"2025-11-03T11:33:20.719778Z","iopub.status.idle":"2025-11-03T11:33:20.724147Z","shell.execute_reply.started":"2025-11-03T11:33:20.719761Z","shell.execute_reply":"2025-11-03T11:33:20.723494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(all_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:33:20.724831Z","iopub.execute_input":"2025-11-03T11:33:20.725065Z","iopub.status.idle":"2025-11-03T11:33:20.875636Z","shell.execute_reply.started":"2025-11-03T11:33:20.725041Z","shell.execute_reply":"2025-11-03T11:33:20.874797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(v_loss2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:33:20.878633Z","iopub.execute_input":"2025-11-03T11:33:20.878806Z","iopub.status.idle":"2025-11-03T11:33:21.001803Z","shell.execute_reply.started":"2025-11-03T11:33:20.878793Z","shell.execute_reply":"2025-11-03T11:33:21.001206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Plot the accuracy**","metadata":{}},{"cell_type":"code","source":"plt.plot(acc2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T11:33:21.002419Z","iopub.execute_input":"2025-11-03T11:33:21.0026Z","iopub.status.idle":"2025-11-03T11:33:21.120996Z","shell.execute_reply.started":"2025-11-03T11:33:21.002586Z","shell.execute_reply":"2025-11-03T11:33:21.120452Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This notebook just trains the model, in order to create a submission to the competition I generated a separate [inference notebook](https://www.kaggle.com/code/bridgeoverwater/fork-of-infer-digit-recognizer-efficientnet) where the trained model / weights are loaded.\n","metadata":{}},{"cell_type":"markdown","source":"[](http://)","metadata":{}}]}