{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n DIGIT RECOGNIZER\n
","metadata":{}},{"cell_type":"markdown","source":"\n Importing Required Libraries\n
","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom matplotlib.image import imread\nimport seaborn as sns\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score, confusion_matrix, f1_score\nfrom sklearn.decomposition import PCA\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, BatchNormalization\nfrom tensorflow.keras import layers, models\nfrom itertools import product\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:34.162685Z","iopub.execute_input":"2025-07-11T13:29:34.163724Z","iopub.status.idle":"2025-07-11T13:29:55.511037Z","shell.execute_reply.started":"2025-07-11T13:29:34.163689Z","shell.execute_reply":"2025-07-11T13:29:55.510114Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Load the dataset\n
","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.512522Z","iopub.execute_input":"2025-07-11T13:29:55.513152Z","iopub.status.idle":"2025-07-11T13:29:55.878107Z","shell.execute_reply.started":"2025-07-11T13:29:55.513128Z","shell.execute_reply":"2025-07-11T13:29:55.874045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.879030Z","iopub.status.idle":"2025-07-11T13:29:55.879457Z","shell.execute_reply.started":"2025-07-11T13:29:55.879261Z","shell.execute_reply":"2025-07-11T13:29:55.879278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_samples = train_data.shape[0]\nprint(f'Number of samples: {num_samples}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.880412Z","iopub.status.idle":"2025-07-11T13:29:55.880744Z","shell.execute_reply.started":"2025-07-11T13:29:55.880593Z","shell.execute_reply":"2025-07-11T13:29:55.880610Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Explore the Dataset (EDA)\n
","metadata":{}},{"cell_type":"markdown","source":"\n Missing Data Check
\n In this case - digit recognizer, it is very important to check if any values are missing. Our data are pixels so we cannot do imputation, because the exact position and value of each pixel matters for image, and filling missing pixels would distort the image information. The only option would be to delete samples with missing data, if any exist.\n
","metadata":{}},{"cell_type":"code","source":"# Show columns with missing values\nmissing_data = train_data.isnull().sum().sum() # First .sum() -> sums missing values per column, Second .sum() -> sums across all columns\nif missing_data > 0:\n print(f'Dataset contains missing values')\nelse:\n print(f'No columns have missing values')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.881450Z","iopub.status.idle":"2025-07-11T13:29:55.881717Z","shell.execute_reply.started":"2025-07-11T13:29:55.881583Z","shell.execute_reply":"2025-07-11T13:29:55.881597Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n From 784 Numbers to an Image 🤯\n
","metadata":{}},{"cell_type":"code","source":"# Check image shape\nfrom typing import Optional\n\ndef check_img_shape(dataset) -> Optional[int]:\n\n num_pixels = train_data.shape[1] - 1 # number of columns minus 1 (label)\n size = int(np.sqrt(num_pixels))\n if size * size == num_pixels:\n print(f'Image is square-shaped: {size}x{size}')\n return size\n else:\n print('Image is not square-shaped')\n return None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.882346Z","iopub.status.idle":"2025-07-11T13:29:55.882646Z","shell.execute_reply.started":"2025-07-11T13:29:55.882481Z","shell.execute_reply":"2025-07-11T13:29:55.882497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_size = check_img_shape(train_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.883837Z","iopub.status.idle":"2025-07-11T13:29:55.884241Z","shell.execute_reply.started":"2025-07-11T13:29:55.884034Z","shell.execute_reply":"2025-07-11T13:29:55.884072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Image save and show\n\ndef show_img(dataset, number_index: int) -> None:\n \n pixels = dataset.iloc[number_index, 1:].values.reshape(image_size, image_size)\n # .values -> bierze same wartości z Series i zapisuje do tablicy numpy 1D; alternatywnie: to_numpy()\n # .reshape() -> przekształca 1D tablicę (784) do 2D obrazu (28x28)\n label = dataset.iloc[number_index, 0]\n \n plt.imshow(pixels, cmap='gray')\n plt.title(f'label = {label}')\n plt.axis('off')\n plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.885040Z","iopub.status.idle":"2025-07-11T13:29:55.885417Z","shell.execute_reply.started":"2025-07-11T13:29:55.885288Z","shell.execute_reply":"2025-07-11T13:29:55.885304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_img(train_data, 36)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.886558Z","iopub.status.idle":"2025-07-11T13:29:55.886866Z","shell.execute_reply.started":"2025-07-11T13:29:55.886703Z","shell.execute_reply":"2025-07-11T13:29:55.886719Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Balance Check 🧮
\n The dataset contains many classes (10), so it is important to check for imbalance. \n As the number of classes increases, the risk of having imbalanced data also increases, because some classes might have very few samples compared to others.\n
","metadata":{}},{"cell_type":"code","source":"label_counts = train_data['label'].value_counts().sort_index()\nprint(label_counts)\n\nprint(f'Majority class: {label_counts.idxmax()}: {label_counts.max()}')\nprint(f'Minority class: {label_counts.idxmin()}: {label_counts.min()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.887853Z","iopub.status.idle":"2025-07-11T13:29:55.888132Z","shell.execute_reply.started":"2025-07-11T13:29:55.887980Z","shell.execute_reply":"2025-07-11T13:29:55.887991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Distribution of Number of Samples (Boxplot) with percentage deviation\n\nmedian_count = label_counts.median()\nprint(f'Median count: {median_count}')\npercent_deviation = ((label_counts - median_count) / median_count) * 100\n\nfig, ax1 = plt.subplots(figsize = (8,6))\n\nsns.boxplot(data=label_counts, color='.8', medianprops={\"color\": \"r\"}, ax=ax1)\nax1.set_ylabel('Number of Samples')\n\nax2 = ax1.twinx()\nax2.set_ylim(\n ((min(label_counts) - median_count) / median_count) * 100,\n ((max(label_counts) - median_count) / median_count) * 100\n)\nax2.set_ylabel('Deviation from Median (%)')\nax2.set_yticks(np.arange(-10, 15, 3))\n\nplt.title('Distribution of Samples per Class (Dual Axis)')\nax1.set_xlabel('All Classes Combined')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.888907Z","iopub.status.idle":"2025-07-11T13:29:55.889204Z","shell.execute_reply.started":"2025-07-11T13:29:55.889074Z","shell.execute_reply":"2025-07-11T13:29:55.889086Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Class Distribution Analysis
\n\n Most classes have between 3800 and 4400 samples \n (with a median of 4157 samples).
\n As we can see in the boxplot above, there's only one outlier, which suggests that our dataset is \n slightly imbalanced — due to the class labeled '1'.
\n This class has about 13% more samples than the median, which is not a significant imbalance, \n so most machine learning models should be able to handle it without major issues.
\n However, I will observe the model’s predictions using a confusion matrix to check if it \n tends to overfit on class '1' or underfit on class '5', which has the lowest sample count.\n
\n","metadata":{}},{"cell_type":"markdown","source":"\n Pixel Value Range Check [0 - 255]
\n I wanna check if all pixels are between 0 and 255. Any deviations suggest errors, so we would delete those samles.\n
","metadata":{}},{"cell_type":"code","source":"pixel_data = train_data.iloc[:,1:]\n\nmin_pixel = pixel_data.min().min()\nmax_pixel = pixel_data.max().max()\nif min_pixel < 0 or max_pixel > 255:\n print('Warning: Some pixel values are out of the range [0, 255]')\nelse:\n print('All pixel values are within the expected range [0, 255]')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.890272Z","iopub.status.idle":"2025-07-11T13:29:55.890540Z","shell.execute_reply.started":"2025-07-11T13:29:55.890417Z","shell.execute_reply":"2025-07-11T13:29:55.890428Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Pixel Intensity Distribution Check
\n Pixel that are too light may not be recognized well by the model, so this is important to check if pixel values need enhancement.\n
","metadata":{}},{"cell_type":"code","source":"pixels_only = train_data.iloc[:, 1:].values\nmean_pixels = pixels_only.mean(axis=0).reshape(image_size, image_size)\n\nsns.heatmap(\n data=mean_pixels,\n cmap='Greys_r',\n cbar=True\n)\nplt.title('Mean Pixel Intensity Across All Images')\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.891753Z","iopub.status.idle":"2025-07-11T13:29:55.892075Z","shell.execute_reply.started":"2025-07-11T13:29:55.891899Z","shell.execute_reply":"2025-07-11T13:29:55.891913Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Mean Pixel Intensity Analysis
\nHeatmap shows that contrast between background and the digit is clear, so there is no need to enhance it. \nTo overall shape in the heatmap looks like a blend of different digits, which is expected and indicates that the digits are well-centered and distinct.\n
","metadata":{}},{"cell_type":"markdown","source":"\n Digit Gallery
\n To visually inspect the dataset I displayed a few examples from each digit class (0-9).\n This helps to verify the digits are clear and check whether any class contains anomalies. \n
","metadata":{}},{"cell_type":"code","source":"digit_classes = list(pd.Series(train_data['label'].unique()).sort_values())\nnumber_digit_to_display = 5\n\nfor digit in digit_classes:\n \n fig, ax = plt.subplots(nrows=1, ncols=number_digit_to_display, figsize=(15,5))\n fig.suptitle(f'Examples of digit: {digit}', fontsize=16)\n \n digits_with_the_same_label = train_data[train_data['label'] == digit].iloc[:,1:]\n random_indexes_list = list(digits_with_the_same_label.sample(n=number_digit_to_display).index)\n\n for ax, index in zip(ax, random_indexes_list):\n image = train_data.iloc[index,1:].values.reshape(image_size, image_size)\n \n sns.heatmap(\n ax=ax,\n data=image,\n cbar=False,\n cmap='Greys_r',\n xticklabels=False,\n yticklabels=False\n )\n ax.set_title(f'Index {index}', fontsize=10)\n \n plt.tight_layout()\n plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.892931Z","iopub.status.idle":"2025-07-11T13:29:55.893205Z","shell.execute_reply.started":"2025-07-11T13:29:55.893076Z","shell.execute_reply":"2025-07-11T13:29:55.893088Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n The images are readable, have good contrast and correct labels.\n
","metadata":{}},{"cell_type":"markdown","source":"\n Preprocessing 🛠️\n
","metadata":{}},{"cell_type":"markdown","source":"\n Train-Test Split\n
","metadata":{}},{"cell_type":"code","source":"# Splitting the data into independent (features) and dependent (target) variables\nX = train_data.drop(['label'], axis=1)\ny = train_data['label']\n\n# Splitting into training and testing sets with stratified sampling\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.894028Z","iopub.status.idle":"2025-07-11T13:29:55.894392Z","shell.execute_reply.started":"2025-07-11T13:29:55.894226Z","shell.execute_reply":"2025-07-11T13:29:55.894241Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Feature Engineering\n
","metadata":{}},{"cell_type":"code","source":"scaler = MinMaxScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.895693Z","iopub.status.idle":"2025-07-11T13:29:55.896007Z","shell.execute_reply.started":"2025-07-11T13:29:55.895870Z","shell.execute_reply":"2025-07-11T13:29:55.895887Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Train Models 📈\n
","metadata":{}},{"cell_type":"markdown","source":"\n Logistic Regression\n
","metadata":{}},{"cell_type":"code","source":"logreg = LogisticRegression(max_iter=1000)\nlogreg.fit(X_train_scaled, y_train)\n\ny_pred_logreg = logreg.predict(X_test_scaled)\naccuracy_logreg = accuracy_score(y_test, y_pred_logreg)\nprint(f'Test set Accuracy: {accuracy_logreg:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.897326Z","iopub.status.idle":"2025-07-11T13:29:55.897622Z","shell.execute_reply.started":"2025-07-11T13:29:55.897473Z","shell.execute_reply":"2025-07-11T13:29:55.897483Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The accuracy of 91% shows that the model performs well on the test data.","metadata":{}},{"cell_type":"code","source":"# Check for overfitting and underfitting\n\ntrain_pred_logreg = logreg.predict(X_train_scaled)\ntrain_accuracy_logreg = accuracy_score(y_train, train_pred_logreg)\nprint(f'Training-set Accuracy: {train_accuracy_logreg:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.898820Z","iopub.status.idle":"2025-07-11T13:29:55.899164Z","shell.execute_reply.started":"2025-07-11T13:29:55.898969Z","shell.execute_reply":"2025-07-11T13:29:55.898984Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The accurasies (Test: 91%, Train: 95%) sugest that the model is slightly overfitting the training data.","metadata":{}},{"cell_type":"code","source":"# Confusion Matrix\n\ndef plot_confusion_matrix(y_true, y_pred, cmap='Blues', title='Confusion Matrix'):\n \n cm = confusion_matrix(y_true, y_pred)\n \n labels = list(pd.Series(y_true.unique()).sort_values())\n \n plt.figure(figsize=(12,8))\n sns.heatmap(cm, annot=True, cmap=cmap, xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Label')\n plt.ylabel('True Label')\n plt.title('Confusion Matrix')\n plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.900332Z","iopub.status.idle":"2025-07-11T13:29:55.900651Z","shell.execute_reply.started":"2025-07-11T13:29:55.900509Z","shell.execute_reply":"2025-07-11T13:29:55.900525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_confusion_matrix(y_test, y_pred_logreg, cmap='Blues', title='Confusion Matrix - Logistic Regression')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.902134Z","iopub.status.idle":"2025-07-11T13:29:55.902776Z","shell.execute_reply.started":"2025-07-11T13:29:55.902581Z","shell.execute_reply":"2025-07-11T13:29:55.902598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Error-Label Count Correlation\n\ncm_logreg = confusion_matrix(y_test, y_pred_logreg)\n\ndef error_label_count_correlation(confusion_matrix):\n\n df_cm = pd.DataFrame(confusion_matrix)\n label_counts = pd.Series(y_test).value_counts().sort_index().values\n \n errors_per_class = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n \n df = pd.DataFrame({\n 'Class': df_cm.value_counts().index.names,\n 'Errors': errors_per_class,\n 'Label Counts': label_counts\n })\n \n correlation = df['Errors'].corr(df['Label Counts'])\n return correlation","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.903621Z","iopub.status.idle":"2025-07-11T13:29:55.903966Z","shell.execute_reply.started":"2025-07-11T13:29:55.903797Z","shell.execute_reply":"2025-07-11T13:29:55.903813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Correlation between errors and label counts: {error_label_count_correlation(cm_logreg):.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.904627Z","iopub.status.idle":"2025-07-11T13:29:55.904935Z","shell.execute_reply.started":"2025-07-11T13:29:55.904762Z","shell.execute_reply":"2025-07-11T13:29:55.904777Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Error-Label Count Correlation Analysis
\n A negative value of correlation (-0.52) betweem errors and label counts suggest that classes with fewer samples tend to have more errors. This meaning that imbalanced data might affect the model's performance.
\n To solve this problem Im going to apply class weight balancing to give more importance to minority classes during training.\n
","metadata":{}},{"cell_type":"code","source":"# Logistic Regression with Class Weight Balancing\n\nlogreg_class_balanced = LogisticRegression(max_iter=1000, class_weight='balanced')\nlogreg_class_balanced.fit(X_train_scaled, y_train)\n\ny_pred_logreg_class_balanced = logreg_class_balanced.predict(X_test_scaled)\n\naccuracy_logreg_class_balanced = accuracy_score(y_test, y_pred_logreg_class_balanced)\nprint(f'Test set Accuracy: {accuracy_logreg_class_balanced:.4f}')\n\ntrain_pred_logreg_class_balanced = logreg_class_balanced.predict(X_train_scaled)\ntrain_accuracy_logreg_class_balanced = accuracy_score(y_train, train_pred_logreg_class_balanced)\nprint(f'Training-set Accuracy: {train_accuracy_logreg_class_balanced:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.906306Z","iopub.status.idle":"2025-07-11T13:29:55.906606Z","shell.execute_reply.started":"2025-07-11T13:29:55.906440Z","shell.execute_reply":"2025-07-11T13:29:55.906454Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n The accuracies after training logistic regression with balanced class weights didn't improve significantly. Maybe balancing class weights alone is not enough to address the class imbalance issue in this dataset.
\n In the next step I'm going to apply dimensionality reduction using PCA to see if this method helps reduce overfitting.\n
","metadata":{}},{"cell_type":"code","source":"# Dimensionality Reduction - PCA\n\npca = PCA(n_components=100)\nX_train_pca = pca.fit_transform(X_train_scaled)\nX_test_pca = pca.transform(X_test_scaled)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.907599Z","iopub.status.idle":"2025-07-11T13:29:55.908014Z","shell.execute_reply.started":"2025-07-11T13:29:55.907779Z","shell.execute_reply":"2025-07-11T13:29:55.907847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Logistic Regression with PCA\n\nlogreg_pca = LogisticRegression(max_iter=1000)\nlogreg_pca.fit(X_train_pca, y_train)\n\ny_pred_logreg_pca = logreg_pca.predict(X_test_pca)\naccuracy_logreg_pca = accuracy_score(y_test, y_pred_logreg_pca)\nprint(f'Accuracy after PCA for Logistic Regression {accuracy_logreg_pca:.4f}')\n\ntrain_pred_logreg_pca = logreg_pca.predict(X_train_pca)\ntrain_accuracy_logreg_pca = accuracy_score(y_train, train_pred_logreg_pca)\nprint(f'Training-set Accuracy: {train_accuracy_logreg_pca:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.909153Z","iopub.status.idle":"2025-07-11T13:29:55.909457Z","shell.execute_reply.started":"2025-07-11T13:29:55.909314Z","shell.execute_reply":"2025-07-11T13:29:55.909330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n The model trained on PCA-reduced features with n_components=100 achieved an accuracy of 91.38% on the test set and 92.51% on the training set. This shows that dimensionality reduction helped reduce overfitting.
\n Now, I will verify if combining two methods - dimensionality reduction and balanced class weights - can improve predictions.\n
","metadata":{}},{"cell_type":"code","source":"# Logistic Regression with PCA-reduced features and balanced class weights\n\nlogreg_pca_balanced = LogisticRegression(max_iter=1000, class_weight='balanced')\nlogreg_pca_balanced.fit(X_train_pca, y_train)\n\ny_pred_logreg_pca_balanced = logreg_pca_balanced.predict(X_test_pca)\naccuracy_logreg_pca_balanced = accuracy_score(y_test, y_pred_logreg_pca_balanced)\nprint(f'Test set Accuracy: {accuracy_logreg_pca:.4f}')\n\ntrain_pred_logreg_pca_balanced = logreg_pca.predict(X_train_pca)\ntrain_accuracy_logreg_balanced = accuracy_score(y_train, train_pred_logreg_pca_balanced)\nprint(f'Training-set Accuracy: {train_accuracy_logreg_balanced:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.910772Z","iopub.status.idle":"2025-07-11T13:29:55.911154Z","shell.execute_reply.started":"2025-07-11T13:29:55.910953Z","shell.execute_reply":"2025-07-11T13:29:55.910969Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Combining PCA and balanced class weights to improve Logistic Regression predictions didn't lead to any change in accuracy. The test and training accuracies are the same as when using PCA alone.\n
","metadata":{}},{"cell_type":"markdown","source":"\n Artificial Neural Network\n
","metadata":{}},{"cell_type":"code","source":"# Create method to train a single ANN model\n\ndef train_ann(X_train, y_train,\n neurons=[128, 64],\n activation='relu',\n loss='sparse_categorical_crossentropy',\n use_batch_norm=False,\n epochs=5,\n batch_size=32,\n seed=42,\n verbose=0):\n\n num_pixels = X_train.shape[1]\n num_classes = len(np.unique(y_train))\n\n model = Sequential()\n \n # Inupt Layer\n model.add(tf.keras.Input(shape=(num_pixels, ))) # (784, ) -> tuple with one element 784\n \n for n in neurons:\n model.add(Dense(n, activation=activation))\n if use_batch_norm:\n model.add(BatchNormalization())\n \n model.add(Dense(num_classes, activation='softmax'))\n \n model.compile(\n optimizer='adam',\n loss=loss,\n metrics=['accuracy']\n )\n \n model.fit(X_train, \n y_train, \n epochs=epochs,\n batch_size=batch_size,\n validation_split=0.1,\n verbose=verbose)\n\n return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.912366Z","iopub.status.idle":"2025-07-11T13:29:55.912712Z","shell.execute_reply.started":"2025-07-11T13:29:55.912535Z","shell.execute_reply":"2025-07-11T13:29:55.912550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create method to return accuracy, F1-score and confusion matrix for a trained model on test data and accuracy on training data\n\ndef evaluate_ann_model(model, X_train, y_train, X_test, y_test):\n\n def error_label_count_correlation(cm):\n \n # List with number of error per class : sum of each row minus diagonal element (true predictions):\n errors_per_class = cm.sum(axis=1) - np.diag(cm)\n\n # List with number of occurrences for each class (true + false predictions):\n true_counts = cm.sum(axis=1)\n\n # Porównanie zależności liczb z dwóch list. Macierz Pearsona:\n corr_matrix = np.corrcoef(errors_per_class, true_counts)\n \n return corr_matrix[0,1]\n # Return correlation coefficient in range [-1, 1]:\n # 1 means perfect positive correlation,\n # -1 means perfect negative correlation,\n # 0 means no linear correlation\n \n y_pred_all = model.predict(X_test, verbose=0)\n y_pred = np.argmax(y_pred_all, axis=1)\n\n train_pred_all = model.predict(X_train, verbose=0)\n train_pred = np.argmax(train_pred_all, axis=1)\n \n f1 = f1_score(y_test, y_pred, average='weighted')\n acc = accuracy_score(y_test, y_pred)\n cm = confusion_matrix(y_test, y_pred)\n train_acc = accuracy_score(y_train, train_pred)\n correlation = error_label_count_correlation(cm)\n \n return acc, f1, cm, train_acc, correlation","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.913744Z","iopub.status.idle":"2025-07-11T13:29:55.914116Z","shell.execute_reply.started":"2025-07-11T13:29:55.913925Z","shell.execute_reply":"2025-07-11T13:29:55.913940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Unified method to train and evaluate an ANN model - training + metrics + confusion matrix\n\ndef train_and_evaluate_ann_full(X_train, y_train, X_test, y_test, **kwargs):\n \n model = train_ann(X_train, y_train, **kwargs)\n \n acc, f1, cm, train_acc, correlation = evaluate_ann_model(\n model, X_train, y_train, X_test, y_test)\n\n df = pd.DataFrame({\n 'Metric': ['F1-score', 'Test Accuracy', 'Training Accuracy', 'Correlation Errors-Label Counts'],\n 'Value': [f1, acc, train_acc, correlation]\n })\n \n return model, f1, df, cm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.915012Z","iopub.status.idle":"2025-07-11T13:29:55.915295Z","shell.execute_reply.started":"2025-07-11T13:29:55.915176Z","shell.execute_reply":"2025-07-11T13:29:55.915187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def grid_search_ann(X_train, y_train, X_test, y_test, param_grid):\n \n keys = list(param_grid.keys())\n values = list(param_grid.values())\n combinations = list(product(*values)) # tworzy wszystkie kombinacje z listy values\n\n best_f1 = 0\n best_model = None\n best_params = None\n best_df = None\n best_cm = None\n \n for combo in combinations:\n params = dict(zip(keys, combo)) # tworzy słownik z dwóch list\n model, f1, df, cm = train_and_evaluate_ann_full(X_train, y_train, X_test, y_test, **params)\n\n if f1 > best_f1:\n best_f1 = f1\n best_model = model\n best_params = params\n best_df = df\n best_cm = cm\n \n return best_model, best_params, best_df, best_cm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.916535Z","iopub.status.idle":"2025-07-11T13:29:55.916910Z","shell.execute_reply.started":"2025-07-11T13:29:55.916709Z","shell.execute_reply":"2025-07-11T13:29:55.916725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_grid = {\n 'neurons': [[128, 64], [256, 128]],\n 'activation': ['relu', 'tanh'],\n 'batch_size': [32],\n 'epochs': [5],\n 'use_batch_norm': [True, False],\n 'verbose': [0]\n}\n\nbest_model, best_params, best_df, best_cm = grid_search_ann(X_train_scaled, y_train, X_test_scaled, y_test, param_grid=param_grid)\n\nprint(\"Best parameters found:\", best_params)\ndisplay(best_df.style.background_gradient(cmap='Greens').set_caption(\"Best Model Metrics\"))\n\nplt.figure(figsize=(6,4))\nsns.heatmap(best_cm, annot=True, cmap='Greens')\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.title('Confusion Matrix')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.918562Z","iopub.status.idle":"2025-07-11T13:29:55.918874Z","shell.execute_reply.started":"2025-07-11T13:29:55.918712Z","shell.execute_reply":"2025-07-11T13:29:55.918724Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Implement Dimensionality Reduction - PCA\n
","metadata":{}},{"cell_type":"code","source":"# Dimensionality Reduction - PCA\n\npca = PCA(n_components=100)\n\nX_train_pca = pca.fit_transform(X_train_scaled) # 100 features\nX_test_pca = pca.transform(X_test_scaled)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.919653Z","iopub.status.idle":"2025-07-11T13:29:55.919950Z","shell.execute_reply.started":"2025-07-11T13:29:55.919832Z","shell.execute_reply":"2025-07-11T13:29:55.919844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_grid = {\n 'neurons': [[128, 64], [256, 128]],\n 'activation': ['relu', 'tanh'],\n 'batch_size': [32],\n 'epochs': [5],\n 'use_batch_norm': [True, False],\n 'verbose': [0]\n}\n\nbest_model_pca, best_params_pca, best_df_pca, best_cm_pca = grid_search_ann(X_train_pca, y_train, X_test_pca, y_test, \n param_grid=param_grid)\n\nprint(\"Best parameters found:\", best_params_pca)\ndisplay(best_df_pca.style.background_gradient(cmap='Greens').set_caption(\"Best Model Metrics\"))\n\nplt.figure(figsize=(6,4))\nsns.heatmap(best_cm_pca, annot=True, cmap='Greens')\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.title('Confusion Matrix')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.921088Z","iopub.status.idle":"2025-07-11T13:29:55.921408Z","shell.execute_reply.started":"2025-07-11T13:29:55.921257Z","shell.execute_reply":"2025-07-11T13:29:55.921272Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Observations and Conclusions\n
","metadata":{}},{"cell_type":"markdown","source":"\n - Logistic Regression with PCA achives good accuracy (91%), showing that dimensionality reduction helps simplify the model while keeping decent performance.
\n - ANN without PCA perform better than Logistic Regression, with test accuracy 96.7% and high F1-score with similar value. This indicates that neural network captures more complex patterns.
\n - The negative correlation between errors and label counts in both ANN models indicates that classes with more samples tend to have fewer errors, which is expected.
\n - Overall, applying PCA before ANN training is beneficial here: it reduces dimensionality, speeds up training, and slightly improves accuracy.\n
","metadata":{}},{"cell_type":"code","source":"# Competition","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.922408Z","iopub.status.idle":"2025-07-11T13:29:55.922630Z","shell.execute_reply.started":"2025-07-11T13:29:55.922523Z","shell.execute_reply":"2025-07-11T13:29:55.922533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_X_test_scaled = scaler.fit_transform(test_data)\ntest_X_test_pca = pca.transform(test_X_test_scaled)\ntest_y_pred_all = best_model_pca.predict(test_X_test_pca)\ntest_y_pred = test_y_pred_all.argmax(axis=1)\n\nsubmission = pd.DataFrame({'ImageId': range(1, len(test_y_pred)+1), 'Label': test_y_pred})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T13:29:55.923554Z","iopub.status.idle":"2025-07-11T13:29:55.923933Z","shell.execute_reply.started":"2025-07-11T13:29:55.923734Z","shell.execute_reply":"2025-07-11T13:29:55.923750Z"}},"outputs":[],"execution_count":null}]}