{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as pyplot \n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-07T11:59:06.357194Z","iopub.execute_input":"2024-12-07T11:59:06.357622Z","iopub.status.idle":"2024-12-07T11:59:06.366367Z","shell.execute_reply.started":"2024-12-07T11:59:06.357586Z","shell.execute_reply":"2024-12-07T11:59:06.365073Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Assessment feedback\n# 5/15 for framing the problem due to not describing the problem in ML terms\n# 7.5/15 for getting the data since I did not discuss what the dataset is or describe it\n# 9/15 for data exploration since I did not calculate and discuss the correlations in the dataset\n# 15/15 for data preparation, model shortlisting, and finetuning.\n# 11.25/15 for summary not refitting the final model to the entirety of the dataset.\n\n# Changelog:\n# added ML problem description \n# added dataset description\n# added correlations and discussion\n# refit model to entirety of data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T11:59:06.368235Z","iopub.execute_input":"2024-12-07T11:59:06.368622Z","iopub.status.idle":"2024-12-07T11:59:06.387176Z","shell.execute_reply.started":"2024-12-07T11:59:06.368585Z","shell.execute_reply":"2024-12-07T11:59:06.386069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Frame the Problem\n","metadata":{}},{"cell_type":"code","source":"# The goal of this problem is to create a model that recognizes an input in the form of a greyscale image of a digit and makes a prediction \n# on what the digit on the image is. As the samples are labeled with what digit is represented in the image, this is a supervised learning \n# classification problem. Accuracy will be used as the model evaluation metric.","metadata":{"execution":{"iopub.status.busy":"2024-12-07T11:59:06.389164Z","iopub.execute_input":"2024-12-07T11:59:06.389658Z","iopub.status.idle":"2024-12-07T11:59:06.398355Z","shell.execute_reply.started":"2024-12-07T11:59:06.389593Z","shell.execute_reply":"2024-12-07T11:59:06.397227Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Get the Data","metadata":{}},{"cell_type":"code","source":"# Description of the data set\n# The data set consists of 42,000 images of handwritten digits, each labelled as 0-9. Each\n# image is square 28 by 28 pixels. Thus, for each image there are 784 features and 1 label, totalling 785 rows.\n# All features are numeric; pixel values indicate the grayscale image intensity. No features are missing.\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T11:59:06.399723Z","iopub.execute_input":"2024-12-07T11:59:06.400683Z","iopub.status.idle":"2024-12-07T11:59:06.408351Z","shell.execute_reply.started":"2024-12-07T11:59:06.400646Z","shell.execute_reply":"2024-12-07T11:59:06.407352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat /kaggle/input/digit-recognizer/train.csv | head -n 5","metadata":{"execution":{"iopub.status.busy":"2024-12-07T11:59:06.410198Z","iopub.execute_input":"2024-12-07T11:59:06.410566Z","iopub.status.idle":"2024-12-07T11:59:06.436296Z","shell.execute_reply.started":"2024-12-07T11:59:06.410520Z","shell.execute_reply":"2024-12-07T11:59:06.435321Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\n    \"/kaggle/input/digit-recognizer/train.csv\",\n)\n\ntest_features_df = pd.read_csv(\n    \"/kaggle/input/digit-recognizer/test.csv\",\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T11:59:06.437596Z","iopub.execute_input":"2024-12-07T11:59:06.437927Z","iopub.status.idle":"2024-12-07T11:59:10.199955Z","shell.execute_reply.started":"2024-12-07T11:59:06.437896Z","shell.execute_reply":"2024-12-07T11:59:10.198706Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Explore the Data","metadata":{}},{"cell_type":"code","source":"# Dataframe information and headers\ntrain_df.info(show_counts=True)\ntrain_df.head()\nprint() # for readability\n\n# Look for missing values \nmissing_values = train_df.isnull().sum()\nprint(\"Missing values in each column:\\n\", missing_values)\nprint()\n\n# no missing values found\n\n# Descriptive statistics\ntrain_df.describe()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T11:59:10.202351Z","iopub.execute_input":"2024-12-07T11:59:10.203082Z","iopub.status.idle":"2024-12-07T11:59:12.378997Z","shell.execute_reply.started":"2024-12-07T11:59:10.203045Z","shell.execute_reply":"2024-12-07T11:59:12.377741Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check Distribution\n\nlabel_counts = train_df[\"label\"].value_counts()\npyplot.figure(figsize=(10, 6))\nsns.barplot(x=label_counts.index, y=label_counts.values, palette='viridis')\npyplot.title('Distribution of Classes')\npyplot.xlabel('Classes (Labels)')\npyplot.ylabel('Number of Images')\npyplot.xticks(rotation=45)\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-07T11:59:12.380439Z","iopub.execute_input":"2024-12-07T11:59:12.380892Z","iopub.status.idle":"2024-12-07T11:59:12.615869Z","shell.execute_reply.started":"2024-12-07T11:59:12.380845Z","shell.execute_reply":"2024-12-07T11:59:12.614457Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show sample images w/label.\n# Code implemented w/help from ChatGPT when prompted to: \"show sample images through a matplotlib pyplot subplot\" \n# then one further prompt to fix the image proccessing by informing ChatGPT that train_df \"consists of a 'label' column and 'pixel0,pixel1...etc' remaining columns\"\n\ndef show_sample_images(train_df, num_images=5): \n    pyplot.figure(figsize=(15, 8))\n    sample_images = train_df.sample(num_images)\n\n    for i, (index, row) in enumerate(sample_images.iterrows()):\n        # image processing\n        img_size = int(np.sqrt(len(row) - 1))  \n        img = row[1:].values.reshape((img_size, img_size))  \n        \n        # plotting\n        pyplot.subplot(1, num_images, i + 1)\n        pyplot.imshow(img, cmap='gray')  \n        pyplot.title(row['label'])\n        pyplot.axis('off')\n    \n    pyplot.show()\n\n\nshow_sample_images(train_df, num_images=20)\n\n# Can see here that we can trim the edges and retain all of the information for most of the images.\n# This can be made more rigorous by checking for correlations.","metadata":{"execution":{"iopub.status.busy":"2024-12-07T11:59:12.617575Z","iopub.execute_input":"2024-12-07T11:59:12.618408Z","iopub.status.idle":"2024-12-07T11:59:13.735497Z","shell.execute_reply.started":"2024-12-07T11:59:12.618358Z","shell.execute_reply":"2024-12-07T11:59:13.734221Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# According to the assessment feedback, I am missing the correlations \n# between features and target as well as among features.\n\n# Correlations between features (grayscale intensity) and target (digit label) \n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ny = train_df.iloc[:, 0]\nX = train_df.iloc[:, 1:]\n\ncorrelations = X.corrwith(y)\n# 0 stdev can cause NaN values, so I replaced them with zeros, then I take the absolute value since sign doesn't matter\ncorrelations = correlations.fillna(0).abs()\n\ncorrelation_image = correlations.values.reshape(28, 28)\n\nplt.figure(figsize=(6, 6))\nplt.imshow(correlation_image, cmap='coolwarm', interpolation='nearest')\nplt.colorbar()\nplt.title('Correlation of Pixel Intensities with Digit Label')\nplt.axis('off')\nplt.show()\n\n# Can see that the features that most strongly correlate with the digit label are the ones in the center\n# can also see that around the edges / away from the center, the pixels start to not carry information.\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T11:59:13.737328Z","iopub.execute_input":"2024-12-07T11:59:13.737790Z","iopub.status.idle":"2024-12-07T11:59:14.749085Z","shell.execute_reply.started":"2024-12-07T11:59:13.737741Z","shell.execute_reply":"2024-12-07T11:59:14.747868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Correlations among the features  \n# plotting all 784 by 784 would look extremely large (784 x 784) + messy and not provide useful information\n# better to plot the total correlations with respect to each feature\n\nimport seaborn as sns #neater heatmap\n\ncorrelation_matrix_features = X.corr()\nsum_of_correlations = correlation_matrix_features.sum(axis=0)\nsum_correlation_matrix = np.array(sum_of_correlations).reshape(28, 28)\nplt.figure(figsize=(8, 6))\nsns.heatmap(np.abs(sum_correlation_matrix), annot=False, cmap='coolwarm', cbar_kws={'label': 'Sum of Feature Correlations'})\nplt.title('Sum of Correlations for Each Pixel')\nplt.show()\n\n#As expected, pixels near the center vary the most with respect to other pixels, since the pixels around the edges dont carry information and thus barely vary.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T11:59:14.750780Z","iopub.execute_input":"2024-12-07T11:59:14.751347Z","iopub.status.idle":"2024-12-07T12:00:24.858610Z","shell.execute_reply.started":"2024-12-07T11:59:14.751295Z","shell.execute_reply":"2024-12-07T12:00:24.857508Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Prepare the Data for ML ","metadata":{}},{"cell_type":"code","source":"# remove label columns for features df and remove pixel columns for training df\ntrain_features_df = train_df.drop(\"label\", axis=1)\ntrain_target_df = train_df.loc[:, \"label\"]\n\ntrain_features_df.info()\nprint()\ntrain_features_df.head()\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:24.861857Z","iopub.execute_input":"2024-12-07T12:00:24.862226Z","iopub.status.idle":"2024-12-07T12:00:24.986691Z","shell.execute_reply.started":"2024-12-07T12:00:24.862192Z","shell.execute_reply":"2024-12-07T12:00:24.985652Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain_target_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:24.987950Z","iopub.execute_input":"2024-12-07T12:00:24.988273Z","iopub.status.idle":"2024-12-07T12:00:24.999036Z","shell.execute_reply.started":"2024-12-07T12:00:24.988241Z","shell.execute_reply":"2024-12-07T12:00:24.997802Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# perform trimming (cropping) operation (improves computational time)\n# reading the dataset documentation from the competition page, the edges are given by:\n# Left edge: i*28 for i=0 to i=27\n# Right edge: i*28+27 i=0 to i=27\n# Top edge:  1,2,...,26\n# Bottom edge: 757, 758, ..., 782\n\n\n\ncolumns_to_drop = (\n    [f\"pixel{i}\" for i in range(0, 784, 28)] +  \n    [f\"pixel{i}\" for i in range(27, 784, 28)] + \n    [f\"pixel{i}\" for i in range(1, 27)] +        \n    [f\"pixel{i}\" for i in range(757, 783)]        \n)\n\n\ntrimmed_df = train_features_df.drop(columns=columns_to_drop,axis=1)\n\n\ntrimmed_df.head()\ntrimmed_df.info() #check that entries-count is 28^2 - (28*2) - (26*2) = 676\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:25.000151Z","iopub.execute_input":"2024-12-07T12:00:25.000491Z","iopub.status.idle":"2024-12-07T12:00:25.146037Z","shell.execute_reply.started":"2024-12-07T12:00:25.000435Z","shell.execute_reply":"2024-12-07T12:00:25.144888Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# In order to be able to do this with a piepline in general, a custom transformer is needed\n# ChatGPT was prompted: \"Create a custom transformer that drops columns to be used in a pipeline\" to help make this\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nclass ColumnDropper(BaseEstimator, TransformerMixin):\n    def __init__(self, columns_to_drop):\n        self.columns_to_drop = columns_to_drop\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        return X.drop(columns=[col for col in self.columns_to_drop if col in X.columns], errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:25.147426Z","iopub.execute_input":"2024-12-07T12:00:25.147862Z","iopub.status.idle":"2024-12-07T12:00:25.154197Z","shell.execute_reply.started":"2024-12-07T12:00:25.147828Z","shell.execute_reply":"2024-12-07T12:00:25.153050Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Perform Train-Test Split\n\nfrom sklearn import model_selection\n\n# x is used for features and y for labels\nx_train, x_test, y_train, y_test = model_selection.train_test_split(trimmed_df, train_target_df, test_size=0.85, random_state=42) # large test size in order to make algorithms run faster (20% of training data)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:25.155536Z","iopub.execute_input":"2024-12-07T12:00:25.155920Z","iopub.status.idle":"2024-12-07T12:00:25.397245Z","shell.execute_reply.started":"2024-12-07T12:00:25.155889Z","shell.execute_reply":"2024-12-07T12:00:25.396173Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train.info()\nprint()\nx_test.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:25.398431Z","iopub.execute_input":"2024-12-07T12:00:25.398770Z","iopub.status.idle":"2024-12-07T12:00:25.634820Z","shell.execute_reply.started":"2024-12-07T12:00:25.398736Z","shell.execute_reply":"2024-12-07T12:00:25.633609Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Explore Different Models","metadata":{}},{"cell_type":"code","source":"# start with logistic regression\nfrom sklearn import linear_model, preprocessing, feature_selection, pipeline, metrics","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:25.636119Z","iopub.execute_input":"2024-12-07T12:00:25.636446Z","iopub.status.idle":"2024-12-07T12:00:25.722707Z","shell.execute_reply.started":"2024-12-07T12:00:25.636414Z","shell.execute_reply":"2024-12-07T12:00:25.721716Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"logreg_pipeline = pipeline.make_pipeline(\n    ColumnDropper(columns_to_drop),\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.StandardScaler(),\n    linear_model.LogisticRegression(\n        max_iter=1000\n    ),\n    verbose=True\n)\n\n_ = logreg_pipeline.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:25.723867Z","iopub.execute_input":"2024-12-07T12:00:25.724195Z","iopub.status.idle":"2024-12-07T12:00:32.990707Z","shell.execute_reply.started":"2024-12-07T12:00:25.724165Z","shell.execute_reply":"2024-12-07T12:00:32.988613Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Report (training data)\n\ntest_predictions = logreg_pipeline.predict(x_train)\n_report = metrics.classification_report(\n    y_train,\n    test_predictions,\n)\nprint(_report)\nprint()\n\n# Report (test data)\n\ntest_predictions = logreg_pipeline.predict(x_test)\n_report = metrics.classification_report(\n    y_test,\n    test_predictions,\n)\nprint(_report)\n\n# Confusion Matrix\n\n_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                y_test,\n                test_predictions\n            )\n)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:32.991977Z","iopub.execute_input":"2024-12-07T12:00:32.992407Z","iopub.status.idle":"2024-12-07T12:00:34.209719Z","shell.execute_reply.started":"2024-12-07T12:00:32.992360Z","shell.execute_reply":"2024-12-07T12:00:34.208552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# try SGDClassifier with hinge loss function\n\n\nsgdhinge_pipeline = pipeline.make_pipeline(\n    ColumnDropper(columns_to_drop),\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.StandardScaler(),\n    linear_model.SGDClassifier(\n        max_iter=1000,\n        verbose=False\n    ),\n    verbose=True,\n)\n\n_ = sgdhinge_pipeline.fit(x_train, y_train)\n\n# running into convergence issues, so will vary the learning rate","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:34.211249Z","iopub.execute_input":"2024-12-07T12:00:34.211727Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Report (training data)\n\ntest_predictions = sgdhinge_pipeline.predict(x_train)\n_report = metrics.classification_report(\n    y_train,\n    test_predictions,\n)\nprint(_report)\nprint()\n\n\n# Report (test data)\n\ntest_predictions = sgdhinge_pipeline.predict(x_test)\n_report = metrics.classification_report(\n    y_test,\n    test_predictions,\n)\nprint(_report)\n\n\n# Confusion Matrix\n_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                y_test,\n                test_predictions\n            )\n)","metadata":{"execution":{"iopub.status.idle":"2024-12-07T12:00:55.697564Z","shell.execute_reply.started":"2024-12-07T12:00:54.541128Z","shell.execute_reply":"2024-12-07T12:00:55.696208Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# try SGDClassifier with perceptron loss function\n\n\nperc_pipeline = pipeline.make_pipeline(\n    ColumnDropper(columns_to_drop),\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.StandardScaler(),\n    linear_model.SGDClassifier(loss = \"perceptron\",\n        max_iter=1000,\n        verbose=False\n    ),\n    verbose=True,\n)\n\n_ = perc_pipeline.fit(x_train, y_train)\n\n# running into convergence issues, so will vary the learning rate","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:00:55.699328Z","iopub.execute_input":"2024-12-07T12:00:55.699884Z","iopub.status.idle":"2024-12-07T12:01:18.624377Z","shell.execute_reply.started":"2024-12-07T12:00:55.699809Z","shell.execute_reply":"2024-12-07T12:01:18.623015Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Report (training data)\n\ntest_predictions = perc_pipeline.predict(x_train)\n_report = metrics.classification_report(\n    y_train,\n    test_predictions,\n)\nprint(_report)\nprint()\n\n# Report (test data)\n\ntest_predictions = perc_pipeline.predict(x_test)\n_report = metrics.classification_report(\n    y_test,\n    test_predictions,\n)\nprint(_report)\n\n\n# Confusion Matrix\n_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                y_test,\n                test_predictions\n            )\n)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:01:18.625710Z","iopub.execute_input":"2024-12-07T12:01:18.626032Z","iopub.status.idle":"2024-12-07T12:01:19.773503Z","shell.execute_reply.started":"2024-12-07T12:01:18.626002Z","shell.execute_reply":"2024-12-07T12:01:19.772380Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# try SVC\n\nfrom sklearn import svm\n\nsvc_pipeline = pipeline.make_pipeline(\n    ColumnDropper(columns_to_drop),\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.StandardScaler(),\n    svm.SVC(kernel = \"linear\"),\n    verbose=True,\n)\n_ = svc_pipeline.fit(x_train, y_train)\n\n# Report (training data)\n\ntest_predictions = svc_pipeline.predict(x_train)\n_report = metrics.classification_report(\n    y_train,\n    test_predictions,\n)\nprint(_report)\nprint()\n\n# Report (test data)\n\ntest_predictions = svc_pipeline.predict(x_test)\n_report = metrics.classification_report(\n    y_test,\n    test_predictions,\n)\nprint(_report)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:01:19.774938Z","iopub.execute_input":"2024-12-07T12:01:19.775293Z","iopub.status.idle":"2024-12-07T12:01:43.472155Z","shell.execute_reply.started":"2024-12-07T12:01:19.775259Z","shell.execute_reply":"2024-12-07T12:01:43.471043Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The SGDClassifier and SVC models seem to do better on test data (slightly, 0.9 vs 0.89), but log_reg and SVC do better on training data (0.95 vs 1.0)\n# All models overfit the data since the training accuracy is significantly better than the test accuracy.\n# Most mistakes happen with predicting that an image shows an 8 when it is not. ","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:01:43.473796Z","iopub.execute_input":"2024-12-07T12:01:43.474212Z","iopub.status.idle":"2024-12-07T12:01:43.479244Z","shell.execute_reply.started":"2024-12-07T12:01:43.474167Z","shell.execute_reply":"2024-12-07T12:01:43.478073Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Fine-tune the Models","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n# Define the parameter grid, perform grid search, \n# then output best score and set of parameters for each model\n# Metric will be accuracy since I do not care about making one type of error more than another\n\nlogreg_param_grid = {\n    'logisticregression__C': [0.001,0.01, 0.1, 1],   \n}\n\n\nlogreg_grid_search = GridSearchCV(logreg_pipeline, param_grid=logreg_param_grid, cv=5, verbose=1)\n_ = logreg_grid_search.fit(x_train, y_train)\n\n\nlogreg_best_params = logreg_grid_search.best_params_\nlogreg_best_score = logreg_grid_search.best_score_\n\n\n\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:01:43.484269Z","iopub.execute_input":"2024-12-07T12:01:43.484650Z","iopub.status.idle":"2024-12-07T12:02:41.540305Z","shell.execute_reply.started":"2024-12-07T12:01:43.484617Z","shell.execute_reply":"2024-12-07T12:02:41.538498Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best parameters for Logistic Regression:\", logreg_best_params)\nprint(\"Best cross-validated accuracy for Logistic Regression:\", logreg_best_score)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:02:41.541549Z","iopub.execute_input":"2024-12-07T12:02:41.542009Z","iopub.status.idle":"2024-12-07T12:02:41.548904Z","shell.execute_reply.started":"2024-12-07T12:02:41.541965Z","shell.execute_reply":"2024-12-07T12:02:41.547646Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsgdhinge_param_grid = {\n    'sgdclassifier__alpha': [0.0001, 0.001, 0.01],\n    'sgdclassifier__learning_rate': ['invscaling', 'adaptive'], \n    'sgdclassifier__eta0': [0.01, 0.1, 1.0]\n}\n\n\nsgdhinge_grid_search = GridSearchCV(sgdhinge_pipeline, param_grid=sgdhinge_param_grid, cv=5, verbose=1)\n_ = sgdhinge_grid_search.fit(x_train, y_train)\n\n\nsgdhinge_best_params = sgdhinge_grid_search.best_params_\nsgdhinge_best_score = sgdhinge_grid_search.best_score_\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:02:41.551194Z","iopub.execute_input":"2024-12-07T12:02:41.551627Z","iopub.status.idle":"2024-12-07T12:08:41.805634Z","shell.execute_reply.started":"2024-12-07T12:02:41.551582Z","shell.execute_reply":"2024-12-07T12:08:41.804435Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best parameters for SGD Hinge:\", sgdhinge_best_params)\nprint(\"Best cross-validated accuracy for SGD Hinge:\", sgdhinge_best_score)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:08:41.807071Z","iopub.execute_input":"2024-12-07T12:08:41.807521Z","iopub.status.idle":"2024-12-07T12:08:41.813244Z","shell.execute_reply.started":"2024-12-07T12:08:41.807451Z","shell.execute_reply":"2024-12-07T12:08:41.811960Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"perc_param_grid = sgdhinge_param_grid\n\n\nperc_grid_search = GridSearchCV(perc_pipeline, param_grid=perc_param_grid, cv=5, verbose=1)\n_ = perc_grid_search.fit(x_train, y_train)\n\n\nperc_best_params = perc_grid_search.best_params_\nperc_best_score = perc_grid_search.best_score_\n\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2024-12-07T12:38:16.161296Z","iopub.execute_input":"2024-12-07T12:38:16.162243Z","iopub.status.idle":"2024-12-07T12:43:33.585278Z","shell.execute_reply.started":"2024-12-07T12:38:16.162201Z","shell.execute_reply":"2024-12-07T12:43:33.584141Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best parameters for Perceptron:\", perc_best_params)\nprint(\"Best cross-validated accuracy for Perceptron:\", perc_best_score)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:38:16.093728Z","iopub.execute_input":"2024-12-07T12:38:16.094130Z","iopub.status.idle":"2024-12-07T12:38:16.101108Z","shell.execute_reply.started":"2024-12-07T12:38:16.094095Z","shell.execute_reply":"2024-12-07T12:38:16.099924Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsvc_param_grid = {\n    'svc__C': [0.001,0.01, 0.1, 1],   \n}\n\n\nsvc_grid_search = GridSearchCV(svc_pipeline, param_grid=svc_param_grid, cv=5, verbose=1)\n_ = svc_grid_search.fit(x_train, y_train)\n\n\nsvc_best_params = svc_grid_search.best_params_\nsvc_best_score = svc_grid_search.best_score_\n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:44:25.612717Z","iopub.execute_input":"2024-12-07T12:44:25.613121Z","iopub.status.idle":"2024-12-07T12:45:18.848989Z","shell.execute_reply.started":"2024-12-07T12:44:25.613088Z","shell.execute_reply":"2024-12-07T12:45:18.847692Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best parameters for SVC:\", svc_best_params)\nprint(\"Best cross-validated accuracy for SVC:\", svc_best_score)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:45:18.858039Z","iopub.execute_input":"2024-12-07T12:45:18.858377Z","iopub.status.idle":"2024-12-07T12:45:18.871871Z","shell.execute_reply.started":"2024-12-07T12:45:18.858338Z","shell.execute_reply":"2024-12-07T12:45:18.870559Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Identify the best model","metadata":{}},{"cell_type":"code","source":"#Best score (CV accuracy) is the tuned SVC model with a CV accuracy score of of 0.92 with C = \n","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:15:05.992708Z","iopub.execute_input":"2024-12-07T12:15:05.993053Z","iopub.status.idle":"2024-12-07T12:15:06.003773Z","shell.execute_reply.started":"2024-12-07T12:15:05.993021Z","shell.execute_reply":"2024-12-07T12:15:06.002783Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Summarize your results","metadata":{}},{"cell_type":"code","source":"# The digit recognizer problem is a computer vision, multi classification problem. In order to create\n# a linear model that best recognizes the digits, four fine tuned models of SGDRegression with hinge loss function, \n# SGDRegression with a perceptron loss function, SVC, and logistic regression were compared. The fine tuning was performed\n# by 5-fold cross validation (CV) and the metric was the CV accuracy. The model with the best score was the \n# SVC with C = 0.01. There was not much room for feature engineering, but in order to train the models more efficiently, the dataset was significantly cut (only 15% used for training)\n# and the edges of the images were removed since they rarely contain useful information.","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:15:06.005049Z","iopub.execute_input":"2024-12-07T12:15:06.005385Z","iopub.status.idle":"2024-12-07T12:15:06.019994Z","shell.execute_reply.started":"2024-12-07T12:15:06.005355Z","shell.execute_reply":"2024-12-07T12:15:06.018802Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Next we refit the fine tuned model to the entirety of the data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T12:15:06.021279Z","iopub.execute_input":"2024-12-07T12:15:06.021644Z","iopub.status.idle":"2024-12-07T12:15:06.037713Z","shell.execute_reply.started":"2024-12-07T12:15:06.021610Z","shell.execute_reply":"2024-12-07T12:15:06.036491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = model_selection.train_test_split(\n    trimmed_df, train_target_df, test_size=0.001, random_state=42) \n\nsvc_pipeline = pipeline.make_pipeline(\n    ColumnDropper(columns_to_drop),\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.StandardScaler(),\n    svm.SVC(kernel = \"linear\", C = 0.01),\n    verbose=True,\n)\n\n_ = svc_pipeline.fit(x_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T13:17:53.740940Z","iopub.execute_input":"2024-12-07T13:17:53.741332Z","iopub.status.idle":"2024-12-07T13:19:36.777575Z","shell.execute_reply.started":"2024-12-07T13:17:53.741298Z","shell.execute_reply":"2024-12-07T13:19:36.776360Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Submit your predictions to the competition","metadata":{}},{"cell_type":"code","source":"predictions = svc_pipeline.predict(test_features_df)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T13:45:52.416852Z","iopub.execute_input":"2024-12-07T13:45:52.417280Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions[:5]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat /kaggle/input/digit-recognizer/sample_submission.csv | head -n 5","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:15:06.059377Z","iopub.status.idle":"2024-12-07T12:15:06.059980Z","shell.execute_reply.started":"2024-12-07T12:15:06.059715Z","shell.execute_reply":"2024-12-07T12:15:06.059746Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = (\n    pd.read_csv(\n        \"/kaggle/input/digit-recognizer/sample_submission.csv\",\n    ).assign(\n        Label=predictions\n    )\n)\n\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:15:06.061453Z","iopub.status.idle":"2024-12-07T12:15:06.061872Z","shell.execute_reply.started":"2024-12-07T12:15:06.061693Z","shell.execute_reply":"2024-12-07T12:15:06.061714Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat /kaggle/working/submission.csv | head -n 5","metadata":{"execution":{"iopub.status.busy":"2024-12-07T12:15:06.063554Z","iopub.status.idle":"2024-12-07T12:15:06.063969Z","shell.execute_reply.started":"2024-12-07T12:15:06.063777Z","shell.execute_reply":"2024-12-07T12:15:06.063796Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}