{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99236,"databundleVersionId":11855598,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:48:31.950279Z","iopub.execute_input":"2025-06-01T08:48:31.950516Z","iopub.status.idle":"2025-06-01T08:48:33.120721Z","shell.execute_reply.started":"2025-06-01T08:48:31.950489Z","shell.execute_reply":"2025-06-01T08:48:33.120053Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 1. Import Required Libraries\n# ==============================","metadata":{}},{"cell_type":"code","source":"import random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\n\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom xgboost import XGBClassifier\n\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom tensorflow.keras.utils import to_categorical\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:48:33.122331Z","iopub.execute_input":"2025-06-01T08:48:33.122669Z","iopub.status.idle":"2025-06-01T08:48:57.096511Z","shell.execute_reply.started":"2025-06-01T08:48:33.122648Z","shell.execute_reply":"2025-06-01T08:48:57.095701Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 2. Load CSV Data\n# ==============================","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/exploring-machine-learning-with-mnist/mnist_train.csv')\ntest_df = pd.read_csv('/kaggle/input/exploring-machine-learning-with-mnist/mnist_test.csv')\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:48:57.097272Z","iopub.execute_input":"2025-06-01T08:48:57.097706Z","iopub.status.idle":"2025-06-01T08:49:01.958914Z","shell.execute_reply.started":"2025-06-01T08:48:57.097688Z","shell.execute_reply":"2025-06-01T08:49:01.958211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 3. Preprocess the Data\n# ==============================","metadata":{}},{"cell_type":"code","source":"X_train = train_df.drop('label', axis=1).values.astype('float32') / 255.0\ny_train = train_df['label'].values\n\nX_test = test_df.drop('label', axis=1).values.astype('float32') / 255.0\ny_test = test_df['label'].values\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:49:01.959854Z","iopub.execute_input":"2025-06-01T08:49:01.960145Z","iopub.status.idle":"2025-06-01T08:49:02.250801Z","shell.execute_reply.started":"2025-06-01T08:49:01.960118Z","shell.execute_reply":"2025-06-01T08:49:02.249989Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 4. Utility Functions\n# ==============================","metadata":{}},{"cell_type":"code","source":"def visualize_samples(X, y_true, y_pred=None, title=\"Sample Images\"):\n    plt.figure(figsize=(12, 8))\n    indices = np.random.choice(len(X), 6, replace=False)\n    for i, idx in enumerate(indices):\n        plt.subplot(2, 3, i + 1)\n        plt.imshow(X[idx].reshape(28, 28), cmap='gray')\n        label = f\"Actual: {y_true[idx]}\"\n        if y_pred is not None:\n            label += f\"\\nPred: {y_pred[idx]}\"\n        plt.title(label)\n        plt.axis('off')\n    plt.suptitle(title)\n    plt.tight_layout()\n    plt.show()\n\ndef plot_confusion_matrix(y_true, y_pred, model_name):\n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(8, 6))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n                xticklabels=range(10), yticklabels=range(10))\n    plt.xlabel('Predicted')\n    plt.ylabel('Actual')\n    plt.title(f'Confusion Matrix - {model_name}')\n    plt.show()\n\ndef evaluate_model(y_true, y_pred, model_name):\n    print(f\"\\nClassification Report for {model_name}:\\n\")\n    print(classification_report(y_true, y_pred))\n    plot_confusion_matrix(y_true, y_pred, model_name)\n    visualize_samples(X_test, y_true, y_pred, title=f\"{model_name} Predictions\")\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:49:02.251660Z","iopub.execute_input":"2025-06-01T08:49:02.251896Z","iopub.status.idle":"2025-06-01T08:49:02.259165Z","shell.execute_reply.started":"2025-06-01T08:49:02.251878Z","shell.execute_reply":"2025-06-01T08:49:02.258558Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 5. Model Training Function\n# ==============================","metadata":{}},{"cell_type":"code","source":"def train_and_evaluate(models, X_train, y_train, X_test, y_test):\n    results = {}\n    for name, model in models.items():\n        print(f\"\\nTraining {name}...\")\n        model.fit(X_train, y_train)\n        y_pred = model.predict(X_test)\n        acc = accuracy_score(y_test, y_pred)\n        print(f\"{name} Accuracy: {acc:.4f}\")\n        results[name] = {'accuracy': acc, 'predictions': y_pred}\n    return results\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:49:02.259930Z","iopub.execute_input":"2025-06-01T08:49:02.260222Z","iopub.status.idle":"2025-06-01T08:49:02.276941Z","shell.execute_reply.started":"2025-06-01T08:49:02.260198Z","shell.execute_reply":"2025-06-01T08:49:02.276477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 6. Define Models\n# ==============================","metadata":{}},{"cell_type":"code","source":"models = {\n    'Logistic Regression': LogisticRegression(max_iter=1000, multi_class='multinomial', n_jobs=-1),\n    'SVM': SVC(),\n    'Random Forest': RandomForestClassifier(n_jobs=-1),\n    'KNN': KNeighborsClassifier(n_jobs=-1),\n    'XGBoost': XGBClassifier(use_label_encoder=False, eval_metric='mlogloss', n_jobs=-1),\n    'Decision Tree': DecisionTreeClassifier(),\n    'Gaussian NB': GaussianNB(),\n    'Gradient Boosting': GradientBoostingClassifier()\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:49:02.279260Z","iopub.execute_input":"2025-06-01T08:49:02.279683Z","iopub.status.idle":"2025-06-01T08:49:02.296988Z","shell.execute_reply.started":"2025-06-01T08:49:02.279665Z","shell.execute_reply":"2025-06-01T08:49:02.296264Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 7. Train and Evaluate All Models\n# ==============================","metadata":{}},{"cell_type":"code","source":"results = train_and_evaluate(models, X_train, y_train, X_test, y_test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:49:02.297762Z","iopub.execute_input":"2025-06-01T08:49:02.297995Z","iopub.status.idle":"2025-06-01T09:36:02.964272Z","shell.execute_reply.started":"2025-06-01T08:49:02.297967Z","shell.execute_reply":"2025-06-01T09:36:02.963588Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 8. Visualize Model Accuracies\n# ==============================","metadata":{}},{"cell_type":"code","source":"\nmodel_scores = pd.DataFrame({\n    'Model': list(results.keys()),\n    'Accuracy': [results[m]['accuracy'] for m in results]\n}).sort_values(by='Accuracy', ascending=False).reset_index(drop=True)\n\nprint(\"\\nModel Performance Summary:\")\nprint(model_scores)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T09:36:02.965179Z","iopub.execute_input":"2025-06-01T09:36:02.965448Z","iopub.status.idle":"2025-06-01T09:36:02.993589Z","shell.execute_reply.started":"2025-06-01T09:36:02.965429Z","shell.execute_reply":"2025-06-01T09:36:02.992849Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 9. Hyperparameter Tuning for Top 3 Models\n# ==============================","metadata":{}},{"cell_type":"code","source":"PARAMS = {\n    'Random Forest': {\n        'model': RandomForestClassifier(random_state=42),\n        'params': {\n            'n_estimators': [50, 100],\n            'max_depth': [10, 20, None],\n            'min_samples_split': [2, 5]\n        }\n    },\n    'SVM': {\n        'model': SVC(),\n        'params': {\n            'C': [0.1, 1, 10],\n            'kernel': ['rbf', 'linear'],\n            'gamma': ['scale', 'auto']\n        }\n    },\n    'XGBoost': {\n        'model': XGBClassifier(use_label_encoder=False, eval_metric='mlogloss', n_jobs=-1),\n        'params': {\n            'n_estimators': [50, 100],\n            'max_depth': [3, 6],\n            'learning_rate': [0.1, 0.2, 0.3]\n        }\n    }\n}\n\ntuned_results = []\n\nfor name, config in PARAMS.items():\n    print(f\"\\nTuning hyperparameters for {name}...\")\n    grid = GridSearchCV(config['model'], config['params'], cv=3, n_jobs=-1, scoring='accuracy', verbose=1)\n    grid.fit(X_train, y_train)\n    best_model = grid.best_estimator_\n    best_accuracy = accuracy_score(y_test, best_model.predict(X_test))\n    print(f\"Best {name} Accuracy: {best_accuracy:.4f}\")\n    tuned_results.append({\n        'Model': name,\n        'Best Params': grid.best_params_,\n        'Accuracy': best_accuracy\n    })\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T09:36:02.994784Z","iopub.execute_input":"2025-06-01T09:36:02.995108Z","iopub.status.idle":"2025-06-01T12:06:22.527784Z","shell.execute_reply.started":"2025-06-01T09:36:02.995058Z","shell.execute_reply":"2025-06-01T12:06:22.527106Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 10. Visualize Tuned Model Accuracies\n# ==============================","metadata":{}},{"cell_type":"code","source":"tuned_df = pd.DataFrame(tuned_results).sort_values(by='Accuracy', ascending=False).reset_index(drop=True)\n\nprint(\"\\nTuned Model Performance Summary:\")\nprint(tuned_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T12:06:22.529769Z","iopub.execute_input":"2025-06-01T12:06:22.529985Z","iopub.status.idle":"2025-06-01T12:06:22.541774Z","shell.execute_reply.started":"2025-06-01T12:06:22.529968Z","shell.execute_reply":"2025-06-01T12:06:22.540984Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 11. Save the Best Model\n# ==============================","metadata":{}},{"cell_type":"code","source":"# Find the best model from the tuned results\nbest_model_entry = max(tuned_results, key=lambda x: x['Accuracy'])\nbest_model_name = best_model_entry['Model']\nbest_model_params = best_model_entry['Best Params']\n\n# Recreate and train the best model\nbest_model = PARAMS[best_model_name]['model'].set_params(**best_model_params)\nbest_model.fit(X_train, y_train)\n\n# Save to disk\njoblib.dump(best_model, f\"{best_model_name.replace(' ', '_').lower()}_best_model.pkl\")\nprint(f\"Saved best model: {best_model_name} with accuracy {best_model_entry['Accuracy']:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T12:06:22.542607Z","iopub.execute_input":"2025-06-01T12:06:22.542922Z","iopub.status.idle":"2025-06-01T12:09:46.621162Z","shell.execute_reply.started":"2025-06-01T12:06:22.542895Z","shell.execute_reply":"2025-06-01T12:09:46.620298Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 12. Load the Best Model\n# ==============================","metadata":{}},{"cell_type":"code","source":"# Load from disk\nloaded_model = joblib.load(f\"{best_model_name.replace(' ', '_').lower()}_best_model.pkl\")\n\nprint(f\"Loaded model: {best_model_name}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T12:09:46.622096Z","iopub.execute_input":"2025-06-01T12:09:46.622446Z","iopub.status.idle":"2025-06-01T12:09:46.661382Z","shell.execute_reply.started":"2025-06-01T12:09:46.622425Z","shell.execute_reply":"2025-06-01T12:09:46.660605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 13. Evaluation for a Selected Model\n# ==============================","metadata":{}},{"cell_type":"code","source":"selected_model = best_model_name\nevaluate_model(y_test, results[selected_model]['predictions'], selected_model)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T12:09:46.662386Z","iopub.execute_input":"2025-06-01T12:09:46.662650Z","iopub.status.idle":"2025-06-01T12:09:48.286627Z","shell.execute_reply.started":"2025-06-01T12:09:46.662632Z","shell.execute_reply":"2025-06-01T12:09:48.285947Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 14. Create Submission File\n# ==============================","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'ID': test_df.index + 1,\n    'Label': results[selected_model]['predictions']\n})\nsubmission.to_csv('submission.csv', index=False)\nprint(f\"\\nSubmission file 'submission.csv' created using model: {selected_model}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T12:09:48.287323Z","iopub.execute_input":"2025-06-01T12:09:48.287615Z","iopub.status.idle":"2025-06-01T12:09:48.320719Z","shell.execute_reply.started":"2025-06-01T12:09:48.287590Z","shell.execute_reply":"2025-06-01T12:09:48.320116Z"}},"outputs":[],"execution_count":null}]}