{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-20T15:48:08.467652Z","iopub.execute_input":"2023-04-20T15:48:08.469131Z","iopub.status.idle":"2023-04-20T15:48:08.515926Z","shell.execute_reply.started":"2023-04-20T15:48:08.469051Z","shell.execute_reply":"2023-04-20T15:48:08.514318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.naive_bayes import MultinomialNB\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:08.519001Z","iopub.execute_input":"2023-04-20T15:48:08.519577Z","iopub.status.idle":"2023-04-20T15:48:10.444007Z","shell.execute_reply.started":"2023-04-20T15:48:08.519519Z","shell.execute_reply":"2023-04-20T15:48:10.442232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename == 'train.csv':\n            train_data = pd.read_csv(os.path.join(dirname, filename))\n        elif filename == 'test.csv':\n            test_data = pd.read_csv(os.path.join(dirname, filename))\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:10.446067Z","iopub.execute_input":"2023-04-20T15:48:10.446593Z","iopub.status.idle":"2023-04-20T15:48:17.788588Z","shell.execute_reply.started":"2023-04-20T15:48:10.446545Z","shell.execute_reply":"2023-04-20T15:48:17.786748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape)\nprint(test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:17.791574Z","iopub.execute_input":"2023-04-20T15:48:17.792143Z","iopub.status.idle":"2023-04-20T15:48:17.801199Z","shell.execute_reply.started":"2023-04-20T15:48:17.792098Z","shell.execute_reply":"2023-04-20T15:48:17.799385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:17.803468Z","iopub.execute_input":"2023-04-20T15:48:17.804005Z","iopub.status.idle":"2023-04-20T15:48:17.849855Z","shell.execute_reply.started":"2023-04-20T15:48:17.803957Z","shell.execute_reply":"2023-04-20T15:48:17.848242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test data set has 784 columns, each column represent value of one pixel. Meaning the picture is 28X28X1\n\nTrain data set has one more column (feature) and that is target variable","metadata":{}},{"cell_type":"markdown","source":"We can try \n1. Logistic Regression\n2. CNN\n\nAs number of features are too many, it is better we don't use tree based algorights like Random Forest, Boosting, Bagging etc.","metadata":{}},{"cell_type":"code","source":"#Model Evaluation\ndef evaluate_model(model, X, y):\n    y_pred = model.predict(X)\n    print('The f1 score:',round(f1_score(y, y_pred, average=\"weighted\"), 2))\n    print(\"\")\n    print('AUC score:', roc_auc_score(y, model.predict_proba(X)[:, 1], average='weighted'))\n    print(\"\")\n    print('Confusion Matrix:')\n    print(confusion_matrix(y, y_pred ))\n    print(\"\")\n    print(classification_report(y, y_pred))\n\ndef print_model_evaluation_report(model, X_train, y_train, X_test, y_test):\n    print(\"***Logistic Regression***\")\n    print(\"Training Performance\")\n    evaluate_model(model, X_train, y_train)\n    print(\"----------------\")\n    print(\"Test Performance\")\n    evaluate_model(model, X_test, y_test)\n    print(\"----------------\")","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:17.851620Z","iopub.execute_input":"2023-04-20T15:48:17.853115Z","iopub.status.idle":"2023-04-20T15:48:17.864092Z","shell.execute_reply.started":"2023-04-20T15:48:17.853055Z","shell.execute_reply":"2023-04-20T15:48:17.862162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_data.label\nX = train_data.drop(['label'], 1)\nX = X.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:17.865861Z","iopub.execute_input":"2023-04-20T15:48:17.866270Z","iopub.status.idle":"2023-04-20T15:48:18.082217Z","shell.execute_reply.started":"2023-04-20T15:48:17.866234Z","shell.execute_reply":"2023-04-20T15:48:18.080502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:18.084287Z","iopub.execute_input":"2023-04-20T15:48:18.085313Z","iopub.status.idle":"2023-04-20T15:48:18.789295Z","shell.execute_reply.started":"2023-04-20T15:48:18.085246Z","shell.execute_reply":"2023-04-20T15:48:18.787419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#logistic_regression = LogisticRegression(solver='liblinear')\n#logistic_regression.fit(X_train, y_train)\n#print_model_evaluation_report(logistic_regression, X_train, y_train, X_test, y_test)\n\n# evaluate multinomial logistic regression model\nfrom numpy import mean\nfrom numpy import std\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom sklearn.linear_model import LogisticRegression\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:18.791061Z","iopub.execute_input":"2023-04-20T15:48:18.791528Z","iopub.status.idle":"2023-04-20T15:48:18.799935Z","shell.execute_reply.started":"2023-04-20T15:48:18.791484Z","shell.execute_reply":"2023-04-20T15:48:18.798048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the multinomial logistic regression model\nmodel = LogisticRegression(multi_class='multinomial', solver='lbfgs')\n# define the model evaluation procedure\ncv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:18.804995Z","iopub.execute_input":"2023-04-20T15:48:18.805747Z","iopub.status.idle":"2023-04-20T15:48:18.815440Z","shell.execute_reply.started":"2023-04-20T15:48:18.805677Z","shell.execute_reply":"2023-04-20T15:48:18.813878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate the model and collect the scores\nn_scores = cross_val_score(model, X_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:48:18.818422Z","iopub.execute_input":"2023-04-20T15:48:18.819047Z","iopub.status.idle":"2023-04-20T15:52:13.910437Z","shell.execute_reply.started":"2023-04-20T15:48:18.818982Z","shell.execute_reply":"2023-04-20T15:52:13.908469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# report the model performance/\nprint('Mean Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:52:13.914888Z","iopub.execute_input":"2023-04-20T15:52:13.916577Z","iopub.status.idle":"2023-04-20T15:52:13.927248Z","shell.execute_reply.started":"2023-04-20T15:52:13.916465Z","shell.execute_reply":"2023-04-20T15:52:13.925377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:52:13.929145Z","iopub.execute_input":"2023-04-20T15:52:13.929736Z","iopub.status.idle":"2023-04-20T15:52:39.630058Z","shell.execute_reply.started":"2023-04-20T15:52:13.929674Z","shell.execute_reply":"2023-04-20T15:52:39.628317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)\nprint('Confusion Matrix:')\nprint(confusion_matrix(y_test, y_pred ))\nprint(\"\")\nprint(classification_report(y_test, y_pred))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:52:39.632646Z","iopub.execute_input":"2023-04-20T15:52:39.633422Z","iopub.status.idle":"2023-04-20T15:52:39.772457Z","shell.execute_reply.started":"2023-04-20T15:52:39.633358Z","shell.execute_reply":"2023-04-20T15:52:39.770637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = test_data\nX = X.to_numpy()\ny_pred = model.predict(X)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T15:53:51.894271Z","iopub.execute_input":"2023-04-20T15:53:51.894860Z","iopub.status.idle":"2023-04-20T15:53:52.126649Z","shell.execute_reply.started":"2023-04-20T15:53:51.894810Z","shell.execute_reply":"2023-04-20T15:53:52.124488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissionDict = {'ImageId' : test_data.index + 1}\nsubmission = pd.DataFrame(submissionDict)\nsubmission['Label'] = y_pred","metadata":{"execution":{"iopub.status.busy":"2023-04-20T16:10:37.324019Z","iopub.execute_input":"2023-04-20T16:10:37.324609Z","iopub.status.idle":"2023-04-20T16:10:37.333842Z","shell.execute_reply.started":"2023-04-20T16:10:37.324557Z","shell.execute_reply":"2023-04-20T16:10:37.332644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-04-20T16:10:38.543794Z","iopub.execute_input":"2023-04-20T16:10:38.545373Z","iopub.status.idle":"2023-04-20T16:10:38.561810Z","shell.execute_reply.started":"2023-04-20T16:10:38.545297Z","shell.execute_reply":"2023-04-20T16:10:38.560060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('/kaggle/working/digit_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T16:10:52.389036Z","iopub.execute_input":"2023-04-20T16:10:52.390152Z","iopub.status.idle":"2023-04-20T16:10:52.428777Z","shell.execute_reply.started":"2023-04-20T16:10:52.390084Z","shell.execute_reply":"2023-04-20T16:10:52.427121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}