{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-02T08:05:44.821208Z","iopub.execute_input":"2023-05-02T08:05:44.822035Z","iopub.status.idle":"2023-05-02T08:05:44.842636Z","shell.execute_reply.started":"2023-05-02T08:05:44.821979Z","shell.execute_reply":"2023-05-02T08:05:44.840907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load and check data","metadata":{}},{"cell_type":"markdown","source":"We know a lot about the data from the description, so we don't have a lot to do here","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:05:48.572028Z","iopub.execute_input":"2023-05-02T08:05:48.573415Z","iopub.status.idle":"2023-05-02T08:05:49.232364Z","shell.execute_reply.started":"2023-05-02T08:05:48.573351Z","shell.execute_reply":"2023-05-02T08:05:49.231140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:05:49.357783Z","iopub.execute_input":"2023-05-02T08:05:49.359833Z","iopub.status.idle":"2023-05-02T08:05:59.061694Z","shell.execute_reply.started":"2023-05-02T08:05:49.359763Z","shell.execute_reply":"2023-05-02T08:05:59.060102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:58:10.304534Z","iopub.execute_input":"2023-05-02T11:58:10.305023Z","iopub.status.idle":"2023-05-02T11:58:10.755623Z","shell.execute_reply.started":"2023-05-02T11:58:10.304980Z","shell.execute_reply":"2023-05-02T11:58:10.753921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:05:59.064554Z","iopub.execute_input":"2023-05-02T08:05:59.065130Z","iopub.status.idle":"2023-05-02T08:05:59.097827Z","shell.execute_reply.started":"2023-05-02T08:05:59.065070Z","shell.execute_reply":"2023-05-02T08:05:59.095988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:05:59.099982Z","iopub.execute_input":"2023-05-02T08:05:59.100612Z","iopub.status.idle":"2023-05-02T08:06:02.335203Z","shell.execute_reply.started":"2023-05-02T08:05:59.100545Z","shell.execute_reply":"2023-05-02T08:06:02.333461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:06:02.339256Z","iopub.execute_input":"2023-05-02T08:06:02.339898Z","iopub.status.idle":"2023-05-02T08:06:02.427122Z","shell.execute_reply.started":"2023-05-02T08:06:02.339835Z","shell.execute_reply":"2023-05-02T08:06:02.425715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No missing values","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(5,5,figsize = (8,8))\nfor i in range(5):\n    for j in range(5):\n        ax[i][j].axis('off')\n        ax[i][j].imshow(train_data.iloc[[i+(j*5)], 1:].to_numpy().astype(np.uint8).reshape(28, 28), cmap='gray')\n\nplt.show()\n#temp = train_data.iloc[:1, 1:].to_numpy().reshape(28,28)\n#plt.imshow(temp, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:06:02.428601Z","iopub.execute_input":"2023-05-02T08:06:02.428988Z","iopub.status.idle":"2023-05-02T08:06:03.477260Z","shell.execute_reply.started":"2023-05-02T08:06:02.428952Z","shell.execute_reply":"2023-05-02T08:06:03.475761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"#### Load packages and split data","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:06:03.478850Z","iopub.execute_input":"2023-05-02T08:06:03.479275Z","iopub.status.idle":"2023-05-02T08:06:03.558774Z","shell.execute_reply.started":"2023-05-02T08:06:03.479232Z","shell.execute_reply":"2023-05-02T08:06:03.557026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Downsample data\n\nMivel jelentős méretű adattáblával dolgozunk, ezért lehetőség van az adathalmazból véletlenszerű mintavétellel - a célváltozó eloszlását megtartva - egy subsample-t létrehozni. \n\nEnnek veszélye, hogy nem minden összefüggés lesz elérhető számunkra a training adatszettben, cserébe spórolhatunk idővel és számítási kapacitással. Amennyiben a hiperparaméter optimalizálás történik a subsample adatszetten a rendes training pedig a teljes adathalmazon, úgy jó esélyünk van optimalizálni a felhasznált információ - szükséges erőforrás skálán.\n\nMivel most nem egy éles környezetben dolgozunk (Azure/AWS felhő helyett kaggle notebook vagy local jupyter) ezért alapvetően limitált erőforrások állnak rendelkezésre, így kifejezetten kis mintán fogom elvégezni a paraméter optimalizálást. Éles projektnél nem ez lenne a helyes döntés, de azt remélem, hogy ebben a háziban ez elfogadható.","metadata":{}},{"cell_type":"code","source":"yX = train_data.copy()\n\ny = yX.pop('label')\nX = yX.copy()\n\nX_train, X_test, y_train, y_test = train_test_split(\n    X, y, test_size=0.9, random_state=42)\n\nX_train.shape, X_test.shape, y_train.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:06:03.560491Z","iopub.execute_input":"2023-05-02T08:06:03.560988Z","iopub.status.idle":"2023-05-02T08:06:04.841892Z","shell.execute_reply.started":"2023-05-02T08:06:03.560943Z","shell.execute_reply":"2023-05-02T08:06:04.840255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Define and run random search","metadata":{}},{"cell_type":"code","source":"model = GradientBoostingClassifier()\n\nparams_dict = {\n    \"learning_rate\" : np.linspace(0.05, 0.3, 6),\n    \"subsample\" : np.linspace(0.3,1,10),\n    \"n_estimators\" : range(10, 40, 5),\n    \"max_depth\" : range(2, 20, 2),\n    \"min_samples_split\" : range(2,20,1),\n}\n\nclf = RandomizedSearchCV(\n    model, \n    params_dict, \n    scoring = \"accuracy\", \n    n_iter = 15, \n    verbose = 2,\n    cv=3\n)\n\nsearch = clf.fit(X_train, y_train)\n\nsearch.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-05-02T08:06:04.845934Z","iopub.execute_input":"2023-05-02T08:06:04.846427Z","iopub.status.idle":"2023-05-02T08:53:07.687954Z","shell.execute_reply.started":"2023-05-02T08:06:04.846374Z","shell.execute_reply":"2023-05-02T08:53:07.685820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Define and run line search operations","metadata":{}},{"cell_type":"code","source":"# learning rate\nlearning_rate_params = {\n    \"learning_rate\" : np.linspace(0.08, 0.12, 5),\n    \"subsample\" : [search.best_params_['subsample']],\n    \"n_estimators\" : [search.best_params_['n_estimators']],\n    \"max_depth\" : [search.best_params_['max_depth']],\n    \"min_samples_split\" : [search.best_params_['min_samples_split']],\n}\n\nlr_clf = GridSearchCV(\n    model, \n    learning_rate_params, \n    scoring = \"accuracy\", \n    verbose = 2, \n    cv = 2)\n\nlr_search = lr_clf.fit(X_train, y_train)\nlr_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-05-02T09:08:59.875941Z","iopub.execute_input":"2023-05-02T09:08:59.876464Z","iopub.status.idle":"2023-05-02T09:21:51.359108Z","shell.execute_reply.started":"2023-05-02T09:08:59.876423Z","shell.execute_reply":"2023-05-02T09:21:51.357485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# subsample\nsubsample_params = {\n    \"learning_rate\" : [lr_search.best_params_['learning_rate']],\n    \"subsample\" : np.linspace(0.6,0.8,5),\n    \"n_estimators\" : [search.best_params_['n_estimators']],\n    \"max_depth\" : [search.best_params_['max_depth']],\n    \"min_samples_split\" : [search.best_params_['min_samples_split']],\n}\n\ns_clf = GridSearchCV(\n    model, \n    subsample_params, \n    scoring = \"accuracy\", \n    verbose = 2, \n    cv = 2)\n\ns_search = s_clf.fit(X_train, y_train)\ns_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-05-02T09:22:23.236892Z","iopub.execute_input":"2023-05-02T09:22:23.237350Z","iopub.status.idle":"2023-05-02T09:35:44.240999Z","shell.execute_reply.started":"2023-05-02T09:22:23.237314Z","shell.execute_reply":"2023-05-02T09:35:44.239408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_estimators\nn_estimators_params = {\n    \"learning_rate\" : [lr_search.best_params_['learning_rate']],\n    \"subsample\" : [s_search.best_params_['subsample']],\n    \"n_estimators\" : range(34, 42, 2),\n    \"max_depth\" : [search.best_params_['max_depth']],\n    \"min_samples_split\" : [search.best_params_['min_samples_split']],\n}\n\nn_clf = GridSearchCV(\n    model, \n    n_estimators_params, \n    scoring = \"accuracy\", \n    verbose = 2, \n    cv = 2)\n\nn_search = n_clf.fit(X_train, y_train)\nn_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-05-02T09:35:44.243631Z","iopub.execute_input":"2023-05-02T09:35:44.244228Z","iopub.status.idle":"2023-05-02T09:48:41.838067Z","shell.execute_reply.started":"2023-05-02T09:35:44.244167Z","shell.execute_reply":"2023-05-02T09:48:41.836625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# max_depth\nmax_params = {\n    \"learning_rate\" : [lr_search.best_params_['learning_rate']],\n    \"subsample\" : [s_search.best_params_['subsample']],\n    \"n_estimators\" : [n_search.best_params_['n_estimators']],\n    \"max_depth\" : range(6, 10, 1),\n    \"min_samples_split\" : [search.best_params_['min_samples_split']],\n}\n\nmax_clf = GridSearchCV(\n    model, \n    max_params, \n    scoring = \"accuracy\", \n    verbose = 2, \n    cv = 2)\n\nmax_search = max_clf.fit(X_train, y_train)\nmax_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-05-02T09:48:41.840729Z","iopub.execute_input":"2023-05-02T09:48:41.841705Z","iopub.status.idle":"2023-05-02T10:00:07.222783Z","shell.execute_reply.started":"2023-05-02T09:48:41.841631Z","shell.execute_reply":"2023-05-02T10:00:07.220741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# min_samples_split\nmin_params = {\n    \"learning_rate\" : [lr_search.best_params_['learning_rate']],\n    \"subsample\" : [s_search.best_params_['subsample']],\n    \"n_estimators\" : [n_search.best_params_['n_estimators']],\n    \"max_depth\" : [max_search.best_params_['max_depth']],\n    \"min_samples_split\" : range(17,21,1),\n}\n\nmin_clf = GridSearchCV(\n    model, \n    min_params, \n    scoring = \"accuracy\", \n    verbose = 2, \n    cv = 2)\n\nmin_search = min_clf.fit(X_train, y_train)\nmin_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-05-02T10:00:07.224762Z","iopub.execute_input":"2023-05-02T10:00:07.225323Z","iopub.status.idle":"2023-05-02T10:09:47.263163Z","shell.execute_reply.started":"2023-05-02T10:00:07.225256Z","shell.execute_reply":"2023-05-02T10:09:47.261467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fit final model","metadata":{}},{"cell_type":"code","source":"yX2 = train_data.copy()\n\ny2 = yX2.pop('label')\nX2 = yX2.copy()\n\nX2_train, X2_test, y2_train, y2_test = train_test_split(\n    X2, y2, test_size=0.4, random_state=42)\n\nX2_train.shape, X2_test.shape, y2_train.shape, y2_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:29:58.805510Z","iopub.execute_input":"2023-05-02T11:29:58.805984Z","iopub.status.idle":"2023-05-02T11:30:00.237861Z","shell.execute_reply.started":"2023-05-02T11:29:58.805948Z","shell.execute_reply":"2023-05-02T11:30:00.236007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XGB_model = GradientBoostingClassifier(\n    learning_rate = lr_search.best_params_['learning_rate'],\n    subsample = s_search.best_params_['subsample'],\n    n_estimators = n_search.best_params_['n_estimators'],\n    max_depth = max_search.best_params_['max_depth'],\n    min_samples_split = min_search.best_params_['min_samples_split'],\n    verbose = 2\n)\n\n\nXGB_classifier = XGB_model.fit(X2_train, y2_train)\nXGB_classifier","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:33:17.611644Z","iopub.execute_input":"2023-05-02T11:33:17.612145Z","iopub.status.idle":"2023-05-02T11:44:56.654411Z","shell.execute_reply.started":"2023-05-02T11:33:17.612107Z","shell.execute_reply":"2023-05-02T11:44:56.652626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Score validation data","metadata":{}},{"cell_type":"code","source":"y2_pred = XGB_classifier.predict(X2_test)\nconfusion_matrix(y2_test, y2_pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:56:07.330681Z","iopub.execute_input":"2023-05-02T11:56:07.331173Z","iopub.status.idle":"2023-05-02T11:56:08.476883Z","shell.execute_reply.started":"2023-05-02T11:56:07.331133Z","shell.execute_reply":"2023-05-02T11:56:08.474934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y2_test, y2_pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:56:36.179900Z","iopub.execute_input":"2023-05-02T11:56:36.180363Z","iopub.status.idle":"2023-05-02T11:56:36.191579Z","shell.execute_reply.started":"2023-05-02T11:56:36.180326Z","shell.execute_reply":"2023-05-02T11:56:36.189946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Score official test data","metadata":{}},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:58:15.883306Z","iopub.execute_input":"2023-05-02T11:58:15.883807Z","iopub.status.idle":"2023-05-02T11:58:15.905645Z","shell.execute_reply.started":"2023-05-02T11:58:15.883763Z","shell.execute_reply":"2023-05-02T11:58:15.903636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_working = test_data.copy()\ntest_id = test_working.pop('id')","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:59:14.665245Z","iopub.execute_input":"2023-05-02T11:59:14.665780Z","iopub.status.idle":"2023-05-02T11:59:14.690885Z","shell.execute_reply.started":"2023-05-02T11:59:14.665740Z","shell.execute_reply":"2023-05-02T11:59:14.689119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = XGB_classifier.predict(test_working)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T11:59:27.216585Z","iopub.execute_input":"2023-05-02T11:59:27.218210Z","iopub.status.idle":"2023-05-02T11:59:27.474974Z","shell.execute_reply.started":"2023-05-02T11:59:27.218046Z","shell.execute_reply":"2023-05-02T11:59:27.473271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_scored = pd.concat([test_id,pd.Series(test_pred, name = 'label')], axis = 1)\ntest_scored.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T12:08:20.358981Z","iopub.execute_input":"2023-05-02T12:08:20.359492Z","iopub.status.idle":"2023-05-02T12:08:20.374301Z","shell.execute_reply.started":"2023-05-02T12:08:20.359443Z","shell.execute_reply":"2023-05-02T12:08:20.373070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_scored.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T12:09:49.411347Z","iopub.execute_input":"2023-05-02T12:09:49.412807Z","iopub.status.idle":"2023-05-02T12:09:49.431500Z","shell.execute_reply.started":"2023-05-02T12:09:49.412755Z","shell.execute_reply":"2023-05-02T12:09:49.429593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}