{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-04-06T12:02:19.951788Z","iopub.execute_input":"2022-04-06T12:02:19.952592Z","iopub.status.idle":"2022-04-06T12:02:19.984469Z","shell.execute_reply.started":"2022-04-06T12:02:19.952489Z","shell.execute_reply":"2022-04-06T12:02:19.983784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\nvalid_data = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:20.058229Z","iopub.execute_input":"2022-04-06T12:02:20.058498Z","iopub.status.idle":"2022-04-06T12:02:25.815046Z","shell.execute_reply.started":"2022-04-06T12:02:20.058469Z","shell.execute_reply":"2022-04-06T12:02:25.814189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nrandom.seed(32)\nnp.random.seed(42)\ntrain_data = train_data.sample(frac=1,random_state=52).reset_index(drop=True)\nvalid_data = valid_data.sample(frac=1,random_state=62).reset_index(drop=True)\nX_train, y_train = (train_data.drop(['label'], axis=1), train_data.label)\nX_valid, y_valid = (valid_data.drop(['label'], axis=1), valid_data.label)\n#X_train = X_train/255. # We don't have to normalize our data\n#X_valid = X_valid/255. # because https://datascience.stackexchange.com/questions/60950/is-it-necessary-to-normalize-data-for-xgboost ","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:25.816932Z","iopub.execute_input":"2022-04-06T12:02:25.817243Z","iopub.status.idle":"2022-04-06T12:02:26.869563Z","shell.execute_reply.started":"2022-04-06T12:02:25.817192Z","shell.execute_reply":"2022-04-06T12:02:26.868927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_train), len(X_valid)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:26.870557Z","iopub.execute_input":"2022-04-06T12:02:26.870801Z","iopub.status.idle":"2022-04-06T12:02:26.87888Z","shell.execute_reply.started":"2022-04-06T12:02:26.870772Z","shell.execute_reply":"2022-04-06T12:02:26.877823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:26.88123Z","iopub.execute_input":"2022-04-06T12:02:26.881538Z","iopub.status.idle":"2022-04-06T12:02:26.893104Z","shell.execute_reply.started":"2022-04-06T12:02:26.881499Z","shell.execute_reply":"2022-04-06T12:02:26.892168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:26.894422Z","iopub.execute_input":"2022-04-06T12:02:26.894647Z","iopub.status.idle":"2022-04-06T12:02:28.100665Z","shell.execute_reply.started":"2022-04-06T12:02:26.89462Z","shell.execute_reply":"2022-04-06T12:02:28.100023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"start = time.time()\nclf = XGBClassifier(max_depth = 10, \n                    n_estimators = 200,\n                    use_label_encoder = False,\n                    eval_metric = 'mlogloss',\n                    num_class = 10,\n                    eta = 0.05,\n                    subsample = 0.9,\n                    colsample_bytree = 0.9,\n                   )#reg_lambda = 0.1)\n\nclf.fit(X_train_small, y_train_small)\nend = time.time()\nprint(f\"Time: {end-start:.2f}s\")","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nimport random","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:28.101561Z","iopub.execute_input":"2022-04-06T12:02:28.102028Z","iopub.status.idle":"2022-04-06T12:02:28.107096Z","shell.execute_reply.started":"2022-04-06T12:02:28.101978Z","shell.execute_reply":"2022-04-06T12:02:28.106227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"start = time.time()\ny_pred = clf.predict(X_valid)\nend = time.time()\nprint(f\"Time: {end-start:.2f}s\")\n\nprint(accuracy_score(y_valid, y_pred))","metadata":{}},{"cell_type":"markdown","source":"# Hyperparameter tuning","metadata":{}},{"cell_type":"markdown","source":"I used https://towardsdatascience.com/xgboost-fine-tune-and-optimize-your-model-23d996fab663\n\nMy idea is to use a multi-phase random training.\n\n1. Train a lot of models on a small training set (random set of the original train.csv) ---> Evaluate them on the validation set (Dig-MNIST.csv) ---> save the best ones\n2. These define a narrower range of parameters. We'll train a lot of models (but less then in the 1st step) on the medium-size dataset. ---> Evaluate them on the validation set (Dig-MNIST.csv) ---> save the best ones\n3. These define a narrower range of parameters. We'll train some models on this parameter-set randomly again and choose the best model for the competition.\n","metadata":{}},{"cell_type":"markdown","source":"We'll tune the following parameters:\n\n* `max_depth`\n* `colsample_bytree`\n* `n_estimators`\n* `learning_rate`\n* `subsample`\n* `reg_lambda`","metadata":{}},{"cell_type":"code","source":"def train(train_set_size,max_depth,colsample_bytree,n_estimators,learning_rate,subsample,reg_lambda):\n    \"\"\"\n    Train an XGBoost classifier with these parameters and returns the trained model\n    \"\"\"\n    #start = time.time()\n    if train_set_size<1.0:\n        train_data_sampled = train_data.sample(frac=train_set_size).reset_index(drop=True)\n    else:\n        train_data_sampled = train_data.copy()\n    #print(len(train_data_sampled))\n    X_train_sampled, y_train_sampled = (train_data_sampled.drop(['label'], axis=1), train_data_sampled.label)\n    \n    clf = XGBClassifier(use_label_encoder = False,\n                        eval_metric = 'mlogloss',\n                        num_class = 10,\n                        max_depth = max_depth, \n                        colsample_bytree = colsample_bytree,\n                        n_estimators = n_estimators,\n                        learning_rate = learning_rate,\n                        subsample = subsample,\n                        reg_lambda = reg_lambda,\n                       )\n    clf.fit(X_train_sampled,y_train_sampled)\n    #end = time.time()\n    #print(f\"T: {end-start:.2f}s\")\n    return clf\n    ","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:28.108369Z","iopub.execute_input":"2022-04-06T12:02:28.108936Z","iopub.status.idle":"2022-04-06T12:02:28.121078Z","shell.execute_reply.started":"2022-04-06T12:02:28.108892Z","shell.execute_reply":"2022-04-06T12:02:28.120309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\nmodel = train(train_set_size = 1.0,\n              max_depth=5,\n              colsample_bytree = 1.0,\n              n_estimators = 10,\n              learning_rate = 0.2,\n              subsample = 1.0,\n              reg_lambda = 100)\nend = time.time()\nprint(f\"Time: {end-start:.2f}s\")             ","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:02:28.122567Z","iopub.execute_input":"2022-04-06T12:02:28.12292Z","iopub.status.idle":"2022-04-06T12:03:44.701087Z","shell.execute_reply.started":"2022-04-06T12:02:28.122887Z","shell.execute_reply":"2022-04-06T12:03:44.700287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\ny_pred = model.predict(X_valid)\nend = time.time()\nprint(f\"Time: {end-start:.2f}s\")\n\nprint(accuracy_score(y_valid, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:03:44.702205Z","iopub.execute_input":"2022-04-06T12:03:44.702413Z","iopub.status.idle":"2022-04-06T12:03:44.91697Z","shell.execute_reply.started":"2022-04-06T12:03:44.702388Z","shell.execute_reply":"2022-04-06T12:03:44.915976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1st round","metadata":{}},{"cell_type":"code","source":"MAX_PARAM_NUM_1 = 80\nMAX_PARAM_NUM_1_BEST = 5\nparameters_1 = pd.DataFrame()\nnp.random.seed(112)\nfor i in range(MAX_PARAM_NUM_1):\n    max_depth = np.random.randint(3,21)\n    n_estimators = np.random.randint(20,200)\n    learning_rate = np.random.rand()*(0.4-0.01)+0.01\n    colsample_bytree = np.random.rand()*(1.0-0.1)+0.1\n    subsample = np.random.rand()*(1.0-0.1)+0.1\n    reg_lambda = np.random.rand()*100.0\n    parameters_1 = parameters_1.append({\"max_depth\":max_depth,\n                                        \"n_estimators\":n_estimators,\n                                        \"learning_rate\":learning_rate,\n                                        \"colsample_bytree\":colsample_bytree,\n                                        \"subsample\":subsample,\n                                        \"reg_lambda\":reg_lambda,\n                                        \"valid_acc\":i},ignore_index=True)\nparameters_1[\"max_depth\"] = parameters_1[\"max_depth\"].astype(int)\nparameters_1[\"n_estimators\"] = parameters_1[\"n_estimators\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:03:44.920785Z","iopub.execute_input":"2022-04-06T12:03:44.921419Z","iopub.status.idle":"2022-04-06T12:03:45.067691Z","shell.execute_reply.started":"2022-04-06T12:03:44.921373Z","shell.execute_reply":"2022-04-06T12:03:45.066785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_accs = []\nstart = time.time()\nfor i in range(MAX_PARAM_NUM_1):\n    params = parameters_1.iloc[i]\n    max_depth = int(params[\"max_depth\"])\n    n_estimators = int(params[\"n_estimators\"])\n    learning_rate = params[\"learning_rate\"]\n    colsample_bytree = params[\"colsample_bytree\"]\n    subsample = params[\"subsample\"]\n    reg_lambda = params[\"reg_lambda\"]\n    #print(f\"{i+1}\\n----\\n\")\n    #print(f\"max_depth {max_depth}\")\n    #print(f\"n_estimators {n_estimators}\")\n    #print(f\"learning_rate {learning_rate}\")\n    #print(f\"colsample_bytree {colsample_bytree}\")\n    #print(f\"subsample {subsample}\")\n    #print(f\"reg_lambda {reg_lambda}\")\n    model = train(train_set_size = 0.05,\n                  max_depth = max_depth,\n                  colsample_bytree = colsample_bytree,\n                  n_estimators = n_estimators,\n                  learning_rate = learning_rate,\n                  subsample = subsample,\n                  reg_lambda = reg_lambda)\n    y_pred = model.predict(X_valid)\n    valid_accs.append(accuracy_score(y_valid, y_pred))\n    end = time.time()\n    print(f\"[{i+1}/{MAX_PARAM_NUM_1}] VA: {accuracy_score(y_valid, y_pred):.4f} | ET: {end-start:.2f}s\")\nparameters_1[\"valid_acc\"]=valid_accs","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:03:45.068868Z","iopub.execute_input":"2022-04-06T12:03:45.069636Z","iopub.status.idle":"2022-04-06T12:30:01.325046Z","shell.execute_reply.started":"2022-04-06T12:03:45.069604Z","shell.execute_reply":"2022-04-06T12:30:01.324184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters_1","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.326836Z","iopub.execute_input":"2022-04-06T12:30:01.327203Z","iopub.status.idle":"2022-04-06T12:30:01.349841Z","shell.execute_reply.started":"2022-04-06T12:30:01.327169Z","shell.execute_reply":"2022-04-06T12:30:01.349138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_parameters_1 = parameters_1.sort_values(\"valid_acc\",ascending=False).head(MAX_PARAM_NUM_1_BEST).min()\nmax_parameters_1 = parameters_1.sort_values(\"valid_acc\",ascending=False).head(MAX_PARAM_NUM_1_BEST).max()\n","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.351189Z","iopub.execute_input":"2022-04-06T12:30:01.351523Z","iopub.status.idle":"2022-04-06T12:30:01.36103Z","shell.execute_reply.started":"2022-04-06T12:30:01.351494Z","shell.execute_reply":"2022-04-06T12:30:01.360078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_parameters_1","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.362663Z","iopub.execute_input":"2022-04-06T12:30:01.362902Z","iopub.status.idle":"2022-04-06T12:30:01.375169Z","shell.execute_reply.started":"2022-04-06T12:30:01.362875Z","shell.execute_reply":"2022-04-06T12:30:01.374178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_parameters_1","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.377713Z","iopub.execute_input":"2022-04-06T12:30:01.378228Z","iopub.status.idle":"2022-04-06T12:30:01.391421Z","shell.execute_reply.started":"2022-04-06T12:30:01.378195Z","shell.execute_reply":"2022-04-06T12:30:01.390622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2nd round","metadata":{}},{"cell_type":"code","source":"MAX_PARAM_NUM_2 = 20\nMAX_PARAM_NUM_2_BEST = 5\nparameters_2 = pd.DataFrame()\nfor i in range(MAX_PARAM_NUM_2):\n    max_depth_MAX = int(max_parameters_1[\"max_depth\"]); max_depth_MIN = int(min_parameters_1[\"max_depth\"])\n    n_estimators_MAX  = int(max_parameters_1[\"n_estimators\"]); n_estimators_MIN = int(min_parameters_1[\"n_estimators\"])\n    learning_rate_MAX = max_parameters_1[\"learning_rate\"]; learning_rate_MIN = min_parameters_1[\"learning_rate\"]\n    colsample_bytree_MAX = max_parameters_1[\"colsample_bytree\"]; colsample_bytree_MIN = min_parameters_1[\"colsample_bytree\"]\n    subsample_MAX = max_parameters_1[\"subsample\"]; subsample_MIN = min_parameters_1[\"subsample\"]\n    reg_lambda_MAX = max_parameters_1[\"reg_lambda\"]; reg_lambda_MIN = min_parameters_1[\"reg_lambda\"]\n    \n    max_depth = np.random.randint(max_depth_MIN,max_depth_MAX+1)\n    n_estimators = np.random.randint(n_estimators_MIN,n_estimators_MAX+1)\n    learning_rate = np.random.rand()*(learning_rate_MAX - learning_rate_MIN)+learning_rate_MIN\n    colsample_bytree = np.random.rand()*(colsample_bytree_MAX - colsample_bytree_MIN)+colsample_bytree_MIN\n    subsample = np.random.rand()*(subsample_MAX - subsample_MIN)+subsample_MIN\n    reg_lambda = np.random.rand()*(reg_lambda_MAX - reg_lambda_MIN)+reg_lambda_MIN\n    \n    parameters_2 = parameters_2.append({\"max_depth\":max_depth,\n                                        \"n_estimators\":n_estimators,\n                                        \"learning_rate\":learning_rate,\n                                        \"colsample_bytree\":colsample_bytree,\n                                        \"subsample\":subsample,\n                                        \"reg_lambda\":reg_lambda,\n                                        \"valid_acc\":i},ignore_index=True)\nparameters_2[\"max_depth\"] = parameters_2[\"max_depth\"].astype(int)\nparameters_2[\"n_estimators\"] = parameters_2[\"n_estimators\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.392918Z","iopub.execute_input":"2022-04-06T12:30:01.393503Z","iopub.status.idle":"2022-04-06T12:30:01.435821Z","shell.execute_reply.started":"2022-04-06T12:30:01.393443Z","shell.execute_reply":"2022-04-06T12:30:01.435126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters_2","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.437266Z","iopub.execute_input":"2022-04-06T12:30:01.437759Z","iopub.status.idle":"2022-04-06T12:30:01.454803Z","shell.execute_reply.started":"2022-04-06T12:30:01.437703Z","shell.execute_reply":"2022-04-06T12:30:01.453591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_accs = []\nstart = time.time()\nfor i in range(MAX_PARAM_NUM_2):\n    params = parameters_2.iloc[i]\n    max_depth = int(params[\"max_depth\"])\n    n_estimators = int(params[\"n_estimators\"])\n    learning_rate = params[\"learning_rate\"]\n    colsample_bytree = params[\"colsample_bytree\"]\n    subsample = params[\"subsample\"]\n    reg_lambda = params[\"reg_lambda\"]\n    #print(f\"{i+1}\\n----\\n\")\n    #print(f\"max_depth {max_depth}\")\n    #print(f\"n_estimators {n_estimators}\")\n    #print(f\"learning_rate {learning_rate}\")\n    #print(f\"colsample_bytree {colsample_bytree}\")\n    #print(f\"subsample {subsample}\")\n    #print(f\"reg_lambda {reg_lambda}\")\n    model = train(train_set_size = 0.15,\n                  max_depth = max_depth,\n                  colsample_bytree = colsample_bytree,\n                  n_estimators = n_estimators,\n                  learning_rate = learning_rate,\n                  subsample = subsample,\n                  reg_lambda = reg_lambda)\n    y_pred = model.predict(X_valid)\n    valid_accs.append(accuracy_score(y_valid, y_pred))\n    end = time.time()\n    print(f\"[{i+1}/{MAX_PARAM_NUM_2}] VA: {accuracy_score(y_valid, y_pred):.4f} | ET: {end-start:.2f}s\")\nparameters_2[\"valid_acc\"]=valid_accs","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:30:01.456309Z","iopub.execute_input":"2022-04-06T12:30:01.456782Z","iopub.status.idle":"2022-04-06T12:53:27.933708Z","shell.execute_reply.started":"2022-04-06T12:30:01.456739Z","shell.execute_reply":"2022-04-06T12:53:27.932713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_parameters_2 = parameters_2.sort_values(\"valid_acc\",ascending=False).head(MAX_PARAM_NUM_2_BEST).min()\nmax_parameters_2 = parameters_2.sort_values(\"valid_acc\",ascending=False).head(MAX_PARAM_NUM_2_BEST).max()","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:53:27.935195Z","iopub.execute_input":"2022-04-06T12:53:27.935515Z","iopub.status.idle":"2022-04-06T12:53:27.94576Z","shell.execute_reply.started":"2022-04-06T12:53:27.935473Z","shell.execute_reply":"2022-04-06T12:53:27.9449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_parameters_2","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:53:27.947377Z","iopub.execute_input":"2022-04-06T12:53:27.947616Z","iopub.status.idle":"2022-04-06T12:53:27.962945Z","shell.execute_reply.started":"2022-04-06T12:53:27.947588Z","shell.execute_reply":"2022-04-06T12:53:27.962068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_parameters_2","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:53:27.9642Z","iopub.execute_input":"2022-04-06T12:53:27.964652Z","iopub.status.idle":"2022-04-06T12:53:27.977954Z","shell.execute_reply.started":"2022-04-06T12:53:27.964609Z","shell.execute_reply":"2022-04-06T12:53:27.976994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3rd round","metadata":{}},{"cell_type":"code","source":"MAX_PARAM_NUM_3 = 16\nMAX_PARAM_NUM_3_BEST = 16\nparameters_3 = pd.DataFrame()\nfor i in range(MAX_PARAM_NUM_3):\n    max_depth_MAX = int(max_parameters_2[\"max_depth\"]); max_depth_MIN = int(min_parameters_2[\"max_depth\"])\n    n_estimators_MAX  = int(max_parameters_2[\"n_estimators\"]); n_estimators_MIN = int(min_parameters_2[\"n_estimators\"])\n    learning_rate_MAX = max_parameters_2[\"learning_rate\"]; learning_rate_MIN = min_parameters_2[\"learning_rate\"]\n    colsample_bytree_MAX = max_parameters_2[\"colsample_bytree\"]; colsample_bytree_MIN = min_parameters_2[\"colsample_bytree\"]\n    subsample_MAX = max_parameters_2[\"subsample\"]; subsample_MIN = min_parameters_2[\"subsample\"]\n    reg_lambda_MAX = max_parameters_2[\"reg_lambda\"]; reg_lambda_MIN = min_parameters_2[\"reg_lambda\"]\n    \n    max_depth = np.random.randint(max_depth_MIN,max_depth_MAX+1)\n    n_estimators = np.random.randint(n_estimators_MIN,n_estimators_MAX+1)\n    learning_rate = np.random.rand()*(learning_rate_MAX - learning_rate_MIN)+learning_rate_MIN\n    colsample_bytree = np.random.rand()*(colsample_bytree_MAX - colsample_bytree_MIN)+colsample_bytree_MIN\n    subsample = np.random.rand()*(subsample_MAX - subsample_MIN)+subsample_MIN\n    reg_lambda = np.random.rand()*(reg_lambda_MAX - reg_lambda_MIN)+reg_lambda_MIN\n    \n    parameters_3 = parameters_3.append({\"max_depth\":max_depth,\n                                        \"n_estimators\":n_estimators,\n                                        \"learning_rate\":learning_rate,\n                                        \"colsample_bytree\":colsample_bytree,\n                                        \"subsample\":subsample,\n                                        \"reg_lambda\":reg_lambda,\n                                        \"valid_acc\":i},ignore_index=True)\nparameters_3[\"max_depth\"] = parameters_3[\"max_depth\"].astype(int)\nparameters_3[\"n_estimators\"] = parameters_3[\"n_estimators\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:53:27.979464Z","iopub.execute_input":"2022-04-06T12:53:27.979761Z","iopub.status.idle":"2022-04-06T12:53:28.00671Z","shell.execute_reply.started":"2022-04-06T12:53:27.979707Z","shell.execute_reply":"2022-04-06T12:53:28.006051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters_3","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:53:28.007941Z","iopub.execute_input":"2022-04-06T12:53:28.008281Z","iopub.status.idle":"2022-04-06T12:53:28.02142Z","shell.execute_reply.started":"2022-04-06T12:53:28.008252Z","shell.execute_reply":"2022-04-06T12:53:28.020762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_accs = []\nstart = time.time()\nmodels = []\nfor i in range(MAX_PARAM_NUM_3):\n    params = parameters_3.iloc[i]\n    max_depth = int(params[\"max_depth\"])\n    n_estimators = int(params[\"n_estimators\"])\n    learning_rate = params[\"learning_rate\"]\n    colsample_bytree = params[\"colsample_bytree\"]\n    subsample = params[\"subsample\"]\n    reg_lambda = params[\"reg_lambda\"]\n    #print(f\"{i+1}\\n----\\n\")\n    #print(f\"max_depth {max_depth}\")\n    #print(f\"n_estimators {n_estimators}\")\n    #print(f\"learning_rate {learning_rate}\")\n    #print(f\"colsample_bytree {colsample_bytree}\")\n    #print(f\"subsample {subsample}\")\n    #print(f\"reg_lambda {reg_lambda}\")\n    model = train(train_set_size = 1.0,\n                  max_depth = max_depth,\n                  colsample_bytree = colsample_bytree,\n                  n_estimators = n_estimators,\n                  learning_rate = learning_rate,\n                  subsample = subsample,\n                  reg_lambda = reg_lambda)\n    models.append(model)\n    y_pred = model.predict(X_valid)\n    valid_accs.append(accuracy_score(y_valid, y_pred))\n    end = time.time()\n    print(f\"[{i+1}/{MAX_PARAM_NUM_3}] VA: {accuracy_score(y_valid, y_pred):.4f} | ET: {end-start:.2f}s\")\nparameters_3[\"valid_acc\"]=valid_accs","metadata":{"execution":{"iopub.status.busy":"2022-04-06T12:53:28.022863Z","iopub.execute_input":"2022-04-06T12:53:28.023163Z","iopub.status.idle":"2022-04-06T13:26:01.746807Z","shell.execute_reply.started":"2022-04-06T12:53:28.023131Z","shell.execute_reply":"2022-04-06T13:26:01.745794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters_3.sort_values(\"valid_acc\",ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:01.74805Z","iopub.execute_input":"2022-04-06T13:26:01.74826Z","iopub.status.idle":"2022-04-06T13:26:01.762418Z","shell.execute_reply.started":"2022-04-06T13:26:01.748232Z","shell.execute_reply":"2022-04-06T13:26:01.761862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters_3","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:01.763354Z","iopub.execute_input":"2022-04-06T13:26:01.764052Z","iopub.status.idle":"2022-04-06T13:26:01.79121Z","shell.execute_reply.started":"2022-04-06T13:26:01.764013Z","shell.execute_reply":"2022-04-06T13:26:01.789722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_ind = parameters_3[\"valid_acc\"].argmax()","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:01.792545Z","iopub.execute_input":"2022-04-06T13:26:01.792873Z","iopub.status.idle":"2022-04-06T13:26:01.806784Z","shell.execute_reply.started":"2022-04-06T13:26:01.79284Z","shell.execute_reply":"2022-04-06T13:26:01.805709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = models[max_ind]","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:01.815816Z","iopub.execute_input":"2022-04-06T13:26:01.816159Z","iopub.status.idle":"2022-04-06T13:26:01.821412Z","shell.execute_reply.started":"2022-04-06T13:26:01.816122Z","shell.execute_reply":"2022-04-06T13:26:01.820386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = clf.predict(X_valid)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:01.823496Z","iopub.execute_input":"2022-04-06T13:26:01.823852Z","iopub.status.idle":"2022-04-06T13:26:02.531153Z","shell.execute_reply.started":"2022-04-06T13:26:01.823812Z","shell.execute_reply":"2022-04-06T13:26:02.530464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_valid, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:02.532562Z","iopub.execute_input":"2022-04-06T13:26:02.533114Z","iopub.status.idle":"2022-04-06T13:26:02.540391Z","shell.execute_reply.started":"2022-04-06T13:26:02.53307Z","shell.execute_reply":"2022-04-06T13:26:02.539775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\n#test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:02.541513Z","iopub.execute_input":"2022-04-06T13:26:02.542351Z","iopub.status.idle":"2022-04-06T13:26:02.894353Z","shell.execute_reply.started":"2022-04-06T13:26:02.542313Z","shell.execute_reply":"2022-04-06T13:26:02.893311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids, test_set = test_data.id, test_data.drop(['id'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:02.895786Z","iopub.execute_input":"2022-04-06T13:26:02.896035Z","iopub.status.idle":"2022-04-06T13:26:02.909843Z","shell.execute_reply.started":"2022-04-06T13:26:02.896006Z","shell.execute_reply":"2022-04-06T13:26:02.909063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_preds = clf.predict(test_set)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:02.911069Z","iopub.execute_input":"2022-04-06T13:26:02.911625Z","iopub.status.idle":"2022-04-06T13:26:03.124952Z","shell.execute_reply.started":"2022-04-06T13:26:02.911588Z","shell.execute_reply":"2022-04-06T13:26:03.12404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(final_preds, index=ids, name='label').to_csv('/kaggle/working/submission.csv')\n#pd.Series(final_preds, index=ids, name='label').to_csv('/kaggle/working/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:26:03.126683Z","iopub.execute_input":"2022-04-06T13:26:03.127327Z","iopub.status.idle":"2022-04-06T13:26:03.146385Z","shell.execute_reply.started":"2022-04-06T13:26:03.127281Z","shell.execute_reply":"2022-04-06T13:26:03.145671Z"},"trusted":true},"execution_count":null,"outputs":[]}]}