{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%bash\n\npip install --upgrade 'scikit-learn<1.6.0'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:49:32.461353Z","iopub.execute_input":"2025-10-05T10:49:32.461691Z","iopub.status.idle":"2025-10-05T10:49:41.412831Z","shell.execute_reply.started":"2025-10-05T10:49:32.461664Z","shell.execute_reply":"2025-10-05T10:49:41.412158Z"}},"outputs":[{"name":"stdout","text":"Requirement already satisfied: scikit-learn<1.6.0 in /usr/local/lib/python3.11/dist-packages (1.2.2)\nCollecting scikit-learn<1.6.0\n Downloading scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\nRequirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<1.6.0) (1.26.4)\nRequirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<1.6.0) (1.15.3)\nRequirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<1.6.0) (1.5.1)\nRequirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<1.6.0) (3.6.0)\nRequirement already satisfied: mkl_fft in /usr/local/lib/python3.11/dist-packages (from numpy>=1.19.5->scikit-learn<1.6.0) (1.3.8)\nRequirement already satisfied: mkl_random in /usr/local/lib/python3.11/dist-packages (from numpy>=1.19.5->scikit-learn<1.6.0) (1.2.4)\nRequirement already satisfied: mkl_umath in /usr/local/lib/python3.11/dist-packages (from numpy>=1.19.5->scikit-learn<1.6.0) (0.1.1)\nRequirement already satisfied: mkl in /usr/local/lib/python3.11/dist-packages (from numpy>=1.19.5->scikit-learn<1.6.0) (2025.2.0)\nRequirement already satisfied: tbb4py in /usr/local/lib/python3.11/dist-packages (from numpy>=1.19.5->scikit-learn<1.6.0) (2022.2.0)\nRequirement already satisfied: mkl-service in /usr/local/lib/python3.11/dist-packages (from numpy>=1.19.5->scikit-learn<1.6.0) (2.4.1)\nRequirement already satisfied: intel-openmp<2026,>=2024 in /usr/local/lib/python3.11/dist-packages (from mkl->numpy>=1.19.5->scikit-learn<1.6.0) (2024.2.0)\nRequirement already satisfied: tbb==2022.* in /usr/local/lib/python3.11/dist-packages (from mkl->numpy>=1.19.5->scikit-learn<1.6.0) (2022.2.0)\nRequirement already satisfied: tcmlib==1.* in /usr/local/lib/python3.11/dist-packages (from tbb==2022.*->mkl->numpy>=1.19.5->scikit-learn<1.6.0) (1.4.0)\nRequirement already satisfied: intel-cmplr-lib-rt in /usr/local/lib/python3.11/dist-packages (from mkl_umath->numpy>=1.19.5->scikit-learn<1.6.0) (2024.2.0)\nRequirement already satisfied: intel-cmplr-lib-ur==2024.2.0 in /usr/local/lib/python3.11/dist-packages (from intel-openmp<2026,>=2024->mkl->numpy>=1.19.5->scikit-learn<1.6.0) (2024.2.0)\nDownloading scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.3 MB)\n ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 13.3/13.3 MB 99.5 MB/s eta 0:00:00\nInstalling collected packages: scikit-learn\n Attempting uninstall: scikit-learn\n Found existing installation: scikit-learn 1.2.2\n Uninstalling scikit-learn-1.2.2:\n Successfully uninstalled scikit-learn-1.2.2\nSuccessfully installed scikit-learn-1.5.2\n","output_type":"stream"},{"name":"stderr","text":"ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ncesium 0.12.4 requires numpy<3.0,>=2.0, but you have numpy 1.26.4 which is incompatible.\n","output_type":"stream"}],"execution_count":2},{"cell_type":"code","source":"import pathlib\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nfrom sklearn import tree, feature_selection, metrics\nfrom sklearn import model_selection, pipeline, preprocessing","metadata":{"id":"e3ea2c52-3ead-4484-8a13-59ff459b0f78","trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:51:27.90877Z","iopub.execute_input":"2025-10-05T10:51:27.909096Z","iopub.status.idle":"2025-10-05T10:51:28.95558Z","shell.execute_reply.started":"2025-10-05T10:51:27.909071Z","shell.execute_reply":"2025-10-05T10:51:28.954728Z"}},"outputs":[],"execution_count":3},{"cell_type":"markdown","source":"# A deep-dive into DecisionTreeClassifier","metadata":{"id":"9Ptcn-IIVEsP"}},{"cell_type":"markdown","source":"## Load the MNIST data","metadata":{"id":"02cc24c9-260e-4840-8872-723f0adc6f41"}},{"cell_type":"code","source":"DATA_DIR = pathlib.Path(\"/kaggle/input/digit-recognizer\")\n\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\ntrain_features_df = train_df.drop(\"label\", axis=1)\ntrain_labels = train_df.loc[:, \"label\"]","metadata":{"id":"c1fded7d-02a7-48c3-99e4-d19d001c8ccf","trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:51:30.682265Z","iopub.execute_input":"2025-10-05T10:51:30.682667Z","iopub.status.idle":"2025-10-05T10:51:34.261335Z","shell.execute_reply.started":"2025-10-05T10:51:30.682645Z","shell.execute_reply":"2025-10-05T10:51:34.260678Z"}},"outputs":[],"execution_count":4},{"cell_type":"markdown","source":"## Data preparation","metadata":{}},{"cell_type":"code","source":"features_preparation_pipeline = pipeline.make_pipeline(\n feature_selection.VarianceThreshold(threshold=0.0),\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:51:34.262525Z","iopub.execute_input":"2025-10-05T10:51:34.262787Z","iopub.status.idle":"2025-10-05T10:51:34.266715Z","shell.execute_reply.started":"2025-10-05T10:51:34.262759Z","shell.execute_reply":"2025-10-05T10:51:34.265817Z"}},"outputs":[],"execution_count":5},{"cell_type":"code","source":"features_preparation_pipeline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:51:34.585384Z","iopub.execute_input":"2025-10-05T10:51:34.586005Z","iopub.status.idle":"2025-10-05T10:51:34.593167Z","shell.execute_reply.started":"2025-10-05T10:51:34.585977Z","shell.execute_reply":"2025-10-05T10:51:34.592285Z"}},"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"Pipeline(steps=[('variancethreshold', VarianceThreshold())])","text/html":"
Pipeline(steps=[('variancethreshold', VarianceThreshold())])
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
"},"metadata":{}}],"execution_count":6},{"cell_type":"markdown","source":"## Start with the default settings\n\nBy default, [DecisionTreeClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html) will consider every feature and find the split for that feature that leads to the biggest improvement in the criterion. The tree keeps splitting in this fashion until every leaf is pure or a node has only one sample.\n\nVisually, this produces a very deep tree with many tiny leaves that often perfectly fits the training data but with jagged, fragmented decision boundaries (high variance, overfitting).","metadata":{}},{"cell_type":"code","source":"decision_tree_classifier = tree.DecisionTreeClassifier(\n criterion='gini',\n splitter='best',\n max_depth=None,\n min_samples_split=2,\n min_samples_leaf=1,\n min_weight_fraction_leaf=0.0,\n max_features=None,\n random_state=None,\n max_leaf_nodes=None,\n min_impurity_decrease=0.0,\n class_weight=None,\n ccp_alpha=0.0,\n monotonic_cst=None,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:51:50.118925Z","iopub.execute_input":"2025-10-05T10:51:50.119241Z","iopub.status.idle":"2025-10-05T10:51:50.123944Z","shell.execute_reply.started":"2025-10-05T10:51:50.119217Z","shell.execute_reply":"2025-10-05T10:51:50.123023Z"}},"outputs":[],"execution_count":7},{"cell_type":"code","source":"classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n decision_tree_classifier,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:52:00.740673Z","iopub.execute_input":"2025-10-05T10:52:00.741004Z","iopub.status.idle":"2025-10-05T10:52:00.745205Z","shell.execute_reply.started":"2025-10-05T10:52:00.740969Z","shell.execute_reply":"2025-10-05T10:52:00.744349Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:52:16.708197Z","iopub.execute_input":"2025-10-05T10:52:16.70891Z","iopub.status.idle":"2025-10-05T10:52:31.134618Z","shell.execute_reply.started":"2025-10-05T10:52:16.708883Z","shell.execute_reply":"2025-10-05T10:52:31.133759Z"}},"outputs":[{"name":"stdout","text":"14.4 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n","output_type":"stream"}],"execution_count":9},{"cell_type":"code","source":"predicted_train_labels = classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T10:52:31.135844Z","iopub.execute_input":"2025-10-05T10:52:31.136591Z","iopub.status.idle":"2025-10-05T10:52:49.196415Z","shell.execute_reply.started":"2025-10-05T10:52:31.136566Z","shell.execute_reply":"2025-10-05T10:52:49.195455Z"}},"outputs":[{"name":"stdout","text":"Accuracy on the training set: 1.0\nCV Accuracy: 0.8530714285714286\n","output_type":"stream"}],"execution_count":10},{"cell_type":"code","source":"# how deep is the unregularized tree?\n(\n classifier_pipeline.named_steps\n .get(\"decisiontreeclassifier\")\n .tree_\n .max_depth\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T08:27:09.615056Z","iopub.execute_input":"2025-10-05T08:27:09.615801Z","iopub.status.idle":"2025-10-05T08:27:09.621496Z","shell.execute_reply.started":"2025-10-05T08:27:09.61577Z","shell.execute_reply":"2025-10-05T08:27:09.620666Z"}},"outputs":[{"execution_count":28,"output_type":"execute_result","data":{"text/plain":"45"},"metadata":{}}],"execution_count":28},{"cell_type":"code","source":"# how many leaf nodes does unregularized tree have?\n(\n classifier_pipeline.named_steps\n .get(\"decisiontreeclassifier\")\n .tree_\n .n_leaves\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T08:28:40.634894Z","iopub.execute_input":"2025-10-05T08:28:40.635271Z","iopub.status.idle":"2025-10-05T08:28:40.642368Z","shell.execute_reply.started":"2025-10-05T08:28:40.635246Z","shell.execute_reply":"2025-10-05T08:28:40.641413Z"}},"outputs":[{"execution_count":31,"output_type":"execute_result","data":{"text/plain":"3055"},"metadata":{}}],"execution_count":31},{"cell_type":"markdown","source":"## Criterion\n\n**Purpose**: Defines the function used to measure the quality of a split.\n\n**Options**: \n\n* `criterion=\"gini\"`: Uses the Gini measure of impurity (default).\n* `criterion=\"entropy\"`: Uses information gain (Shannon entropy).\n* `criterion=\"log_loss\"`: Uses cross-entropy loss\n\n**Effect:** Determines how the algorithm decides which feature and threshold to split on at each node. Different criteria can lead to different tree structures, but often have similar performance.\n\n**Which to use?**:\n\nUse `criterion=\"log_loss\"` when you care about having well-calibrated class probability predictions (not just hard class predictions!) as the other impurity measures often lead to poorly calibrated probabilities. Also consider setting `criterion=\"log_loss\"` when you plan to tune `max_leaf_nodes` or `ccp_alpha`.","metadata":{}},{"cell_type":"code","source":"param_grid = [\n {\n \"decisiontreeclassifier__criterion\": [\n \"gini\",\n \"entropy\",\n \"log_loss\",\n ],\n }\n]\n\ncriterion_tuning = model_selection.GridSearchCV(\n classifier_pipeline,\n param_grid,\n cv=4,\n n_jobs=-1,\n refit=True,\n scoring=\"accuracy\",\n verbose=0,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T11:00:13.491498Z","iopub.execute_input":"2025-10-05T11:00:13.491832Z","iopub.status.idle":"2025-10-05T11:00:13.497298Z","shell.execute_reply.started":"2025-10-05T11:00:13.491808Z","shell.execute_reply":"2025-10-05T11:00:13.496442Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = criterion_tuning.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T11:00:15.230222Z","iopub.execute_input":"2025-10-05T11:00:15.230538Z","iopub.status.idle":"2025-10-05T11:01:13.066348Z","shell.execute_reply.started":"2025-10-05T11:00:15.230512Z","shell.execute_reply":"2025-10-05T11:01:13.065502Z"}},"outputs":[{"name":"stdout","text":"57.8 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"criterion_tuning.best_params_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T11:01:13.067662Z","iopub.execute_input":"2025-10-05T11:01:13.067914Z","iopub.status.idle":"2025-10-05T11:01:13.073Z","shell.execute_reply.started":"2025-10-05T11:01:13.067893Z","shell.execute_reply":"2025-10-05T11:01:13.072182Z"}},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"{'decisiontreeclassifier__criterion': 'entropy'}"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"criterion_tuning.best_score_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T11:01:13.073846Z","iopub.execute_input":"2025-10-05T11:01:13.074111Z","iopub.status.idle":"2025-10-05T11:01:13.094322Z","shell.execute_reply.started":"2025-10-05T11:01:13.074087Z","shell.execute_reply":"2025-10-05T11:01:13.093518Z"}},"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"0.8635"},"metadata":{}}],"execution_count":14},{"cell_type":"code","source":"(\n pd.DataFrame\n .from_dict(\n criterion_tuning.cv_results_\n ).sort_values(\n by=\"rank_test_score\"\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T11:01:13.096071Z","iopub.execute_input":"2025-10-05T11:01:13.096429Z","iopub.status.idle":"2025-10-05T11:01:13.133092Z","shell.execute_reply.started":"2025-10-05T11:01:13.096408Z","shell.execute_reply":"2025-10-05T11:01:13.132378Z"}},"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":" mean_fit_time std_fit_time mean_score_time std_score_time \\\n1 14.006975 0.138306 0.070279 0.009018 \n2 13.913872 0.181808 0.052736 0.006028 \n0 15.059634 0.306907 0.065511 0.007208 \n\n param_decisiontreeclassifier__criterion \\\n1 entropy \n2 log_loss \n0 gini \n\n params split0_test_score \\\n1 {'decisiontreeclassifier__criterion': 'entropy'} 0.865333 \n2 {'decisiontreeclassifier__criterion': 'log_loss'} 0.862571 \n0 {'decisiontreeclassifier__criterion': 'gini'} 0.854952 \n\n split1_test_score split2_test_score split3_test_score mean_test_score \\\n1 0.854857 0.867810 0.866000 0.863500 \n2 0.855619 0.866381 0.868190 0.863190 \n0 0.849810 0.852952 0.854381 0.853024 \n\n std_test_score rank_test_score \n1 0.005072 1 \n2 0.004819 2 \n0 0.001994 3 ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
mean_fit_timestd_fit_timemean_score_timestd_score_timeparam_decisiontreeclassifier__criterionparamssplit0_test_scoresplit1_test_scoresplit2_test_scoresplit3_test_scoremean_test_scorestd_test_scorerank_test_score
114.0069750.1383060.0702790.009018entropy{'decisiontreeclassifier__criterion': 'entropy'}0.8653330.8548570.8678100.8660000.8635000.0050721
213.9138720.1818080.0527360.006028log_loss{'decisiontreeclassifier__criterion': 'log_loss'}0.8625710.8556190.8663810.8681900.8631900.0048192
015.0596340.3069070.0655110.007208gini{'decisiontreeclassifier__criterion': 'gini'}0.8549520.8498100.8529520.8543810.8530240.0019943
\n
"},"metadata":{}}],"execution_count":15},{"cell_type":"markdown","source":"## Depth and size controls: \"how deep can the tree grow?\"\n\n| Parameter | Intuition | Visual Effect |\n| ----------------------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------- |\n| **`max_depth`** | The maximum number of levels (root-to-leaf). | 🌲 Shallower tree → fewer rectangular regions; smoother boundaries. |\n| **`max_leaf_nodes`** | Directly limits the number of terminal leaves (regions). | 🌿 Controls the number of distinct class regions. |\n| **`min_samples_split`** | Requires a node to have at least this many samples before attempting a split. | 🚫 Prevents small branches from forming. |\n| **`min_samples_leaf`** | Ensures each leaf has a minimum number of samples. | 🧩 Merges small isolated regions into larger, smoother ones. |\n","metadata":{}},{"cell_type":"code","source":"_display = (\n model_selection.ValidationCurveDisplay\n .from_estimator(\n criterion_tuning.best_estimator_,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n param_name=\"decisiontreeclassifier__max_depth\",\n param_range=[0, 1, 2, 4, 8, 16, 32, 64],\n score_type=\"both\",\n score_name=\"Accuracy\",\n scoring=\"accuracy\",\n )\n)\n\n_display.ax_.set_title(\"Validation Curve for Decision Tree\")\n_display.ax_.set_xlabel(r\"max_depth\")\n_display.ax_.set_ylim(0.0, 1.1)\n_display.ax_.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T11:10:28.382344Z","iopub.execute_input":"2025-10-05T11:10:28.382621Z","iopub.status.idle":"2025-10-05T11:11:31.00053Z","shell.execute_reply.started":"2025-10-05T11:10:28.3826Z","shell.execute_reply":"2025-10-05T11:11:30.999694Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAAjcAAAHLCAYAAAA0kLlRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABigElEQVR4nO3dd3gUVd/G8e/uplcCIQk9ofcWiiAKKkVQFBBFQCkqYkFUrPgoiKjYUFTwUVC6FMEuRYryoILSi/TeSejpbXfePxb2JSRAAkkm2dyf68oFe+bMzG+XZHIzc+aMxTAMAxERERE3YTW7ABEREZG8pHAjIiIibkXhRkRERNyKwo2IiIi4FYUbERERcSsKNyIiIuJWFG5ERETErSjciIiIiFtRuBERERG3onAjxdL+/fuxWCxMnjzZ1fb6669jsVhytL7FYuH111/P05ratGlDmzZt8nSbxV1CQgKPPPIIERERWCwWnnnmGbNLylf9+vUjMjIyV+ssW7YMi8XCsmXL8qUmETMo3Eihd9ddd+Hn50d8fPxl+/Tu3RsvLy9OnTpVgJXl3tatW3n99dfZv3+/2aVkERMTw/PPP0/NmjXx8/PD39+f6Oho3nzzTc6ePWt2edfk7bffZvLkyTz++ONMmzaNBx98MF/3FxkZicViwWKxYLVaKVGiBPXq1ePRRx/ln3/+ydd9FyWTJ092fU5X+sptUBO5wKJnS0lhN3v2bO6//36mTJlCnz59sixPSkoiLCyMW2+9lZ9++ilH29y/fz9RUVFMmjSJfv36AZCRkUFGRgY+Pj5XXd9isTB8+PBcn72ZO3cu9957L7///nuWszRpaWkAeHl55WqbeWH16tV06tSJhIQEHnjgAaKjowFYs2YNs2bNomXLlixatKjA67peN9xwAx4eHvz5558Fsr/IyEhCQkJ47rnnAIiPj2fbtm3MmTOH48eP8+yzz/Lhhx/m2/7T09NxOBx4e3vneB2Hw0FaWhpeXl5YrQXz/929e/eyYsWKTG2PPPIIzZo149FHH3W1BQQE0KVLlwKpSdyLh9kFiFzNXXfdRWBgIDNmzMg23Pz4448kJibSu3fv69qPh4cHHh7m/UiYEWoAzp49S9euXbHZbKxfv56aNWtmWv7WW28xYcKEPNlXYmIi/v7+ebKtnIiNjaV27dp5tr2MjAwcDscV/63KlSvHAw88kKnt3XffpVevXnz00UdUq1aNxx9/PM9qupinp2eu17FarTkK9HmpcuXKVK5cOVPbY489RuXKlbN8dhfLyecvArosJUWAr68v3bp1Y+nSpcTGxmZZPmPGDAIDA7nrrrs4ffo0zz//PPXq1SMgIICgoCA6duzIxo0br7qf7MbcpKam8uyzz1K6dGnXPg4fPpxl3QMHDvDEE09Qo0YNfH19KVWqFPfee2+my0+TJ0/m3nvvBeCWW25xnXq/MNYhuzE3sbGxPPzww4SHh+Pj40ODBg2YMmVKpj4Xxg998MEHjB8/nipVquDt7U3Tpk1ZvXr1Vd/3F198wZEjR/jwww+zBBuA8PBwXn31Vdfry403ioyMdJ0Fu/B+LRYL//vf/3jiiScICwujfPnyzJ0719WeXS0Wi4V///3X1bZ9+3a6d+9OyZIl8fHxoUmTJlc9Q3dhHMm+ffuYN2+e67O+8O+R2891zJgxrs9169atV9x3dnx9fZk2bRolS5bkrbfe4uIT5g6HgzFjxlCnTh18fHwIDw9n4MCBnDlzJst2FixYQOvWrQkMDCQoKIimTZsyY8YM1/LsxtzMmjWL6Oho1zr16tXj448/zvJZXTrmZs6cOURHR+Pr60toaCgPPPAAR44cydSnX79+BAQEcOTIEbp06UJAQAClS5fm+eefx2635/pzutjVPv+cfl+cPXuWZ555hgoVKuDt7U3VqlV59913cTgc11WfFG46cyNFQu/evZkyZQrffPMNgwYNcrWfPn2aX3/9lZ49e+Lr68uWLVv44YcfuPfee4mKiiImJoYvvviC1q1bs3XrVsqWLZur/T7yyCNMnz6dXr160bJlS3777TfuuOOOLP1Wr17NihUruP/++ylfvjz79+/nv//9L23atGHr1q34+flx8803M3jwYD755BNeeeUVatWqBeD681LJycm0adOG3bt3M2jQIKKiopgzZw79+vXj7NmzPP3005n6z5gxg/j4eAYOHIjFYuG9996jW7du7N2794r/o//pp5/w9fWle/fuufpscuqJJ56gdOnSDBs2jMTERO644w4CAgL45ptvaN26daa+s2fPpk6dOtStWxeALVu2cOONN1KuXDlefvll/P39+eabb+jSpQvffvstXbt2zXaftWrVYtq0aTz77LOUL1/edZmodOnSuf5cJ02aREpKCo8++ije3t6ULFnymj6HgIAAunbtyldffcXWrVupU6cOAAMHDmTy5Mn079+fwYMHs2/fPsaOHcv69ev566+/XP92kydP5qGHHqJOnToMHTqUEiVKsH79ehYuXEivXr2y3efixYvp2bMnt912G++++y4A27Zt46+//sryPi92oZ6mTZsyatQoYmJi+Pjjj/nrr79Yv349JUqUcPW12+106NCB5s2b88EHH7BkyRJGjx5NlSpV8uQMVXaff06/L5KSkmjdujVHjhxh4MCBVKxYkRUrVjB06FCOHTvGmDFjrrs+KaQMkSIgIyPDKFOmjNGiRYtM7Z9//rkBGL/++qthGIaRkpJi2O32TH327dtneHt7G2+88UamNsCYNGmSq2348OHGxT8SGzZsMADjiSeeyLS9Xr16GYAxfPhwV1tSUlKWmleuXGkAxtSpU11tc+bMMQDj999/z9K/devWRuvWrV2vx4wZYwDG9OnTXW1paWlGixYtjICAACMuLi7TeylVqpRx+vRpV98ff/zRAIyff/45y74uFhISYjRo0OCKfS526Xu/oFKlSkbfvn1drydNmmQARqtWrYyMjIxMfXv27GmEhYVlaj927JhhtVoz/TvddtttRr169YyUlBRXm8PhMFq2bGlUq1btqrVWqlTJuOOOOzK15fZzDQoKMmJjY6+6r8vt72IfffSRARg//vijYRiG8ccffxiA8fXXX2fqt3DhwkztZ8+eNQIDA43mzZsbycnJmfo6HA7X3/v27WtUqlTJ9frpp582goKCsnz+F/v9998zfU+mpaUZYWFhRt26dTPt65dffjEAY9iwYZn2B2T6NzMMw2jUqJERHR192X1mx9/fP9P3z5U+/5x+X4wcOdLw9/c3du7cmWn9l19+2bDZbMbBgwdzVaMUHbosJUWCzWbj/vvvZ+XKlZku9cyYMYPw8HBuu+02ALy9vV2DIu12O6dOnSIgIIAaNWqwbt26XO1z/vz5AAwePDhTe3a3E/v6+rr+np6ezqlTp6hatSolSpTI9X4v3n9ERAQ9e/Z0tXl6ejJ48GASEhKyXNbp0aMHISEhrtc33XQT4By8eSVxcXEEBgZeU405MWDAAGw2W6a2Hj16EBsbm+lSyNy5c3E4HPTo0QNwnpX77bffuO+++4iPj+fkyZOcPHmSU6dO0aFDB3bt2pXlMklO5PZzveeeeyhdunSu95OdgIAAANedf3PmzCE4OJh27dq53t/JkyeJjo4mICCA33//HXCegYmPj+fll1/OMj7mStMXlChRgsTERBYvXpzjGtesWUNsbCxPPPFEpn3dcccd1KxZk3nz5mVZ57HHHsv0+qabbrrq911OXfr55+b7Ys6cOdx0002EhIRk+nzbtm2L3W5n+fLleVKjFD4KN1JkXBgwfGGMweHDh/njjz+4//77Xb88HQ6Ha9Cmt7c3oaGhlC5dmk2bNnHu3Llc7e/AgQNYrVaqVKmSqb1GjRpZ+iYnJzNs2DDXdf0L+z179myu93vx/qtVq5blDpYLl7EOHDiQqb1ixYqZXl8IOtmN3bhYUFDQFW+zv15RUVFZ2m6//XaCg4OZPXu2q2327Nk0bNiQ6tWrA7B7924Mw+C1116jdOnSmb6GDx8OkO0YrKvJ7eeaXf3XKiEhAcAVJnft2sW5c+cICwvL8h4TEhJc72/Pnj0Arst1OfXEE09QvXp1OnbsSPny5XnooYdYuHDhFde58P6z+z6vWbNmls/Hx8cnS/gLCQm56vddTl36+efm+2LXrl0sXLgwS7+2bdtm6ifuR2NupMiIjo6mZs2azJw5k1deeYWZM2diGEamu6TefvttXnvtNR566CFGjhxJyZIlsVqtPPPMM/k6gPCpp55i0qRJPPPMM7Ro0YLg4GAsFgv3339/gQ1cvPTsyAXGVWZ7qFmzJhs2bHDdDnytLjeA9OKzWhd4e3vTpUsXvv/+ez777DNiYmL466+/ePvtt119Lnxuzz//PB06dMh221WrVr3menMqu/qv1YWB0hfqdjgchIWF8fXXX2fb/3rPGIWFhbFhwwZ+/fVXFixYwIIFC5g0aRJ9+vTJMoD6Wl3u+y6vXPr55+b7wuFw0K5dO1588cVs+10I0uJ+FG6kSOnduzevvfYamzZtYsaMGVSrVo2mTZu6ls+dO5dbbrmFr776KtN6Z8+eJTQ0NFf7qlSpEg6Hgz179mT6X+yOHTuy9J07dy59+/Zl9OjRrraUlJQsk9/ldAbkC/vftGkTDocj01mG7du3u5bnhc6dO7Ny5Uq+/fbbTJdqLickJCTL+0pLS+PYsWO52m+PHj2YMmUKS5cuZdu2bRiG4bokBbhuFfb09HT9TzsvFNTneqmEhAS+//57KlSo4DpLVKVKFZYsWcKNN954xRB14ezhv//+m+tA5+XlRefOnencuTMOh4MnnniCL774gtdeey3bbV14/zt27ODWW2/NtGzHjh359vnkVG6+L6pUqUJCQkKefv9I0aDLUlKkXDhLM2zYMDZs2JBlbhubzZblTMWcOXOuaWxGx44dAfjkk08ytWd3h0V2+/3000+znM24MMdLTmb87dSpE8ePH8906SYjI4NPP/2UgICALHcaXavHHnuMMmXK8Nxzz7Fz584sy2NjY3nzzTddr6tUqZJlrML48eNzfetv27ZtKVmyJLNnz2b27Nk0a9Ys0yWIsLAw2rRpwxdffJFtcDpx4kSu9ndBQX2uF0tOTubBBx/k9OnT/Oc//3GF3Pvuuw+73c7IkSOzrJORkeH6Pmnfvj2BgYGMGjWKlJSUTP2udGbu0hm7rVYr9evXB5zTHGSnSZMmhIWF8fnnn2fqs2DBArZt25bt3YIFKTffF/fddx8rV67k119/zdLv7NmzZGRk5GutYh6duZEiJSoqipYtW/Ljjz8CZAk3d955J2+88Qb9+/enZcuWbN68ma+//jrLhGE50bBhQ3r27Mlnn33GuXPnaNmyJUuXLmX37t1Z+t55551MmzaN4OBgateuzcqVK1myZAmlSpXKsk2bzca7777LuXPn8Pb25tZbbyUsLCzLNh999FG++OIL+vXrx9q1a4mMjGTu3Ln89ddfjBkzJs8GAYeEhPD999/TqVMnGjZsmGmG4nXr1jFz5kxatGjh6v/II4/w2GOPcc8999CuXTs2btzIr7/+muszY56ennTr1o1Zs2aRmJjIBx98kKXPuHHjaNWqFfXq1WPAgAFUrlyZmJgYVq5cyeHDh3M0f9Gl8vtzPXLkCNOnTwecZ2u2bt3qmqH4ueeeY+DAga6+rVu3ZuDAgYwaNYoNGzbQvn17PD092bVrF3PmzOHjjz+me/fuBAUF8dFHH/HII4/QtGlTevXqRUhICBs3biQpKemyl5geeeQRTp8+za233kr58uU5cOAAn376KQ0bNrzsFASenp68++679O/fn9atW9OzZ0/XreCRkZE8++yz1/X55IWcfl+88MIL/PTTT9x5553069eP6OhoEhMT2bx5M3PnzmX//v25/r6VIsK8G7VErs24ceMMwGjWrFmWZSkpKcZzzz1nlClTxvD19TVuvPFGY+XKlVlus87JreCGYRjJycnG4MGDjVKlShn+/v5G586djUOHDmW5HfrMmTNG//79jdDQUCMgIMDo0KGDsX379iy3RxuGYUyYMMGoXLmyYbPZMt2Ce2mNhmEYMTExru16eXkZ9erVy1Tzxe/l/fffz/J5XFrnlRw9etR49tlnjerVqxs+Pj6Gn5+fER0dbbz11lvGuXPnXP3sdrvx0ksvGaGhoYafn5/RoUMHY/fu3Ze9FXz16tWX3efixYsNwLBYLMahQ4ey7bNnzx6jT58+RkREhOHp6WmUK1fOuPPOO425c+de9T1d7tbs6/1cr7Q/wPWegoKCjDp16hgDBgww/vnnn8uuN378eCM6Otrw9fU1AgMDjXr16hkvvviicfTo0Uz9fvrpJ6Nly5aGr6+vERQUZDRr1syYOXOma/mlt4LPnTvXaN++vREWFmZ4eXkZFStWNAYOHGgcO3bM1efSW8EvmD17ttGoUSPD29vbKFmypNG7d2/j8OHDmfr07dvX8Pf3z/J+svtZuprL3Qp+uc8/p98X8fHxxtChQ42qVasaXl5eRmhoqNGyZUvjgw8+MNLS0nJVoxQderaUiIiIuBWNuRERERG3onAjIiIibkXhRkRERNyKwo2IiIi4FYUbERERcSvFbp4bh8PB0aNHCQwMzNVssSIiImIewzCIj4+nbNmyWZ4Nd6liF26OHj1KhQoVzC5DRERErsGhQ4coX778FfsUu3BzYfbRQ4cOERQUlKfbTk9PZ9GiRa5ZRkVEckvHEZHsxcXFUaFChRzNIl7sws2FS1FBQUH5Em78/PwICgrSQUlEromOIyJXlpMhJRpQLCIiIm5F4UZERETcisKNiIiIuJViN+Ymp+x2O+np6blaJz09HQ8PD1JSUrDb7flUWdHg6emJzWYzuwwRESmGFG4uYRgGx48f5+zZs9e0bkREBIcOHdIcOkCJEiWIiIjQZyEiIgVK4eYSF4JNWFgYfn5+ufrF7HA4SEhIICAg4KoTDLkzwzBISkoiNjYWgDJlyphckYiIFCcKNxex2+2uYFOqVKlcr+9wOEhLS8PHx6dYhxsAX19fAGJjYwkLC9MlKhERKTDF+zfwJS6MsfHz8zO5Evdw4XPM7dglERGR66Fwkw2NEckb+hxFRMQMCjciIiLiVhRuRERExK0o3MhlRUZGMmbMGLPLEBERyRWFGzdgsViu+PX6669f03ZXr17No48+mrfFioiI5DPdCu4Gjh075vr77NmzGTZsGDt27HC1BQQEuP5uGAZ2ux0Pj6v/05cuXTpvCxURESkAOnNzFYZhkJSWkeOv5DR7rvpf6cswjBzVGBER4foKDg7GYrG4Xm/fvp3AwEAWLFhAdHQ03t7e/Pnnn+zZs4e7776b8PBwAgICaNq0KUuWLMm03UsvS1ksFr788ku6du2Kn58f1apV46effsrLj1tEROS66czNVSSn26k97FdT9r31jQ74eeXNP9HLL7/MBx98QOXKlQkJCeHQoUN06tSJt956C29vb6ZOnUrnzp3ZsWMHFStWvOx2RowYwXvvvcf777/Pp59+Su/evTlw4AAlS5bMkzpFRESul87cFBNvvPEG7dq1o0qVKpQsWZIGDRowcOBA6tatS7Vq1Rg5ciRVqlS56pmYfv360bNnT6pWrcrbb79NQkICq1atKqB3ISIicnU6c3MVvp42tr7RIUd9HQ4H8XHxBAYF5snjF3w98+6RBU2aNMn0OiEhgddff5158+Zx7NgxMjIySE5O5uDBg1fcTv369V1/9/f3JygoyPUMKRERkcJA4eYqLBZLji8NORwOMrxs+Hl5FLpnS/n7+2d6/fzzz7N48WI++OADqlatiq+vL927dyctLe2K2/H09Mz02mKx4HA48rxeERGRa6VwU0z99ddf9OvXj65duwLOMzn79+83tygREZE8ULhOL0iBqVatGt999x0bNmxg48aN9OrVS2dgRETELSjcFFMffvghISEhtGzZks6dO9OhQwcaN25sdlkiIiLXzWLkdDIVNxEXF0dwcDDnzp0jKCgo07KUlBT27dtHVFQUPj4+ud62w+EgLi6OoKCgQjfmxgzX+3mKFEfp6enMnz+fTp06ZRnjJlKcXen396X0G1hERETcisKNiIiIuBWFGxEREXErCjciIiLiVkwNN8uXL6dz586ULVsWi8XCDz/8cNV1li1bRuPGjfH29qZq1apMnjw53+sUERGRosPUcJOYmEiDBg0YN25cjvrv27ePO+64g1tuuYUNGzbwzDPP8Mgjj/Drr+Y82FJEREQKH1NnKO7YsSMdO3bMcf/PP/+cqKgoRo8eDUCtWrX4888/+eijj+jQIWfPfxIREcktwzBwGBf9iYFhgGGAwzBwGAbFal6Vq7BZLPh7mxcxitTjF1auXEnbtm0ztXXo0IFnnnnmsuukpqaSmprqeh0XFwc455JIT0/P1Dc9Pd35jetwXNNsvRemDLqwjeLO4XBgGAbp6enYbHn3EFARd3bhuHTp8amwSstwMGnFAbYcjcPA+YveOB8CMr/GFQAchgGZXp/vn20fZ5BwOLJu02EAOMNG1v1mv80LtRhcCCsXAkvW9S5eR3KnUYVgvnm0eZ5uMzc/E0Uq3Bw/fpzw8PBMbeHh4cTFxZGcnIyvr2+WdUaNGsWIESOytC9atAg/P79MbR4eHkRERJCQkHDVB0heSXx8/DWv607S0tJITk5m+fLlZGRkmF2OSJGyePFis0u4qthkmLrLxqFEi9mlSCFz5sxZ5s+fn6fbTEpKynHfIhVursXQoUMZMmSI63VcXBwVKlSgffv22c5QfOjQIQICAq5pRl3DMIiPjycwMBCLRT/sKSkp+Pr6cvPNN2uGYpEcSk9PZ/HixbRr167QzlBsGAbfbzjKhz9vJzndTrCvJwNaReLvbcNisWABrBYLFgtYLWRp+//X2bVdeH3xNpzLLBdt03r+GJu1T+Y2rlKL1WKBi7aZeT//X+elbZfWYsG5cx35nSwW8LTl7bDeC1decqJIhZuIiAhiYmIytcXExBAUFJTtWRsAb29vvL29s7R7enpmOXDY7XYsFgtWq/WaHp9w4VLUhW0UlKsFqeHDh/P6669f87a///57unTpkut1rVYrFosl289aRK6ssP7cxKWk8+r3W/hp41EAbqhcko96NKRMcPbHYJG8kpufhyIVblq0aJHlNNfixYtp0aKFSRUVDseOHXP9ffbs2QwbNowdO3a42gICAswoS0TczLqDZ3h65noOnUnGZrUwpF11HmtdBZtV5yukcDH1VvCEhAQ2bNjAhg0bAOet3hs2bODgwYOA85JSnz59XP0fe+wx9u7dy4svvsj27dv57LPP+Oabb3j22Wfzr0jDgLTEnH+lJ+Wu/5W+cjiKLSIiwvUVHByMxWLJ1DZr1ixq1aqFj48PNWvW5LPPPnOtm5aWxqBBgyhTpgw+Pj5UqlSJUaNGARAZGQlA165dsVgsrtciUrzYHQbjft/NvZ+v5NCZZMqH+PLNwBY8eUtVBRsplEw9c7NmzRpuueUW1+sLY2P69u3L5MmTOXbsmCvoAERFRTFv3jyeffZZPv74Y8qXL8+XX36Zv7eBpyfB22Vz1NUKlMjLfb9yFLz8r2sTX3/9NcOGDWPs2LE0atSI9evXM2DAAPz9/enbty+ffPIJP/30E9988w0VK1bk0KFDHDp0CIDVq1cTFhbGpEmTuP3223XHk0gxFBOXwrOzN7BizykA7qxfhre71SPIp/BdMhO5wNRw06ZNG9ft09nJbvbhNm3asH79+nysyr0MHz6c0aNH061bN8AZELdu3coXX3xB3759OXjwINWqVaNVq1ZYLBYqVarkWrd06dIAlChRgoiICFPqFxHzLNkawwtzN3ImKR0/Lxuv31WHe6PL64YJKfSK1JgbU3j6Oc+g5IDD4SAuPp6gwMC8GVDs6Xf1PleQmJjInj17ePjhhxkwYICrPSMjg+DgYAD69etHu3btqFGjBrfffjt33nkn7du3v679ikjRlpJuZ9T8bUxZeQCAOmWD+LRnIyqX1vg9KRoUbq7GYsn5pSGHAzztzv4FeLfU5SQkJAAwYcIEmjfPPJnShUtMjRs3Zt++fSxYsIAlS5Zw33330bZtW+bOnVvg9YqI+XbFxPPUzPVsP+6cr+uRVlG8cHsNvD10WVqKDoUbNxYeHk7ZsmXZu3cvvXv3vmy/oKAgevToQY8ePejevTu33347p0+fpmTJknh6emK32wuwahExg2EYzFx1iDd+2UJKuoNS/l6Mvq8BbWqEmV2aSK4p3Li5ESNGMHjwYIKDg7n99ttJTU1lzZo1nDlzhiFDhvDhhx9SpkwZGjVqhNVqZc6cOURERFCiRAnAecfU0qVLufHGG/H29iYkJMTcNyQiee5sUhovf7uZhVuOA3BTtVBG39eAsEBNvilFk8KNm3vkkUfw8/Pj/fff54UXXsDf35969eq5nscVGBjIe++9x65du7DZbDRt2pT58+e7xgyNHj2aIUOGMGHCBMqVK8f+/fvNezMikuf+2XuKZ2Zv4Ni5FDxtFl7oUINHWlXGqlu8pQizGFe6XckNxcXFERwczLlz57J9/MK+ffuIioq6pscFOBwO4uLiCAoKKtAZigur6/08RYqj9PR05s+fT6dOnfJ1huIMu4NPf9vNp7/twmFAZCk/Pu3ZmHrlg/NtnyLX40q/vy+lMzciIsXM4TNJPDNrA2sOnAHgnsblGXF3HQK89StB3IO+k0VEipH5m4/x8rebiEvJwN/bxttd63F3w3JmlyWSpxRuRESKgaS0DEb+spWZq5wzkDeoUIJP729ExVLXN5+WSGGkcCMi4ua2Ho3jqZnr2HMiEQvweJsqPNuuOp42jQ0U96Rwkw2Hw2F2CW5Bn6OIuQzDYMqK/bw1fxvpdoOwQG/G9GhIy6qhZpcmkq8Ubi7i5eWF1Wrl6NGjlC5dGi8vr1w9Q8XhcJCWlkZKSkqxvlvKMAzS0tI4ceIEVqsVLy8vs0sSKXZOJaTy4txNLN0eC8BtNcN4/94GlPTXz6O4P4Wbi1itVqKiojh27BhHj+bseVIXMwyD5ORkfH199WA5wM/Pj4oVKxbroCdihr92n+TZ2RuIjU/Fy8PKfzrVok+LSjouSbGhcHMJLy8vKlasSEZGRq4fO5Cens7y5cu5+eab83V+iqLAZrPh4eGhg6lIAUq3Oxi9aCdf/G8PBlA1LIBPezaiVpkrzwki4m4UbrJhsVjw9PTMdUCx2WxkZGTg4+NT7MONiBSsA6cSGTxrAxsPnQWgZ7OKDLuzNr5eeuClFD8KNyIiRdwP64/wnx82k5hqJ8jHg3fvqU/HemXMLkvENAo3IiJFVEJqBsN+/Jfv1h0BoFlkST66vyHlSviaXJmIuRRuRESKoE2HzzJ45nr2n0rCaoHBt1Vj0C1V8dDcNSIKNyIiRYnDYTDhj728/+sOMhwGZYJ9+KRnI5pGljS7NJFCQ+FGRKSIiI1P4blvNvLHrpMAdKwbwTvd6hPspxsYRC6mcCMiUgT8vj2W5+Zs5HRiGj6eVoZ3rsP9TStougWRbCjciIgUYqkZdt5dsIOJf+0DoGZEIGN7NaJqWKDJlYkUXgo3IiKF1J4TCTw1Yz1bj8UB0K9lJC93rImPp+auEbkShRsRkULGMGDO2iOMnLed5HQ7IX6evN+9AW1rh5tdmkiRoHAjIlKIxCWnM2WXlfV/bwGgZZVSfNSjIeFBPiZXJlJ0KNyIiBQSaw+cZvDM9Rw5a8XDamFI++oMvLkKNqsGDYvkhsKNiIjJ7A6Dz37fzZglu7AbBqW8DT7v24ymlUubXZpIkaRwIyJiomPnknlm1gb+2XcagM71I2jlfZiGFUqYW5hIEaZwIyJikl+3HOeluZs4m5yOn5eNkXfXpXO9MBYsOGx2aSJFmsKNiEgBS0m38+a8rUz/+yAA9coF80nPRkSF+pOenm5ydSJFn8KNiEgB2nE8nqdmrmNnTAIAA2+uzHPta+DloQdeiuQVhRsRkQJgGAbT/znIm79sJTXDQWiAFx/e15Cbq2vQsEheU7gREclnZxLTeOnbTSzaGgNA6+qlGX1fA0IDvE2uTMQ9KdyIiOSjlXtO8czs9cTEpeJhtfByx5o8dGMUVs1dI5JvFG5ERPJBht3Bx0t3Mfa33RhAVKg/n/ZsRN1ywWaXJuL2FG5ERPLYodNJPD1rPesOngXgviblGd65Dv7eOuSKFAT9pImI5KGfNx7lle83E5+SQYC3B6O61aNzg7JmlyVSrCjciIjkgaS0DF7/aQvfrHFOwNe4Ygk+vr8RFUr6mVxZPnA4IPEEnN4DB/+GYxsh+QxYrOe/LGCxnf8zu9cXf9kuWn6+r9V60evzX1YbcP7PTO2X9LXanK+tVsDmfH1xH+slf1o8nMuttov62zLvz2oFq8f5bVy03GJztruWn6/hqi4ab2W5zNiry7VfaVuX7ZKTbeVwfznZltUDPMwdLK9wIyJynf49co6nZq5n38lELMCgW6vy9G3V8LC5wdw1huEMLqcuBJn1cGIHxB+DpFNmVyd5Io8Ht1ssUKE5PLQwb7ebCwo3IiLXyDAMJv61n3cWbCPdbhAe5M2YHo1oUaWU2aVdu5Q4OL0PDq6AI+vhxHaIO3I+yBhZ+3sHQmAZ8Al2LjfOf3HJn4YBOC6/3PUnV1l+0Z/Xs26WP7nKvtxZHr8/wwCHPW+3mUsKNyIi18DuMBg8cz3zNh8DoF3tcN67pz4h/l4mV5YLaUlwZh8cWAlH1sKJbXDuMCSeJNtfeF4BziBTsjJE1HX+77x0TQiMuLbLEJkCCf+/zzx7fa3rZLPc4QDDAYbd+afDgTOs2f//l7nhuNK7zWbbV+yUh9vK4fbyals2bwgwd3JKhRsRkWvw0eKdzNt8DE+bhWF31uaBGyphyfE4CRNkpMGZ/XBwJRxeDbHb4OyBKwQZ//NBpgpE1HMGmfDaEBAONs+8qenC51WYPzcpkhRuRERyacHmY4z9fTcA73dvQJdG5Uyu6BIOO5w95Ly0dGgVxGw5H2ROZH92wdMPAstCaBWIaAAVmkF4XfAvDTb9mpCiR9+1IiK5sON4PEPmbATgkVZR5gcbw4C4o3DgL+cZmeObnWNmEmMvH2SCykKpqhBR33lGJqKeM8hY3WAAtAgKNyIiOXYuKZ0BU9eQnGbnxqqleLljzYIvIj4GDqyAQ/9AzGbnXUwJsc6xH5fy8HUGmdBqziBT8QbnmRn/UroUJG5N4UZEJAfsDoOnZq3n4Okkyof48mnPxvl/q3fSaWeQObjSeUbm1C5nuMk2yPhAULn/DzKVWkCZhuBXMn9rFCmEFG5ERHLgg0U7WL7zBD4eVsY/2ISSeX1XVGo8HFiBZf8K6h/ajO3T/0D80csEGe/zQaY6lGkAFVtA2UbgWyJvaxIpohRuRESu4pdNR/nvsj0AvHdvA2qXDbq+DaannD8jswKOroeTO53jZhwZeABRF/e1eUNweecZmTINnWdkykaDT+D11SDixhRuRESuYNuxOJ4/P4B44M2VuSu3z4nKSHOOjznwFxxd55zd99wRcKRn7WvzwhFUgb2e1YmsWR+PyJZQvqnztmwRyTGFGxGRyziTmMaAqWtISXdwU7VQXrz9KgOIHXY4vAb2/wFH1p2fFO8Q2LMPMs4zMjWgbEOo5AwydjzYMn8+lW7qBJ55NJ+MSDGjcCMiko0Mu4OnZq7n8JlkKoT48mnPRtisl9xhlHQa1k2DI2sgdiucPQj2tKwbs3meDzI1zweZG51zyWQ3q296NkFIRHJF4UZEJBvv/7qDP3efxMfTyoS+TSjhd8kA4rMH4YvWkHw6c7v1fJApXdM5yDfyRijfDDyK0GMZRIo4hRsRkUv8uOEIXyzfC8DoextSM+KSAcSJp2BSR2ew8S3pnAivbCOo1AoqNFWQETGZwo2IyEW2HD3Hi3M3AfB4myrcUb9M5g5piTDlTucDJv1C4bE/nBPliUihobm2RUTOO31+AHFqhoPW1UvzfPsamTvYM+Dr+5zja7z8od88BRuRQkjhRkQE5wDiJ79ex9GzKVQs6ccn918ygNgw4LtH4MCfzgHCvedCmAmPXxCRq1K4EREBRi3Yzsq9p/D1tPFl3yYE+11yG/avr8CW7wEL3DPReeu2iBRKpoebcePGERkZiY+PD82bN2fVqlVX7D9mzBhq1KiBr68vFSpU4NlnnyUlJaWAqhURd/T9+sN89ec+AD7q0YDq4ZfM/vvXJ/D3Z86/d3wPat9VwBWKSG6YGm5mz57NkCFDGD58OOvWraNBgwZ06NCB2NjYbPvPmDGDl19+meHDh7Nt2za++uorZs+ezSuvvFLAlYuIu/j3yDle+nYzAE/dWpXb614ygHjjN7B4mPPvrZ6F5o8WcIUiklum3i314YcfMmDAAPr37w/A559/zrx585g4cSIvv/xylv4rVqzgxhtvpFevXgBERkbSs2dP/vnnn8vuIzU1ldTUVNfruLg4ANLT00nP48myLmwvr7crIvnjVEIqD09ZTVqGg9bVQxnUOirTz69l7/+w/fgYFgzs9XrguPmVfJ9kT8cRkezl5mfCtHCTlpbG2rVrGTp0qKvNarXStm1bVq5cme06LVu2ZPr06axatYpmzZqxd+9e5s+fz4MPPnjZ/YwaNYoRI0ZkaV+0aBF+fn7X/0aysXjx4nzZrojkHbsDPttmIybOQmkfg9uDj7Nw4QLX8qDkg7Ta+RYWh52jwU1YbesICxZcYYt5S8cRkcySkpJy3Ne0cHPy5Ensdjvh4eGZ2sPDw9m+fXu26/Tq1YuTJ0/SqlUrDMMgIyODxx577IqXpYYOHcqQIUNcr+Pi4qhQoQLt27cnKOg6n+x7ifT0dBYvXky7du3w1DNhRAq1kfO2sTvuEH5eNqYOaE7VsID/X3j2IB5fPY3FkYwjvB6l+/1AJw+fAqlLxxGR7F248pITRWoSv2XLlvH222/z2Wef0bx5c3bv3s3TTz/NyJEjee2117Jdx9vbG2/vrM9v8fT0zLcDR35uW0Su35w1h5j69yEAxvRoSK1yIf+/MOk0TOsMKWegRCWs/X7G6ht4mS3lHx1HRDLLzc+DaeEmNDQUm81GTExMpvaYmBgiIiKyXee1117jwQcf5JFHHgGgXr16JCYm8uijj/Kf//wHq9X0m79EpJDbeOgsr3zvHED89G3VaF/nouNNWhJM6gRxR8CvFPRfAL4hl9mSiBRWpqUBLy8voqOjWbp0qavN4XCwdOlSWrRoke06SUlJWQKMzWYDwDCM/CtWRNzCifhUBkxdQ7rd4LaaYTx9W7X/X2jPgOn3wIlt4OkHfX+B4HLmFSsi18zUy1JDhgyhb9++NGnShGbNmjFmzBgSExNdd0/16dOHcuXKMWrUKAA6d+7Mhx9+SKNGjVyXpV577TU6d+7sCjkiItlJy3Dw2PS1xManEhXqz5j7G2K9MAOxYcC3D8HBFc6neveaA+G1zS1YRK6ZqeGmR48enDhxgmHDhnH8+HEaNmzIwoULXYOMDx48mOlMzauvvorFYuHVV1/lyJEjlC5dms6dO/PWW2+Z9RZEpIgY+ctW1h44g7+XcwbiQJ+Lrt8vfAm2/ghYoNt4iGplWp0icv0sRjG7nhMXF0dwcDDnzp3Ll7ul5s+fT6dOnTQQUKQQ+Wb1IV78dhMW4Mu+Tbit1kV3af45BpYMd/69w9vQ4kkzSnTRcUQke7n5/a0RuCLi1tYfPOMaQPxM22qZg83GWbDkdeffWwwyPdiISN5QuBERtxUbn8KAqWvIcBi0qx3OU7deNIB491L48UnAgLr3QPs3TatTRPKWwo2IuKW0DAePTl3LyYQ0Kof681GPiwYQH90Is3qDIwMib4JuE8BiMbdgEckzCjci4paG/fgvGw6dxd/bxlf9mhLgff7+iTMHYOpdkJEMYXWg9xyw6m5LEXeicCMibufrfw4wa/UhLMDYXo2JCvV3Lkg8BRNvh5SzEFwe+s8DT18zSxWRfKBwIyJuZe2B0wz7cQsAQ9pX55YaYc4FaUkwuSPEH3XOOtx/oWYfFnFTCjci4jZi4lIYMHUtdodBhzrhDLqlqnOBPQOmdYUTO/5/9uESFcwtVkTyjcKNiLiF1Aw7j0xZw+nENKqU9ufD+xpisVicsw/P6QeH/garB9w/EyLqml2uiOQjhRsRKfIMw+CV7/5l85Fz+HvbmNivKf4XBhDPfwG2/wxYoMtnUKWNmaWKSAFQuBGRIm/aygN8u+4wFgt81juaSqXODyBePhpWT3D+vd0IqN/DvCJFpMAo3IhIkbZ6/2lG/LwVgBc61KB19dLOBeu/ht9GOv/e/DG48WmTKhSRgqZwIyJF1rFzyQyYuga7YXB7nQgeb13FuWDnIvh5MGBArbvh9ndMrVNECpbCjYgUSSnpdh6avJqzSenOAcQ9GjgHEB9ZB9886Jx9uGJLuHeSZh8WKWYUbkSkyDEMgxe/3cS2Y/EEeHswuX8z/Lw84PQ+5y3fGSlQugY8+J1mHxYphhRuRKTImfjXfn7acBSLBT5/IJoKJf2csw9P6uicfTiorHOSPs0+LFIsKdyISJGycs9J3prnHED8YoeatKoW6px9eNLtEH8MfEpAv4XgV9LcQkXENAo3IlJkHDmbzMBpa3EY0LFuBI+1ruycfXjq3XByJ3j4Qp8foGQls0sVERMp3IhIkZCSbqf/pFXEpWRQtXSAcwZicA4ePrzKOftwj2lQtpHZpYqIyRRuRKTQMwyDId9sYGdMAgHeHkzq3xRfLxv8MgR2zAcs0PkTqNbO7FJFpBBQuBGRQm/88r3M33wcqwW+eLCxcwDxsndh7URnh1v/A416m1ukiBQaCjciUqj9tesk7y7cDsBLt9fkxqqlYe0UWDbK2aHJw3DzCyZWKCKFjcKNiBRah04nMXC6cwBxp3oRPHpzZdixAOYNAQyo0QnuGG12mSJSyCjciEihlJxmp+/EVSSkZlAt7PwA4iNrYE4/5+zD5ZvCfdM0+7CIZKFwIyKFjmEYDJ61nr0nEwn08WDyQ83widsP0+9xzj5cqir0+QlsHmaXKiKFkMKNiBQ6437fzeKtMVgtMP6BaMp5JMCkTpByDgIj4KFfwcvP7DJFpJDSf3tEpFBZtiOW0Yt2AvByx5q0qOAL42+GhOPgEwz95oN/qMlVikhhpnAjIoXGodNJPPn1OgycA4gH3FgRJnaAU7vBwwd6fwulqphdpogUcgo3IlIoJKVl0PvLf0hMszsHEN/bAMusXnBkrfPJ3vdOhgpNzS5TRIoAjbkREdMZhsETX6/j4OkkAn08mPpwM3wWPAu7fgUs0Gk01OhodpkiUkQo3IiI6T5cvJNlO044BxA/GE2ZtR/C+qnOhTe/CE36m1ugiBQpCjciYqrFW44z9rfdAAztWIsWp3+E5e87FzZ6EG59xcTqRKQoUrgREdPsO5HA4FkbMIA76pXhkdAtsOBFwICqbeGuT80uUUSKIIUbETFFYmoGD3y1iuR0O9XCA/ioRQqW7x5xzj5ctjH0nKXZh0XkmuhuKREpcIZh8Oi0NRw5m0yQjwcz7g7B65s7nbMPl6wMfX8Cm6fZZYpIEaVwIyIF7p2F2/lr9ylsFgtfdq9I6e/uds4+HBAO/ReAd6DZJYpIEabLUiJSoOZtOsoX/9sLwH/aVaTZb70hIQa8g5zPiwqMMLlCESnqFG5EpMDsionnuTkbAbirbmke2j0ITu9xzj7cczaE1TS5QhFxBwo3IlIgElIzeHDiKlLSHVQr7c9Hjnfg6Hrn7MNdv4DIlmaXKCJuQuFGRPKdw2Hw0KTVHD+XQpCvB9+Vn4lt71LnwvZvQ50uptYnIu5F4UZE8t3IeVtZtf80NquFH2r/QeC2Wc4FNz4DNzxmam0i4n4UbkQkX32//jCT/toPwBd1tlJ5y1jngvo9oN0I8woTEbelcCMi+Wbr0XO8/O1mAAZHHqTtnlGAAVFtnONsRETygcKNiOSLc0lp9Ju0mtQMB+1CYnj25OvO2Ycj6kPvOZp9WETyjcKNiOQ5h8Og76TVxManUs3nLJ/bX8eSkQIhkdD3Z/DwMrtEEXFjCjcikuf+88NmNhw6S4g1gR99RmBLOwf+paHvL+BbwuzyRMTNKdyISJ6ateogM1cdwps0Fga9g19KjPNxCg98ByUqmF2eiBQDCjcikmc2HDrDsB+3YMXB3KAxhKfsBQ9vuG86lKlvdnkiUkzowZkikidOJaTy8OQ1pNntfOb3JfXSNoDFBneNhSptzC5PRIqRXJ+5iYyM5I033uDgwYP5UY+IFEEOh0Hfias4lZjGS17f0smxzLngtuFQ/z5TaxOR4ifX4eaZZ57hu+++o3LlyrRr145Zs2aRmpqaH7WJSBHx/NyN/Hs0jt62pTxu/c7Z2PxxaPW0uYWJSLF0TeFmw4YNrFq1ilq1avHUU09RpkwZBg0axLp16/KjRhEpxCav2Md3647Q1rqWkZ4TnY11ukLHd8wtTESKrWseUNy4cWM++eQTjh49yvDhw/nyyy9p2rQpDRs2ZOLEiRiGkZd1ikghtGrfKd6at43Glp381+tjrBgQ2Qq6fWV2aSJSjF3zgOL09HS+//57Jk2axOLFi7nhhht4+OGHOXz4MK+88gpLlixhxowZeVmriBQiMXEpPDZ9HZUch5ji/R6eZEB4Heg9F2w2s8sTkWIs1+Fm3bp1TJo0iZkzZ2K1WunTpw8fffQRNWvWdPXp2rUrTZs2zdNCRaTwyLA76DtxFV6Jx5ju/Q6BJEGJitDnJ/D0Nbs8ESnmch1umjZtSrt27fjvf/9Lly5d8PT0zNInKiqK+++/P08KFJHC5+XvNnP0+DHmeL1LhOU0+IVCnx/BP9Ts0kREch9u9u7dS6VKla7Yx9/fn0mTJl1zUSJSeC3acpyf1+5lqteH1LAeBq8A6P0NlKxsdmkiIsA1DCiOjY3ln3/+ydL+zz//sGbNmlwXMG7cOCIjI/Hx8aF58+asWrXqiv3Pnj3Lk08+SZkyZfD29qZ69erMnz8/1/sVkdw7k5jGi3M28p7neJpbtztnH77nSygXbXZpIiIuuQ43Tz75JIcOHcrSfuTIEZ588slcbWv27NkMGTKE4cOHs27dOho0aECHDh2IjY3Ntn9aWhrt2rVj//79zJ07lx07djBhwgTKlSuX27chItdg0Ix1tEtfwt22FRgWK3R8H2p0NLssEZFMcn1ZauvWrTRu3DhLe6NGjdi6dWuutvXhhx8yYMAA+vfvD8Dnn3/OvHnzmDhxIi+//HKW/hMnTuT06dOsWLHCNdYnMjLyivtITU3NNMlgXFwc4LzbKz09PVf1Xs2F7eX1dkUKg+n/HOTI3n8Z7zUFAEfDB3HU7wX6fs9TOo6IZC83PxO5Djfe3t7ExMRQuXLm6+vHjh3DwyPnm0tLS2Pt2rUMHTrU1Wa1Wmnbti0rV67Mdp2ffvqJFi1a8OSTT/Ljjz9SunRpevXqxUsvvYTtMreejho1ihEjRmRpX7RoEX5+fjmuNzcWL16cL9sVMcvpVPhoo8E0z3H4W1I5EVCLFcYtoEvC+UbHEZHMkpKSctw31+Gmffv2DB06lB9//JHg4GDAOQ7mlVdeoV27djnezsmTJ7Hb7YSHh2dqDw8PZ/v27dmus3fvXn777Td69+7N/Pnz2b17N0888QTp6ekMHz4823WGDh3KkCFDXK/j4uKoUKEC7du3JygoKMf15kR6ejqLFy+mXbt22d5FJlIU2R0Gd41bwWOWr2hg3Yvh4UuJB6bQqZQGEOcHHUdEsnfhyktO5DrcfPDBB9x8881UqlSJRo0aAbBhwwbCw8OZNm1abjeXKw6Hg7CwMMaPH4/NZiM6OpojR47w/vvvXzbceHt74+3tnaXd09Mz3w4c+bltkYL2wfxtlDq5isc8fwbA0uFNPCNqmFyV+9NxRCSz3Pw85DrclCtXjk2bNvH111+zceNGfH196d+/Pz179szVjkNDQ7HZbMTExGRqj4mJISIiItt1ypQpg6enZ6ZLULVq1eL48eOkpaXh5eWV27cjIlew7sAZ5vyxkflen2G1GFCtPTR9xOyyRESu6Joev+Dv78+jjz56XTv28vIiOjqapUuX0qVLF8B5Zmbp0qUMGjQo23VuvPFGZsyYgcPhwGp13ui1c+dOypQpo2AjksdS0u08MX0tb3t8SYTlDIZ/GJZ7JppdlojIVV3zs6W2bt3KwYMHSUtLy9R+11135XgbQ4YMoW/fvjRp0oRmzZoxZswYEhMTXXdP9enTh3LlyjFq1CgAHn/8ccaOHcvTTz/NU089xa5du3j77bcZPHjwtb4NEbmMF+du4pak+dzuuRrDYsNy7yTwCTS7LBGRq7qmGYq7du3K5s2bsVgsrqd/WywWAOx2e4631aNHD06cOMGwYcM4fvw4DRs2ZOHCha5BxgcPHnSdoQGoUKECv/76K88++yz169enXLlyPP3007z00ku5fRsicgXzNx9jy6bV/OLlHEdnafGE82nfIiJFQK7DzdNPP01UVBRLly4lKiqKVatWcerUKZ577jk++OCDXBcwaNCgy16GWrZsWZa2Fi1a8Pfff+d6PyKSM6cSUnn127VM8xyLryUNwutB2zfMLktEJMdyHW5WrlzJb7/9RmhoKFarFavVSqtWrRg1ahSDBw9m/fr1+VGniBQAwzB4/Ot1PJ7xNXU8DmB4+mPpOROsuZ7MXETENLk+YtntdgIDndfdQ0NDOXr0KACVKlVix44deVudiBSoiX/uw/vAMgZ4OCfns9wxGkpUMLkqEZHcyfWZm7p167Jx40aioqJo3rw57733Hl5eXowfPz7LrMUiUnTsOZHA+IWr+dnzc2dD7buhYU9zixIRuQa5DjevvvoqiYmJALzxxhvceeed3HTTTZQqVYrZs2fneYEikv8y7A4GTlnNW9b/EmY5ixFYFkvXL8wuS0TkmuQ63HTo0MH196pVq7J9+3ZOnz5NSEiI644pESla3lmwnRZnfqCt53oMiweWHtPA09fsskRErkmuxtykp6fj4eHBv//+m6m9ZMmSCjYiRdSqfaf4c8VyXvX4GgDLzUOgfBOTqxIRuXa5OnPj6elJxYoVczWXjYgUXgmpGTz39d9M8BiLtyUdykVDm1fMLktE5Lrk+m6p//znP7zyyiucPn06P+oRkQL04tyNPJQyhZrWQzi8AuH+maCzsCJSxOV6zM3YsWPZvXs3ZcuWpVKlSvj7+2davm7dujwrTkTyz08bjpK8ZQH9vX4FwHr3WAgMN7kqEZHrl+twc+EhlyJSdMXGpTD6u+V863n+jqj690OdLqbWJCKSV3IdboYPH54fdYhIATEMgyemr+ENYxyh1jgcwRWxdv7Y7LJERPKM5lQXKWa+WL6X+kdm0dq2CYfVE2vPmeDpY3ZZIiJ5JtdnbqxW6xVv+9adVCKF147jccxb9CtzPWYCYL31VYioa3JVIiJ5K9fh5vvvv8/0Oj09nfXr1zNlyhRGjBiRZ4WJSN5KzbDz9LSVjLV9irclA6NiSyw3Pm12WSIieS7X4ebuu+/O0ta9e3fq1KnD7Nmzefjhh/OkMBHJW2/P28YD576gqsdR7N7B2HpM123fIuKW8mzMzQ033MDSpUvzanMikof+2n2SY//M5QEP58+ordt48C9lclUiIvkj12duspOcnMwnn3xCuXLl8mJzIpKHziWnM3LGEmZ6TnA2RPeDGrebWpOISH7Kdbi59AGZhmEQHx+Pn58f06dPz9PiROT6vfDNel5L/4QQWwL2kCrYOr5vdkkiIvkq1+Hmo48+yhRurFYrpUuXpnnz5oSEhORpcSJyfb5dd5ionRO50XMLdqsXtp4zwcPL7LJERPJVrsNNv3798qEMEclrh88kMfP7H5np8Q0AtvYjIayGyVWJiOS/XA8onjRpEnPmzMnSPmfOHKZMmZInRYnI9bE7DJ6b/hfvWT7B02LHUfkWaD7Q7LJERApErsPNqFGjCA0NzdIeFhbG22+/nSdFicj1+e+y3XSLGUtl63EyfEpi7T5Rt32LSLGR63Bz8OBBoqKisrRXqlSJgwcP5klRInLt/j1yju1Lp9HDYxkGFjzumwR+Jc0uS0SkwOQ63ISFhbFp06Ys7Rs3bqRUKc2bIWKm5DQ7w6cu5C2P87d9Nx8IlduYWpOISEHLdbjp2bMngwcP5vfff8dut2O32/ntt994+umnuf/++/OjRhHJoZE/b+LF5A8JtiSRUaomlvZvml2SiEiBy/XdUiNHjmT//v3cdttteHg4V3c4HPTp00djbkRM9Pv2GELWfUZzz+1kWH3w6D0TbJ5mlyUiUuByHW68vLyYPXs2b775Jhs2bMDX15d69epRqVKl/KhPRHLgVEIqE2fPYZLHXAA87ngPSlY2uSoREXNc8+MXqlWrRrVq1fKyFhG5BoZh8MqsFbxp/xgPqwN7tduxNe5jdlkiIqbJ9Zibe+65h3fffTdL+3vvvce9996bJ0WJSM7NXHWI9gc+oJI1ljTf0tjuGa/bvkWkWMt1uFm+fDmdOnXK0t6xY0eWL1+eJ0WJSM7sO5nI2l++4B7bnziw4NVjKvgEm12WiIipcn1ZKiEhAS+vrM+m8fT0JC4uLk+KEpGrS7c7eGPqfD62fgWApeXTENnS5KpERMyX6zM39erVY/bs2VnaZ82aRe3atfOkKBG5uk+XbGPQ2XcJsiSTVroeltteM7skEZFCIddnbl577TW6devGnj17uPXWWwFYunQpM2bMYO7cuXleoIhktfbAGax/fEC0xy7Sbb549foabNd8f4CIiFvJ9dGwc+fO/PDDD7z99tvMnTsXX19fGjRowG+//UbJkpriXSS/JaRmMGH614yzfQ+A511jIERTMYiIXHBN/9W74447uOOOOwCIi4tj5syZPP/886xduxa73Z6nBYpIZqO+W8mraR9hsxik1bgbrwaaGVxE5GK5HnNzwfLly+nbty9ly5Zl9OjR3Hrrrfz99995WZuIXGLh5qPcsPVNyltOkuJXBq9u48wuSUSk0MnVmZvjx48zefJkvvrqK+Li4rjvvvtITU3lhx9+0GBikXwWE5fCH99+ylu2v3FgxafXdPAONLssEZFCJ8dnbjp37kyNGjXYtGkTY8aM4ejRo3z66af5WZuInOdwGLzz9TyGGhOdr29+Eco3MbkqEZHCKcdnbhYsWMDgwYN5/PHH9dgFkQI25c9d9D32JgHWFJLDGuPb5kWzSxIRKbRyfObmzz//JD4+nujoaJo3b87YsWM5efJkftYmIsDOmHhSFr9JQ+teUm3++PaeBlab2WWJiBRaOQ43N9xwAxMmTODYsWMMHDiQWbNmUbZsWRwOB4sXLyY+Pj4/6xQpllIz7IyfOoWB1p8A8OoyFoLLm1yViEjhluu7pfz9/XnooYf4888/2bx5M8899xzvvPMOYWFh3HXXXflRo0ixNXbeap5LGI3VYpBc614s9bqZXZKISKF3zbeCA9SoUYP33nuPw4cPM3PmzLyqSUSAFbtPUHvNq5SxnCbRrzy+XcaYXZKISJFwXeHmApvNRpcuXfjpp5/yYnMixd65pHR+nzmajrbVZGDD/4Hp4B1gdlkiIkVCnoQbEck7hmEwZvY8ns04f9t361egbCOTqxIRKToUbkQKmZ/X7ueefcPxs6QSH94cr9ZDzC5JRKRIUbgRKUQOn0ni5M+vUde6n2RbIIG9J4NVP6YiIrmho6ZIIWF3GEycOpGHLD8D4HXP5xBU1uSqRESKHoUbkUJiypI1DDz9PgDxtXthq32nyRWJiBRNCjcihcC/h89S8c8XCbec5ZxfJQK7jDa7JBGRIkvhRsRkyWl2Fk8bRVvrOtLxIOjB6eDlZ3ZZIiJFlsKNiMm+/HYej6d8BUD6LcOwlKlvckUiIkWbwo2Iif635RBtt72CjyWd0+E34nfTU2aXJCJS5CnciJjkZEIqR+a+RC3rIRJtwZR8cLJu+xYRyQM6koqYwDAMpk/7kl7GPAA8u4+HgDCTqxIRcQ8KNyIm+OHP9fQ+/g4Ap2r3xavW7SZXJCLiPgpFuBk3bhyRkZH4+PjQvHlzVq1alaP1Zs2ahcVioUuXLvlboEge2ncinlKLn6G0JY7TfpUp1fU9s0sSEXErpoeb2bNnM2TIEIYPH866deto0KABHTp0IDY29orr7d+/n+eff56bbrqpgCoVuX7pdgdLJr/BzdaNpOFFiT7TwdPH7LJERNyK6eHmww8/ZMCAAfTv35/atWvz+eef4+fnx8SJEy+7jt1up3fv3owYMYLKlSsXYLUi12fmzwvok+D83k665XWsEXVMrkhExP14mLnztLQ01q5dy9ChQ11tVquVtm3bsnLlysuu98YbbxAWFsbDDz/MH3/8ccV9pKamkpqa6nodFxcHQHp6Ounp6df5DjK7sL283q64h/V7j3PDuhfwtmZwPOwmSrUYoO8VyULHEZHs5eZnwtRwc/LkSex2O+Hh4Znaw8PD2b59e7br/Pnnn3z11Vds2LAhR/sYNWoUI0aMyNK+aNEi/PzyZxbYxYsX58t2pehKsYNl41Tusx7hjCWY9WE9SFuwwOyypBDTcUQks6SkpBz3NTXc5FZ8fDwPPvggEyZMIDQ0NEfrDB06lCFDhrhex8XFUaFCBdq3b09QUFCe1peens7ixYtp164dnp6eebptKdqmTf2ChyxLALB1+5y2NduZXJEUVjqOiGTvwpWXnDA13ISGhmKz2YiJicnUHhMTQ0RERJb+e/bsYf/+/XTu3NnV5nA4APDw8GDHjh1UqVIl0zre3t54e3tn2Zanp2e+HTjyc9tS9Py2agNdDo4CCxyr/TBl6nUyuyQpAnQcEcksNz8Ppg4o9vLyIjo6mqVLl7raHA4HS5cupUWLFln616xZk82bN7NhwwbX11133cUtt9zChg0bqFChQkGWL3JVMeeS8J0/iJKWBGJ8q1Gm2yizSxIRcXumX5YaMmQIffv2pUmTJjRr1owxY8aQmJhI//79AejTpw/lypVj1KhR+Pj4ULdu3UzrlyhRAiBLu4jZHA6DpROH0YvNpOBFSN/p4JH1LKKIiOQt08NNjx49OHHiBMOGDeP48eM0bNiQhQsXugYZHzx4EKuetyNF0M8LF9D97ESwQFzrkYRF1DS7JBGRYsH0cAMwaNAgBg0alO2yZcuWXXHdyZMn531BItdp56EY6v0zBC+LnYNht1KxzUCzSxIRKTZ0SkQkj6Vm2NkzbRCVLcc4bStFhb5fgsVidlkiIsWGwo1IHvtl1ud0TFuEAwvWbhOw+JcyuyQRkWJF4UYkD63ZuInbdr0FwIGaAyhR5zaTKxIRKX4UbkTyyLmEFCw/PEYJSyKHfWsQ1f0ts0sSESmWFG5E8oBhGCyb/B+ijS0k40OpvtPAw8vsskREiiWFG5E88L/fF9DpxCQATtw0Et+IGiZXJCJSfCnciFynw8djqfy/Z/C02NkV2o6Ktw4wuyQRkWJN4UbkOtgdBrsnP05FSwwnrKWJ6jdet32LiJhM4UbkOiyZ8xltUpZgNyw4uozHI6Ck2SWJiBR7Cjci12j7tn9psfVNAHZUH0h4/VtNrkhEREDhRuSaJKekkj7nEYIsSez1rkWt+980uyQRETlP4UbkGqycPJR6jm0k4kupPlOx2DzNLklERM5TuBHJpXV/LKD1sYkAHGr5JsHlqptckYiIXEzhRiQXTpyMJXzpU9gsBptKdqBm+0fMLklERC7hYXYBIoVdeoadjf8sJWnV19Q9u5RylniOWcKp3v8Ls0sTEZFsKNyIZMMwDLZv3cjxP6dS5dg8mnDcucACJyylSO/2FT6BIeYWKSIi2VK4EbnIkSOH2PnbNML2fU8dx05qnW9PxpudJdsQ2Kw3UU07aQCxiEghpnAjxd7Zc+fY9NtsfLfNpWHqGspZ7ADYDQs7/ZtgNOhB9Zt70MA3yORKRUQkJxRupFhKTU9n4x+/kLZ+Fg3i/sfNlmTnAgvs9axGfPVuVL21L7VKlTO3UBERyTWFGyk2HA6DLRv+5tTKadQ8sZBmnHIusECMpTRHK95FhTZ9qRzVwNxCRUTkuijciNvbt283B5ZNptzBn6ln7He1x+PHntLtCGn5IJUa3Ea4VTMjiIi4A4UbcUsnT51iy9LpBO38jgbpG4myGACkGza2B7XEs3FPqt/YjYZeviZXKiIieU3hRtxGUnIyG//3PZZNs2mQuILWljTnAgvs9K5LSq3uVL/1QeoFhZpbqIiI5CuFGynS7HYHm1b9RsKqr6l9egktLHHOBRY4YitHbFQXItv0o3p5PSJBRKS4ULiRIscwDHbt2MzR5VOJOjqPRhx1LrDAaYLZX6Yj4a36Uq52C8pZLOYWKyIiBU7hRoqMY8eOsPO3aZTa+wN17du4cC4mGS92hrTGr0kvqt7QmZKaYE9EpFhTuJFCLS4hnn9/+wbPrXNokLyK1hdPsOfXGHu9+6jeuicN/INNrlRERAoLhRspdNLSM9j01wJS182k3rnfaWlJci6wwD6Pypyt1o0qt/SlVlhFcwsVEZFCSeFGCgXDMNi2aTUnV0ylWswCmnDSucACsZZQDle4k3I39yWqamNzCxURkUJP4UZMdejAXvYum0KZAz9R27HX1Z6AL7tC2xJ8w4NENW5LmNVmYpUiIlKUKNxIgTt95jRbf/sa/+3fUT9tPRXOT7CXZtjYEXgDtob3U/2m7jTy9jO5UhERKYoUbqRApKSmsul/P+LYOIv6CX/SypLqXGCBnV61SarZnWq3PEi9kDBzCxURkSJP4UbyjcPu4N+1y4n7Zzq1Ti6imeWcc4EFjljLcjzybiJv6Uf1CjXNLVRERNyKwo3kub27tnJ4+RQqHv6Z+sYRZ6MFzhDE3ojbKd3yQSrWu0kT7ImISL5QuJE8cSL2ONt/m0bI7u+om7GVyufbUwxPtpe4Gd/onlRrcTfRnl6m1ikiIu5P4UauWWJiIpt//wbbljnUT/qHmywZADgMC9v9GpFRuzvVb+lFw4AQkysVEZHiROFGciUjI4PNK38lee1M6pz5jRssic4FFtjvEcWpKl2pcks/akdUMrdQEREpthRu5KoMw2DXlnXE/DmVysfn04hY5wILnLCU4kC5Oyh7Ux8iazQl0tRKRUREFG7kCo4ePsCe36cQtu9Hajh2ux5UmYAvO0veSlDz3lRp0oHSNn0biYhI4aHfSpLJubNn2fL7DHy3f0u9lHWUtTgASDdsbA9ohqVBD2rcfB+NffxNrlRERCR7CjdCWloam/74CfuGmdSN+4OWF02wt8uzFvE1ulH91gepV7KMuYWKiIjkgMJNMWU4HGxb/yenV06nxslfacJZ5wILHLVGcLTiXVRs049qkXVMrVNERCS3FG6KmYN7tnPgf1Mof+hnahuHXO1nCWRPWHtKtuxDVIPWlNUEeyIiUkQp3BQDp07Gsn3pNIJ2f0+99M1UPN+eaniyLbgVXo17UuPGLkR7eptap4iISF5QuHFTycnJbF42B+vmb6iXuJIbL5pgb4dPA1Jq30PNWx6kYZAm2BMREfeicONG7HYHW/5ZRMLqGdQ+vZRmlgTnAgscsFXiRFQXKt/aj1plK195QyIiIkWYwo0b2Lt9PUeXTyXq6C/Uv3iCPUqyv2wnIm7qQ6WazaikcTQiIlIMKNwUUbHHDrHrtymE7v2BGvZdrgdVJuLD9pBbCGjai+rNO2mCPRERKXb0m68ISUiI49+lM/HeNpd6yWu48fwEexmGlW3+TXHUvY+at/Qg2jfQ5EpFRETMo3BTyKWnp/PvXz+Ttm4Wdc/9jxssKc4FFtjtWZ2zVbtR/da+1Ctd1txCRURECgmFm0LIcDjYuWklp1ZMo1rsQhpxxrnAAkct4Ryu0JkKrftStUp9cwsVEREphBRuCpGjB3ax7/fJlDn4MzUcB1zt5whgV2g7SrR4gCqNbqWs1WpilSIiIoWbwo3Jzp45ybal0wjc8R210zZT1mIAzgn2tga1xKPh/dS6qRtNvHxMrlRERKRoULgxQWpqMv/+7zuMjbOpl7CCFpZ05wILbPOuT1KNe6hx24M0Ci5lbqEiIiJFkMJNAXHYHWxf8xvnVk2n5qklRBPvXGCBg9YKHI+8m6hb+1GrfDVzCxURESniFG7y2YGdmziyfDKVjvxCbSPG1X6SEuyN6ERYqz5E1rmBippgT0REJE8UipGp48aNIzIyEh8fH5o3b86qVasu23fChAncdNNNhISEEBISQtu2ba/Y3wynY4+ycuYotr/ZjEozbqLl4a8oZ8SQZHiztkQHttw2mZKv7qbZY/8lsm4LULARERHJM6afuZk9ezZDhgzh888/p3nz5owZM4YOHTqwY8cOwsLCsvRftmwZPXv2pGXLlvj4+PDuu+/Svn17tmzZQrly5Ux4B05JifFsXjqTiI0zKLluE+EWOwB2w8JWvyZk1LmXWrf0JNo/yLQaRUREigOLYRiGmQU0b96cpk2bMnbsWAAcDgcVKlTgqaee4uWXX77q+na7nZCQEMaOHUufPn2u2j8uLo7g4GDOnTtHUFDeBY218yYQvfp51+vdHlU5XbUbVW95kJLhFfNsPyLi3tLT05k/fz6dOnXC09PT7HJECo3c/P429cxNWloaa9euZejQoa42q9VK27ZtWblyZY62kZSURHp6OiVLlsx2eWpqKqmpqa7XcXFxgPMAkp6efh3VZ1a91T3sW/sp27wbUr3TE1SqGU2l88vycj8i4t4uHC903BDJLDc/E6aGm5MnT2K32wkPD8/UHh4ezvbt23O0jZdeeomyZcvStm3bbJePGjWKESNGZGlftGgRfn5+uS/6ShqMBGDL3hi27J2ft9sWkWJl8eLFZpcgUqgkJSXluK/pY26uxzvvvMOsWbNYtmwZPj7ZT3I3dOhQhgwZ4nodFxdHhQoVaN++fZ5elgJnqly8eDHt2rXT6WQRuSY6johk78KVl5wwNdyEhoZis9mIiYnJ1B4TE0NERMQV1/3ggw945513WLJkCfXrX/4ZS97e3nh7e2dp9/T0zLcDR35uW0SKBx1HRDLLzc+DqbeCe3l5ER0dzdKlS11tDoeDpUuX0qJFi8uu99577zFy5EgWLlxIkyZNCqJUERERKSJMvyw1ZMgQ+vbtS5MmTWjWrBljxowhMTGR/v37A9CnTx/KlSvHqFGjAHj33XcZNmwYM2bMIDIykuPHjwMQEBBAQECAae9DRERECgfTw02PHj04ceIEw4YN4/jx4zRs2JCFCxe6BhkfPHgQ60VPwf7vf/9LWloa3bt3z7Sd4cOH8/rrrxdk6SIiIlIImR5uAAYNGsSgQYOyXbZs2bJMr/fv35//BYmIiEiRVSgevyAiIiKSVxRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErhSLcjBs3jsjISHx8fGjevDmrVq26Yv85c+ZQs2ZNfHx8qFevHvPnzy+gSkVERKSwMz3czJ49myFDhjB8+HDWrVtHgwYN6NChA7Gxsdn2X7FiBT179uThhx9m/fr1dOnShS5duvDvv/8WcOUiIiJSGJkebj788EMGDBhA//79qV27Np9//jl+fn5MnDgx2/4ff/wxt99+Oy+88AK1atVi5MiRNG7cmLFjxxZw5SIiIlIYeZi587S0NNauXcvQoUNdbVarlbZt27Jy5cps11m5ciVDhgzJ1NahQwd++OGHbPunpqaSmprqen3u3DkATp8+TXp6+nW+g8zS09NJSkri1KlTeHp65um2RaR40HFEJHvx8fEAGIZx1b6mhpuTJ09it9sJDw/P1B4eHs727duzXef48ePZ9j9+/Hi2/UeNGsWIESOytEdFRV1j1SIiImKW+Ph4goODr9jH1HBTEIYOHZrpTI/D4eD06dOUKlUKi8WSp/uKi4ujQoUKHDp0iKCgoDzdtogUDzqOiGTPMAzi4+MpW7bsVfuaGm5CQ0Ox2WzExMRkao+JiSEiIiLbdSIiInLV39vbG29v70xtJUqUuPaicyAoKEgHJRG5LjqOiGR1tTM2F5g6oNjLy4vo6GiWLl3qanM4HCxdupQWLVpku06LFi0y9QdYvHjxZfuLiIhI8WL6ZakhQ4bQt29fmjRpQrNmzRgzZgyJiYn0798fgD59+lCuXDlGjRoFwNNPP03r1q0ZPXo0d9xxB7NmzWLNmjWMHz/ezLchIiIihYTp4aZHjx6cOHGCYcOGcfz4cRo2bMjChQtdg4YPHjyI1fr/J5hatmzJjBkzePXVV3nllVeoVq0aP/zwA3Xr1jXrLbh4e3szfPjwLJfBRERySscRketnMXJyT5WIiIhIEWH6JH4iIiIieUnhRkRERNyKwo2IiIi4FYUbERERcSsKNyIiIuJWFG4KyC+//EKNGjWoVq0aX375pdnliEgR1bVrV0JCQujevbvZpYgUWroVvABkZGRQu3Ztfv/9d4KDg4mOjmbFihWUKlXK7NJEpIhZtmwZ8fHxTJkyhblz55pdjkihpDM3BWDVqlXUqVOHcuXKERAQQMeOHVm0aJHZZYlIEdSmTRsCAwPNLkOkUFO4yYHly5fTuXNnypYti8Vi4YcffsjSZ9y4cURGRuLj40Pz5s1ZtWqVa9nRo0cpV66c63W5cuU4cuRIQZQuIoXI9R5LRCRnFG5yIDExkQYNGjBu3Lhsl8+ePZshQ4YwfPhw1q1bR4MGDejQoQOxsbEFXKmIFGY6logUDIWbHOjYsSNvvvkmXbt2zXb5hx9+yIABA+jfvz+1a9fm888/x8/Pj4kTJwJQtmzZTGdqjhw5QtmyZQukdhEpPK73WCIiOaNwc53S0tJYu3Ytbdu2dbVZrVbatm3LypUrAWjWrBn//vsvR44cISEhgQULFtChQwezShaRQignxxIRyRnTnwpe1J08eRK73e56ivkF4eHhbN++HQAPDw9Gjx7NLbfcgsPh4MUXX9SdUiKSSU6OJQBt27Zl48aNJCYmUr58eebMmUOLFi0KulyRQk3hpoDcdddd3HXXXWaXISJF3JIlS8wuQaTQ02Wp6xQaGorNZiMmJiZTe0xMDBERESZVJSJFjY4lInlH4eY6eXl5ER0dzdKlS11tDoeDpUuX6lSxiOSYjiUieUeXpXIgISGB3bt3u17v27ePDRs2ULJkSSpWrMiQIUPo27cvTZo0oVmzZowZM4bExET69+9vYtUiUtjoWCJSQAy5qt9//90Asnz17dvX1efTTz81KlasaHh5eRnNmjUz/v77b/MKFpFCSccSkYKhZ0uJiIiIW9GYGxEREXErCjciIiLiVhRuRERExK0o3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjci4tYmT55MiRIlCmRf/fr1o0uXLgWyLxG5PIUbEZFc2r9/PxaLhQ0bNphdiohkQ+FGRERE3IrCjYhcszZt2vDUU0/xzDPPEBISQnh4OBMmTCAxMZH+/fsTGBhI1apVWbBgAQB2u52HH36YqKgofH19qVGjBh9//LFreykpKdSpU4dHH33U1bZnzx4CAwOZOHFijmqaPHkyFStWxM/Pj65du3Lq1KksfX788UcaN26Mj48PlStXZsSIEWRkZLiWWywW/vvf/9KxY0d8fX2pXLkyc+fOdS2PiooCoFGjRlgsFtq0aZNp+x988AFlypShVKlSPPnkk6Snp+eodhHJI2Y/llxEiq7WrVsbgYGBxsiRI42dO3caI0eONGw2m9GxY0dj/Pjxxs6dO43HH3/cKFWqlJGYmGikpaUZw4YNM1avXm3s3bvXmD59uuHn52fMnj3btc3169cbXl5exg8//GBkZGQYN9xwg9G1a9cc1fP3338bVqvVePfdd40dO3YYH3/8sVGiRAkjODjY1Wf58uVGUFCQMXnyZGPPnj3GokWLjMjISOP111939QGMUqVKGRMmTDB27NhhvPrqq4bNZjO2bt1qGIZhrFq1ygCMJUuWGMeOHTNOnTplGIZh9O3b1wgKCjIee+wxY9u2bcbPP/9s+Pn5GePHj8+DT1tEckrhRkSuWevWrY1WrVq5XmdkZBj+/v7Ggw8+6Go7duyYARgrV67MdhtPPvmkcc8992Rqe++994zQ0FBj0KBBRpkyZYyTJ0/mqJ6ePXsanTp1ytTWo0ePTOHmtttuM95+++1MfaZNm2aUKVPG9RowHnvssUx9mjdvbjz++OOGYRjGvn37DMBYv359pj59+/Y1KlWqZGRkZLja7r33XqNHjx45ql9E8oYuS4nIdalfv77r7zabjVKlSlGvXj1XW3h4OACxsbEAjBs3jujoaEqXLk1AQADjx4/n4MGDmbb53HPPUb16dcaOHcvEiRMpVapUjmrZtm0bzZs3z9TWokWLTK83btzIG2+8QUBAgOtrwIABHDt2jKSkpMuu16JFC7Zt23bVGurUqYPNZnO9LlOmjOu9i0jB8DC7ABEp2jw9PTO9tlgsmdosFgsADoeDWbNm8fzzzzN69GhatGhBYGAg77//Pv/880+mbcTGxrJz505sNhu7du3i9ttvz7N6ExISGDFiBN26dcuyzMfH57q3n93n4XA4rnu7IpJzCjciUmD++usvWrZsyRNPPOFq27NnT5Z+Dz30EPXq1ePhhx9mwIABtG3bllq1al11+7Vq1coSlP7+++9Mrxs3bsyOHTuoWrXqFbf1999/06dPn0yvGzVqBICXlxfgHCAtIoWPwo2IFJhq1aoxdepUfv31V6Kiopg2bRqrV6923X0EzstWK1euZNOmTVSoUIF58+bRu3dv/v77b1eouJzBgwdz44038sEHH3D33Xfz66+/snDhwkx9hg0bxp133knFihXp3r07VquVjRs38u+///Lmm2+6+s2ZM4cmTZrQqlUrvv76a1atWsVXX30FQFhYGL6+vixcuJDy5cvj4+NDcHBwHn5SInI9NOZGRArMwIED6datGz169KB58+acOnUq01mc7du388ILL/DZZ59RoUIFAD777DNOnjzJa6+9dtXt33DDDUyYMIGPP/6YBg0asGjRIl599dVMfTp06MAvv/zCokWLaNq0KTfccAMfffQRlSpVytRvxIgRzJo1i/r16zN16lRmzpxJ7dq1AfDw8OCTTz7hiy++oGzZstx9993X+9GISB6yGIZhmF2EiEhhYrFY+P777/UoBZEiSmduRERExK0o3IhIkdGxY8dMt3Bf/PX222+bXZ6IFBK6LCUiRcaRI0dITk7OdlnJkiUpWbJkAVckIoWRwo2IiIi4FV2WEhEREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRERFxKwo3IiIi4lYUbkRERMSt/B9N75ULNcTVtAAAAABJRU5ErkJggg==\n"},"metadata":{}}],"execution_count":21},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nparam_distributions = [\n {\n \"decisiontreeclassifier__max_depth\": stats.randint(1, 42),\n \"decisiontreeclassifier__max_leaf_nodes\": stats.randint(10, 3055),\n \"decisiontreeclassifier__min_samples_split\": stats.beta(a=2, b=98),\n \"decisiontreeclassifier__min_samples_leaf\": stats.beta(a=1, b=99),\n }\n]\n\ntree_growth_tuning = model_selection.RandomizedSearchCV(\n criterion_tuning.best_estimator_,\n param_distributions,\n cv=4,\n n_iter=128,\n n_jobs=-1,\n refit=True,\n scoring=\"accuracy\",\n verbose=0,\n random_state=prng,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T12:09:24.02979Z","iopub.execute_input":"2025-10-05T12:09:24.030147Z","iopub.status.idle":"2025-10-05T12:09:24.03925Z","shell.execute_reply.started":"2025-10-05T12:09:24.030123Z","shell.execute_reply":"2025-10-05T12:09:24.038304Z"}},"outputs":[],"execution_count":39},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = tree_growth_tuning.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T12:09:28.147531Z","iopub.execute_input":"2025-10-05T12:09:28.147816Z","execution_failed":"2025-10-05T12:58:40.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tree_growth_tuning.best_params_","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-05T12:58:40.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tree_growth_tuning.best_score_","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-05T12:58:40.962Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Class weights\n\n* `class_weight=None`: Parameter controls how much importance each class has in the loss function. By default, all classes are treated equally (no class weighting).\n* `class_weight=\"balanced\"`: gives the classes weights that are inversely proportionaly to their frequency. This makes rare classes get larger weights, balancing their influence.\n* You can also manually give classes weights by passing a dictionary mapping the class labels to their weights.\n\n### When to use?\n\n* With imbalanced classes (e.g. fraud detection, medical diagnosis) \"balanced\" is a good first try.\n* Custom weights if domain knowledge tells you one class is much more important.\n\n### When not to use?\n\n* With symmetric tasks (roughly equal class sizes), keep `class_weight=None`.\n* `class_weight=\"balanced\"` can sometimes hurt if classes are already balanced.","metadata":{}},{"cell_type":"code","source":"sgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=\"balanced\",\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Parallelism and reproducibility\n\n* `n_jobs=None`: By default, training runs on a single core (or default system settings). Set to an integer to control the number of cores directly, or set to `-1` to use all available cores.\n* `random_state=None`: Randomness (e.g., shuffling, weight initialization) is not fixed, so results may vary run to run unless you set a seed.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(train_features_df, train_labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Choosing the loss function","metadata":{}},{"cell_type":"markdown","source":"### `loss=\"hinge\"`\n\n* Trains a [linear Support Vector Machine (SVM)](https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html) using SGD.\n* Use when you care about margins and separating hyperplanes.\n* Good for high-dimensional sparse data (e.g. text classification).\n* Doesn’t provide probabilities.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n \n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### `loss=\"log_loss\"`\n\n* Trains logistic/softmax regression.\n* Use when you want probabilities, interpretability, or threshold tuning.\n* Often better with imbalanced data, since probabilities can be calibrated.\n","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"log_loss\",\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### loss=\"modified_huber\"\n\n* A smooth variant of hinge loss.\n* Like `hinge` this loss is robust to outliers.\n* Gives **probability-like** scores (but not calibrated probabilities).\n* Nice compromise between `hinge` and `log_loss`.","metadata":{}},{"cell_type":"markdown","source":"#### Exercise:\n\nModify the code above to train and evaluate a model with the modified Huber loss function.","metadata":{}},{"cell_type":"code","source":"# INSERT YOUR CODE HERE!","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Controlling training iterations\n\n* `max_iter=1000`: The maximum number of passes over the training data (epochs).\n* `tol=0.001`: Training stops early if the objective does not improve by at least `tol` for `n_iter_no_change` consecutive epochs.\n* `shuffle=True`: Data is shuffled after each epoch to avoid cycles and improve convergence.\n* `verbose=0`: No progress messages are printed during training. Values larger than 0 will print more detailed information about training progress.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Regularization\n\nRegularization helps improve models that are overfitting the training data. There are two general strategies for regularizing models.\n\n1. **Early-stopping**: reduces over-fitting to model iterations.\n2. **Penalizing model parameters**: reduces over-fitting to model features. ","metadata":{}},{"cell_type":"markdown","source":"### Avoid overtraining with `early_stopping=True`\n\n* `early_stopping=False`: By default, SGDClassifier does not reserve validation data or stop early based on validation score. Training runs until convergence or max epochs.\n* `validation_fraction=0.1`: Fraction of data reserved for validation when `early_stopping=True`. By default `early_stopping=False` and this is ignored.\n* `n_iter_no_change=5`: Training will stop if the validation score (or training score, if no validation split) does not improve for `n_iter_no_change` iterations in a row.\n\nWhen `early_stopping=True`, the training data is split into a training and validation set. After each epoch, performance is checked on the validation set. If the score doesn’t improve for 1n_iter_no_change1 epochs by at least `tol`, training stops early. This helps avoid training too long (a form of overfitting), especially when `max_iter` is large.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n early_stopping=True,\n validation_fraction=0.10,\n penalty=None,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Penalizing model parameters\n\n* `penalty=\"l2\"`: L2 regularization is applied to prevent overfitting. This penalizes large weights by their squared magnitude.\n* `alpha=0.0001`: Regularization strength (λ in optimization). Higher values increase regularization (smaller coefficients), lower values allow more complex models.\n* `l1_ratio=0.15`: Controls the L1/L2 mix when using `penalty=\"elasticnet\"`. Since the default is `l2`, this value has no effect unless you change penalty.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n penalty=\"l2\",\n alpha=0.0001,\n l1_ratio=0.15,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Exercise:\n\nModify the code above to train and evaluate a model using an ElasticNet penalty. Can you find a value for the `l1_ratio` hyperparameter to mimic an `l2` penalty? Can you find a value for the `l1_ratio` that mimics an `l1` penalty? ","metadata":{}},{"cell_type":"code","source":"# INSERT YOUR CODE HERE!","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Combine both types of regularization\n\nFor the most part you will want to always use early stopping in combination with explicit regularization via penalties.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n early_stopping=True,\n validation_fraction=0.10,\n penalty=\"l2\",\n alpha=0.0001,\n l1_ratio=0.15,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Controlling the learning rate","metadata":{}},{"cell_type":"markdown","source":"### `learning_rate=\"optimal\"`\n\nUses a theoretically motivated adaptive learning rate that sets the learning rate as follows\n\n`eta = 1.0 / (alpha * t)` \n\nwhere `t` is the iteration count and `alpha` is the amount of parameter regularization.","metadata":{}},{"cell_type":"code","source":"alpha = 1e-3\nts = np.linspace(1, 1000, 1000)\n_ = plt.plot(ts, 1.0 / (alpha * ts))\n_ = plt.title(\"learning_rate='optimal'\")\n_ = plt.ylabel(\"$\\eta_t$\")\n_ = plt.yscale(\"log\")\n_ = plt.xlabel(\"Iteration, $t$\")\n_ = plt.grid(True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n early_stopping=True,\n validation_fraction=0.10,\n penalty=\"l2\",\n alpha=0.0001,\n l1_ratio=0.15,\n learning_rate=\"optimal\",\n eta0=1e-3,\n power_t=0.5,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### `learning_rate=\"constant\"`\n\n* `eta0=0.0`: Controls the initial learning rate which is them held constant for the remainder of training.","metadata":{}},{"cell_type":"code","source":"eta0 = 1e3\nts = np.linspace(1, 1000, 1000)\n_ = plt.plot(ts, eta0 * np.ones(1000))\n_ = plt.title(\"learning_rate='constant'\")\n_ = plt.ylabel(\"$\\eta_t$\")\n_ = plt.xlabel(\"Iteration, $t$\")\n_ = plt.grid(True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Exercise\n\nModify the code above to use a constant learning rate strategy with a learning rate of 1e-6. Train and evaluate the model. Repeat the experiment with a learning rate of 1e3. What impact does a constant and very small or a constant and very large learning rate have on model performance?","metadata":{}},{"cell_type":"code","source":"# INSERT YOUR CODE HERE!","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### `learning_rate=\"invscaling\"`\n\nUses an adaptive learning rate that sets the learning rate as follows\n\n`eta = eta0 / t**power_t`\n\nWhere `eta0` is the initial learning rate and `power_t` controls the decay of the learning rate. Values of `power_t` closer to 1 will lead to faster decay of the learning rate; values of `power_t` closer to 0 will lead to slower decay of the learning rate.","metadata":{}},{"cell_type":"code","source":"eta0 = 1e3\npower_t = 0.5\nts = np.linspace(1, 1000, 1000)\n_ = plt.plot(ts, eta0 / ts**power_t)\n_ = plt.title(\"learning_rate='invscaling'\")\n_ = plt.ylabel(\"$\\eta_t$\")\n_ = plt.yscale(\"log\")\n_ = plt.xlabel(\"Iteration, $t$\")\n_ = plt.grid(True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n early_stopping=True,\n validation_fraction=0.10,\n penalty=\"elasticnet\",\n alpha=0.0001,\n l1_ratio=0.15,\n learning_rate=\"invscaling\",\n eta0=1e3,\n power_t=0.5,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### `learning_rate=\"adaptive\"`\n\nStart with the initial learning rate `eta0` (default = 0.0, but here you must set it, e.g. `eta0=0.01` — otherwise training won’t move).\n\n* After each epoch, check if the objective (loss) improves.\n* If improvement is observed, then keep the same learning rate.\n* If no improvement for `n_iter_no_change` consecutive epochs, then divide the learning rate by 5.\n\nSetting `learning_rate=\"adaptive\"` adjusts the learning rate in a fashion that is similar to learning rate scheduling in deep learning frameworks:\n\n* Stay aggressive while making progress.\n* Automatically slow down if the model stops improving.\n\nHelps balance speed (large steps early) and stability (smaller steps when near a minimum).","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n early_stopping=True,\n validation_fraction=0.10,\n penalty=\"elasticnet\",\n alpha=0.0001,\n l1_ratio=0.15,\n learning_rate=\"adaptive\",\n eta0=1e-2,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Averaging parameters\n\nThe `average` hyperparameter in `SGDClassifier` controls whether the model uses **Polyak–Ruppert averaging** of the parameters during training. This is a technique from stochastic optimization that can improve stability and generalization.\n\n### What normally happens (`average=False`)\n\n* In vanilla SGD, parameters are updated after each sample (or mini-batch).\n* At the end of training, the model keeps the **last parameters**.\n\n### What happens with `average=True`\n\n* Instead of just keeping the final parameters, the model computes the **running average of all past weights**:\n\n$$ \\bar{w}_T = \\frac{1}{T}\\sum_{t=1}^T w_t $$\n\n* This smoothed weight vector is used for predictions.\n* The idea: averaging cancels out the “noise” of individual updates, leading to more stable convergence.\n\n### If you set `average=int`\n\n* The model will **start averaging only after that many updates**.\n* Example: `average=1000` → first 1000 updates are standard SGD, after that averaging kicks in.\n* This is useful as a **warm-up** so the model can move quickly at the start, then stabilize.\n\n### Why averaging helps\n\nSGD updates are noisy because each step is based on a single (or small batch) sample. Averaging parameters:\n\n* Reduces variance in the parameters.\n* Improves generalization (especially in noisy, high-dimensional, or sparse problems).\n* It’s particularly effective for **linear models** trained with SGD.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n max_iter=1000,\n n_iter_no_change=5,\n shuffle=True,\n tol=0.001,\n verbose=0,\n early_stopping=True,\n validation_fraction=0.10,\n penalty=\"elasticnet\",\n alpha=0.0001,\n l1_ratio=0.15,\n learning_rate=\"adaptive\",\n eta0=1e-2,\n average=42_000 * 3, # only start averaging after this many updates (~3 epochs!)\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier_pipeline.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_train_labels = sgd_classifier_pipeline.predict(\n train_features_df\n)\n\ntrain_accuracy = metrics.accuracy_score(\n train_labels,\n predicted_train_labels,\n)\n\nprint(f\"Accuracy on the training set: {train_accuracy}\")\n\ncv_accuracy_scores = model_selection.cross_val_score(\n sgd_classifier_pipeline,\n train_features_df,\n train_labels,\n cv=4,\n n_jobs=-1,\n scoring=\"accuracy\",\n verbose=0,\n)\n\nprint(f\"CV Accuracy: {cv_accuracy_scores.mean()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Hyperparameter tuning strategy","metadata":{}},{"cell_type":"markdown","source":"### Step 1: Choose your loss function\n\nChoose the loss function most appropriate for your problem and set up early stopping. In addition to helping regularize our models, early stopping will significantly speed up hyperparameter tuning runs.","metadata":{}},{"cell_type":"code","source":"prng = np.random.RandomState(42)\n\nsgd_classifier_pipeline = pipeline.make_pipeline(\n features_preparation_pipeline,\n linear_model.SGDClassifier(\n fit_intercept=True,\n class_weight=None,\n n_jobs=-1,\n random_state=prng,\n loss=\"hinge\",\n early_stopping=True,\n validation_fraction=0.10,\n n_iter_no_change=5,\n tol=0.001,\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_grid = [\n {\n \"sgdclassifier__loss\": [\n \"hinge\",\n \"log_loss\",\n \"modified_huber\",\n \"squared_hinge\",\n \"perceptron\",\n ],\n }\n]\n\nloss_function_tuning = model_selection.GridSearchCV(\n sgd_classifier_pipeline,\n param_grid,\n cv=4,\n n_jobs=-1,\n refit=True,\n scoring=\"accuracy\",\n verbose=0,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"loss_function_tuning","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_ = loss_function_tuning.fit(\n train_features_df,\n train_labels,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"loss_function_tuning.best_params_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"loss_function_tuning.best_score_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n pd.DataFrame\n .from_dict(\n loss_function_tuning.cv_results_\n ).sort_values(\n by=\"rank_test_score\"\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Step 2: Tune regularization hyperparameters","metadata":{}},{"cell_type":"code","source":"param_grid = [\n {\n \"sgdclassifier__penalty\": [\"elasticnet\"],\n \"sgdclassifier__alpha\": np.logspace(-6, -1, 6),\n \"sgdclassifier__l1_ratio\": [0, 0.15, 0.3, 0.45, 0.6, 0.85, 1.0],\n \"sgdclassifier__n_iter_no_change\": [5, 10, 15]\n }\n]\n\nregularization_tuning = model_selection.RandomizedSearchCV(\n loss_function_tuning.best_estimator_,\n param_grid,\n cv=4,\n n_iter=25,\n n_jobs=-1,\n refit=True,\n scoring=\"accuracy\",\n verbose=0,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"regularization_tuning","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_ = regularization_tuning.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# accessing the best hyperparameters\nregularization_tuning.best_params_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# accessing the score for the best hyperparameters\nregularization_tuning.best_score_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n pd.DataFrame\n .from_dict(\n regularization_tuning.cv_results_\n ).sort_values(\n by=\"rank_test_score\"\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Step 3: Tune the Learning Rate","metadata":{}},{"cell_type":"code","source":"param_distributions = [\n {\n \"sgdclassifier__learning_rate\": [\"optimal\"],\n },\n {\n \"sgdclassifier__learning_rate\": [\"adaptive\"],\n \"sgdclassifier__eta0\": [1e-3, 1e-2, 1e0, 1e2, 1e3],\n \"sgdclassifier__n_iter_no_change\": [5, 10, 15, 20]\n },\n {\n \"sgdclassifier__learning_rate\": [\"invscaling\"],\n \"sgdclassifier__eta0\": [1e-3, 1e-2, 1e0, 1e2, 1e3],\n \"sgdclassifier__power_t\": [0.25, 0.5, 0.75],\n }\n]\n\nlearning_rate_tuning = model_selection.RandomizedSearchCV(\n regularization_tuning.best_estimator_,\n param_distributions,\n cv=4,\n n_jobs=-1,\n n_iter=25,\n refit=True,\n scoring=\"accuracy\",\n verbose=0,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_ = learning_rate_tuning.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_rate_tuning.best_params_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_rate_tuning.best_score_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n pd.DataFrame\n .from_dict(\n learning_rate_tuning.cv_results_\n ).sort_values(\n by=\"rank_test_score\"\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Step 4: Try averaging parameters","metadata":{}},{"cell_type":"code","source":"param_grid = [\n {\n \"sgdclassifier__average\": [\n False,\n True,\n 42_000 * 5,\n 42_000 * 10,\n 42_000 * 15\n ],\n },\n]\n\naverage_tuning = model_selection.GridSearchCV(\n learning_rate_tuning.best_estimator_,\n param_grid,\n cv=4,\n n_jobs=-1,\n refit=True,\n scoring=\"accuracy\",\n verbose=0,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_ = average_tuning.fit(\n train_features_df,\n train_labels\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"average_tuning.best_params_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"average_tuning.best_score_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n pd.DataFrame\n .from_dict(\n average_tuning.cv_results_\n ).sort_values(\n by=\"rank_test_score\"\n )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submitting to the competition","metadata":{}},{"cell_type":"markdown","source":"### Generate test predictions","metadata":{}},{"cell_type":"code","source":"test_features_df = pd.read_csv(DATA_DIR / \"test.csv\")\ntest_predictions = average_tuning.predict(test_features_df)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create submission file","metadata":{}},{"cell_type":"code","source":"_ = (\n pd.read_csv(\n \"/kaggle/input/digit-recognizer/sample_submission.csv\",\n index_col=\"ImageId\"\n ).assign(\n Label=test_predictions\n ).to_csv(\"submission.csv\", index=True)\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat submission.csv | head -n 5","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}