{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Importing Packages and Collecting Data, Defining Evaluation ","metadata":{}},{"cell_type":"markdown","source":"## 1.1 Importing Packages","metadata":{}},{"cell_type":"code","source":"'''Importing Data Manipulattion Modules'''\nimport re\nimport numpy as np\nimport pandas as pd\nfrom pandas.core.dtypes.dtypes import CategoricalDtype\nfrom scipy.stats import norm, skew\nfrom scipy.special import boxcox1p\npd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))\npd.set_option(\"display.max_columns\", 81)\npd.set_option(\"display.max_rows\", 101)\npd.set_option(\"display.max_colwidth\", 100)\n\n'''Seaborn and Matplotlib Visualization'''\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('bmh')                    \nsns.set_style({'axes.grid':False}) \nsns.set_style('whitegrid')\n%matplotlib inline\n\n'''Validation'''\nfrom sklearn.model_selection import KFold, cross_val_score\n\n'''Ignore deprecation and future, and user warnings.'''\nimport warnings as wrn\nwrn.filterwarnings('ignore', category = DeprecationWarning) \nwrn.filterwarnings('ignore', category = FutureWarning) \nwrn.filterwarnings('ignore', category = UserWarning) ","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:17.111711Z","iopub.execute_input":"2023-04-11T01:50:17.111974Z","iopub.status.idle":"2023-04-11T01:50:18.427342Z","shell.execute_reply.started":"2023-04-11T01:50:17.111929Z","shell.execute_reply":"2023-04-11T01:50:18.426397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 Collecting Data","metadata":{}},{"cell_type":"code","source":"'''Check the files'''\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input/grupo-bimbo-inventory-demand/\"]).decode(\"utf8\"))","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:18.429776Z","iopub.execute_input":"2023-04-11T01:50:18.430081Z","iopub.status.idle":"2023-04-11T01:50:18.448433Z","shell.execute_reply.started":"2023-04-11T01:50:18.430025Z","shell.execute_reply":"2023-04-11T01:50:18.446950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### town_state","metadata":{}},{"cell_type":"code","source":"dtype = {'Agencia_ID': 'int16'}\ntown_state_df = pd.read_csv(\"../input/grupo-bimbo-inventory-demand/town_state.csv.zip\", dtype=dtype)\ntown_state_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:18.449849Z","iopub.execute_input":"2023-04-11T01:50:18.450141Z","iopub.status.idle":"2023-04-11T01:50:18.490872Z","shell.execute_reply.started":"2023-04-11T01:50:18.450041Z","shell.execute_reply":"2023-04-11T01:50:18.489989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df['Town_ID'] = town_state_df.Town.apply(lambda x: x.split(' ')[0]).astype('int16')\ntown_state_df['Town_name'] = town_state_df.Town.apply(lambda x: ' '.join(x.split(' ')[1:]))\n\nprint(f'town_state\\'s shape: {town_state_df.shape}')\ntown_state_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:18.492354Z","iopub.execute_input":"2023-04-11T01:50:18.492665Z","iopub.status.idle":"2023-04-11T01:50:18.510961Z","shell.execute_reply.started":"2023-04-11T01:50:18.492606Z","shell.execute_reply":"2023-04-11T01:50:18.509959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:18.513330Z","iopub.execute_input":"2023-04-11T01:50:18.513636Z","iopub.status.idle":"2023-04-11T01:50:18.527342Z","shell.execute_reply.started":"2023-04-11T01:50:18.513578Z","shell.execute_reply":"2023-04-11T01:50:18.526356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product","metadata":{}},{"cell_type":"code","source":"dtype = {'Producto_ID': 'int32'}\nproduct_df = pd.read_csv(\"../input/grupo-bimbo-inventory-demand/producto_tabla.csv.zip\", dtype=dtype)\nproduct_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:18.529051Z","iopub.execute_input":"2023-04-11T01:50:18.529396Z","iopub.status.idle":"2023-04-11T01:50:18.553768Z","shell.execute_reply.started":"2023-04-11T01:50:18.529343Z","shell.execute_reply":"2023-04-11T01:50:18.553097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df['popular_name'] = product_df.NombreProducto.str.extract(r'^(.*?)(\\d*\\s\\d+(kg|Kg|g|G|ml| ml|p|Reb)\\s)', expand=False)[0]\nproduct_df['property'] = product_df.NombreProducto.str.extract(r'^.*\\d+(kg|Kg|g|G|ml| ml|p|Reb)\\s(.*?)\\s\\d+$', expand=False)[1]\nproduct_df['unit'] = product_df.NombreProducto.str.extract(r'(\\d*\\s\\d+(kg|Kg|g|G|ml| ml))', expand=False)[0]\nproduct_df['pieces'] =  product_df.NombreProducto.str.extract('(\\d+(p|Reb)) ', expand=False)[0]\n\nprint(f'product\\'s shape: {product_df.shape}')\nproduct_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:25.364161Z","iopub.execute_input":"2023-04-11T01:50:25.364565Z","iopub.status.idle":"2023-04-11T01:50:25.412753Z","shell.execute_reply.started":"2023-04-11T01:50:25.364517Z","shell.execute_reply":"2023-04-11T01:50:25.412130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:41.438444Z","iopub.execute_input":"2023-04-11T01:50:41.438842Z","iopub.status.idle":"2023-04-11T01:50:41.450871Z","shell.execute_reply.started":"2023-04-11T01:50:41.438800Z","shell.execute_reply":"2023-04-11T01:50:41.449801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### client","metadata":{}},{"cell_type":"code","source":"dtype = {'Cliente_ID': 'int32'}\nclient_df = pd.read_csv(\"../input/grupo-bimbo-inventory-demand/cliente_tabla.csv.zip\", dtype=dtype)\n\ndup_sr = client_df.groupby('Cliente_ID')['Cliente_ID'].count().astype('int8')\ndup_sr.name = 'dup_num'\nclient_df = pd.merge(client_df, pd.DataFrame(dup_sr).reset_index())\ndel dup_sr\n\nprint(f'client\\'s shape: {client_df.shape}')\nclient_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:48.321874Z","iopub.execute_input":"2023-04-11T01:50:48.322290Z","iopub.status.idle":"2023-04-11T01:50:49.387856Z","shell.execute_reply.started":"2023-04-11T01:50:48.322240Z","shell.execute_reply":"2023-04-11T01:50:49.386800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:06:54.263882Z","iopub.execute_input":"2023-03-27T15:06:54.264297Z","iopub.status.idle":"2023-03-27T15:06:54.334274Z","shell.execute_reply.started":"2023-03-27T15:06:54.264221Z","shell.execute_reply":"2023-03-27T15:06:54.333381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### test","metadata":{}},{"cell_type":"code","source":"dtype = {\n    'id': 'int32',\n    'Semana': 'int8',\n    'Agencia_ID': 'int16',\n    'Canal_ID': 'int8',\n    'Ruta_SAK': 'int16',\n    'Cliente_ID': 'int32',\n    'Producto_ID': 'int32',\n}\ntest_df = pd.read_csv(\"../input/grupo-bimbo-inventory-demand/test.csv.zip\", dtype=dtype)\n\nprint(f'test\\'s shape: {test_df.shape}')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:50:52.084191Z","iopub.execute_input":"2023-04-11T01:50:52.084511Z","iopub.status.idle":"2023-04-11T01:50:57.352618Z","shell.execute_reply.started":"2023-04-11T01:50:52.084422Z","shell.execute_reply":"2023-04-11T01:50:57.351951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:51:03.431448Z","iopub.execute_input":"2023-04-11T01:51:03.431905Z","iopub.status.idle":"2023-04-11T01:51:03.438835Z","shell.execute_reply.started":"2023-04-11T01:51:03.431858Z","shell.execute_reply":"2023-04-11T01:51:03.438171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train","metadata":{}},{"cell_type":"code","source":"dtype = {\n    'Semana': 'int8',\n    'Agencia_ID': 'int16',\n    'Canal_ID': 'int8',\n    'Ruta_SAK': 'int16',\n    'Cliente_ID': 'int32',\n    'Producto_ID': 'int32',\n    'Venta_uni_hoy': 'int16',\n    'Venta_hoy': 'float32',\n    'Dev_uni_proxima': 'int32',\n    'Dev_proxima': 'float32',\n    'Demanda_uni_equil': 'int16',\n}\ntrain_df = pd.read_csv(\"../input/grupo-bimbo-inventory-demand/train.csv.zip\", dtype=dtype)\n\nprint(f'train\\'s shape: {train_df.shape}')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:51:06.166691Z","iopub.execute_input":"2023-04-11T01:51:06.166986Z","iopub.status.idle":"2023-04-11T01:51:56.484974Z","shell.execute_reply.started":"2023-04-11T01:51:06.166936Z","shell.execute_reply":"2023-04-11T01:51:56.484072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('max: %s' % train_df['Demanda_uni_equil'].max())\nprint('min: %s' % train_df['Demanda_uni_equil'].min())\nprint('avg: %s' % train_df['Demanda_uni_equil'].mean())","metadata":{"execution":{"iopub.status.busy":"2023-04-11T02:00:20.115470Z","iopub.execute_input":"2023-04-11T02:00:20.115769Z","iopub.status.idle":"2023-04-11T02:00:20.383473Z","shell.execute_reply.started":"2023-04-11T02:00:20.115715Z","shell.execute_reply":"2023-04-11T02:00:20.382356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:51:56.486542Z","iopub.execute_input":"2023-04-11T01:51:56.486928Z","iopub.status.idle":"2023-04-11T01:51:56.494138Z","shell.execute_reply.started":"2023-04-11T01:51:56.486888Z","shell.execute_reply":"2023-04-11T01:51:56.493138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product_price","metadata":{}},{"cell_type":"code","source":"sale_price_sr = (train_df.Venta_hoy / train_df.Venta_uni_hoy)\nreturn_price_sr = (train_df.Dev_proxima / train_df.Dev_uni_proxima)\nproduct_price_df = pd.DataFrame({'Producto_ID': train_df.Producto_ID, 'sale_price': sale_price_sr, 'return_price': return_price_sr})\n\ndel sale_price_sr\ndel return_price_sr\n\nprint(f'product price\\'s shape: {product_price_df.shape}')\nproduct_price_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:51:56.495574Z","iopub.execute_input":"2023-04-11T01:51:56.495974Z","iopub.status.idle":"2023-04-11T01:52:00.359267Z","shell.execute_reply.started":"2023-04-11T01:51:56.495912Z","shell.execute_reply":"2023-04-11T01:52:00.358490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_price_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.614724Z","iopub.execute_input":"2023-03-27T15:08:15.615059Z","iopub.status.idle":"2023-03-27T15:08:15.622225Z","shell.execute_reply.started":"2023-03-27T15:08:15.614965Z","shell.execute_reply":"2023-03-27T15:08:15.621521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3 Defining Evaluation","metadata":{}},{"cell_type":"code","source":"'''KFold for cross validation'''\nkf = KFold(n_splits=3, shuffle=True, random_state=2)\n\n'''Define the validation function'''\ndef rmsle_cv(model, X, y, cv=kf):\n    rmsle = np.sqrt(\n        -cross_val_score(\n            model,\n            X, y,\n            scoring=\"neg_mean_squared_log_error\",\n            cv=cv,\n        )\n    )\n    return(rmsle)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.623568Z","iopub.execute_input":"2023-03-27T15:08:15.624075Z","iopub.status.idle":"2023-03-27T15:08:15.644953Z","shell.execute_reply.started":"2023-03-27T15:08:15.624027Z","shell.execute_reply":"2023-03-27T15:08:15.643769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Adjusting Data","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Cleansing","metadata":{}},{"cell_type":"markdown","source":"### town_state","metadata":{}},{"cell_type":"code","source":"town_state_df['Town'] = town_state_df['Town'].str.upper()\ntown_state_df['Town_name'] = town_state_df['Town_name'].str.upper()\ntown_state_df['State'] = town_state_df['State'].str.upper()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.646544Z","iopub.execute_input":"2023-03-27T15:08:15.646952Z","iopub.status.idle":"2023-03-27T15:08:15.666826Z","shell.execute_reply.started":"2023-03-27T15:08:15.646875Z","shell.execute_reply":"2023-03-27T15:08:15.665732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df.groupby(['Town_name'])['Town_ID'].nunique().sort_values(ascending=False)[:6]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.668548Z","iopub.execute_input":"2023-03-27T15:08:15.668937Z","iopub.status.idle":"2023-03-27T15:08:15.687570Z","shell.execute_reply.started":"2023-03-27T15:08:15.668861Z","shell.execute_reply":"2023-03-27T15:08:15.686655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df[(town_state_df['Town_name'].isin(['LOS MOCHIS', 'PINOTEPA']))].sort_values(by='Town_name')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.689257Z","iopub.execute_input":"2023-03-27T15:08:15.690130Z","iopub.status.idle":"2023-03-27T15:08:15.708624Z","shell.execute_reply.started":"2023-03-27T15:08:15.689860Z","shell.execute_reply":"2023-03-27T15:08:15.707489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df.loc[498, 'Town_ID'] = 2561","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.710262Z","iopub.execute_input":"2023-03-27T15:08:15.710585Z","iopub.status.idle":"2023-03-27T15:08:15.721564Z","shell.execute_reply.started":"2023-03-27T15:08:15.710530Z","shell.execute_reply":"2023-03-27T15:08:15.720378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.722862Z","iopub.execute_input":"2023-03-27T15:08:15.724087Z","iopub.status.idle":"2023-03-27T15:08:15.741059Z","shell.execute_reply.started":"2023-03-27T15:08:15.724016Z","shell.execute_reply":"2023-03-27T15:08:15.740136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df.groupby('Town_ID')['Town'].nunique().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.742477Z","iopub.execute_input":"2023-03-27T15:08:15.742795Z","iopub.status.idle":"2023-03-27T15:08:15.760391Z","shell.execute_reply.started":"2023-03-27T15:08:15.742729Z","shell.execute_reply":"2023-03-27T15:08:15.759206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df[town_state_df.Town_ID.isin([2561, 2169, 2152])].sort_values(by='Town_ID')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.762165Z","iopub.execute_input":"2023-03-27T15:08:15.762619Z","iopub.status.idle":"2023-03-27T15:08:15.780328Z","shell.execute_reply.started":"2023-03-27T15:08:15.762411Z","shell.execute_reply":"2023-03-27T15:08:15.779543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df['Town_ID'].max()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.781555Z","iopub.execute_input":"2023-03-27T15:08:15.781985Z","iopub.status.idle":"2023-03-27T15:08:15.788899Z","shell.execute_reply.started":"2023-03-27T15:08:15.781938Z","shell.execute_reply":"2023-03-27T15:08:15.788101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df.loc[199, 'Town_ID'] = 3217\ntown_state_df.loc[311, 'Town_ID'] = 3218","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.790480Z","iopub.execute_input":"2023-03-27T15:08:15.790934Z","iopub.status.idle":"2023-03-27T15:08:15.805964Z","shell.execute_reply.started":"2023-03-27T15:08:15.790867Z","shell.execute_reply":"2023-03-27T15:08:15.804561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df['Town_ID'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.807794Z","iopub.execute_input":"2023-03-27T15:08:15.808371Z","iopub.status.idle":"2023-03-27T15:08:15.816451Z","shell.execute_reply.started":"2023-03-27T15:08:15.808080Z","shell.execute_reply":"2023-03-27T15:08:15.815403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product_price","metadata":{}},{"cell_type":"code","source":"sale_prices_df = product_price_df.drop('return_price', axis=1).dropna().rename(columns={'sale_price': 'price'})\nreturn_prices_df = product_price_df.drop('sale_price', axis=1).dropna().rename(columns={'return_price': 'price'})\nprices_df = pd.concat([sale_prices_df, return_prices_df])\nprices_df = prices_df.groupby('Producto_ID')['price'].median().reset_index()\nprices_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:15.817943Z","iopub.execute_input":"2023-03-27T15:08:15.818297Z","iopub.status.idle":"2023-03-27T15:08:33.180641Z","shell.execute_reply.started":"2023-03-27T15:08:15.818197Z","shell.execute_reply":"2023-03-27T15:08:33.179618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:33.182062Z","iopub.execute_input":"2023-03-27T15:08:33.182320Z","iopub.status.idle":"2023-03-27T15:08:33.187632Z","shell.execute_reply.started":"2023-03-27T15:08:33.182276Z","shell.execute_reply":"2023-03-27T15:08:33.186958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product","metadata":{}},{"cell_type":"code","source":"product_df['in_train'] = 0\nproduct_df['in_test'] = 0\nproduct_df.loc[product_df['Producto_ID'].isin(test_df['Producto_ID'].unique()), 'in_test'] = 1\nproduct_df.loc[product_df['Producto_ID'].isin(train_df['Producto_ID'].unique()), 'in_train'] = 1\nproduct_df = product_df[(product_df['in_test'] == 1) | (product_df['in_train'] == 1)]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:33.188911Z","iopub.execute_input":"2023-03-27T15:08:33.189411Z","iopub.status.idle":"2023-03-27T15:08:34.216303Z","shell.execute_reply.started":"2023-03-27T15:08:33.189364Z","shell.execute_reply":"2023-03-27T15:08:34.214971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df[product_df['property'].isnull()]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.218084Z","iopub.execute_input":"2023-03-27T15:08:34.218426Z","iopub.status.idle":"2023-03-27T15:08:34.236100Z","shell.execute_reply.started":"2023-03-27T15:08:34.218362Z","shell.execute_reply":"2023-03-27T15:08:34.235069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df.loc[117, 'popular_name'] = 'Donas'\nproduct_df.loc[117, 'property'] = 'Prom BIM'\nproduct_df.loc[117, 'unit'] = None\nproduct_df.loc[117, 'pieces'] = '6p'\n\nproduct_df.loc[190, 'popular_name'] = 'Paletina para Cafe'\nproduct_df.loc[190, 'property'] = 'NES'\nproduct_df.loc[190, 'unit'] = None\nproduct_df.loc[190, 'pieces'] = None\n\nproduct_df.loc[381, 'popular_name'] = 'Camioncitos Bimbo'\nproduct_df.loc[381, 'property'] = 'BIM'\nproduct_df.loc[381, 'unit'] = None\nproduct_df.loc[381, 'pieces'] = None\n\nproduct_df.loc[1152, 'popular_name'] = 'Burrito Vaporero FrijolChorizo'\nproduct_df.loc[1152, 'property'] = 'CU LON'\nproduct_df.loc[1152, 'unit'] = '90g'\nproduct_df.loc[1152, 'pieces'] = None\n\nproduct_df.loc[1677, 'popular_name'] = 'Tarima Twin Pack Thins Multig'\nproduct_df.loc[1677, 'property'] = 'CU ORO'\nproduct_df.loc[1677, 'unit'] = None\nproduct_df.loc[1677, 'pieces'] = None\n\nproduct_df.loc[1888, 'popular_name'] = 'Deliciosas Chochochispas'\nproduct_df.loc[1888, 'property'] = 'Prom MTA LAR'\nproduct_df.loc[1888, 'unit'] = '204g'\nproduct_df.loc[1888, 'pieces'] = None\n\nproduct_df.loc[1889, 'popular_name'] = 'Deliciosas Chochochispas'\nproduct_df.loc[1889, 'property'] = 'Prom LAR'\nproduct_df.loc[1889, 'unit'] = '204g'\nproduct_df.loc[1889, 'pieces'] = None\n\nproduct_df.loc[2449, 'popular_name'] = 'Galleta Granel Classics Chocolate'\nproduct_df.loc[2449, 'property'] = 'GBI'\nproduct_df.loc[2449, 'unit'] = None\nproduct_df.loc[2449, 'pieces'] = None","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.245055Z","iopub.execute_input":"2023-03-27T15:08:34.245394Z","iopub.status.idle":"2023-03-27T15:08:34.286161Z","shell.execute_reply.started":"2023-03-27T15:08:34.245342Z","shell.execute_reply":"2023-03-27T15:08:34.285164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df[product_df['popular_name'].isnull()]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.288341Z","iopub.execute_input":"2023-03-27T15:08:34.288715Z","iopub.status.idle":"2023-03-27T15:08:34.312195Z","shell.execute_reply.started":"2023-03-27T15:08:34.288657Z","shell.execute_reply":"2023-03-27T15:08:34.311108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df.loc[877, 'popular_name'] = 'Tortilla Hna Chihuahua'\nproduct_df.loc[877, 'unit'] = '535g'\nproduct_df.loc[877, 'pieces'] = '10p'\n\nproduct_df.loc[1585, 'popular_name'] = 'Principe Cho Bco MG'\nproduct_df.loc[1585, 'unit'] = '110g'\nproduct_df.loc[1585, 'pieces'] = '10p'\n\nproduct_df.loc[1748, 'popular_name'] = 'Combo Salma mas Levite'\nproduct_df.loc[1748, 'unit'] = '1360g'\nproduct_df.loc[1748, 'pieces'] = None","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.314126Z","iopub.execute_input":"2023-03-27T15:08:34.314523Z","iopub.status.idle":"2023-03-27T15:08:34.331504Z","shell.execute_reply.started":"2023-03-27T15:08:34.314467Z","shell.execute_reply":"2023-03-27T15:08:34.330510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df['pieces'] = product_df['pieces'].str.extract(r'(\\d+)(p|Reb)')[0]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.333356Z","iopub.execute_input":"2023-03-27T15:08:34.333745Z","iopub.status.idle":"2023-03-27T15:08:34.353484Z","shell.execute_reply.started":"2023-03-27T15:08:34.333675Z","shell.execute_reply":"2023-03-27T15:08:34.352544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df['weight'] = product_df['unit'].str.strip()\nproduct_df['weight'] = product_df['weight'].str.replace(' ', '.')\nproduct_df['weight'] = product_df['weight'].str.upper()\nw = product_df['weight'].str.extract('(.+?)(KG|G|ML)', expand=True)\nproduct_df['weight'] = w[0].astype('float') * w[1].map({'KG':1000, 'G':1, 'ML':1})","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.354727Z","iopub.execute_input":"2023-03-27T15:08:34.355241Z","iopub.status.idle":"2023-03-27T15:08:34.380975Z","shell.execute_reply.started":"2023-03-27T15:08:34.355190Z","shell.execute_reply":"2023-03-27T15:08:34.379547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df = pd.merge(product_df, prices_df, how='left')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.382589Z","iopub.execute_input":"2023-03-27T15:08:34.382952Z","iopub.status.idle":"2023-03-27T15:08:34.402536Z","shell.execute_reply.started":"2023-03-27T15:08:34.382888Z","shell.execute_reply":"2023-03-27T15:08:34.401604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Imputing Missing Data","metadata":{}},{"cell_type":"markdown","source":"### product","metadata":{}},{"cell_type":"code","source":"product_df['pieces'] = product_df['pieces'].fillna(1)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.403801Z","iopub.execute_input":"2023-03-27T15:08:34.404218Z","iopub.status.idle":"2023-03-27T15:08:34.409794Z","shell.execute_reply.started":"2023-03-27T15:08:34.404177Z","shell.execute_reply":"2023-03-27T15:08:34.408837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df[product_df['weight'].isnull()]['price'].max()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.411171Z","iopub.execute_input":"2023-03-27T15:08:34.411621Z","iopub.status.idle":"2023-03-27T15:08:34.428355Z","shell.execute_reply.started":"2023-03-27T15:08:34.411531Z","shell.execute_reply":"2023-03-27T15:08:34.427299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_df[product_df['price'].isnull()]['weight'].max()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.431059Z","iopub.execute_input":"2023-03-27T15:08:34.431365Z","iopub.status.idle":"2023-03-27T15:08:34.441563Z","shell.execute_reply.started":"2023-03-27T15:08:34.431315Z","shell.execute_reply":"2023-03-27T15:08:34.440269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = product_df.dropna()\ndf = df[(df['price'] <= 311) & (df['weight'] <= 1880)]\nplt.figure(figsize=(16,8))\nsns.scatterplot(x='weight', y='price', data=df)\ndel df","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:34.442870Z","iopub.execute_input":"2023-03-27T15:08:34.443316Z","iopub.status.idle":"2023-03-27T15:08:35.080053Z","shell.execute_reply.started":"2023-03-27T15:08:34.443243Z","shell.execute_reply":"2023-03-27T15:08:35.078674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\ndf = product_df.dropna()\ndf = df[(df['price'] <= 100) & (df['weight'] <= 1880)]\n\n# predict missing prices\nlf = LinearRegression()\nlf.fit(df['weight'].values.reshape(-1, 1), df['price'])\n\nprices = lf.predict(product_df[product_df['price'].isnull()]['weight'].values.reshape(-1, 1))\n\nproduct_df.loc[product_df['price'].isnull(), 'price'] = prices\n\n# predict missing weights\nlf = LinearRegression()\nlf.fit(df['price'].values.reshape(-1, 1), df['weight'])\n\nweights = lf.predict(product_df[product_df['weight'].isnull()]['price'].values.reshape(-1, 1))\n\nproduct_df.loc[product_df['weight'].isnull(), 'weight'] = weights\n\ndel df\ndel prices\ndel weights","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:35.081986Z","iopub.execute_input":"2023-03-27T15:08:35.082420Z","iopub.status.idle":"2023-03-27T15:08:35.251313Z","shell.execute_reply.started":"2023-03-27T15:08:35.082342Z","shell.execute_reply":"2023-03-27T15:08:35.249996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(product_df.drop(['unit'], axis=1).isnull().sum() == 0).all()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:35.253112Z","iopub.execute_input":"2023-03-27T15:08:35.253856Z","iopub.status.idle":"2023-03-27T15:08:35.266636Z","shell.execute_reply.started":"2023-03-27T15:08:35.253772Z","shell.execute_reply":"2023-03-27T15:08:35.265477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.3 Transforming Data Type","metadata":{}},{"cell_type":"code","source":"product_df['pieces'] = product_df['pieces'].astype('int16')\nproduct_df['in_train'] = product_df['in_train'].astype('bool')\nproduct_df['in_test'] = product_df['in_test'].astype('bool')\nproduct_df['weight'] = product_df['weight'].astype('float32')\nproduct_df['price'] = product_df['price'].astype('float32')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:35.268168Z","iopub.execute_input":"2023-03-27T15:08:35.268688Z","iopub.status.idle":"2023-03-27T15:08:35.281416Z","shell.execute_reply.started":"2023-03-27T15:08:35.268640Z","shell.execute_reply":"2023-03-27T15:08:35.280111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"town_state_df['State'] = town_state_df['State'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:35.283158Z","iopub.execute_input":"2023-03-27T15:08:35.283620Z","iopub.status.idle":"2023-03-27T15:08:35.300357Z","shell.execute_reply.started":"2023-03-27T15:08:35.283532Z","shell.execute_reply":"2023-03-27T15:08:35.299501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Canal_ID'] = train_df['Canal_ID'].astype('category')\ntest_df['Canal_ID'] = test_df['Canal_ID'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:35.301995Z","iopub.execute_input":"2023-03-27T15:08:35.302565Z","iopub.status.idle":"2023-03-27T15:08:37.780578Z","shell.execute_reply.started":"2023-03-27T15:08:35.302502Z","shell.execute_reply":"2023-03-27T15:08:37.779627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.4 Dropping Features","metadata":{}},{"cell_type":"code","source":"train_df.drop(['Venta_uni_hoy', 'Venta_hoy', 'Dev_uni_proxima', 'Dev_proxima'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:37.782184Z","iopub.execute_input":"2023-03-27T15:08:37.782864Z","iopub.status.idle":"2023-03-27T15:08:39.146496Z","shell.execute_reply.started":"2023-03-27T15:08:37.782779Z","shell.execute_reply":"2023-03-27T15:08:39.145476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.5 Merging Data","metadata":{}},{"cell_type":"markdown","source":"### town_state","metadata":{}},{"cell_type":"code","source":"train_df = pd.merge(train_df, town_state_df[['Agencia_ID', 'Town_ID']], how='left')\ntest_df = pd.merge(test_df, town_state_df[['Agencia_ID', 'Town_ID']], how='left')\ntrain_df.drop('Agencia_ID', axis=1, inplace=True)\ntest_df.drop('Agencia_ID', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:08:39.148014Z","iopub.execute_input":"2023-03-27T15:08:39.148514Z","iopub.status.idle":"2023-03-27T15:09:00.448937Z","shell.execute_reply.started":"2023-03-27T15:08:39.148448Z","shell.execute_reply":"2023-03-27T15:09:00.447743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product","metadata":{}},{"cell_type":"code","source":"train_df = pd.merge(\n    train_df,\n    product_df[[\n        'Producto_ID', 'popular_name', 'property',\n        'pieces', 'weight', 'price'\n    ]], how='left')\ntest_df = pd.merge(\n    test_df,\n    product_df[[\n        'Producto_ID', 'popular_name', 'property',\n        'pieces', 'weight', 'price'\n    ]], how='left')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:00.450790Z","iopub.execute_input":"2023-03-27T15:09:00.451404Z","iopub.status.idle":"2023-03-27T15:09:22.004755Z","shell.execute_reply.started":"2023-03-27T15:09:00.451163Z","shell.execute_reply":"2023-03-27T15:09:22.003617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:22.006442Z","iopub.execute_input":"2023-03-27T15:09:22.006749Z","iopub.status.idle":"2023-03-27T15:09:22.027581Z","shell.execute_reply.started":"2023-03-27T15:09:22.006699Z","shell.execute_reply":"2023-03-27T15:09:22.026446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.6 Bin-Counting","metadata":{}},{"cell_type":"markdown","source":"### Semana","metadata":{}},{"cell_type":"code","source":"semana_med_s = train_df.groupby('Semana')['Demanda_uni_equil'].median()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:22.029177Z","iopub.execute_input":"2023-03-27T15:09:22.029490Z","iopub.status.idle":"2023-03-27T15:09:29.211906Z","shell.execute_reply.started":"2023-03-27T15:09:22.029433Z","shell.execute_reply":"2023-03-27T15:09:29.210806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"semana_med_s","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:29.214020Z","iopub.execute_input":"2023-03-27T15:09:29.214506Z","iopub.status.idle":"2023-03-27T15:09:29.223385Z","shell.execute_reply.started":"2023-03-27T15:09:29.214368Z","shell.execute_reply":"2023-03-27T15:09:29.222028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop('Semana', axis=1, inplace=True)\ntest_df.drop('Semana', axis=1, inplace=True)\ndel semana_med_s","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:29.225548Z","iopub.execute_input":"2023-03-27T15:09:29.225992Z","iopub.status.idle":"2023-03-27T15:09:33.449352Z","shell.execute_reply.started":"2023-03-27T15:09:29.225913Z","shell.execute_reply":"2023-03-27T15:09:33.448483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cliente_ID","metadata":{}},{"cell_type":"code","source":"client_med_s = train_df.groupby('Cliente_ID')['Demanda_uni_equil'].median().astype('int16')\nclient_med_s.name = 'client_med'","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:33.451360Z","iopub.execute_input":"2023-03-27T15:09:33.451772Z","iopub.status.idle":"2023-03-27T15:09:42.779601Z","shell.execute_reply.started":"2023-03-27T15:09:33.451698Z","shell.execute_reply":"2023-03-27T15:09:42.778686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### popular_name","metadata":{}},{"cell_type":"code","source":"popular_name_med_s = train_df.groupby('popular_name')['Demanda_uni_equil'].median().astype('int16')\npopular_name_med_s.name = 'popular_name_med'","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:42.781277Z","iopub.execute_input":"2023-03-27T15:09:42.782108Z","iopub.status.idle":"2023-03-27T15:09:53.516596Z","shell.execute_reply.started":"2023-03-27T15:09:42.781743Z","shell.execute_reply":"2023-03-27T15:09:53.515306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Town_ID","metadata":{}},{"cell_type":"code","source":"town_id_med_s = train_df.groupby('Town_ID')['Demanda_uni_equil'].median().astype('int16')\ntown_id_med_s.name = 'town_id_med'","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:53.518148Z","iopub.execute_input":"2023-03-27T15:09:53.518409Z","iopub.status.idle":"2023-03-27T15:09:59.526396Z","shell.execute_reply.started":"2023-03-27T15:09:53.518368Z","shell.execute_reply":"2023-03-27T15:09:59.525221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ruta_SAK","metadata":{}},{"cell_type":"code","source":"ruta_id_med_s = train_df.groupby('Ruta_SAK')['Demanda_uni_equil'].median().astype('int16')\nruta_id_med_s.name = 'ruta_id_med'","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:09:59.528259Z","iopub.execute_input":"2023-03-27T15:09:59.528603Z","iopub.status.idle":"2023-03-27T15:10:05.511351Z","shell.execute_reply.started":"2023-03-27T15:09:59.528541Z","shell.execute_reply":"2023-03-27T15:10:05.510229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.7 Merging Bin-Counting Data","metadata":{}},{"cell_type":"code","source":"test_df = pd.merge(test_df, client_med_s.reset_index(), how='left')\ntest_df = pd.merge(test_df, popular_name_med_s.reset_index(), how='left')\ntest_df = pd.merge(test_df, town_id_med_s.reset_index(), how='left')\ntest_df = pd.merge(test_df, ruta_id_med_s.reset_index(), how='left')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:10:05.513094Z","iopub.execute_input":"2023-03-27T15:10:05.513498Z","iopub.status.idle":"2023-03-27T15:10:16.288987Z","shell.execute_reply.started":"2023-03-27T15:10:05.513425Z","shell.execute_reply":"2023-03-27T15:10:16.287710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(train_df, client_med_s.reset_index(), how='left')\ntrain_df = pd.merge(train_df, popular_name_med_s.reset_index(), how='left')\ntrain_df = pd.merge(train_df, town_id_med_s.reset_index(), how='left')\ntrain_df = pd.merge(train_df, ruta_id_med_s.reset_index(), how='left')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:10:16.290668Z","iopub.execute_input":"2023-03-27T15:10:16.291125Z","iopub.status.idle":"2023-03-27T15:11:44.353137Z","shell.execute_reply.started":"2023-03-27T15:10:16.291051Z","shell.execute_reply":"2023-03-27T15:11:44.351971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.8 Imputing Test Missing Data","metadata":{}},{"cell_type":"code","source":"test_df['popular_name_med'] = test_df['popular_name_med'].fillna(test_df['popular_name_med'].mean())\ntest_df['client_med'] = test_df['client_med'].fillna(test_df['client_med'].mean())\ntest_df['ruta_id_med'] = test_df['ruta_id_med'].fillna(test_df['ruta_id_med'].mean())","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:44.355184Z","iopub.execute_input":"2023-03-27T15:11:44.355779Z","iopub.status.idle":"2023-03-27T15:11:44.561108Z","shell.execute_reply.started":"2023-03-27T15:11:44.355534Z","shell.execute_reply":"2023-03-27T15:11:44.560006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.9 Transforming Data Type","metadata":{}},{"cell_type":"code","source":"train_df['client_med'] = train_df['client_med'].astype('int16')\ntrain_df['popular_name_med'] = train_df['popular_name_med'].astype('int16')\ntrain_df['town_id_med'] = train_df['town_id_med'].astype('int16')\ntrain_df['ruta_id_med'] = train_df['ruta_id_med'].astype('int16')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:44.562676Z","iopub.execute_input":"2023-03-27T15:11:44.563008Z","iopub.status.idle":"2023-03-27T15:11:45.905955Z","shell.execute_reply.started":"2023-03-27T15:11:44.562951Z","shell.execute_reply":"2023-03-27T15:11:45.904718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['client_med'] = test_df['client_med'].astype('int16')\ntest_df['popular_name_med'] = test_df['popular_name_med'].astype('int16')\ntest_df['town_id_med'] = test_df['town_id_med'].astype('int16')\ntest_df['ruta_id_med'] = test_df['ruta_id_med'].astype('int16')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:45.907914Z","iopub.execute_input":"2023-03-27T15:11:45.908244Z","iopub.status.idle":"2023-03-27T15:11:46.000991Z","shell.execute_reply.started":"2023-03-27T15:11:45.908188Z","shell.execute_reply":"2023-03-27T15:11:45.999740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.10 Dropping Features","metadata":{}},{"cell_type":"code","source":"train_df.drop(\n    ['Ruta_SAK', 'Cliente_ID', 'Producto_ID', 'Town_ID', 'popular_name', 'property', 'pieces'],\n    axis=1, inplace=True)\ntest_df.drop(\n    ['Ruta_SAK', 'Cliente_ID', 'Producto_ID', 'Town_ID', 'popular_name', 'property', 'pieces'],\n    axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:46.002764Z","iopub.execute_input":"2023-03-27T15:11:46.003195Z","iopub.status.idle":"2023-03-27T15:11:50.727594Z","shell.execute_reply.started":"2023-03-27T15:11:46.003119Z","shell.execute_reply":"2023-03-27T15:11:50.726586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:50.729114Z","iopub.execute_input":"2023-03-27T15:11:50.729441Z","iopub.status.idle":"2023-03-27T15:11:50.744719Z","shell.execute_reply.started":"2023-03-27T15:11:50.729382Z","shell.execute_reply":"2023-03-27T15:11:50.743582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:50.746449Z","iopub.execute_input":"2023-03-27T15:11:50.747083Z","iopub.status.idle":"2023-03-27T15:11:50.769084Z","shell.execute_reply.started":"2023-03-27T15:11:50.746778Z","shell.execute_reply":"2023-03-27T15:11:50.767897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Take a glance at all variables","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:50.770997Z","iopub.execute_input":"2023-03-27T15:11:50.771415Z","iopub.status.idle":"2023-03-27T15:11:50.785922Z","shell.execute_reply.started":"2023-03-27T15:11:50.771340Z","shell.execute_reply":"2023-03-27T15:11:50.784617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:50.787868Z","iopub.execute_input":"2023-03-27T15:11:50.788351Z","iopub.status.idle":"2023-03-27T15:11:50.801886Z","shell.execute_reply.started":"2023-03-27T15:11:50.788288Z","shell.execute_reply":"2023-03-27T15:11:50.800764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:11:50.803619Z","iopub.execute_input":"2023-03-27T15:11:50.803979Z","iopub.status.idle":"2023-03-27T15:12:09.223026Z","shell.execute_reply.started":"2023-03-27T15:11:50.803910Z","shell.execute_reply":"2023-03-27T15:12:09.222161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:12:09.224477Z","iopub.execute_input":"2023-03-27T15:12:09.225012Z","iopub.status.idle":"2023-03-27T15:12:10.900066Z","shell.execute_reply.started":"2023-03-27T15:12:09.224962Z","shell.execute_reply":"2023-03-27T15:12:10.898738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Plot histogram of numerical variables to validate pandas intuition.'''\ndef draw_histograms(df, variables, n_rows, n_cols, size):\n    fig=plt.figure()\n    for i, var_name in enumerate(variables):\n        ax=fig.add_subplot(n_rows, n_cols, i+1)\n        df[var_name].hist(bins=40, ax=ax, color='skyblue', alpha=0.8, figsize=size)\n        ax.set_title(var_name, fontsize=43)\n        ax.tick_params(axis='both', which='major', labelsize=35)\n        ax.tick_params(axis='both', which='minor', labelsize=35)\n        ax.set_xlabel('')\n    fig.tight_layout(rect=[0, 0.03, 1, 0.95])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:12:10.902198Z","iopub.execute_input":"2023-03-27T15:12:10.902840Z","iopub.status.idle":"2023-03-27T15:12:10.912142Z","shell.execute_reply.started":"2023-03-27T15:12:10.902548Z","shell.execute_reply":"2023-03-27T15:12:10.911163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewness = train_df.select_dtypes(include=['int8', 'int16', 'int32', 'int64', 'float32', 'float64']).apply(lambda x: skew(x))\nskew_index = skewness[abs(skewness) >= 0.75].index\nskewness[skew_index].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:12:10.913738Z","iopub.execute_input":"2023-03-27T15:12:10.914081Z","iopub.status.idle":"2023-03-27T15:12:24.637254Z","shell.execute_reply.started":"2023-03-27T15:12:10.914018Z","shell.execute_reply":"2023-03-27T15:12:24.636151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewness = test_df.select_dtypes(include=['int8', 'int16', 'int32', 'int64', 'float32', 'float64']).apply(lambda x: skew(x))\nskew_index = skewness[abs(skewness) >= 0.75].index\nskewness[skew_index].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:12:24.638856Z","iopub.execute_input":"2023-03-27T15:12:24.639170Z","iopub.status.idle":"2023-03-27T15:12:27.075162Z","shell.execute_reply.started":"2023-03-27T15:12:24.639122Z","shell.execute_reply":"2023-03-27T15:12:27.074423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 BoxCox Transform","metadata":{}},{"cell_type":"code","source":"'''BoxCox Transform'''\nlam = 0.01\nfor column in skew_index:\n    train_df[column] = boxcox1p(train_df[column], lam)\n    test_df[column] = boxcox1p(test_df[column], lam)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:12:27.076274Z","iopub.execute_input":"2023-03-27T15:12:27.076663Z","iopub.status.idle":"2023-03-27T15:12:56.102662Z","shell.execute_reply.started":"2023-03-27T15:12:27.076621Z","shell.execute_reply":"2023-03-27T15:12:56.101627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:12:56.103967Z","iopub.execute_input":"2023-03-27T15:12:56.104404Z","iopub.status.idle":"2023-03-27T15:13:00.561203Z","shell.execute_reply.started":"2023-03-27T15:12:56.104359Z","shell.execute_reply":"2023-03-27T15:13:00.560396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:00.562350Z","iopub.execute_input":"2023-03-27T15:13:00.562785Z","iopub.status.idle":"2023-03-27T15:13:00.959124Z","shell.execute_reply.started":"2023-03-27T15:13:00.562740Z","shell.execute_reply":"2023-03-27T15:13:00.958100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.3 Transforming Data Type","metadata":{}},{"cell_type":"code","source":"train_df['client_med'] = train_df['client_med'].astype('float32')\ntrain_df['popular_name_med'] = train_df['popular_name_med'].astype('float32')\ntrain_df['town_id_med'] = train_df['town_id_med'].astype('float32')\ntrain_df['ruta_id_med'] = train_df['ruta_id_med'].astype('float32')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:00.961080Z","iopub.execute_input":"2023-03-27T15:13:00.961716Z","iopub.status.idle":"2023-03-27T15:13:06.726428Z","shell.execute_reply.started":"2023-03-27T15:13:00.961460Z","shell.execute_reply":"2023-03-27T15:13:06.725031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['client_med'] = test_df['client_med'].astype('float32')\ntest_df['popular_name_med'] = test_df['popular_name_med'].astype('float32')\ntest_df['town_id_med'] = test_df['town_id_med'].astype('float32')\ntest_df['ruta_id_med'] = test_df['ruta_id_med'].astype('float32')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:06.728808Z","iopub.execute_input":"2023-03-27T15:13:06.729686Z","iopub.status.idle":"2023-03-27T15:13:07.184031Z","shell.execute_reply.started":"2023-03-27T15:13:06.729398Z","shell.execute_reply":"2023-03-27T15:13:07.182855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Exploratory Data Analysis ","metadata":{}},{"cell_type":"markdown","source":"## 4.1 Analyzing Target ","metadata":{}},{"cell_type":"code","source":"sample_train_df = train_df.sample(n=10000)\nsample_train_df['log_target'] = np.log1p(sample_train_df['Demanda_uni_equil'])","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:07.185657Z","iopub.execute_input":"2023-03-27T15:13:07.186053Z","iopub.status.idle":"2023-03-27T15:13:16.423453Z","shell.execute_reply.started":"2023-03-27T15:13:07.185989Z","shell.execute_reply":"2023-03-27T15:13:16.422276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:16.425058Z","iopub.execute_input":"2023-03-27T15:13:16.425758Z","iopub.status.idle":"2023-03-27T15:13:16.442005Z","shell.execute_reply.started":"2023-03-27T15:13:16.425583Z","shell.execute_reply":"2023-03-27T15:13:16.440839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''correlation matrix'''\nplt.subplots(figsize=(20, 16))\nk = 20 #number of variables for heatmap\ncorrmat = sample_train_df.corr()\ncols = corrmat.nlargest(k, 'log_target')['log_target'].index\n\ncm = np.corrcoef(sample_train_df[cols].values.T)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True,\n                 fmt='.2f', annot_kws={'size': 10}, cmap='Blues',\n                 yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:16.444227Z","iopub.execute_input":"2023-03-27T15:13:16.444556Z","iopub.status.idle":"2023-03-27T15:13:17.473181Z","shell.execute_reply.started":"2023-03-27T15:13:16.444497Z","shell.execute_reply":"2023-03-27T15:13:17.468639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Check feature inportance by applying LightGBM'''\nimport lightgbm as lgb\nmodel_lgb = lgb.LGBMRegressor(num_leaves=1000,\n                              max_depth=5,\n                              learning_rate=0.1,\n                              random_state=2)\nmodel_lgb.fit(sample_train_df.drop(['Demanda_uni_equil', 'log_target'], axis=1), sample_train_df['log_target'])","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:17.474863Z","iopub.execute_input":"2023-03-27T15:13:17.475194Z","iopub.status.idle":"2023-03-27T15:13:17.702965Z","shell.execute_reply.started":"2023-03-27T15:13:17.475142Z","shell.execute_reply":"2023-03-27T15:13:17.702002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(model_lgb.feature_importances_,\n             index=sample_train_df.drop(['Demanda_uni_equil', 'log_target'], axis=1).columns,\n             columns=['importance']).sort_values('importance', ascending=False)\ndf[df.importance > 10]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:17.707607Z","iopub.execute_input":"2023-03-27T15:13:17.709099Z","iopub.status.idle":"2023-03-27T15:13:17.723301Z","shell.execute_reply.started":"2023-03-27T15:13:17.709021Z","shell.execute_reply":"2023-03-27T15:13:17.722566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weight\nplt.figure(figsize=(16, 8))\nsns.scatterplot(x='weight', y='log_target', data=sample_train_df, palette='Blues_d')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:17.724453Z","iopub.execute_input":"2023-03-27T15:13:17.724866Z","iopub.status.idle":"2023-03-27T15:13:18.364478Z","shell.execute_reply.started":"2023-03-27T15:13:17.724810Z","shell.execute_reply":"2023-03-27T15:13:18.363209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# price\nplt.figure(figsize=(16, 8))\nsns.scatterplot(x='price', y='log_target', data=sample_train_df, palette='Blues_d')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:18.366137Z","iopub.execute_input":"2023-03-27T15:13:18.366473Z","iopub.status.idle":"2023-03-27T15:13:18.979932Z","shell.execute_reply.started":"2023-03-27T15:13:18.366411Z","shell.execute_reply":"2023-03-27T15:13:18.978939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# client_med\nplt.figure(figsize=(16, 8))\nsns.scatterplot(x='client_med', y='log_target', data=sample_train_df, palette='Blues_d')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:18.981495Z","iopub.execute_input":"2023-03-27T15:13:18.981882Z","iopub.status.idle":"2023-03-27T15:13:19.587186Z","shell.execute_reply.started":"2023-03-27T15:13:18.981731Z","shell.execute_reply":"2023-03-27T15:13:19.586036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# popular_name_med\nplt.figure(figsize=(16, 8))\nsns.scatterplot(x='popular_name_med', y='log_target', data=sample_train_df, palette='Blues_d')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:19.588699Z","iopub.execute_input":"2023-03-27T15:13:19.589070Z","iopub.status.idle":"2023-03-27T15:13:20.212156Z","shell.execute_reply.started":"2023-03-27T15:13:19.588985Z","shell.execute_reply":"2023-03-27T15:13:20.210988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# town_id_med\nplt.figure(figsize=(16, 8))\nsns.scatterplot(x='town_id_med', y='log_target', data=sample_train_df, palette='Blues_d')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:20.213895Z","iopub.execute_input":"2023-03-27T15:13:20.214244Z","iopub.status.idle":"2023-03-27T15:13:20.827164Z","shell.execute_reply.started":"2023-03-27T15:13:20.214188Z","shell.execute_reply":"2023-03-27T15:13:20.825860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ruta_id_med\nplt.figure(figsize=(16, 8))\nsns.scatterplot(x='ruta_id_med', y='log_target', data=sample_train_df, palette='Blues_d')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:20.828712Z","iopub.execute_input":"2023-03-27T15:13:20.829082Z","iopub.status.idle":"2023-03-27T15:13:21.466092Z","shell.execute_reply.started":"2023-03-27T15:13:20.829025Z","shell.execute_reply":"2023-03-27T15:13:21.465126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Model Building and Evaluation","metadata":{}},{"cell_type":"markdown","source":"## 5.1 Importing Packages","metadata":{}},{"cell_type":"code","source":"'''Importing Modeling Interested Modules'''\nfrom sklearn.base import BaseEstimator\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.linear_model import LinearRegression, LassoCV, RidgeCV, ElasticNetCV\nfrom sklearn.svm import SVR\nfrom sklearn.kernel_ridge import KernelRidge\nfrom lightgbm import LGBMRegressor","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:21.468016Z","iopub.execute_input":"2023-03-27T15:13:21.468364Z","iopub.status.idle":"2023-03-27T15:13:21.478140Z","shell.execute_reply.started":"2023-03-27T15:13:21.468302Z","shell.execute_reply":"2023-03-27T15:13:21.476973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.2 Preparation before Building Models ","metadata":{}},{"cell_type":"code","source":"'''Adjust dataframe for modeling'''\ntrain_y = train_df['Demanda_uni_equil']\ntrain_df.drop(['Demanda_uni_equil'], axis=1, inplace=True)\ntrain_X = train_df\ntest_X = test_df.drop('id', axis=1)\n\n'''Transform categorical features to dummy variables'''\ntrain_X = pd.get_dummies(train_X)\ntest_X = pd.get_dummies(test_X)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:21.479289Z","iopub.execute_input":"2023-03-27T15:13:21.479751Z","iopub.status.idle":"2023-03-27T15:13:30.553916Z","shell.execute_reply.started":"2023-03-27T15:13:21.479518Z","shell.execute_reply":"2023-03-27T15:13:30.552621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df['Demanda_uni_equil'] = np.expm1(sample_train_df['log_target']).astype('int32')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.555555Z","iopub.execute_input":"2023-03-27T15:13:30.555941Z","iopub.status.idle":"2023-03-27T15:13:30.563094Z","shell.execute_reply.started":"2023-03-27T15:13:30.555879Z","shell.execute_reply":"2023-03-27T15:13:30.561984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.564505Z","iopub.execute_input":"2023-03-27T15:13:30.564788Z","iopub.status.idle":"2023-03-27T15:13:30.589257Z","shell.execute_reply.started":"2023-03-27T15:13:30.564741Z","shell.execute_reply":"2023-03-27T15:13:30.587720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Prepare sample train for the fast training'''\nsample_train_y = sample_train_df['Demanda_uni_equil']\nsample_train_df.drop(['Demanda_uni_equil', 'log_target'], axis=1, inplace=True)\nsample_train_X = sample_train_df\n\nsample_train_X = pd.get_dummies(sample_train_X)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.591132Z","iopub.execute_input":"2023-03-27T15:13:30.591592Z","iopub.status.idle":"2023-03-27T15:13:30.612919Z","shell.execute_reply.started":"2023-03-27T15:13:30.591508Z","shell.execute_reply":"2023-03-27T15:13:30.611972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''We should use the log transform of the target value'''\nclass MyEstimator(BaseEstimator):\n    def __init__(self, model):\n        self.model = model\n        \n    def fit(self, X, y):\n        self.model.fit(X, np.log1p(y))\n        return self \n\n    def predict(self, X):\n        predicts = np.expm1(self.model.predict(X))\n        mask = (predicts <= 0)\n        predicts[mask] = 0\n        return predicts","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.614700Z","iopub.execute_input":"2023-03-27T15:13:30.615133Z","iopub.status.idle":"2023-03-27T15:13:30.623419Z","shell.execute_reply.started":"2023-03-27T15:13:30.615064Z","shell.execute_reply":"2023-03-27T15:13:30.622403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Define evaluation function for Convienience'''\ndef evaluation_model(model, train_X, train_y, test_X):\n    cv = rmsle_cv(model, train_X, train_y)\n    cv_mean = np.round(cv.mean(), 5)\n    cv_std = np.round(cv.std(), 5)\n    sample_prediction = model.predict(test_X.loc[:3, :])\n    return {'cv_mean': cv_mean, 'cv_std': cv_std, 'sample_prediction': sample_prediction}","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.625100Z","iopub.execute_input":"2023-03-27T15:13:30.625403Z","iopub.status.idle":"2023-03-27T15:13:30.635267Z","shell.execute_reply.started":"2023-03-27T15:13:30.625356Z","shell.execute_reply":"2023-03-27T15:13:30.634337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Define Hyperparameters Tuning Function'''\ndef tune_hyperparameters(model, param_grid, train_X, train_y):\n    grid = GridSearchCV(\n        model, param_grid, \n        scoring='neg_mean_squared_log_error',\n        cv=3, n_jobs=-1,\n    )\n    grid.fit(train_X, train_y)\n    best_params = grid.best_params_ \n    best_score = np.round(np.sqrt(-1 * grid.best_score_), 5)\n    return best_params, best_score","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.636939Z","iopub.execute_input":"2023-03-27T15:13:30.637299Z","iopub.status.idle":"2023-03-27T15:13:30.645423Z","shell.execute_reply.started":"2023-03-27T15:13:30.637181Z","shell.execute_reply":"2023-03-27T15:13:30.644491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.3 Building Models","metadata":{}},{"cell_type":"markdown","source":"### LinearRegression","metadata":{}},{"cell_type":"code","source":"model = make_pipeline(\n    RobustScaler(),\n    LinearRegression(),\n)\nlr_model = MyEstimator(model)\nlr_model.fit(sample_train_X, sample_train_y)\nlr_eval = evaluation_model(lr_model, sample_train_X, sample_train_y, test_X)\nprint(lr_eval)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.646530Z","iopub.execute_input":"2023-03-27T15:13:30.646844Z","iopub.status.idle":"2023-03-27T15:13:30.822686Z","shell.execute_reply.started":"2023-03-27T15:13:30.646777Z","shell.execute_reply":"2023-03-27T15:13:30.821635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LassoCV","metadata":{}},{"cell_type":"code","source":"model = make_pipeline(\n    RobustScaler(),\n    LassoCV(\n        alphas=(0.00001, 0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10),\n    ),\n)\nlasso_cv_model = MyEstimator(model)\nlasso_cv_model.fit(sample_train_X, sample_train_y)\nlasso_cv_eval = evaluation_model(lasso_cv_model, sample_train_X, sample_train_y, test_X)\nprint(lasso_cv_eval)\n\nopt_alpha = lasso_cv_model.model.steps[1][1].alpha_\nprint(f'\\nopt_alpha: {opt_alpha}')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:30.824633Z","iopub.execute_input":"2023-03-27T15:13:30.825313Z","iopub.status.idle":"2023-03-27T15:13:31.425259Z","shell.execute_reply.started":"2023-03-27T15:13:30.825249Z","shell.execute_reply":"2023-03-27T15:13:31.424098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##test","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### RidgeCV","metadata":{}},{"cell_type":"code","source":"model = make_pipeline(\n    RobustScaler(),\n    RidgeCV(\n        alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10),\n    ),\n)\nridge_cv_model = MyEstimator(model)\nridge_cv_model.fit(sample_train_X, sample_train_y)\nridge_cv_eval = evaluation_model(ridge_cv_model, sample_train_X, sample_train_y, test_X)\nprint(ridge_cv_eval)\n\nopt_alpha = ridge_cv_model.model.steps[1][1].alpha_\nprint(f'\\nopt_alpha: {opt_alpha}')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:31.427270Z","iopub.execute_input":"2023-03-27T15:13:31.428031Z","iopub.status.idle":"2023-03-27T15:13:31.727943Z","shell.execute_reply.started":"2023-03-27T15:13:31.427951Z","shell.execute_reply":"2023-03-27T15:13:31.726880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ElasticNetCV","metadata":{}},{"cell_type":"code","source":"model = make_pipeline(\n    RobustScaler(),\n    ElasticNetCV(\n        alphas=(0.00001, 0.0001, 0.0002, 0.0003), \n        l1_ratio=(0.001, 0.01, 0.1, 0.2, 0.3, 0.4, 0.5),\n    ),\n)\nelsnt_cv_model = MyEstimator(model)\nelsnt_cv_model.fit(sample_train_X, sample_train_y)\nelsnt_cv_eval = evaluation_model(elsnt_cv_model, sample_train_X, sample_train_y, test_X)\nprint(elsnt_cv_eval)\n\nopt_alpha = elsnt_cv_model.model.steps[1][1].alpha_\nopt_l1_ratio = elsnt_cv_model.model.steps[1][1].l1_ratio_\nprint(f'\\nopt_alpha: {opt_alpha} opt_l1_ratio: {opt_l1_ratio}')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:31.729845Z","iopub.execute_input":"2023-03-27T15:13:31.730505Z","iopub.status.idle":"2023-03-27T15:13:33.470048Z","shell.execute_reply.started":"2023-03-27T15:13:31.730443Z","shell.execute_reply":"2023-03-27T15:13:33.465845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SVR","metadata":{}},{"cell_type":"code","source":"# {'cv_mean': 0.57025, 'cv_std': 0.00775, 'sample_prediction': array([3.85650165, 1.24517075, 3.45866592])}\n\n# grid best_params: {'model__svr__C': 10, 'model__svr__epsilon': 0.1, 'model__svr__gamma': 0.01}\n\n# ### build basemodel\nmodel = make_pipeline(\n    RobustScaler(),\n    SVR(),\n)\nsvr_model = MyEstimator(model)\n# \n# ### optimize hyperparameters\nparam_grid = {'model__svr__C': [1, 10, 20],\n              'model__svr__epsilon': [0.001, 0.01, 0.1],\n              'model__svr__gamma': [0.0001, 0.001, 0.01]}\nbest_params, best_score = \\\n    tune_hyperparameters(svr_model, param_grid, sample_train_X, sample_train_y)\n# \n# ### fit using best_params\nsvr_model.set_params(**best_params)\nsvr_model.fit(sample_train_X, sample_train_y)\nsvr_eval = evaluation_model(svr_model, sample_train_X, sample_train_y, test_X)\nprint(svr_eval)\n# \nprint(f'\\ngrid best_params: {best_params}')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:34:34.230717Z","iopub.execute_input":"2023-03-27T15:34:34.231489Z","iopub.status.idle":"2023-03-27T15:37:30.256756Z","shell.execute_reply.started":"2023-03-27T15:34:34.231408Z","shell.execute_reply":"2023-03-27T15:37:30.255427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### KernelRidge","metadata":{}},{"cell_type":"code","source":"# {'cv_mean': 0.56925, 'cv_std': 0.00546, 'sample_prediction': array([4.15061605, 1.21544493, 3.65678685, 1.81862787])}\n\n# grid best_params: {'model__kernelridge__alpha': 0.5, 'model__kernelridge__coef0': 3, 'model__kernelridge__degree': 2, 'model__kernelridge__kernel': 'polynomial'}\n\n# ### build basemodel\n# model = make_pipeline(\n#     RobustScaler(),\n#     KernelRidge(),\n# )\n# kr_model = MyEstimator(model)\n# \n# ### optimize hyperparameters\n# param_grid = {'model__kernelridge__alpha': [0.01, 0.1, 0.5, 1],\n#               'model__kernelridge__kernel': ['linear', 'polynomial'],\n#               'model__kernelridge__degree': [1, 1.5, 2, 3],\n#               'model__kernelridge__coef0': [3, 4, 5]}\n# best_params, best_score = \\\n#     tune_hyperparameters(kr_model, param_grid, sample_train_X, sample_train_y)\n# \n# ### fit using best_params\n# kr_model.set_params(**best_params)\n# kr_model.fit(sample_train_X, sample_train_y)\n# kr_eval = evaluation_model(kr_model, sample_train_X, sample_train_y, test_X)\n# print(kr_eval)\n# \n# print(f'\\ngrid best_params: {best_params}')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:33.495624Z","iopub.execute_input":"2023-03-27T15:13:33.505288Z","iopub.status.idle":"2023-03-27T15:13:33.517142Z","shell.execute_reply.started":"2023-03-27T15:13:33.505129Z","shell.execute_reply":"2023-03-27T15:13:33.515459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LightGBM","metadata":{}},{"cell_type":"code","source":"model = LGBMRegressor(learning_rate=0.01, n_estimators=3000,\n                      num_leaves=5,\n                      objective='regression',\n                      max_bin=55, bagging_fraction=0.8,\n                      bagging_freq=5, feature_fraction=0.2319,\n                      feature_fraction_seed=9, bagging_seed=9,\n                      min_data_in_leaf=6, min_sum_hessian_in_leaf=11)\nlgb_model = MyEstimator(model)\nlgb_model.fit(sample_train_X, sample_train_y)\nlgb_eval = evaluation_model(lgb_model, sample_train_X, sample_train_y, test_X)\nprint(lgb_eval)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:33.519914Z","iopub.execute_input":"2023-03-27T15:13:33.520762Z","iopub.status.idle":"2023-03-27T15:13:38.976199Z","shell.execute_reply.started":"2023-03-27T15:13:33.520683Z","shell.execute_reply":"2023-03-27T15:13:38.975033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.4 Submission","metadata":{}},{"cell_type":"code","source":"def output_submission_file(model, test_X, filename='submission.csv'):\n    prediction = model.predict(test_X)\n    df = pd.DataFrame({'id': test_df['id'], 'Demanda_uni_equil': prediction})\n    print(f'{df.shape}')\n    print(f'{df.head(5)}')\n    df.to_csv(filename, index=False)\n    df.to_csv(filename + '.gz', index=False, compression='gzip')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:38.981499Z","iopub.execute_input":"2023-03-27T15:13:38.984073Z","iopub.status.idle":"2023-03-27T15:13:38.992629Z","shell.execute_reply.started":"2023-03-27T15:13:38.983989Z","shell.execute_reply":"2023-03-27T15:13:38.991728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Submission'''\noutput_submission_file(lgb_model, test_X)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T15:13:38.994350Z","iopub.execute_input":"2023-03-27T15:13:38.994740Z","iopub.status.idle":"2023-03-27T15:20:35.837711Z","shell.execute_reply.started":"2023-03-27T15:13:38.994676Z","shell.execute_reply":"2023-03-27T15:20:35.836471Z"},"trusted":true},"execution_count":null,"outputs":[]}]}