{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":103436,"databundleVersionId":12476472,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:46:53.831613Z","iopub.execute_input":"2025-06-17T15:46:53.831965Z","iopub.status.idle":"2025-06-17T15:46:54.207359Z","shell.execute_reply.started":"2025-06-17T15:46:53.831940Z","shell.execute_reply":"2025-06-17T15:46:54.206347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('ggplot')\npd.set_option('display.max_columns', 200)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:46:54.208800Z","iopub.execute_input":"2025-06-17T15:46:54.209245Z","iopub.status.idle":"2025-06-17T15:46:55.202499Z","shell.execute_reply.started":"2025-06-17T15:46:54.209215Z","shell.execute_reply":"2025-06-17T15:46:55.201574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"delay_4_6 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv', na_values=[' ', '', '   '], low_memory=False)\nnot_delay_4_6 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/not_delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv', na_values=[' ', '', '   '], low_memory=False)\ndelay_7_9 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv', na_values=[' ', '', '   '], low_memory=False)\nnot_delay_7_9 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/not_delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv', na_values=[' ', '', '   '], low_memory=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:46:55.203437Z","iopub.execute_input":"2025-06-17T15:46:55.203833Z","iopub.status.idle":"2025-06-17T15:47:11.642509Z","shell.execute_reply.started":"2025-06-17T15:46:55.203809Z","shell.execute_reply":"2025-06-17T15:47:11.641668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_4_6 = pd.concat([delay_4_6, not_delay_4_6], axis = 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:11.644636Z","iopub.execute_input":"2025-06-17T15:47:11.644990Z","iopub.status.idle":"2025-06-17T15:47:11.784650Z","shell.execute_reply.started":"2025-06-17T15:47:11.644969Z","shell.execute_reply":"2025-06-17T15:47:11.783831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_7_9 = pd.concat([delay_7_9, not_delay_7_9], axis = 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:11.785515Z","iopub.execute_input":"2025-06-17T15:47:11.785780Z","iopub.status.idle":"2025-06-17T15:47:12.137189Z","shell.execute_reply.started":"2025-06-17T15:47:11.785760Z","shell.execute_reply":"2025-06-17T15:47:12.136180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_4_6.drop(columns=['HEAVY_FLG', 'EXPENSIVE_FLG', 'ACTUAL_SHIP_DAYS',\n                       'SPECIFY_PRODUCTION_DAYS', 'SPECIFY_SHIP_DAYS',\n                       'HAZARD_FLG', 'IO_UNFIT_FLG', 'WEIGHT_UNIT',\n                       'SUPPLIER_CATEGORY_CD', 'PRODUCT_ASSORT'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:12.138321Z","iopub.execute_input":"2025-06-17T15:47:12.138576Z","iopub.status.idle":"2025-06-17T15:47:12.298123Z","shell.execute_reply.started":"2025-06-17T15:47:12.138556Z","shell.execute_reply":"2025-06-17T15:47:12.297101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_4_6.head(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:12.298972Z","iopub.execute_input":"2025-06-17T15:47:12.299204Z","iopub.status.idle":"2025-06-17T15:47:12.335882Z","shell.execute_reply.started":"2025-06-17T15:47:12.299185Z","shell.execute_reply":"2025-06-17T15:47:12.334843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_7_9.head(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:12.336689Z","iopub.execute_input":"2025-06-17T15:47:12.336948Z","iopub.status.idle":"2025-06-17T15:47:12.359088Z","shell.execute_reply.started":"2025-06-17T15:47:12.336927Z","shell.execute_reply":"2025-06-17T15:47:12.358302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ship_days_expect(data):\n    data['Order date'] = pd.to_datetime(data['Order date'])\n    data['VSD'] = pd.to_datetime(data['VSD'])\n    \n    data['SHIP_DAYS_EXPECT'] = (data['VSD'] - data['Order date']).dt.days\n    return data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:12.359942Z","iopub.execute_input":"2025-06-17T15:47:12.360192Z","iopub.status.idle":"2025-06-17T15:47:12.374277Z","shell.execute_reply.started":"2025-06-17T15:47:12.360173Z","shell.execute_reply":"2025-06-17T15:47:12.373210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_4_6 = ship_days_expect(data_4_6)\ndata_7_9 = ship_days_expect(data_7_9)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:12.378055Z","iopub.execute_input":"2025-06-17T15:47:12.378793Z","iopub.status.idle":"2025-06-17T15:47:12.809230Z","shell.execute_reply.started":"2025-06-17T15:47:12.378765Z","shell.execute_reply":"2025-06-17T15:47:12.808253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = data_7_9.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:12.810498Z","iopub.execute_input":"2025-06-17T15:47:12.810825Z","iopub.status.idle":"2025-06-17T15:47:13.657108Z","shell.execute_reply.started":"2025-06-17T15:47:12.810800Z","shell.execute_reply":"2025-06-17T15:47:13.656300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.drop(columns=['Order date', 'VSD', 'REASON_CD', 'SOUF_RCV_NO', 'QTUF_RCV_NO'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:13.658033Z","iopub.execute_input":"2025-06-17T15:47:13.658262Z","iopub.status.idle":"2025-06-17T15:47:13.829544Z","shell.execute_reply.started":"2025-06-17T15:47:13.658244Z","shell.execute_reply":"2025-06-17T15:47:13.828248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = data.rename(columns={'Sales order line number': 'SALES_ORDER_LINE_NUMBER',\n                            'Stock class': 'STOCK_CLASS',\n                            'Consider count hodiday Saturday': 'CONSIDER_COUNT_HOLIDAY_SATURDAY',\n                            'SO QTY': 'SO_QTY',\n                            'OTHER AREA SHIP DIV': 'OTHER_AREA_SHIP_DIV',\n                            'ALLOCATION QTY': 'ALLOCATION_QTY',\n                            'SUPPLIER INV AMOUNT': 'SUPPLIER_INV_AMOUNT',\n                            'PACKING RANK': 'PACKING_RANK',\n                            'PRODUCT ATTRIBUTION': 'PRODUCT_ATTRIBUTION',\n                            'SPECIAL DIV': 'SPECIALDIV',\n                            'LOGICAL PLANT': 'LOGICAL_PLANT',\n                            'PURCHASE AMOUNT': 'PURCHASE_AMOUNT',\n                            'DIRECT SHIP FLG': 'DIRECT_SHIP_FLG',\n                            'Ship Mode': 'SHIP_MODE',\n                            'SHIP DECISION NO': 'SHIP_DECISION_NO',\n                            'PACK QTY': 'PACK_QTY',\n                            'WEIGHT PER PIECE': 'WEIGHT_PER_PIECE'})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:13.830528Z","iopub.execute_input":"2025-06-17T15:47:13.830814Z","iopub.status.idle":"2025-06-17T15:47:14.077477Z","shell.execute_reply.started":"2025-06-17T15:47:13.830793Z","shell.execute_reply":"2025-06-17T15:47:14.076616Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing values","metadata":{}},{"cell_type":"code","source":"missing_cols = data.isna().sum()[data.isna().sum() > 0]\nmissing_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:14.078870Z","iopub.execute_input":"2025-06-17T15:47:14.079106Z","iopub.status.idle":"2025-06-17T15:47:15.233167Z","shell.execute_reply.started":"2025-06-17T15:47:14.079088Z","shell.execute_reply":"2025-06-17T15:47:15.232214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"**Missing ratio**\")\nfor col in missing_cols.index:\n  missing_ratio = data[col].isna().mean() * 100\n  dtype = data[col].dtype\n  print(f\"{col:<33}- {dtype} - \\t: {missing_ratio:.5f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.234320Z","iopub.execute_input":"2025-06-17T15:47:15.234671Z","iopub.status.idle":"2025-06-17T15:47:15.311779Z","shell.execute_reply.started":"2025-06-17T15:47:15.234641Z","shell.execute_reply":"2025-06-17T15:47:15.310817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = data.dropna(subset=['SHIP_MODE', 'SUPPLIER_DIV', 'CONSIDER_COUNT_HOLIDAY_SATURDAY'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.312854Z","iopub.execute_input":"2025-06-17T15:47:15.313187Z","iopub.status.idle":"2025-06-17T15:47:15.659019Z","shell.execute_reply.started":"2025-06-17T15:47:15.313159Z","shell.execute_reply":"2025-06-17T15:47:15.658161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.OTHER_AREA_SHIP_DIV.fillna(0, inplace=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.659881Z","iopub.execute_input":"2025-06-17T15:47:15.660193Z","iopub.status.idle":"2025-06-17T15:47:15.675024Z","shell.execute_reply.started":"2025-06-17T15:47:15.660160Z","shell.execute_reply":"2025-06-17T15:47:15.673791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Duplicated values","metadata":{}},{"cell_type":"code","source":"(data['SPECIAL_DIV'] == data['SPECIALDIV']).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.676037Z","iopub.execute_input":"2025-06-17T15:47:15.676373Z","iopub.status.idle":"2025-06-17T15:47:15.688118Z","shell.execute_reply.started":"2025-06-17T15:47:15.676346Z","shell.execute_reply":"2025-06-17T15:47:15.687245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(data.STOCK_CLASS == data.PRODUCT_ATTRIBUTION).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.688959Z","iopub.execute_input":"2025-06-17T15:47:15.689261Z","iopub.status.idle":"2025-06-17T15:47:15.706381Z","shell.execute_reply.started":"2025-06-17T15:47:15.689234Z","shell.execute_reply":"2025-06-17T15:47:15.705594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.drop(columns=['SPECIALDIV', 'STOCK_CLASS', 'SUBSIDIARY_CD'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.707452Z","iopub.execute_input":"2025-06-17T15:47:15.708193Z","iopub.status.idle":"2025-06-17T15:47:15.945077Z","shell.execute_reply.started":"2025-06-17T15:47:15.708170Z","shell.execute_reply":"2025-06-17T15:47:15.944122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.duplicated().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:15.946183Z","iopub.execute_input":"2025-06-17T15:47:15.946419Z","iopub.status.idle":"2025-06-17T15:47:17.618497Z","shell.execute_reply.started":"2025-06-17T15:47:15.946400Z","shell.execute_reply":"2025-06-17T15:47:17.617746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.drop(columns=['GLOBAL_NO', 'INNER_CD', 'PRODUCT_CD', 'SHIP_DECISION_NO', 'SALES_ORDER_LINE_NUMBER'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:17.619619Z","iopub.execute_input":"2025-06-17T15:47:17.619952Z","iopub.status.idle":"2025-06-17T15:47:17.755515Z","shell.execute_reply.started":"2025-06-17T15:47:17.619927Z","shell.execute_reply":"2025-06-17T15:47:17.754587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.info()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:17.756510Z","iopub.execute_input":"2025-06-17T15:47:17.756850Z","iopub.status.idle":"2025-06-17T15:47:18.070055Z","shell.execute_reply.started":"2025-06-17T15:47:17.756824Z","shell.execute_reply":"2025-06-17T15:47:18.069139Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# OUTLIERS","metadata":{}},{"cell_type":"code","source":"def z_score_outlier_detection(data, columns, threshold=3):\n  outliers_dict = {}\n  for column in columns:\n    Z_scores = (data[column] - np.mean(data[column])) / np.std(data[column])\n    outliers = np.where(np.abs(Z_scores) > threshold)[0]\n    outliers_dict[column] = outliers\n    total_outliers = len(outliers)\n    print(f\"Total outliers {column:<40}: {total_outliers}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:18.071025Z","iopub.execute_input":"2025-06-17T15:47:18.071276Z","iopub.status.idle":"2025-06-17T15:47:18.076860Z","shell.execute_reply.started":"2025-06-17T15:47:18.071257Z","shell.execute_reply":"2025-06-17T15:47:18.075821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_cols = data.select_dtypes(include=['number']).columns\nz_score_outlier_detection(data, numeric_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:18.077787Z","iopub.execute_input":"2025-06-17T15:47:18.078085Z","iopub.status.idle":"2025-06-17T15:47:18.456516Z","shell.execute_reply.started":"2025-06-17T15:47:18.078058Z","shell.execute_reply":"2025-06-17T15:47:18.455485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.loc[data['SO_QTY'] > 400, 'SO_QTY'] = int(data['SO_QTY'].mean())\ndata.loc[data['ALLOCATION_QTY'] > 400, 'ALLOCATION_QTY'] = int(data['ALLOCATION_QTY'].mean())\ndata.loc[data['SUPPLIER_INV_AMOUNT'] > 94000, 'SUPPLIER_INV_AMOUNT'] = int(data['SUPPLIER_INV_AMOUNT'].mean())\ndata.loc[data['PURCHASE_AMOUNT'] > 94000, 'PURCHASE_AMOUNT'] = int(data['PURCHASE_AMOUNT'].mean())\ndata.loc[data['PACK_QTY'] > 300, 'PACK_QTY'] = int(data['PACK_QTY'].mean())\ndata.loc[data['WEIGHT_PER_PIECE'] >18000, 'WEIGHT_PER_PIECE'] = int(data['WEIGHT_PER_PIECE'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:18.457857Z","iopub.execute_input":"2025-06-17T15:47:18.458201Z","iopub.status.idle":"2025-06-17T15:47:18.511963Z","shell.execute_reply.started":"2025-06-17T15:47:18.458169Z","shell.execute_reply":"2025-06-17T15:47:18.510815Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# CATEGORY","metadata":{}},{"cell_type":"code","source":"cat_features = ['BRAND_CD', 'CLASSIFY_CD', 'SUPPLIER_CD']\n\nfor col in cat_features:\n  data[col] = data[col].astype('category')\n  counts = data[col].value_counts()\n  threshold = len(data) * 0.02\n  common_data = counts[counts > threshold].index\n  data[col] = data[col].apply(lambda x: x if x in common_data else 'Other').astype('category')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:18.513023Z","iopub.execute_input":"2025-06-17T15:47:18.513312Z","iopub.status.idle":"2025-06-17T15:47:18.945940Z","shell.execute_reply.started":"2025-06-17T15:47:18.513288Z","shell.execute_reply":"2025-06-17T15:47:18.945167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['CUST_CD'] = data['CUST_CD'].astype('category')\ncounts = data['CUST_CD'].value_counts()\nthreshold = 10000\ncommon_data = counts[counts > threshold].index\ndata['CUST_CD'] = data['CUST_CD'].apply(lambda x: x if x in common_data else 'Other').astype('category')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:18.951235Z","iopub.execute_input":"2025-06-17T15:47:18.951513Z","iopub.status.idle":"2025-06-17T15:47:19.299873Z","shell.execute_reply.started":"2025-06-17T15:47:18.951491Z","shell.execute_reply":"2025-06-17T15:47:19.299091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_cols = ['CLASSIFY_CD', 'CUST_CD', 'BRAND_CD', 'SUPPLIER_CD', 'DIRECT_SHIP_FLG',\n                   'OTHER_AREA_SHIP_DIV', 'PRODUCT_ATTRIBUTION', 'LOGICAL_PLANT',\n                   'SHIP_MODE', 'DELI_DIV', 'PACKING_RANK', 'SO_DAY_OF_MONTH',\n                   'SO_DAY_OF_WEEK', 'SUPPLIER_DIV', 'SPECIAL_DIV']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:19.300683Z","iopub.execute_input":"2025-06-17T15:47:19.300917Z","iopub.status.idle":"2025-06-17T15:47:19.305420Z","shell.execute_reply.started":"2025-06-17T15:47:19.300900Z","shell.execute_reply":"2025-06-17T15:47:19.304473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.SUPPLIER_DIV.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:19.306314Z","iopub.execute_input":"2025-06-17T15:47:19.306562Z","iopub.status.idle":"2025-06-17T15:47:19.338585Z","shell.execute_reply.started":"2025-06-17T15:47:19.306543Z","shell.execute_reply":"2025-06-17T15:47:19.337927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"one_hot_cols = ['CLASSIFY_CD', 'CUST_CD', 'SUPPLIER_CD', 'LOGICAL_PLANT',\n                'SHIP_MODE', 'PACKING_RANK', 'DELI_DIV', 'SUPPLIER_DIV']\n\ndata = pd.get_dummies(data, columns=one_hot_cols, drop_first=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:19.339573Z","iopub.execute_input":"2025-06-17T15:47:19.339895Z","iopub.status.idle":"2025-06-17T15:47:19.840269Z","shell.execute_reply.started":"2025-06-17T15:47:19.339864Z","shell.execute_reply":"2025-06-17T15:47:19.839371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['BRAND_CD'] = (data['BRAND_CD'] == 'MSM1').astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:19.841260Z","iopub.execute_input":"2025-06-17T15:47:19.841721Z","iopub.status.idle":"2025-06-17T15:47:19.850915Z","shell.execute_reply.started":"2025-06-17T15:47:19.841668Z","shell.execute_reply":"2025-06-17T15:47:19.850027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:19.851836Z","iopub.execute_input":"2025-06-17T15:47:19.852087Z","iopub.status.idle":"2025-06-17T15:47:20.001770Z","shell.execute_reply.started":"2025-06-17T15:47:19.852066Z","shell.execute_reply":"2025-06-17T15:47:20.000953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = data.drop(columns=['label'])\nY_train = data['label']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:20.002923Z","iopub.execute_input":"2025-06-17T15:47:20.003239Z","iopub.status.idle":"2025-06-17T15:47:20.088255Z","shell.execute_reply.started":"2025-06-17T15:47:20.003210Z","shell.execute_reply":"2025-06-17T15:47:20.087326Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TEST DATA","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/PILOT_10.csv', na_values=[' ', '', '   '], low_memory=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:20.089090Z","iopub.execute_input":"2025-06-17T15:47:20.089368Z","iopub.status.idle":"2025-06-17T15:47:21.307663Z","shell.execute_reply.started":"2025-06-17T15:47:20.089341Z","shell.execute_reply":"2025-06-17T15:47:21.306795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"id = test['ID']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.308817Z","iopub.execute_input":"2025-06-17T15:47:21.309085Z","iopub.status.idle":"2025-06-17T15:47:21.313964Z","shell.execute_reply.started":"2025-06-17T15:47:21.309065Z","shell.execute_reply":"2025-06-17T15:47:21.312929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.314801Z","iopub.execute_input":"2025-06-17T15:47:21.315046Z","iopub.status.idle":"2025-06-17T15:47:21.422711Z","shell.execute_reply.started":"2025-06-17T15:47:21.315028Z","shell.execute_reply":"2025-06-17T15:47:21.421796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = ship_days_expect(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.423898Z","iopub.execute_input":"2025-06-17T15:47:21.424227Z","iopub.status.idle":"2025-06-17T15:47:21.463699Z","shell.execute_reply.started":"2025-06-17T15:47:21.424196Z","shell.execute_reply":"2025-06-17T15:47:21.462950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.drop(columns=['ID', 'Order date', 'SUBSIDIARY_CD', 'Stock class', 'VSD', 'GLOBAL_NO', \n                   'INNER_CD', 'PRODUCT_CD', 'SHIP DECISION NO', 'Sales order line number', \n                   'SPECIAL DIV','REASON_CD', 'SOUF_RCV_NO', 'QTUF_RCV_NO'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.464844Z","iopub.execute_input":"2025-06-17T15:47:21.465265Z","iopub.status.idle":"2025-06-17T15:47:21.486492Z","shell.execute_reply.started":"2025-06-17T15:47:21.465241Z","shell.execute_reply":"2025-06-17T15:47:21.485680Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = test.rename(columns={'Consider count hodiday Saturday': 'CONSIDER_COUNT_HOLIDAY_SATURDAY',\n                            # 'Sales order line number': 'SALES_ORDER_LINE_NUMBER',\n                            'SO QTY': 'SO_QTY',\n                            'OTHER AREA SHIP DIV': 'OTHER_AREA_SHIP_DIV',\n                            'ALLOCATION QTY': 'ALLOCATION_QTY',\n                            'SUPPLIER INV AMOUNT': 'SUPPLIER_INV_AMOUNT',\n                            'PACKING RANK': 'PACKING_RANK',\n                            'PRODUCT ATTRIBUTION': 'PRODUCT_ATTRIBUTION',\n                            'LOGICAL PLANT': 'LOGICAL_PLANT',\n                            'PURCHASE AMOUNT': 'PURCHASE_AMOUNT',\n                            'DIRECT SHIP FLG': 'DIRECT_SHIP_FLG',\n                            'Ship Mode': 'SHIP_MODE',\n                            # 'SHIP DECISION NO': 'SHIP_DECISION_NO',\n                            'PACK QTY': 'PACK_QTY',\n                            'WEIGHT PER PIECE': 'WEIGHT_PER_PIECE'})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.487405Z","iopub.execute_input":"2025-06-17T15:47:21.487674Z","iopub.status.idle":"2025-06-17T15:47:21.513331Z","shell.execute_reply.started":"2025-06-17T15:47:21.487653Z","shell.execute_reply":"2025-06-17T15:47:21.512345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.OTHER_AREA_SHIP_DIV.fillna(0, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.514809Z","iopub.execute_input":"2025-06-17T15:47:21.515630Z","iopub.status.idle":"2025-06-17T15:47:21.521370Z","shell.execute_reply.started":"2025-06-17T15:47:21.515604Z","shell.execute_reply":"2025-06-17T15:47:21.520615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.get_dummies(test, columns=one_hot_cols, drop_first=False)\ntest = test.reindex(columns=X_train.columns, fill_value=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:21.522241Z","iopub.execute_input":"2025-06-17T15:47:21.522568Z","iopub.status.idle":"2025-06-17T15:47:24.555900Z","shell.execute_reply.started":"2025-06-17T15:47:21.522539Z","shell.execute_reply":"2025-06-17T15:47:24.555058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['BRAND_CD'] = (test['BRAND_CD'] == 'MSM1').astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:24.556724Z","iopub.execute_input":"2025-06-17T15:47:24.557001Z","iopub.status.idle":"2025-06-17T15:47:24.573748Z","shell.execute_reply.started":"2025-06-17T15:47:24.556981Z","shell.execute_reply":"2025-06-17T15:47:24.572784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:24.574672Z","iopub.execute_input":"2025-06-17T15:47:24.574932Z","iopub.status.idle":"2025-06-17T15:47:24.619234Z","shell.execute_reply.started":"2025-06-17T15:47:24.574913Z","shell.execute_reply":"2025-06-17T15:47:24.618167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if list(X_train.columns) == list(test.columns):\n    print(\"True\")\nelse:\n    print(\"False\")\n    print(\"X_test.columns:\", list(X_train.columns))\n    print(\"test.columns   :\", list(test.columns))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:24.620095Z","iopub.execute_input":"2025-06-17T15:47:24.620317Z","iopub.status.idle":"2025-06-17T15:47:24.626433Z","shell.execute_reply.started":"2025-06-17T15:47:24.620299Z","shell.execute_reply":"2025-06-17T15:47:24.625321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MODELING","metadata":{}},{"cell_type":"markdown","source":"**XGBOOST**","metadata":{}},{"cell_type":"code","source":"# pip install xgboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:24.627308Z","iopub.execute_input":"2025-06-17T15:47:24.627526Z","iopub.status.idle":"2025-06-17T15:47:24.644359Z","shell.execute_reply.started":"2025-06-17T15:47:24.627508Z","shell.execute_reply":"2025-06-17T15:47:24.643534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\n\nmodelXGB = xgb.XGBClassifier(\n    enable_categorical=True,\n    n_estimators=100,\n    max_depth=3,\n    learning_rate=0.1,\n    use_label_encoder=False,\n    eval_metric='logloss'\n)\n\nmodelXGB.fit(X_train, Y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:24.645296Z","iopub.execute_input":"2025-06-17T15:47:24.645675Z","iopub.status.idle":"2025-06-17T15:47:41.534680Z","shell.execute_reply.started":"2025-06-17T15:47:24.645644Z","shell.execute_reply":"2025-06-17T15:47:41.533625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_XGB = modelXGB.predict(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:41.535665Z","iopub.execute_input":"2025-06-17T15:47:41.535997Z","iopub.status.idle":"2025-06-17T15:47:42.366190Z","shell.execute_reply.started":"2025-06-17T15:47:41.535974Z","shell.execute_reply":"2025-06-17T15:47:42.365552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_XGB = pd.DataFrame(Y_pred_XGB, columns=['label'])\nY_pred_XGB['ID'] = id\nY_pred_XGB = Y_pred_XGB[['ID', 'label']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:42.367398Z","iopub.execute_input":"2025-06-17T15:47:42.367702Z","iopub.status.idle":"2025-06-17T15:47:42.375291Z","shell.execute_reply.started":"2025-06-17T15:47:42.367681Z","shell.execute_reply":"2025-06-17T15:47:42.374618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_XGB['label'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:42.376521Z","iopub.execute_input":"2025-06-17T15:47:42.377133Z","iopub.status.idle":"2025-06-17T15:47:42.397652Z","shell.execute_reply.started":"2025-06-17T15:47:42.377110Z","shell.execute_reply":"2025-06-17T15:47:42.396712Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**LIGHTGBM**","metadata":{}},{"cell_type":"code","source":"# !pip install lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:42.398679Z","iopub.execute_input":"2025-06-17T15:47:42.398995Z","iopub.status.idle":"2025-06-17T15:47:42.416036Z","shell.execute_reply.started":"2025-06-17T15:47:42.398964Z","shell.execute_reply":"2025-06-17T15:47:42.415026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\n\nmodelLGB = lgb.LGBMClassifier(\n    n_estimators=100,\n    learning_rate=0.1,\n    # class_weight={0: 2, 1: 5},\n    num_leaves=31,\n    random_state=42\n)\nmodelLGB.fit(X_train, Y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:42.417110Z","iopub.execute_input":"2025-06-17T15:47:42.417478Z","iopub.status.idle":"2025-06-17T15:47:57.496284Z","shell.execute_reply.started":"2025-06-17T15:47:42.417455Z","shell.execute_reply":"2025-06-17T15:47:57.495384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_LGB=modelLGB.predict(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:57.497087Z","iopub.execute_input":"2025-06-17T15:47:57.497713Z","iopub.status.idle":"2025-06-17T15:47:57.903597Z","shell.execute_reply.started":"2025-06-17T15:47:57.497678Z","shell.execute_reply":"2025-06-17T15:47:57.902671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_LGB = pd.DataFrame(Y_pred_LGB, columns=['label'])\nY_pred_LGB['ID'] = id\nY_pred_LGB = Y_pred_LGB[['ID', 'label']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:57.904610Z","iopub.execute_input":"2025-06-17T15:47:57.904880Z","iopub.status.idle":"2025-06-17T15:47:57.911792Z","shell.execute_reply.started":"2025-06-17T15:47:57.904860Z","shell.execute_reply":"2025-06-17T15:47:57.910975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_LGB['label'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:57.912793Z","iopub.execute_input":"2025-06-17T15:47:57.913101Z","iopub.status.idle":"2025-06-17T15:47:57.932684Z","shell.execute_reply.started":"2025-06-17T15:47:57.913074Z","shell.execute_reply":"2025-06-17T15:47:57.931824Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**RANDOM FOREST**","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nmodelRF = RandomForestClassifier(\n    n_estimators=100, \n    max_depth=None, \n    random_state=42\n)\n\nmodelRF.fit(X_train, Y_train)\n\nY_pred_RF = modelRF.predict(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:47:57.934174Z","iopub.execute_input":"2025-06-17T15:47:57.934806Z","iopub.status.idle":"2025-06-17T15:54:00.928840Z","shell.execute_reply.started":"2025-06-17T15:47:57.934775Z","shell.execute_reply":"2025-06-17T15:54:00.927829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_RF = pd.DataFrame(Y_pred_RF, columns=['label'])\nY_pred_RF['ID'] = id\nY_pred_RF = Y_pred_RF[['ID', 'label']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:54:00.929989Z","iopub.execute_input":"2025-06-17T15:54:00.930264Z","iopub.status.idle":"2025-06-17T15:54:00.939393Z","shell.execute_reply.started":"2025-06-17T15:54:00.930242Z","shell.execute_reply":"2025-06-17T15:54:00.938245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_RF['label'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:54:00.940496Z","iopub.execute_input":"2025-06-17T15:54:00.940841Z","iopub.status.idle":"2025-06-17T15:54:00.965105Z","shell.execute_reply.started":"2025-06-17T15:54:00.940817Z","shell.execute_reply":"2025-06-17T15:54:00.964135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred_RF.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:54:00.966133Z","iopub.execute_input":"2025-06-17T15:54:00.966386Z","iopub.status.idle":"2025-06-17T15:54:01.097099Z","shell.execute_reply.started":"2025-06-17T15:54:00.966366Z","shell.execute_reply":"2025-06-17T15:54:01.095967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\n# Confusion matrix\ncm = confusion_matrix(Y_pred_RF['label'], Y_pred_LGB['label'])\nplt.figure(figsize=(6, 4))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Oranges', cbar=False)\nplt.title('Confusion Matrix - LightGBM')\nplt.xlabel('Dự đoán')\nplt.ylabel('Thực tế')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T15:59:39.284600Z","iopub.execute_input":"2025-06-17T15:59:39.286052Z","iopub.status.idle":"2025-06-17T15:59:39.431877Z","shell.execute_reply.started":"2025-06-17T15:59:39.286010Z","shell.execute_reply":"2025-06-17T15:59:39.430835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\n# Confusion matrix\ncm = confusion_matrix(Y_pred_RF['label'], Y_pred_XGB['label'])\nplt.figure(figsize=(6, 4))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Oranges', cbar=False)\nplt.title('Confusion Matrix - LightGBM')\nplt.xlabel('Dự đoán')\nplt.ylabel('Thực tế')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:06:45.010653Z","iopub.execute_input":"2025-06-17T16:06:45.011644Z","iopub.status.idle":"2025-06-17T16:06:45.129127Z","shell.execute_reply.started":"2025-06-17T16:06:45.011609Z","shell.execute_reply":"2025-06-17T16:06:45.128196Z"}},"outputs":[],"execution_count":null}]}