{"metadata":{"kernelspec":{"display_name":"base","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.7"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":103436,"databundleVersionId":12476472,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# file dealy và not_delay_7_9\ndata_df=pd.read_csv(\"delay_and_not_7_9.csv\")\nprint(data_df.dtypes)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.shape","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.columns","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.sample(100)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.info()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.isnull().sum()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.duplicated().sum()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df['Order date']=pd.to_datetime(data_df['Order date'], errors= 'coerce')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_data_df= data_df.select_dtypes(include=[\"float64\",\"int64\"])\ncorr_matrix = numeric_data_df.corr()\ncorr_matrix","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,8))\nplt.imshow(corr_matrix, cmap='coolwarm', interpolation='nearest')\nplt.colorbar()\nplt.xticks(ticks=np.arange(len(corr_matrix.columns)), labels=corr_matrix.columns, rotation=90)\nplt.yticks(ticks=np.arange(len(corr_matrix.columns)), labels=corr_matrix.columns)\nplt.title('Correlation Matrix')\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = numeric_data_df.corr()\ncorr_unstack= corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1).astype(bool))\nhigh_corr=corr_unstack.stack().reset_index()\nhigh_corr.columns=['feature 1','feature 2','Correlation']\nhigh_corr=high_corr[high_corr['Correlation'].abs()>0.5]\nhigh_corr","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df = data_df.rename(columns={'Order date': 'Order_date'})\ndata_df = data_df.rename(columns={'DIRECT SHIP FLG': 'DIRECT_SHIP_FLG'})\ndata_df = data_df.rename(columns={'Sales order line number': 'Sales_order_line_number'})\ndata_df = data_df.rename(columns={'SO QTY': 'SO_QTY'})\ndata_df = data_df.rename(columns={'Consider count hodiday Saturday': 'Consider_count_hodiday_Saturday'})","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df=data_df.drop(['PRODUCT ATTRIBUTION','ALLOCATION QTY', 'SPECIAL DIV','PURCHASE AMOUNT','CLASSIFY_CD','SUPPLIER_DIV'],axis=1)\nprint(data_df.shape)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.isnull().sum()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df=data_df.drop(['SOUF_RCV_NO','QTUF_RCV_NO','OTHER AREA SHIP DIV'],axis=1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_data_df= data_df.select_dtypes(include= ['object'])\ncategorical_data_df = categorical_data_df.dropna() ","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_data_df.dtypes","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for column in categorical_data_df.columns:\n categorical_data_df[column] = categorical_data_df[column].astype('category')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_data_df.dtypes","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.stats import chi2_contingency\n\nresults = []\nselected_columns=[column for column in categorical_data_df.columns if categorical_data_df[column].nunique()<50]\nchi2_matrix=pd.DataFrame(np.zeros((len(selected_columns), len(selected_columns))),\n columns=selected_columns,\n index=selected_columns)\nfor i, column1 in enumerate(selected_columns):\n for j, column2 in enumerate(selected_columns[i+1:]):\n frequency=pd.crosstab(categorical_data_df[column1], categorical_data_df[column2])\n chi2, p, dof, expected = chi2_contingency(frequency)\n if p < 0.05:\n results.append((column1, column2, chi2, p))\n chi2_matrix.iloc[i, j+i+1] = chi2\n chi2_matrix.iloc[j+i+1, i] = chi2 \nresults_df=pd.DataFrame(results, columns=['Column1', 'Column2', 'Chi2', 'p_value'])\nsignificant_relations = results_df.sort_values('p_value')\nsignificant_relations","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 10))\nsns.heatmap(chi2_matrix, cmap='coolwarm', fmt='.2f', xticklabels=chi2_matrix.columns, yticklabels=chi2_matrix.columns)\nplt.title('Chi2 Square Matrix')\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Xử lý Missing values","metadata":{}},{"cell_type":"code","source":"ship_mode=data_df['Ship Mode'].mode()[0]\ndata_df['Ship Mode'].replace(['NaN', ' ', ''], np.nan, inplace=True)\ndata_df['Ship Mode']=data_df['Ship Mode'].fillna(ship_mode)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.isnull().sum()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df['SHIP DECISION NO']","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df=data_df.drop(columns='REASON_CD', axis=1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"median_ship_decision=data_df['SHIP DECISION NO'].median()\ndata_df['SHIP DECISION NO']=data_df['SHIP DECISION NO'].fillna(median_ship_decision)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.isnull().sum()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df = data_df.rename(columns={'WEIGHT PER PIECE': 'WEIGHT_PER_PIECE'})\ndata_df = data_df.rename(columns={'PACK QTY': 'PACK_QTY'})\ndata_df = data_df.rename(columns={'Stock class': 'Stock_class'})\ndata_df = data_df.rename(columns={'SUPPLIER INV AMOUNT': 'SUPPLIER_INV_AMOUNT'})\ndata_df = data_df.rename(columns={'PACKING RANK': 'PACKING_RANK'})\ndata_df = data_df.rename(columns={'LOGICAL PLANT': 'LOGICAL_PLANT'})\ndata_df = data_df.rename(columns={'Ship Mode': 'Ship_Mode'})\ndata_df = data_df.rename(columns={'SHIP DECISION NO': 'SHIP_DECISION_NO'})\ndata_df = data_df.rename(columns={'PACK QTY': 'PACK_QTY'})\ndata_df = data_df.rename(columns={'WEIGHT PER PIECE': 'WEIGHT_PER_PIECE'})","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Xử lý giá trị ngoại lai","metadata":{}},{"cell_type":"code","source":"numeric_data_df=numeric_data_df.drop(columns='label')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outlier={}\nnumeric_data_df=[col for col in numeric_data_df if col in data_df.columns]\n\nfor column in numeric_data_df:\n Q1=data_df[column].quantile(0.25)\n Q3=data_df[column].quantile(0.75)\n IQR=Q3-Q1\n lower_bound=Q1-1.5*IQR\n upper_bound=Q3+1.5*IQR\n\n outliers=data_df[(data_df[column]upper_bound)]\n outlier_percentage=len(outliers)/len(data_df)*100\n outlier[column]={\n 'Number Outliers':len(outliers),\n 'Outlier Percentage':outlier_percentage\n }\noutlier_summary=pd.DataFrame(outlier).T\nprint(outlier_summary.sort_values('Number Outliers', ascending=False))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 10))\nsns.barplot(data=outlier_summary.sort_values('Number Outliers', ascending=False),\n x='Number Outliers', y=outlier_summary.index,\n palette='Reds_r')\nplt.title('Số lượng giá trị ngoại lai theo thuộc tính')\nplt.xlabel('Số lượng ngoại lai')\nplt.ylabel('Thuộc tính')\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for column in numeric_data_df:\n Q1=data_df[column].quantile(0.25)\n Q3=data_df[column].quantile(0.75)\n IQR=Q3-Q1\n lower_bound=Q1-1.5*IQR\n upper_bound=Q3+1.5*IQR\n outliers=data_df[(data_df[column]upper_bound)]\n outlier_percentage=len(outliers)/len(data_df)*100\n if outlier_percentage>0.1:\n lower_threshold = data_df[column].quantile(0.05)\n upper_threshold = data_df[column].quantile(0.95)\n data_df[column] = data_df[column].clip(lower=lower_threshold, upper=upper_threshold)\n elif outlier_percentage >0.05 and outlier_percentage<0.1:\n data_df[column]=np.log1p(data_df[column])\nprint(data_df.head())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Chuẩn hóa ","metadata":{}},{"cell_type":"code","source":"data_df['label'].value_counts()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n#chuan hoa du lieu nhieu ngoai lai\nscaler=MinMaxScaler() \nnumeric_data_df=data_df.select_dtypes(include=['float64', 'int64']).columns.tolist()\ndata_df[numeric_data_df]=scaler.fit_transform(data_df[numeric_data_df])\nprint(data_df.head())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Encoding label","metadata":{}},{"cell_type":"code","source":"categorical_data_df.columns","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df['label'].value_counts()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.columns","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder\n\ncategorical_column= ['SUBSIDIARY_CD', 'GLOBAL_NO', 'BRAND_CD', 'INNER_CD', 'SUPPLIER_CD',\n 'PACKING_RANK', 'PRODUCT_CD', 'VSD', 'DELI_DIV', 'Ship_Mode'\n]\ndata_df[categorical_column]=data_df[categorical_column].astype(str)\nencoder=OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)\ndata_df[categorical_column]=encoder.fit_transform(data_df[categorical_column])\ndata_encoded=pd.concat([data_df[categorical_column]], axis=1)\nprint(data_encoded.head())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df.describe()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_df=data_df.drop(columns=['SPECIAL_DIV','SUBSIDIARY_CD','SPECIAL_DIV','SUBSIDIARY_CD','Stock_class','SHIP_DECISION_NO','PACK_QTY','SPECIAL_DIV'], axis=1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom xgboost import XGBClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import f1_score","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drop_columns = ['label','ID', 'Order_date', 'GLOBAL_NO', 'WEIGHT_UNIT','HAZARD_FLG','SPECIFY_SHIP_DAYS','PRODUCT_ASSORT'] \nx = data_df.drop(columns=drop_columns, errors='ignore')\ny = data_df['label']\n\nle_dict = {}\nfor col in x.select_dtypes(include='object').columns:\n le = LabelEncoder()\n x[col] = le.fit_transform(x[col].astype(str))\n le_dict[col] = le\n\n# chia train/test\nx_train, x_test, y_train, y_test = train_test_split(x, y, stratify=y, test_size=0.2, random_state=42)\nprint('Train:', x_train.shape)\nprint('Test:', x_test.shape)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Train label distribution:\")\nprint(y_train.value_counts(normalize=True)) \nprint(\"\\test label distribution:\")\nprint(y_test.value_counts(normalize=True))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\nsmote = SMOTE(random_state=42)\nx_train, y_train = smote.fit_resample(x_train, y_train)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"neg = (y_train == 0).sum()\npos = (y_train == 1).sum()\nratio = neg / pos\nmodel = XGBClassifier(scale_pos_weight=ratio, random_state=42)\nmodel.fit(x_train, y_train)\ny_train_pred = model.predict(x_train)\ny_test_pred = model.predict(x_test)\n\nprint(\"F1-Score train:\", f1_score(y_train, y_train_pred))\nprint(\"F1-Score val :\", f1_score(y_test, y_test_pred))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Total samples:\", len(x))\nprint(\"Unique labels:\", y.nunique())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_columns = x.columns.tolist()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# nghiệm thu","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(\"PILOT_10.csv\")\ndf_test['Order date']=pd.to_datetime(df_test['Order date'], errors= 'coerce')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = df_test.rename(columns={'Order date': 'Order_date'})\ndf_test = df_test.rename(columns={'DIRECT SHIP FLG': 'DIRECT_SHIP_FLG'})\ndf_test = df_test.rename(columns={'Sales order line number': 'Sales_order_line_number'})\ndf_test = df_test.rename(columns={'SO QTY': 'SO_QTY'})\ndf_test = df_test.rename(columns={'Consider count hodiday Saturday': 'Consider_count_hodiday_Saturday'})\ndf_test=df_test.drop(['PRODUCT ATTRIBUTION','ALLOCATION QTY', 'SPECIAL DIV','PURCHASE AMOUNT','CLASSIFY_CD','SUPPLIER_DIV'],axis=1)\ndf_test=df_test.drop(['SOUF_RCV_NO','QTUF_RCV_NO','OTHER AREA SHIP DIV'],axis=1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_df_test= df_test.select_dtypes(include= ['object'])\ncategorical_df_test = categorical_df_test.dropna() \ncategorical_df_test.dtypes\nfor column in categorical_df_test.columns:\n categorical_df_test[column] = categorical_df_test[column].astype('category')\ncategorical_df_test.dtypes\nresults = []\nselected_columns=[column for column in categorical_df_test.columns if categorical_df_test[column].nunique()<50]\n\nchi2_matrix=pd.DataFrame(np.zeros((len(selected_columns), len(selected_columns))),\n columns=selected_columns,\n index=selected_columns)\nfor i, column1 in enumerate(selected_columns):\n for j, column2 in enumerate(selected_columns[i+1:]):\n frequency=pd.crosstab(categorical_df_test[column1], categorical_df_test[column2])\n chi2, p, dof, expected = chi2_contingency(frequency)\n if p < 0.05:\n results.append((column1, column2, chi2, p))\n chi2_matrix.iloc[i, j+i+1] = chi2\n chi2_matrix.iloc[j+i+1, i] = chi2 \nresults_df=pd.DataFrame(results, columns=['Column1', 'Column2', 'Chi2', 'p_value'])\nsignificant_relations = results_df.sort_values('p_value')\nsignificant_relations","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ship_mode=df_test['Ship Mode'].mode()[0]\ndf_test['Ship Mode'].replace(['NaN', ' ', ''], np.nan, inplace=True)\ndf_test['Ship Mode']=df_test['Ship Mode'].fillna(ship_mode)\ndf_test=df_test.drop(columns='REASON_CD', axis=1)\nmedian_ship_decision=df_test['SHIP DECISION NO'].median()\ndf_test['SHIP DECISION NO']=df_test['SHIP DECISION NO'].fillna(median_ship_decision)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = df_test.rename(columns={'WEIGHT PER PIECE': 'WEIGHT_PER_PIECE'})\ndf_test = df_test.rename(columns={'PACK QTY': 'PACK_QTY'})\ndf_test = df_test.rename(columns={'Stock class': 'Stock_class'})\ndf_test = df_test.rename(columns={'SUPPLIER INV AMOUNT': 'SUPPLIER_INV_AMOUNT'})\ndf_test = df_test.rename(columns={'PACKING RANK': 'PACKING_RANK'})\ndf_test = df_test.rename(columns={'LOGICAL PLANT': 'LOGICAL_PLANT'})\ndf_test = df_test.rename(columns={'Ship Mode': 'Ship_Mode'})\ndf_test = df_test.rename(columns={'SHIP DECISION NO': 'SHIP_DECISION_NO'})\ndf_test = df_test.rename(columns={'PACK QTY': 'PACK_QTY'})\ndf_test = df_test.rename(columns={'WEIGHT PER PIECE': 'WEIGHT_PER_PIECE'})\n\noutlier={}\nnumeric_data_df=[col for col in numeric_data_df if col in df_test.columns]\nfor column in numeric_data_df:\n Q1=df_test[column].quantile(0.25)\n Q3=df_test[column].quantile(0.75)\n IQR=Q3-Q1\n lower_bound=Q1-1.5*IQR\n upper_bound=Q3+1.5*IQR\n outliers=df_test[(df_test[column]upper_bound)]\n outlier_percentage=len(outliers)/len(df_test)*100\n outlier[column]={\n 'Number Outliers':len(outliers),\n 'Outlier Percentage':outlier_percentage\n }\noutlier_summary=pd.DataFrame(outlier).T\nprint(outlier_summary.sort_values('Number Outliers', ascending=False))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for column in numeric_data_df:\n Q1=df_test[column].quantile(0.25)\n Q3=df_test[column].quantile(0.75)\n IQR=Q3-Q1\n lower_bound=Q1-1.5*IQR\n upper_bound=Q3+1.5*IQR\n outliers=df_test[(df_test[column]upper_bound)]\n outlier_percentage=len(outliers)/len(df_test)*100\n if outlier_percentage>0.1:\n lower_threshold = df_test[column].quantile(0.05)\n upper_threshold = df_test[column].quantile(0.95)\n df_test[column] = df_test[column].clip(lower=lower_threshold, upper=upper_threshold)\n elif outlier_percentage >0.05 and outlier_percentage<0.1:\n df_test[column]=np.log1p(df_test[column])\nprint(df_test.head())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler=MinMaxScaler() \nnumeric_data_df = df_test.select_dtypes(include=['float64', 'int64']).drop(columns=['ID'], errors='ignore').columns.tolist()\ndf_test[numeric_data_df]=scaler.fit_transform(df_test[numeric_data_df])\nprint(df_test.head())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_column= ['SUBSIDIARY_CD', 'GLOBAL_NO', 'BRAND_CD', 'INNER_CD', 'SUPPLIER_CD',\n 'PACKING_RANK', 'PRODUCT_CD', 'VSD', 'DELI_DIV', 'Ship_Mode'\n]\ndf_test[categorical_column]=df_test[categorical_column].astype(str)\nencoder=OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)\ndf_test[categorical_column]=encoder.fit_transform(df_test[categorical_column])\ndata_encoded=pd.concat([df_test[categorical_column]], axis=1)\nprint(data_encoded.head())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = df_test.rename(columns={'Order date': 'Order_date'})\ndf_test = df_test.rename(columns={'DIRECT SHIP FLG': 'DIRECT_SHIP_FLG'})\ndf_test = df_test.rename(columns={'Sales order line number': 'Sales_order_line_number'})\ndf_test = df_test.rename(columns={'SO QTY': 'SO_QTY'})\ndf_test = df_test.rename(columns={'Consider count hodiday Saturday': 'Consider_count_hodiday_Saturday'})","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test=df_test.drop(columns=['SPECIAL_DIV','SUBSIDIARY_CD','Stock_class','SHIP_DECISION_NO','PACK_QTY','SPECIAL_DIV'], axis=1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drop_columns = ['ID', 'Order_date', 'GLOBAL_NO', 'WEIGHT_UNIT','HAZARD_FLG','SPECIFY_SHIP_DAYS','PRODUCT_ASSORT'] \nx_pilot = df_test.drop(columns=[col for col in drop_columns if col in df_test.columns], errors='ignore').copy()\nfor col in x_pilot.select_dtypes(include='object').columns:\n if col in le_dict:\n le = le_dict[col]\n x_pilot[col] = x_pilot[col].map(\n lambda x: le.transform([x])[0] if x in le.classes_ else -1\n )\n else:\n x_pilot[col] = -1 \nfor col in train_columns:\n if col not in x_pilot.columns:\n x_pilot[col] = -1\nx_pilot = x_pilot[train_columns] \ny_pred_pilot = model.predict(x_pilot).astype(int)\nresult = pd.DataFrame({\n 'ID': df_test['ID'],\n 'label': y_pred_pilot\n})\nresult","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result.to_csv(\"submission.csv\", index=False)","metadata":{},"outputs":[],"execution_count":null}]}