{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-25T17:53:33.780316Z","iopub.execute_input":"2022-04-25T17:53:33.780927Z","iopub.status.idle":"2022-04-25T17:53:33.790044Z","shell.execute_reply.started":"2022-04-25T17:53:33.78087Z","shell.execute_reply":"2022-04-25T17:53:33.788784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Challenge \n* Many dimensions of feature vector \n* Sparse (many NULL values) \n* Extremely imbalanced abnormal / normal data ratio (anomaly detection problem) \n* Special evaluation function (Matthews correlation coefficient) ) \n\n### 2. Solution \n* Dimension reduction \n* Creation of feature vector using only S32 data \n* Random with less information or attribute deletion biased to a specific value \n* Features with high importance based on decision tree Only the amount is adopted \n* PCA \n* Numericalization of category variables + low bit (reading with the minimum required low-bit type specification) \n* Use of XGBoost, LightGBM, etc. (strong against sparse data) \n* Sparse estimation by Lasso regression is ?? \n* Divide the training data to generate multiple models and create an ensemble model \n* Study online learning (especially the category attribute part) \n\n### 3. Baseline development \n* Data loader: 100,000 normal data + 6000 abnormal data (Excluding time series data) \n* Model: LightGBM \n* Tuning: Baysian Optimization * Evaluation index: Matthews correlation coefficient \n\n\n### 4. Challenge \n* Since there are many dimensions and there is little data with a large amount of information, the model is overfitted. It ends up. (Even if the training data is highly accurate, the test data is not accurate at all) \n\n\n### 5. TODO \n* Dimension deletion by reducing unnecessary variables (construction of feature quantity vector centered on S32) \n* Adversarial Validation- Compare the distribution of training data and test data-Create a model by creating validation data similar to the test data \n* EDA-based feature comparison between defect data and normal data + study of feature quantities","metadata":{}},{"cell_type":"code","source":"# Seaborn and matplotlib\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Plotly library\nfrom plotly.offline import init_notebook_mode, iplot\nimport plotly.graph_objs as go\nfrom plotly import tools\n\n# Sklearn and lightgbm\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgbm\nimport sklearn\nfrom sklearn.tree import DecisionTreeClassifier\n\n# Set some configurations\nimport gc\nimport time\nimport sys\n\n# Import MCC and F1 score\nfrom sklearn.metrics import matthews_corrcoef\nfrom sklearn.metrics import f1_score\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:53:33.79204Z","iopub.execute_input":"2022-04-25T17:53:33.792772Z","iopub.status.idle":"2022-04-25T17:53:33.803619Z","shell.execute_reply.started":"2022-04-25T17:53:33.79271Z","shell.execute_reply":"2022-04-25T17:53:33.802579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# All categorical variables contained in the training and test data. 107 in total (excluding NaN). \n#Set nan = 0, map to a value from 0 to 108, and import with int8.\n\nall_cats = ['NaN','T-18748192','T-2147481664','T-21474819','T-2147482176','T-2147482432',\n            'T-21474825','T-2147482688','T-2147482816','T-2147482944','T-2147483646',\n            'T-2147483647','T-2147483648','T-21474872','T1','T11141888','T1132',\n            'T113776','T1152','T12','T12582912','T128','T1310','T132','T134217728',\n            'T1372','T143','T145','T16','T16384','T16512','T16777216','T16777232',\n            'T16777248','T16777472','T16777557','T16779428','T16793941','T178258',\n            'T18436','T2','T24','T2516','T25165824','T256','T262144','T262656',\n            'T268','T26808','T268435456','T3','T32','T32896','T331648','T33554432',\n            'T33554448','T33554944','T36992','T393216','T3942','T4','T41944',\n            'T4325376','T43968','T4718592','T48','T48576','T488','T492','T5','T512',\n            'T514','T518','T52','T524288', 'T524544','T544','T55424','T56',\n            'T589824','T6','T618624','T63616','T64','T6553','T65536','T678864',\n            'T7','T748928','T7808','T786432','T786944','T8','T83888','T8389632',\n            'T8651776','T86752','T8768','T8912896','T9','T917','T9174','T9175552',\n            'T91764','T96','T96112','T97','T98']\n\n\nle = LabelEncoder()\nle = le.fit(all_cats)\n\ndef transformCategoricalDF(df, le):\n    matrix = []\n    for row in df.itertuples(name=None, index=None):\n        #print(row)\n        row = le.transform(row)\n        matrix.append(row)\n        del row\n    out_df = pd.DataFrame(matrix,columns=df.columns).astype('int8')\n    return out_df\n    ","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:53:33.844729Z","iopub.execute_input":"2022-04-25T17:53:33.845465Z","iopub.status.idle":"2022-04-25T17:53:33.855387Z","shell.execute_reply.started":"2022-04-25T17:53:33.845425Z","shell.execute_reply":"2022-04-25T17:53:33.854455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s32_cat_cols = ['Id','L3_S32_F3851','L3_S32_F3853','L3_S32_F3854']\ns32_num_cols = ['Id','Response','L3_S32_F3850']\ns32_date_cols = ['Id','L3_S32_D3852']","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:53:33.888795Z","iopub.execute_input":"2022-04-25T17:53:33.889444Z","iopub.status.idle":"2022-04-25T17:53:33.895645Z","shell.execute_reply.started":"2022-04-25T17:53:33.889396Z","shell.execute_reply":"2022-04-25T17:53:33.894282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Memory size display function\ndef getMmSize(obj):\n    print(str(round(sys.getsizeof(obj)/1000000,2))+' MB')\n\n# Numeric data frame compression function\ndef compressNumDF(df):\n    id_r_df = df.loc[:,['Id','Response']]\n    num_df = df.drop(['Id','Response'],axis=1).astype('float16')\n    df = pd.concat([id_r_df, num_df],axis=1)\n    return df\n\ndef lgb_mcc_score(y_hat, data):\n    y_true = data.get_label() # ground truth\n    y_hat = np.round(y_hat) # prediction\n    return 'mcc', matthews_corrcoef(y_true, y_hat), True\n\n# def lgb_mcc(preds, dtrain):\n#     THRESHOLD = 0.5\n#     labels = dtrain.get_label()\n#     return 'mcc', matthews_corrcoef(labels, preds >= THRESHOLD)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:53:33.941894Z","iopub.execute_input":"2022-04-25T17:53:33.9423Z","iopub.status.idle":"2022-04-25T17:53:33.949725Z","shell.execute_reply.started":"2022-04-25T17:53:33.94226Z","shell.execute_reply":"2022-04-25T17:53:33.948274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sys.exit()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:53:33.970562Z","iopub.execute_input":"2022-04-25T17:53:33.971272Z","iopub.status.idle":"2022-04-25T17:53:33.975351Z","shell.execute_reply.started":"2022-04-25T17:53:33.971228Z","shell.execute_reply":"2022-04-25T17:53:33.974453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n#Learning data reading process (limited to features related to Station32)\n#\nstart_tm=time.time()\n# Load categorical training data\ntrain_cat = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip', dtype=str,usecols=s32_cat_cols, nrows=300000, low_memory=False)\ntrain_num = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip', nrows=300000,usecols=s32_num_cols,low_memory=False)\ntrain_date = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip', nrows=300000,usecols=s32_date_cols,low_memory=False)\n#result_df_reader = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',usecols=['Id','Response'], chunksize=100000)\nend_tm=time.time()\n\nelapse_time = end_tm - start_tm\nprint (\"elapsed_time:{0}\".format(elapse_time) + \"[sec]\")\n","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:53:34.019731Z","iopub.execute_input":"2022-04-25T17:53:34.020089Z","iopub.status.idle":"2022-04-25T17:54:11.679098Z","shell.execute_reply.started":"2022-04-25T17:53:34.020056Z","shell.execute_reply":"2022-04-25T17:54:11.678295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_date.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:11.680461Z","iopub.execute_input":"2022-04-25T17:54:11.680974Z","iopub.status.idle":"2022-04-25T17:54:11.692453Z","shell.execute_reply.started":"2022-04-25T17:54:11.680941Z","shell.execute_reply":"2022-04-25T17:54:11.691412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n#Category variable data label encoding + compression processing\n#\ntmp_df=train_cat.drop(['Id'],axis=1).fillna('NaN') #Delete Id column + Set NaN to null value\ntrain_cat = transformCategoricalDF(tmp_df, le)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:11.69419Z","iopub.execute_input":"2022-04-25T17:54:11.694704Z","iopub.status.idle":"2022-04-25T17:54:53.977965Z","shell.execute_reply.started":"2022-04-25T17:54:11.694671Z","shell.execute_reply":"2022-04-25T17:54:53.976607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([train_cat,train_num,train_date.drop(['Id'],axis=1)], axis=1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:53.979947Z","iopub.execute_input":"2022-04-25T17:54:53.980361Z","iopub.status.idle":"2022-04-25T17:54:54.00114Z","shell.execute_reply.started":"2022-04-25T17:54:53.980315Z","shell.execute_reply":"2022-04-25T17:54:53.999851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:54.002599Z","iopub.execute_input":"2022-04-25T17:54:54.003147Z","iopub.status.idle":"2022-04-25T17:54:54.01101Z","shell.execute_reply.started":"2022-04-25T17:54:54.003096Z","shell.execute_reply":"2022-04-25T17:54:54.009705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model building routine\n","metadata":{}},{"cell_type":"code","source":"#Separate training data into label and data body\ny = train_df['Response']\nX = train_df.drop(['Id','Response'],axis=1)\n\nlgb_params = {\"objective\":\"binary\", \n              #\"metric\":\"binary_logloss\",\n              \"metric\": \"None\", #Need to set \"None\" to evaluate with MCC\n              \"num_iterations\":100,\n              \"learning_rate\":0.1,\n              \"num_leaves\":31,\n              \"seed\":134,\n              \"max_depth\":5,\n              \"min_data_in_leaf\":20,\n              \"early_stopping_round\":100,\n              \"verbosity\": -1}\nevals_result = {}\n\n# Divide into training data and verification data\nX_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True, random_state=42)\n\n#print(X_train);\n#print(y_train);\n#LightGBM \nlgb_train = lgbm.Dataset(X_train, y_train)\nlgb_eval = lgbm.Dataset(X_test, y_test, reference=lgb_train)\n\nmodel = lgbm.train(lgb_params,\n            lgb_train,\n# The dataset on which the metric is tracked\n            valid_sets=[lgb_eval, lgb_train],\n# Name above\n            valid_names=['eval', 'train'],\n            num_boost_round=200,\n#Object that keeps the history of metrics\n            evals_result=evals_result,\n# Function to calculate original metric (set MCC)\n            feval=lgb_mcc_score\n          )","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:58:44.475751Z","iopub.execute_input":"2022-04-25T17:58:44.476317Z","iopub.status.idle":"2022-04-25T17:59:16.801262Z","shell.execute_reply.started":"2022-04-25T17:58:44.47627Z","shell.execute_reply":"2022-04-25T17:59:16.800082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:54.072939Z","iopub.execute_input":"2022-04-25T17:54:54.073212Z","iopub.status.idle":"2022-04-25T17:54:54.080294Z","shell.execute_reply.started":"2022-04-25T17:54:54.073186Z","shell.execute_reply":"2022-04-25T17:54:54.079203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:54.083495Z","iopub.execute_input":"2022-04-25T17:54:54.083938Z","iopub.status.idle":"2022-04-25T17:54:54.151362Z","shell.execute_reply.started":"2022-04-25T17:54:54.083891Z","shell.execute_reply":"2022-04-25T17:54:54.149467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\n\ndef get_evaluate(y_test, predict):\n\n    fpr, tpr, thr_arr = metrics.roc_curve(y_test, predict)\n\n    auc = metrics.auc(fpr, tpr)\n    precision = metrics.precision_score(y_test, predict)\n    recall = metrics.recall_score(y_test, predict)      \n\n    return auc, precision, recall\n\npredict_proba = model.predict(X_test, num_iteration=model.best_iteration)\n#If predict_probability is 0.5 or more, set it to 1.\npredict = [0 if i < 0.5 else 1 for i in predict_proba]\n\nauc, precision, recall = get_evaluate(y_test, predict)\n\nprint('AUC:{}, precision:{}, recall:{}'.format(auc, precision, recall))","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:59:27.061005Z","iopub.execute_input":"2022-04-25T17:59:27.061701Z","iopub.status.idle":"2022-04-25T17:59:27.316731Z","shell.execute_reply.started":"2022-04-25T17:59:27.06166Z","shell.execute_reply":"2022-04-25T17:59:27.315583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df = pd.DataFrame({'Response':predict})\npredict_df.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:59:35.291718Z","iopub.execute_input":"2022-04-25T17:59:35.292048Z","iopub.status.idle":"2022-04-25T17:59:35.320818Z","shell.execute_reply.started":"2022-04-25T17:59:35.292019Z","shell.execute_reply":"2022-04-25T17:59:35.319722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Calculation of feature importance (acquired in data frame)\ncols = list(X.columns)         #List of feature names\nf_importance = np.array(model.feature_importance()) #Calculation of feature importance\nf_importance = f_importance / np.sum(f_importance)  #Normalization (comment out if not needed)\ndf_importance = pd.DataFrame({'feature':cols, 'importance':f_importance})\ndf_importance = df_importance.sort_values('importance', ascending=False) # Descending sort\ndf_importance.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:59:36.415383Z","iopub.execute_input":"2022-04-25T17:59:36.415751Z","iopub.status.idle":"2022-04-25T17:59:36.429004Z","shell.execute_reply.started":"2022-04-25T17:59:36.415719Z","shell.execute_reply":"2022-04-25T17:59:36.427885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s32_cat_cols = ['Id','L3_S32_F3851','L3_S32_F3853','L3_S32_F3854']\ns32_num_cols = ['L3_S32_F3850']\ns32_date_cols = ['L3_S32_D3852']\n\n#Read test data and create evaluation results by model\n# Read 10000 lines at a time, judge normal / abnormal in the model, and combine the results into the DataFrame for output.\ndf_cat_reader = pd.read_csv('../input/bosch-production-line-performance/test_categorical.csv.zip', dtype=\"str\", usecols=s32_cat_cols, chunksize=10000)\ndf_num_reader = pd.read_csv('../input/bosch-production-line-performance/test_numeric.csv.zip',usecols=s32_num_cols, chunksize=10000)\ndf_date_reader = pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',usecols=s32_date_cols, chunksize=10000)\n# 出力用DataFrame\ndf_test_out = pd.DataFrame()\ni=0\nj=0\nk=0\nfor df_cat in df_cat_reader:\n    j=j+1\n    print(\"Iteration j={0}\".format(j))\n    for df_num in df_num_reader:\n        k=k+1\n        print(\"Iteration k={0}\".format(k))\n        for df_date in df_date_reader:\n            df_num = df_num.reset_index(drop=True)\n            df_date = df_date.reset_index(drop=True)\n            i = i+1\n            print(\"Iteration {0}\".format(i))\n            start_tm=time.time()\n            tmp_df = df_cat['Id'].reset_index(drop=True)\n            df_a = df_cat.drop(['Id'],axis=1).fillna('NaN')\n            end_tm=time.time()\n            print(\"elapsed time:{0}\".format(end_tm-start_tm))\n            start_tm=time.time()\n            df_b = transformCategoricalDF(df_a, le)\n            end_tm=time.time()\n            print(\"elapsed time:{0}\".format(end_tm-start_tm))\n            start_tm=time.time()\n            test_df = pd.concat([df_b,df_num,df_date],axis=1)\n            predict_proba = model.predict(test_df, num_iteration=model.best_iteration)\n            predict = [0 if i < 0.03 else 1 for i in predict_proba]\n            predict_df = pd.DataFrame({'Response':predict})\n            tmp_df = pd.concat([tmp_df,predict_df],axis=1)\n            print(tmp_df.shape)\n            df_test_out = pd.concat([df_test_out,tmp_df])\n            end_tm=time.time()\n            print(\"elapsed time:{0}\".format(end_tm-start_tm))\n            del tmp_df, df_a, df_b, test_df, predict_df\n            break;\n        break;\n\ndf_test_out = df_test_out.reset_index(drop=True)\ndf_test_out.to_csv(\"submission.csv\", index=False)\n\n#del df_test_out\n#gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:59:37.773301Z","iopub.execute_input":"2022-04-25T17:59:37.773694Z","iopub.status.idle":"2022-04-25T18:03:49.455555Z","shell.execute_reply.started":"2022-04-25T17:59:37.773656Z","shell.execute_reply":"2022-04-25T18:03:49.45455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_tm=time.time()\n\nelapse_time = end_tm - start_tm\nprint (\"elapsed_time:{0}\".format(elapse_time) + \"[sec]\")\n\nsys.exit()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T18:04:01.038346Z","iopub.execute_input":"2022-04-25T18:04:01.038893Z","iopub.status.idle":"2022-04-25T18:04:01.045353Z","shell.execute_reply.started":"2022-04-25T18:04:01.038853Z","shell.execute_reply":"2022-04-25T18:04:01.044184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_out['Response'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T18:04:03.380017Z","iopub.execute_input":"2022-04-25T18:04:03.380716Z","iopub.status.idle":"2022-04-25T18:04:03.403814Z","shell.execute_reply.started":"2022-04-25T18:04:03.380661Z","shell.execute_reply":"2022-04-25T18:04:03.402622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_tm=time.time()\n\nelapse_time = end_tm - start_tm\nprint (\"elapsed_time:{0}\".format(elapse_time) + \"[sec]\")\n\nsys.exit()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T18:04:05.402709Z","iopub.execute_input":"2022-04-25T18:04:05.403357Z","iopub.status.idle":"2022-04-25T18:04:05.409968Z","shell.execute_reply.started":"2022-04-25T18:04:05.403318Z","shell.execute_reply":"2022-04-25T18:04:05.408793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# --- Draw the learning result in a graph ---\nplt.figure(figsize=(8,6))\ntrain_metric = y_test\nplt.plot(train_metric, label='train mcc')\neval_metric = predict\nplt.plot(eval_metric, label='eval mcc')\nplt.grid()\nplt.legend()\nplt.xlabel('rounds')\nplt.ylabel('mcc')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T18:23:32.141274Z","iopub.execute_input":"2022-04-25T18:23:32.14166Z","iopub.status.idle":"2022-04-25T18:23:32.614544Z","shell.execute_reply.started":"2022-04-25T18:23:32.141625Z","shell.execute_reply":"2022-04-25T18:23:32.613405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.exit()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T17:54:54.164843Z","iopub.status.idle":"2022-04-25T17:54:54.165201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test data label encoding Data generation + storage \nSince it takes time to label test data, use the encoded data. (Processing time: about 120 minutes)\n","metadata":{}},{"cell_type":"code","source":"df_reader = pd.read_csv('../input/bosch-production-line-performance/test_categorical.csv.zip', dtype=\"str\", chunksize=10000)\ndf_test_out = pd.DataFrame()\ni=0\nfor df in df_reader:\n    i = i+1\n    start_tm=time.time()\n    tmp_df = df['Id'].reset_index(drop=True)\n    df_a = df.drop(['Id'],axis=1).fillna('NaN')\n    df_b = transformCategoricalDF(df_a, le)\n    tmp_df = pd.concat([tmp_df,df_b],axis=1)\n    df_test_out = pd.concat([df_test_out,tmp_df])\n    end_tm=time.time()\n    print(\"Iter{0}: {1} sec\".format(i, round(end_tm-start_tm,2)), end=\", \")\n    del tmp_df, df_a, df_b\n    # if i==2: break;\ndf_test_out = df_test_out.reset_index(drop=True)\n#df_test_out.to_csv(\"test_cat_encoded.csv\", index=False)\ndf_test_out.to_csv(\"test_cat_encoded.csv.zip\", index=False, compression=\"zip\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T18:04:49.91542Z","iopub.execute_input":"2022-04-25T18:04:49.915821Z","iopub.status.idle":"2022-04-25T18:15:43.702428Z","shell.execute_reply.started":"2022-04-25T18:04:49.915783Z","shell.execute_reply":"2022-04-25T18:15:43.700762Z"},"trusted":true},"execution_count":null,"outputs":[]}]}