{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-04T11:58:33.792Z","iopub.execute_input":"2021-10-04T11:58:33.792441Z","iopub.status.idle":"2021-10-04T11:58:33.806299Z","shell.execute_reply.started":"2021-10-04T11:58:33.792346Z","shell.execute_reply":"2021-10-04T11:58:33.805358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. 課題\n* 特徴量ベクトルの次元数が多い\n* スパース（NULL値が多い）\n* 異常・正常のデータ割合が極端に不均衡（異常検知問題）\n* 評価関数が特殊(マシューズ相関係数)\n\n### 2. 解決案\n* 次元の削減\n    * S32のデータのみを用いた特徴量ベクトルの作成\n    * 情報量の少ないランダム or 特定の値に偏った属性削除\n    * 決定木ベースで重要度の高い特徴量のみ採用\n    * PCA\n* カテゴリ変数の数値化＋低ビット化(必要最低限の低ビットな型指定での読み込み)\n* XGBoost, LightGBMなどの利用（スパースなデータに強い）※Lasso回帰によるスパース推定は？\n* 学習データを分割して複数モデルを生成し、アンサンブルモデルを作成\n* オンライン学習の検討（特にカテゴリー属性の部分）\n\n### 3. ベースライン開発\n* データローダー：正常データ100000件＋異常データ6000件(時系列データは除く)\n* モデル：LightGBM\n* チューニング：Baysian Optimization\n* 評価指標：マシューズ相関係数\n\n### 4. 課題\n* 次元数が多く情報量のあるデータが少ないため、モデルがオーバーフィッティングしてしまう。  \n(学習データでは高精度が出てもテストデータでは全く精度が出ない)\n\n### 5. TODO\n* 不要な変数の削減による次元削除（S32を中心とした特徴量ベクトルの構成）\n* Adversarial Validation\n    - 訓練データとテストデータの分布を比較\n    - テストデータに類似のバリデーションデータを作ってモデルを構築\n* EDAによる不具合データと正常データ間の特徴比較＋特徴量の検討\n\n<br>","metadata":{}},{"cell_type":"code","source":"# Seaborn and matplotlib\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Plotly library\nfrom plotly.offline import init_notebook_mode, iplot\nimport plotly.graph_objs as go\nfrom plotly import tools\n\n# Sklearn and lightgbm\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgbm\n\n# Set some configurations\nimport gc\nimport time\nimport sys\n\n# Import MCC and F1 score\nfrom sklearn.metrics import matthews_corrcoef\nfrom sklearn.metrics import f1_score\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:58:33.809275Z","iopub.execute_input":"2021-10-04T11:58:33.809747Z","iopub.status.idle":"2021-10-04T11:58:36.296558Z","shell.execute_reply.started":"2021-10-04T11:58:33.809692Z","shell.execute_reply":"2021-10-04T11:58:36.295515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# 学習データとテストデータに含まれるすべてのカテゴリ変数。合計107個(NaN除く)。\n# nan=0として、0～108の値にマッピングし、int8で取り込む形にする。\nall_cats = ['NaN','T-18748192','T-2147481664','T-21474819','T-2147482176','T-2147482432',\n            'T-21474825','T-2147482688','T-2147482816','T-2147482944','T-2147483646',\n            'T-2147483647','T-2147483648','T-21474872','T1','T11141888','T1132',\n            'T113776','T1152','T12','T12582912','T128','T1310','T132','T134217728',\n            'T1372','T143','T145','T16','T16384','T16512','T16777216','T16777232',\n            'T16777248','T16777472','T16777557','T16779428','T16793941','T178258',\n            'T18436','T2','T24','T2516','T25165824','T256','T262144','T262656',\n            'T268','T26808','T268435456','T3','T32','T32896','T331648','T33554432',\n            'T33554448','T33554944','T36992','T393216','T3942','T4','T41944',\n            'T4325376','T43968','T4718592','T48','T48576','T488','T492','T5','T512',\n            'T514','T518','T52','T524288', 'T524544','T544','T55424','T56',\n            'T589824','T6','T618624','T63616','T64','T6553','T65536','T678864',\n            'T7','T748928','T7808','T786432','T786944','T8','T83888','T8389632',\n            'T8651776','T86752','T8768','T8912896','T9','T917','T9174','T9175552',\n            'T91764','T96','T96112','T97','T98']\n\n\nle = LabelEncoder()\nle = le.fit(all_cats)\n\ndef transformCategoricalDF(df, le):\n    matrix = []\n    for row in df.itertuples(name=None, index=None):\n        #print(row)\n        row = le.transform(row)\n        matrix.append(row)\n        del row\n    out_df = pd.DataFrame(matrix,columns=df.columns).astype('int8')\n    return out_df\n    ","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:58:36.299095Z","iopub.execute_input":"2021-10-04T11:58:36.299531Z","iopub.status.idle":"2021-10-04T11:58:36.313481Z","shell.execute_reply.started":"2021-10-04T11:58:36.299478Z","shell.execute_reply":"2021-10-04T11:58:36.312417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s32_cat_cols = ['Id','L3_S32_F3851','L3_S32_F3853','L3_S32_F3854']\ns32_num_cols = ['Id','Response','L3_S32_F3850']\ns32_date_cols = ['Id','L3_S32_D3852']","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:58:36.316114Z","iopub.execute_input":"2021-10-04T11:58:36.316743Z","iopub.status.idle":"2021-10-04T11:58:36.335634Z","shell.execute_reply.started":"2021-10-04T11:58:36.316691Z","shell.execute_reply":"2021-10-04T11:58:36.334256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# メモリサイズ表示関数\ndef getMmSize(obj):\n    print(str(round(sys.getsizeof(obj)/1000000,2))+' MB')\n\n# 数値データフレームの圧縮関数\ndef compressNumDF(df):\n    id_r_df = df.loc[:,['Id','Response']]\n    num_df = df.drop(['Id','Response'],axis=1).astype('float16')\n    df = pd.concat([id_r_df, num_df],axis=1)\n    return df\n\ndef lgb_mcc_score(y_hat, data):\n    y_true = data.get_label() # ground truth\n    y_hat = np.round(y_hat) # prediction\n    return 'mcc', matthews_corrcoef(y_true, y_hat), True\n\n# def lgb_mcc(preds, dtrain):\n#     THRESHOLD = 0.5\n#     labels = dtrain.get_label()\n#     return 'mcc', matthews_corrcoef(labels, preds >= THRESHOLD)","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:58:36.337519Z","iopub.execute_input":"2021-10-04T11:58:36.337852Z","iopub.status.idle":"2021-10-04T11:58:36.348656Z","shell.execute_reply.started":"2021-10-04T11:58:36.337818Z","shell.execute_reply":"2021-10-04T11:58:36.347829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sys.exit()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:58:36.350483Z","iopub.execute_input":"2021-10-04T11:58:36.351117Z","iopub.status.idle":"2021-10-04T11:58:36.366683Z","shell.execute_reply.started":"2021-10-04T11:58:36.351071Z","shell.execute_reply":"2021-10-04T11:58:36.365537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# 学習データの読み込み処理（Station32に関連する特徴量に限定）\n#\nstart_tm=time.time()\n# Load categorical training data\ntrain_cat = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip', dtype=str,usecols=s32_cat_cols, nrows=300000, low_memory=False)\ntrain_num = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip', nrows=300000,usecols=s32_num_cols,low_memory=False)\ntrain_date = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip', nrows=300000,usecols=s32_date_cols,low_memory=False)\n#result_df_reader = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',usecols=['Id','Response'], chunksize=100000)\nend_tm=time.time()\n\nelapse_time = end_tm - start_tm\nprint (\"elapsed_time:{0}\".format(elapse_time) + \"[sec]\")\n","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:58:36.367982Z","iopub.execute_input":"2021-10-04T11:58:36.368323Z","iopub.status.idle":"2021-10-04T11:59:24.868086Z","shell.execute_reply.started":"2021-10-04T11:58:36.368289Z","shell.execute_reply":"2021-10-04T11:59:24.866961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_date.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:59:24.869552Z","iopub.execute_input":"2021-10-04T11:59:24.86987Z","iopub.status.idle":"2021-10-04T11:59:24.894567Z","shell.execute_reply.started":"2021-10-04T11:59:24.869838Z","shell.execute_reply":"2021-10-04T11:59:24.893547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# カテゴリ変数データのラベルエンコーディング＋圧縮処理\n#\ntmp_df=train_cat.drop(['Id'],axis=1).fillna('NaN') # Id列の削除＋null値にNaNを設定\ntrain_cat = transformCategoricalDF(tmp_df, le)","metadata":{"execution":{"iopub.status.busy":"2021-10-04T11:59:24.895942Z","iopub.execute_input":"2021-10-04T11:59:24.896291Z","iopub.status.idle":"2021-10-04T12:00:22.769823Z","shell.execute_reply.started":"2021-10-04T11:59:24.896256Z","shell.execute_reply":"2021-10-04T12:00:22.768561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([train_cat,train_num,train_date.drop(['Id'],axis=1)], axis=1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:00:22.772779Z","iopub.execute_input":"2021-10-04T12:00:22.773203Z","iopub.status.idle":"2021-10-04T12:00:22.79961Z","shell.execute_reply.started":"2021-10-04T12:00:22.773152Z","shell.execute_reply":"2021-10-04T12:00:22.798521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:00:22.801496Z","iopub.execute_input":"2021-10-04T12:00:22.801824Z","iopub.status.idle":"2021-10-04T12:00:22.808744Z","shell.execute_reply.started":"2021-10-04T12:00:22.801791Z","shell.execute_reply":"2021-10-04T12:00:22.807553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデル構築ルーチン","metadata":{}},{"cell_type":"code","source":"# 学習データをラベルとデータ本体に分離\ny = train_df['Response']\nX = train_df.drop(['Id','Response'],axis=1)\n\nlgb_params = {\"objective\":\"binary\", \n              #\"metric\":\"binary_logloss\",\n              \"metric\": \"None\", # MCCで評価するために\"None\"を設定する必要あり\n              \"num_iterations\":100,\n              \"learning_rate\":0.1,\n              \"num_leaves\":31,\n              \"seed\":134,\n              \"max_depth\":5,\n              \"min_data_in_leaf\":20,\n              \"early_stopping_round\":100,\n              \"verbosity\": -1}\nevals_result = {}\n\n# 学習用データと検証用データに分割する\nX_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True, random_state=42)\n\n#print(X_train);\n#print(y_train);\n#LightGBM のデータセットの表現に直す\nlgb_train = lgbm.Dataset(X_train, y_train)\nlgb_eval = lgbm.Dataset(X_test, y_test, reference=lgb_train)\n\nmodel = lgbm.train(lgb_params,\n            lgb_train,\n            # メトリックを追跡する対象のデータセット\n            valid_sets=[lgb_eval, lgb_train],\n            # 上記の名前\n            valid_names=['eval', 'train'],\n            num_boost_round=200,\n            # メトリックの履歴を残すオブジェクト\n            evals_result=evals_result,\n            # 独自メトリックを計算する関数(MCCを設定)\n            feval=lgb_mcc_score\n          )","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:00:22.811045Z","iopub.execute_input":"2021-10-04T12:00:22.811663Z","iopub.status.idle":"2021-10-04T12:01:14.671731Z","shell.execute_reply.started":"2021-10-04T12:00:22.811612Z","shell.execute_reply":"2021-10-04T12:01:14.669721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\n\ndef get_evaluate(y_test, predict):\n\n    fpr, tpr, thr_arr = metrics.roc_curve(y_test, predict)\n\n    auc = metrics.auc(fpr, tpr)\n    precision = metrics.precision_score(y_test, predict)\n    recall = metrics.recall_score(y_test, predict)      \n\n    return auc, precision, recall\n\npredict_proba = model.predict(X_test, num_iteration=model.best_iteration)\n# predict_probaが0.5以上なら1とする\npredict = [0 if i < 0.5 else 1 for i in predict_proba]\n\nauc, precision, recall = get_evaluate(y_test, predict)\n\nprint('AUC:{}, precision:{}, recall:{}'.format(auc, precision, recall))","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:01:14.673423Z","iopub.execute_input":"2021-10-04T12:01:14.67387Z","iopub.status.idle":"2021-10-04T12:01:15.06613Z","shell.execute_reply.started":"2021-10-04T12:01:14.673822Z","shell.execute_reply":"2021-10-04T12:01:15.064944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df = pd.DataFrame({'Response':predict})\npredict_df.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:01:15.067755Z","iopub.execute_input":"2021-10-04T12:01:15.06842Z","iopub.status.idle":"2021-10-04T12:01:15.11878Z","shell.execute_reply.started":"2021-10-04T12:01:15.068372Z","shell.execute_reply":"2021-10-04T12:01:15.11787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 特徴量重要度の算出 (データフレームで取得)\ncols = list(X.columns)         # 特徴量名のリスト\nf_importance = np.array(model.feature_importance()) # 特徴量重要度の算出\nf_importance = f_importance / np.sum(f_importance)  # 正規化(必要ない場合はコメントアウト)\ndf_importance = pd.DataFrame({'feature':cols, 'importance':f_importance})\ndf_importance = df_importance.sort_values('importance', ascending=False) # 降順ソート\n\ndf_importance.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:01:15.120304Z","iopub.execute_input":"2021-10-04T12:01:15.120637Z","iopub.status.idle":"2021-10-04T12:01:15.137215Z","shell.execute_reply.started":"2021-10-04T12:01:15.120605Z","shell.execute_reply":"2021-10-04T12:01:15.136072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s32_cat_cols = ['Id','L3_S32_F3851','L3_S32_F3853','L3_S32_F3854']\ns32_num_cols = ['L3_S32_F3850']\ns32_date_cols = ['L3_S32_D3852']\n\n# テストデータの読み込みとモデルによる評価結果の作成\n# 10000行ずつ読み込んで、モデルで正常・異常判定を行い、結果を出力用のDataFrameに結合\ndf_cat_reader = pd.read_csv('../input/bosch-production-line-performance/test_categorical.csv.zip', dtype=\"str\", usecols=s32_cat_cols, chunksize=10000)\ndf_num_reader = pd.read_csv('../input/bosch-production-line-performance/test_numeric.csv.zip',usecols=s32_num_cols, chunksize=10000)\ndf_date_reader = pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',usecols=s32_date_cols, chunksize=10000)\n# 出力用DataFrame\ndf_test_out = pd.DataFrame()\ni=0\nj=0\nk=0\nfor df_cat in df_cat_reader:\n    j=j+1\n    print(\"Iteration j={0}\".format(j))\n    for df_num in df_num_reader:\n        k=k+1\n        print(\"Iteration k={0}\".format(k))\n        for df_date in df_date_reader:\n            df_num = df_num.reset_index(drop=True)\n            df_date = df_date.reset_index(drop=True)\n            i = i+1\n            print(\"Iteration {0}\".format(i))\n            start_tm=time.time()\n            tmp_df = df_cat['Id'].reset_index(drop=True)\n            df_a = df_cat.drop(['Id'],axis=1).fillna('NaN')\n            end_tm=time.time()\n            print(\"elapsed time:{0}\".format(end_tm-start_tm))\n            start_tm=time.time()\n            df_b = transformCategoricalDF(df_a, le)\n            end_tm=time.time()\n            print(\"elapsed time:{0}\".format(end_tm-start_tm))\n            start_tm=time.time()\n            test_df = pd.concat([df_b,df_num,df_date],axis=1)\n            predict_proba = model.predict(test_df, num_iteration=model.best_iteration)\n            # predict_probaが0.5以上なら1とする\n            predict = [0 if i < 0.03 else 1 for i in predict_proba]\n            predict_df = pd.DataFrame({'Response':predict})\n            tmp_df = pd.concat([tmp_df,predict_df],axis=1)\n            print(tmp_df.shape)\n            df_test_out = pd.concat([df_test_out,tmp_df])\n            end_tm=time.time()\n            print(\"elapsed time:{0}\".format(end_tm-start_tm))\n            del tmp_df, df_a, df_b, test_df, predict_df\n            break;\n        break;\n\n# Indexを修正\ndf_test_out = df_test_out.reset_index(drop=True)\n\n# 提出用のCSVファイルに結果を保存\ndf_test_out.to_csv(\"submission.csv\", index=False)\n\n#del df_test_out\n#gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:01:15.13893Z","iopub.execute_input":"2021-10-04T12:01:15.139417Z","iopub.status.idle":"2021-10-04T12:07:05.757979Z","shell.execute_reply.started":"2021-10-04T12:01:15.139369Z","shell.execute_reply":"2021-10-04T12:07:05.756712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_tm=time.time()\n\nelapse_time = end_tm - start_tm\nprint (\"elapsed_time:{0}\".format(elapse_time) + \"[sec]\")\n\nsys.exit()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:07:05.759665Z","iopub.execute_input":"2021-10-04T12:07:05.760009Z","iopub.status.idle":"2021-10-04T12:07:05.769808Z","shell.execute_reply.started":"2021-10-04T12:07:05.759972Z","shell.execute_reply":"2021-10-04T12:07:05.767825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_out['Response'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:07:05.771228Z","iopub.status.idle":"2021-10-04T12:07:05.771941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_tm=time.time()\n\nelapse_time = end_tm - start_tm\nprint (\"elapsed_time:{0}\".format(elapse_time) + \"[sec]\")\n\nsys.exit()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:07:05.773595Z","iopub.status.idle":"2021-10-04T12:07:05.774043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# --- 学習結果をグラフで描画 ---\nplt.figure(figsize=(8,6))\ntrain_metric = evals_result['train']['mcc']\nplt.plot(train_metric, label='train mcc')\neval_metric = evals_result['eval']['mcc']\nplt.plot(eval_metric, label='eval mcc')\nplt.grid()\nplt.legend()\nplt.xlabel('rounds')\nplt.ylabel('mcc')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:07:05.776715Z","iopub.status.idle":"2021-10-04T12:07:05.777172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.exit()","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:07:05.779583Z","iopub.status.idle":"2021-10-04T12:07:05.780471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## テストデータのラベルエンコーディングデータ生成＋保存\nテストデータのラベルエンコーディングに時間がかかるため、エンコード処理したデータを利用する形にする。(処理時間：約120分)","metadata":{}},{"cell_type":"code","source":"# 10000行ずつ読み込んで、モデルで正常・異常判定を行い、結果を出力用のDataFrameに結合\ndf_reader = pd.read_csv('../input/bosch-production-line-performance/test_categorical.csv.zip', dtype=\"str\", chunksize=10000)\n# 出力用DataFrame\ndf_test_out = pd.DataFrame()\ni=0\nfor df in df_reader:\n    i = i+1\n    start_tm=time.time()\n    tmp_df = df['Id'].reset_index(drop=True)\n    df_a = df.drop(['Id'],axis=1).fillna('NaN')\n    df_b = transformCategoricalDF(df_a, le)\n    tmp_df = pd.concat([tmp_df,df_b],axis=1)\n    df_test_out = pd.concat([df_test_out,tmp_df])\n    end_tm=time.time()\n    print(\"Iter{0}: {1} sec\".format(i, round(end_tm-start_tm,2)), end=\", \")\n    del tmp_df, df_a, df_b\n    # if i==2: break;\n\n# Indexを修正\ndf_test_out = df_test_out.reset_index(drop=True)\n\n\n# 提出用のCSVファイルに結果を保存, 2.6GB\n#df_test_out.to_csv(\"test_cat_encoded.csv\", index=False)\ndf_test_out.to_csv(\"test_cat_encoded.csv.zip\", index=False, compression=\"zip\")","metadata":{"execution":{"iopub.status.busy":"2021-10-04T12:07:05.781911Z","iopub.status.idle":"2021-10-04T12:07:05.782603Z"},"trusted":true},"execution_count":null,"outputs":[]}]}