{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup #","metadata":{}},{"cell_type":"code","source":"# import sys\n# !cp ../input/rapids/rapids.21.06 /opt/conda/envs/rapids.tar.gz\n# !cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\n# sys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\n# sys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\n# sys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path \n# !cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:21:14.743094Z","iopub.execute_input":"2022-06-22T11:21:14.743444Z","iopub.status.idle":"2022-06-22T11:21:14.769615Z","shell.execute_reply.started":"2022-06-22T11:21:14.743364Z","shell.execute_reply":"2022-06-22T11:21:14.768877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport shutil\nfrom glob import glob\nimport multiprocessing as mp\nimport gc\nfrom pathlib import Path\nfrom scipy import stats\nfrom scipy.special import boxcox, softmax\nfrom scipy import sparse\n\nfrom multiprocessing import cpu_count\nimport copy\nimport pickle\nimport warnings\nfrom datetime import datetime, timedelta\nfrom time import time, sleep, mktime\nfrom matplotlib import font_manager as fm, rc, rcParams\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport re\nimport random as rnd\nimport psutil\nfrom optuna import Trial, create_study\nfrom optuna.samplers import TPESampler\n\nimport numpy as np\nfrom numpy import array, nan, random as np_rnd, where\nimport pandas as pd\nfrom pandas import DataFrame as dataframe, Series as series, isna, read_csv\nfrom pandas.tseries.offsets import DateOffset\n\nfrom sklearn.model_selection import train_test_split as tts, StratifiedKFold, StratifiedShuffleSplit\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, MinMaxScaler, RobustScaler, KBinsDiscretizer\nfrom sklearn import metrics\nfrom sklearn.compose import ColumnTransformer\n# config_missingpy(); from missingpy import MissForest\nfrom sklearn.impute import KNNImputer\nfrom optuna import Trial, create_study\nfrom sklearn.model_selection import GroupKFold, GroupShuffleSplit, StratifiedGroupKFold\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\ntry:\n    import cudf as cd\n    import cupy as cp\n    from cuml.cluster import KMeans\n    from cuml.neighbors import NearestNeighbors\n    from cuml.metrics.cluster import silhouette_score\nexcept:\n    print(\"RAPIDS Import ERROR\")\n\nimport catboost as cat\nimport xgboost as xgb\n\n# # ===== tensorflow =====\n# import tensorflow as tf\n# from tensorflow import random as tf_rnd\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.models import Sequential\n# from tensorflow.keras import layers\n# from tensorflow.keras import activations\n# from tensorflow.keras import optimizers\n# from tensorflow.keras import metrics as tf_metrics\n# from tensorflow.keras import callbacks as tf_callbacks\n# from tqdm.keras import TqdmCallback\n# import tensorflow_addons as tfa\n# from tensorflow.keras.utils import plot_model\n# from keras.utils.layer_utils import count_params\n\n# import keras_tuner as kt\n# from keras_tuner import HyperModel\n# import tensorflow_hub as tf_hub\n# import tensorflow_recommenders as tfrs\n\n# # GPU check\n# if tf.test.gpu_device_name() != '/device:GPU:0':\n#     print('GPU device not found')\n# else:\n#     print('Found GPU')\n\n# # GPU memory setting\n# gpus = tf.config.list_physical_devices('GPU')\n# if gpus:\n#   try:\n#     tf.config.experimental.set_memory_growth(gpus[0], True)\n#   except RuntimeError as e:\n#     print(e)\n\nwarnings.filterwarnings(action='ignore')\nrcParams['axes.unicode_minus'] = False\npd.set_option('display.max_columns', 100)\npd.set_option('display.max_rows', 100)\npd.set_option('display.width', 1000)\npd.set_option('max_colwidth', 200)\n# plt.rc('font', family='NanumSquareB')\n\ndata_dir = Path('../input/AI4Code')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:21:14.771267Z","iopub.execute_input":"2022-06-22T11:21:14.771597Z","iopub.status.idle":"2022-06-22T11:21:20.607424Z","shell.execute_reply.started":"2022-06-22T11:21:14.771562Z","shell.execute_reply":"2022-06-22T11:21:20.606625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ===== utility functions =====\n# label encoding for categorical column with excepting na value\ndef seed_everything(seed=42):\n    # python random module\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n#     # tf random\n#     tf_rnd.set_seed(seed)\n    # RAPIDS random\n    try:\n        cp.random.seed(seed)\n    except:\n        pass\ndef which(bool_list):\n    return where(bool_list)[0]\ndef easyIO(x=None, path=None, op=\"r\"):\n    tmp = None\n    if op == \"r\":\n        with open(path, \"rb\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    elif op == \"w\":\n        with open(path, \"wb\") as f:\n            pickle.dump(x, f)\n    else:\n        print(\"Unknown operation type\")\ndef diff(first, second):\n    second = set(second)\n    return [item for item in first if item not in second]\ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\ndef orderElems(for_order, using_ref):\n    return [i for i in using_ref if i in for_order]\n# concatenate by row\ndef cbr(df1, df2):\n    if type(df1) == series:\n        tmp_concat = series(pd.concat([dataframe(df1), dataframe(df2)], axis=0, ignore_index=True).iloc[:,0])\n        tmp_concat.reset_index(drop=True, inplace=True)\n    elif type(df1) == dataframe:\n        tmp_concat = pd.concat([df1, df2], axis=0, ignore_index=True)\n        tmp_concat.reset_index(drop=True, inplace=True)\n    elif type(df1) == np.ndarray:\n        tmp_concat = np.concatenate([df1, df2], axis=0)\n    else:\n        print(\"Unknown Type: return 1st argument\")\n        tmp_concat = df1\n    return tmp_concat\ndef change_width(ax, new_value):\n    for patch in ax.patches :\n        current_width = patch.get_width()\n        adj_value = current_width - new_value\n        # we change the bar width\n        patch.set_width(new_value)\n        # we recenter the bar\n        patch.set_x(patch.get_x() + adj_value * .5)\ndef week_of_month(date):\n    month = date.month\n    week = 0\n    while date.month == month:\n        week += 1\n        date -= timedelta(days=7)\n    return week\ndef getSeason(date):\n    month = date.month\n    if month in [3, 4, 5]:\n        return \"Spring\"\n    elif month in [6, 7, 8]:\n        return \"Summer\"\n    elif month in [9, 10, 11]:\n        return \"Fall\"\n    else:\n        return \"Winter\"\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n# def softmax(x):\n#     max = np.max(x, axis=1, keepdims=True)  # returns max of each row and keeps same dims\n#     e_x = np.exp(x - max)  # subtracts each row with its max value\n#     sum = np.sum(e_x, axis=1, keepdims=True)  # returns sum of each row and keeps same dims\n#     f_x = e_x / sum\n#     return f_x\ndef sigmoid(x):\n    return 1/(1 + np.exp(-x))\ndef dispPerformance(result_dic):\n    perf_table = dataframe()\n    index_names = []\n    for k, v in result_dic.items():\n        index_names.append(k)\n        perf_table = pd.concat([perf_table, series(v[\"performance\"]).to_frame().T], ignore_index=True, axis=0)\n    perf_table.index = index_names\n    perf_table.sort_values(perf_table.columns[0], inplace=True)\n    print(perf_table)\n    return perf_table\ndef powspace(start, stop, power, num):\n    start = np.power(start, 1/float(power))\n    stop = np.power(stop, 1/float(power))\n    return np.power(np.linspace(start, stop, num=num), power)\ndef xgb_custom_lossfunction(alpha = 1):\n    def support_under_mse(label, pred):\n        # grad : 1차 미분\n        # hess : 2차 미분\n        residual = (label - pred).astype(\"float\")\n        grad = np.where(residual > 0, -2 * alpha * residual, -2 * residual)\n        hess = np.where(residual > 0, 2 * alpha, 2.0)\n        return grad, hess\n    return support_under_mse\ndef pd_flatten(df):\n    df = df.unstack()\n    df.index = [str(i) + \"_\" + str(j) for i, j in df.index]\n    return df\ndef tf_losses_rmse(y_true, y_pred, sample_weight=None):\n    return tf.sqrt(tf.reduce_mean((y_true - y_pred) ** 2)) if sample_weight is None else tf.sqrt(tf.reduce_mean(((y_true - y_pred) ** 2) * sample_weight))\ndef tf_loss_nmae(y_true, y_pred, sample_weight=False):\n    mae = tf.reduce_mean(tf.math.abs(y_true - y_pred))\n    score = tf.math.divide(mae, tf.reduce_mean(tf.math.abs(y_true)))\n    return score\ndef text_extractor(string, lang=\"eng\", spacing=True):\n    # # 괄호를 포함한 괄호 안 문자 제거 정규식\n    # re.sub(r'\\([^)]*\\)', '', remove_text)\n    # # <>를 포함한 <> 안 문자 제거 정규식\n    # re.sub(r'\\<[^)]*\\>', '', remove_text)\n    if lang == \"eng\":\n        text_finder = re.compile('[^ A-Za-z]') if spacing else re.compile('[^A-Za-z]')\n    elif lang == \"kor\":\n        text_finder = re.compile('[^ ㄱ-ㅣ가-힣+]') if spacing else re.compile('[^ㄱ-ㅣ가-힣+]')\n    # default : kor + eng\n    else:\n        text_finder = re.compile('[^ A-Za-zㄱ-ㅣ가-힣+]') if spacing else re.compile('[^A-Za-zㄱ-ㅣ가-힣+]')\n    return text_finder.sub('', string)\ndef memory_usage(message='debug'):\n    # current process RAM usage\n    p = psutil.Process()\n    rss = p.memory_info().rss / 2 ** 20 # Bytes to MB\n    print(f\"[{message}] memory usage: {rss: 10.3f} MB\")\n    return rss\nclass MyLabelEncoder:\n    def __init__(self, preset={}):\n        # dic_cat format -> {\"col_name\": {\"value\": replace}}\n        self.dic_cat = preset\n    def fit_transform(self, data_x, col_names):\n        tmp_x = copy.deepcopy(data_x)\n        for i in col_names:\n            # if key is not in dic, update dic\n            if i not in self.dic_cat.keys():\n                tmp_dic = dict.fromkeys(sorted(set(tmp_x[i]).difference([nan])))\n                label_cnt = 0\n                for j in tmp_dic.keys():\n                    tmp_dic[j] = label_cnt\n                    label_cnt += 1\n                self.dic_cat[i] = tmp_dic\n            # transform value which is not in dic to nan\n            tmp_x[i] = tmp_x[i].astype(\"object\")\n            conv = tmp_x[i].replace(self.dic_cat[i])\n            for conv_idx, j in enumerate(conv):\n                if j not in self.dic_cat[i].values():\n                    conv[conv_idx] = nan\n            # final return\n            tmp_x[i] = conv.astype(\"float\")\n        return tmp_x\n    def transform(self, data_x):\n        tmp_x = copy.deepcopy(data_x)\n        for i in self.dic_cat.keys():\n            # transform value which is not in dic to nan\n            tmp_x[i] = tmp_x[i].astype(\"object\")\n            conv = tmp_x[i].replace(self.dic_cat[i])\n            for conv_idx, j in enumerate(conv):\n                if j not in self.dic_cat[i].values():\n                    conv[conv_idx] = nan\n            # final return\n            tmp_x[i] = conv.astype(\"float\")\n        return tmp_x\n    def clear(self):\n        self.dic_cat = {}\nclass MyOneHotEncoder:\n    def __init__(self, label_preset={}):\n        self.dic_cat = {}\n        self.label_preset = label_preset\n    def fit_transform(self, data_x, col_names):\n        tmp_x = dataframe()\n        for i in data_x:\n            if i not in col_names:\n                tmp_x = pd.concat([tmp_x, dataframe(data_x[i])], axis=1)\n            else:\n                if not ((data_x[i].dtype.name == \"object\") or (data_x[i].dtype.name == \"category\")):\n                    print(F\"WARNING : {i} is not object or category\")\n                self.dic_cat[i] = OneHotEncoder(sparse=False, handle_unknown=\"ignore\")\n                conv = self.dic_cat[i].fit_transform(dataframe(data_x[i])).astype(\"int\")\n                col_list = []\n                for j in self.dic_cat[i].categories_[0]:\n                    if i in self.label_preset.keys():\n                        for k, v in self.label_preset[i].items():\n                            if v == j:\n                                col_list.append(str(i) + \"_\" + str(k))\n                    else:\n                        col_list.append(str(i) + \"_\" + str(j))\n                conv = dataframe(conv, columns=col_list)\n                tmp_x = pd.concat([tmp_x, conv], axis=1)\n        return tmp_x\n    def transform(self, data_x):\n        tmp_x = dataframe()\n        for i in data_x:\n            if not i in list(self.dic_cat.keys()):\n                tmp_x = pd.concat([tmp_x, dataframe(data_x[i])], axis=1)\n            else:\n                if not ((data_x[i].dtype.name == \"object\") or (data_x[i].dtype.name == \"category\")):\n                    print(F\"WARNING : {i} is not object or category\")\n                conv = self.dic_cat[i].transform(dataframe(data_x[i])).astype(\"int\")\n                col_list = []\n                for j in self.dic_cat[i].categories_[0]:\n                    if i in self.label_preset.keys():\n                        for k, v in self.label_preset[i].items():\n                            if v == j: col_list.append(str(i) + \"_\" + str(k))\n                    else:\n                        col_list.append(str(i) + \"_\" + str(j))\n                conv = dataframe(conv, columns=col_list)\n                tmp_x = pd.concat([tmp_x, conv], axis=1)\n        return tmp_x\n    def clear(self):\n        self.dic_cat = {}\n        self.label_preset = {}\nclass MyKNNImputer:\n    def __init__(self, k=5):\n        self.imputer = KNNImputer(n_neighbors=k)\n        self.dic_cat = {}\n    def fit_transform(self, x, cat_vars=None):\n        if cat_vars is None:\n            x_imp = dataframe(self.imputer.fit_transform(x), columns=x.columns)\n        else:\n            naIdx = dict.fromkeys(cat_vars)\n            for i in cat_vars:\n                self.dic_cat[i] = diff(list(sorted(set(x[i]))), [nan])\n                naIdx[i] = list(which(array(x[i].isna())))\n            x_imp = dataframe(self.imputer.fit_transform(x), columns=x.columns)\n\n            # if imputed categorical value are not in the range, adjust the value\n            for i in cat_vars:\n                x_imp[i] = x_imp[i].apply(lambda x: int(round(x, 0)))\n                for j in naIdx[i]:\n                    if x_imp[i][j] not in self.dic_cat[i]:\n                        if x_imp[i][j] < self.dic_cat[i][0]:\n                            x_imp[i][naIdx[i]] = self.dic_cat[i][0]\n                        elif x_imp[i][j] > self.dic_cat[i][0]:\n                            x_imp[i][naIdx[i]] = self.dic_cat[i][len(self.dic_cat[i]) - 1]\n        return x_imp\n    def transform(self, x):\n        if len(self.dic_cat.keys()) == 0:\n            x_imp = dataframe(self.imputer.transform(x), columns=x.columns)\n        else:\n            naIdx = dict.fromkeys(self.dic_cat.keys())\n            for i in self.dic_cat.keys():\n                naIdx[i] = list(which(array(x[i].isna())))\n            x_imp = dataframe(self.imputer.transform(x), columns=x.columns)\n\n            # if imputed categorical value are not in the range, adjust the value\n            for i in self.dic_cat.keys():\n                x_imp[i] = x_imp[i].apply(lambda x: int(round(x, 0)))\n                for j in naIdx[i]:\n                    if x_imp[i][j] not in self.dic_cat[i]:\n                        if x_imp[i][j] < self.dic_cat[i][0]:\n                            x_imp[i][naIdx[i]] = self.dic_cat[i][0]\n                        elif x_imp[i][j] > self.dic_cat[i][0]:\n                            x_imp[i][naIdx[i]] = self.dic_cat[i][len(self.dic_cat[i]) - 1]\n        return x_imp\n    def clear(self):\n        self.imputer = None\n        self.dic_cat = {}\ndef remove_outlier(df, std=3, mode=\"remove\"):\n    tmp_df = df.copy()\n    if mode == \"remove\":\n        outlier_mask = (np.abs(stats.zscore(tmp_df)) > std).all(axis=1)\n        print(\"found outlier :\", outlier_mask.sum())\n        tmp_df = tmp_df[~outlier_mask]\n    elif mode == \"interpolate\":\n        tmp_outlier = []\n        for i in tmp_df:\n            outlier_mask = (np.abs(stats.zscore(tmp_df[i])) > std)\n            tmp_outlier.append(outlier_mask.sum())\n            if tmp_outlier[-1] == 0:\n                continue\n            tmp_df[i][outlier_mask] = np.nan\n            tmp_df[i] = tmp_df[i].interpolate(method='linear').bfill()\n        print(\"found outlier :\", np.sum(outlier_mask))\n    return tmp_df\n\nseed_everything()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:21:20.609466Z","iopub.execute_input":"2022-06-22T11:21:20.609738Z","iopub.status.idle":"2022-06-22T11:21:20.888252Z","shell.execute_reply.started":"2022-06-22T11:21:20.609693Z","shell.execute_reply":"2022-06-22T11:21:20.887363Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data #","metadata":{}},{"cell_type":"code","source":"# define function reading the train data\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\ndef update_rawdata_dic(x, proc_id):\n    rawdata = [read_notebook(path) for path in x]\n    tmp_dic.update({proc_id: rawdata})\n    print(\"job finished :\", proc_id, \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:20.891331Z","iopub.execute_input":"2022-06-22T11:21:20.891558Z","iopub.status.idle":"2022-06-22T11:21:20.897738Z","shell.execute_reply.started":"2022-06-22T11:21:20.891533Z","shell.execute_reply":"2022-06-22T11:21:20.896639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read one by one alogn with the notebook ids\n# random select train dataset\npaths_train = list((data_dir / 'train').glob('*.json'))\nnp_rnd.seed(1)\npaths_train = list(array(paths_train)[np_rnd.randint(0, len(paths_train)-1, size=10000, dtype=int)])\n# paths_train = list(array(paths_train)[np_rnd.randint(0, len(paths_train)-1, size=100, dtype=int)])\n\n# ===== original code =====\nstart_time = time()\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\nprint(time() - start_time)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-22T11:21:20.900168Z","iopub.execute_input":"2022-06-22T11:21:20.900893Z","iopub.status.idle":"2022-06-22T11:21:26.352277Z","shell.execute_reply.started":"2022-06-22T11:21:20.900846Z","shell.execute_reply":"2022-06-22T11:21:26.35159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concatenate the data by row\ndf_full_raw = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\nprint(df_full_raw)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:26.353456Z","iopub.execute_input":"2022-06-22T11:21:26.357833Z","iopub.status.idle":"2022-06-22T11:21:26.415409Z","shell.execute_reply.started":"2022-06-22T11:21:26.357788Z","shell.execute_reply":"2022-06-22T11:21:26.414707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_test = list((data_dir / 'test').glob('*.json'))\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:26.419804Z","iopub.execute_input":"2022-06-22T11:21:26.421674Z","iopub.status.idle":"2022-06-22T11:21:26.484457Z","shell.execute_reply.started":"2022-06-22T11:21:26.421618Z","shell.execute_reply":"2022-06-22T11:21:26.483722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:26.485771Z","iopub.execute_input":"2022-06-22T11:21:26.486459Z","iopub.status.idle":"2022-06-22T11:21:26.499579Z","shell.execute_reply.started":"2022-06-22T11:21:26.486417Z","shell.execute_reply":"2022-06-22T11:21:26.498651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del notebooks_train, notebooks_test, paths_train, paths_test","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:26.504006Z","iopub.execute_input":"2022-06-22T11:21:26.505868Z","iopub.status.idle":"2022-06-22T11:21:26.51572Z","shell.execute_reply.started":"2022-06-22T11:21:26.505825Z","shell.execute_reply":"2022-06-22T11:21:26.514322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ordering the Cells #","metadata":{}},{"cell_type":"code","source":"def get_ranks(orderd, unordered, normalize=False, score_by_position=True):\n    if score_by_position:\n        scores = pd.Series([orderd.index(d) for d in unordered]) + 1\n        return (scores / scores.max()).tolist() if normalize else scores.tolist()\n    else:\n        scores = pd.Series([len(orderd)-1 - orderd.index(d) for d in unordered]) + 1\n        return (scores / scores.max()).tolist() if normalize else scores.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:26.520243Z","iopub.execute_input":"2022-06-22T11:21:26.522434Z","iopub.status.idle":"2022-06-22T11:21:26.531491Z","shell.execute_reply.started":"2022-06-22T11:21:26.522394Z","shell.execute_reply":"2022-06-22T11:21:26.530849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list\n\nprint(df_orders)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-22T11:21:26.535177Z","iopub.execute_input":"2022-06-22T11:21:26.535762Z","iopub.status.idle":"2022-06-22T11:21:29.619316Z","shell.execute_reply.started":"2022-06-22T11:21:26.535726Z","shell.execute_reply":"2022-06-22T11:21:29.618587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the data including correct order\n# join the unordered raw train data to the correctly ordered data\ndf_ranks = df_orders.to_frame().join(\n    df_full_raw.reset_index('cell_id').groupby('id')['cell_id'].apply(list), how='right',\n)\n# columns order : NB_id, correct cell order, incorrect cell order\nprint(df_ranks.head())","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.623025Z","iopub.execute_input":"2022-06-22T11:21:29.623649Z","iopub.status.idle":"2022-06-22T11:21:29.72253Z","shell.execute_reply.started":"2022-06-22T11:21:29.623612Z","shell.execute_reply":"2022-06-22T11:21:29.721709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the correct rank score and save to the dictionary\nranks = {}\nfor id_, cell_order, cell_id in df_ranks.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id, normalize=True, score_by_position=True)}","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.723833Z","iopub.execute_input":"2022-06-22T11:21:29.724214Z","iopub.status.idle":"2022-06-22T11:21:29.809533Z","shell.execute_reply.started":"2022-06-22T11:21:29.724177Z","shell.execute_reply":"2022-06-22T11:21:29.808733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full = (\n    # create dataframe from the dictionary including rank score   \n    pd.DataFrame.from_dict(ranks, orient='index')\n    # rename the index\n    .rename_axis('id')\n    # flatten to the row from the list-typed correctly ordered cell_id object \n    .apply(pd.Series.explode)\n    # set 2nd index as cell_id\n    .set_index('cell_id', append=True)\n# join the unordered dataframe to the ordered dataframe\n).join(df_full_raw, on=[\"id\", \"cell_id\"], how=\"right\")\n\ndf_full = df_full.sort_values([\"id\", \"rank\"], ascending=[True, False])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.810829Z","iopub.execute_input":"2022-06-22T11:21:29.811226Z","iopub.status.idle":"2022-06-22T11:21:29.884863Z","shell.execute_reply.started":"2022-06-22T11:21:29.811192Z","shell.execute_reply":"2022-06-22T11:21:29.883931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def restruct_rank_values(x):\n    input_df = x.copy()\n    for i in input_df.index.get_level_values(0).unique():\n        tmp_df = input_df.loc[i].copy()\n        tmp_md = []\n        last_code_rank_value = 0\n        interval_rank_value = tmp_df.loc[(tmp_df[\"cell_type\"] == \"code\"), \"rank\"].diff().min()\n        for idx, value in enumerate(tmp_df[\"cell_type\"]):\n            if value == \"markdown\":\n                tmp_md.append(idx)\n            elif value == \"code\":\n                if len(tmp_md) > 0:\n                    tmp_df[\"rank\"].iloc[tmp_md] = np.linspace(last_code_rank_value, tmp_df[\"rank\"].iloc[idx], len(tmp_md)+2)[1:-1]\n                    tmp_md = []\n                else:\n                    pass\n                last_code_rank_value = tmp_df[\"rank\"].iloc[idx]\n        # if markdown is last cell\n        if len(tmp_md) > 0:\n            tmp_df[\"rank\"].iloc[tmp_md] = last_code_rank_value + interval_rank_value\n            tmp_md = []\n        input_df.loc[i] = tmp_df.values\n    return input_df","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.886409Z","iopub.execute_input":"2022-06-22T11:21:29.886927Z","iopub.status.idle":"2022-06-22T11:21:29.898566Z","shell.execute_reply.started":"2022-06-22T11:21:29.88688Z","shell.execute_reply":"2022-06-22T11:21:29.89773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.900209Z","iopub.execute_input":"2022-06-22T11:21:29.900774Z","iopub.status.idle":"2022-06-22T11:21:29.928476Z","shell.execute_reply.started":"2022-06-22T11:21:29.9007Z","shell.execute_reply":"2022-06-22T11:21:29.927521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full[\"rank\"] = df_full.groupby([\"id\", \"cell_type\"]).cumcount()\ndf_full[\"rank\"] = df_full.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True) * 100","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.929717Z","iopub.execute_input":"2022-06-22T11:21:29.930072Z","iopub.status.idle":"2022-06-22T11:21:29.950516Z","shell.execute_reply.started":"2022-06-22T11:21:29.930038Z","shell.execute_reply":"2022-06-22T11:21:29.949869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.loc[(df_full[\"cell_type\"] == \"markdown\"), \"rank\"] = 0","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.954471Z","iopub.execute_input":"2022-06-22T11:21:29.956315Z","iopub.status.idle":"2022-06-22T11:21:29.963798Z","shell.execute_reply.started":"2022-06-22T11:21:29.956275Z","shell.execute_reply":"2022-06-22T11:21:29.963104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get positional target value on markdown cell\ndf_full = restruct_rank_values(df_full)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:29.966318Z","iopub.execute_input":"2022-06-22T11:21:29.966742Z","iopub.status.idle":"2022-06-22T11:21:31.56856Z","shell.execute_reply.started":"2022-06-22T11:21:29.966705Z","shell.execute_reply":"2022-06-22T11:21:31.567589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full = df_full.sort_values([\"id\", \"rank\"], ascending=[True, False])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.569914Z","iopub.execute_input":"2022-06-22T11:21:31.570277Z","iopub.status.idle":"2022-06-22T11:21:31.580852Z","shell.execute_reply.started":"2022-06-22T11:21:31.570241Z","shell.execute_reply":"2022-06-22T11:21:31.579979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.582289Z","iopub.execute_input":"2022-06-22T11:21:31.582647Z","iopub.status.idle":"2022-06-22T11:21:31.598959Z","shell.execute_reply.started":"2022-06-22T11:21:31.582613Z","shell.execute_reply":"2022-06-22T11:21:31.597831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.600143Z","iopub.execute_input":"2022-06-22T11:21:31.600522Z","iopub.status.idle":"2022-06-22T11:21:31.617216Z","shell.execute_reply.started":"2022-06-22T11:21:31.600487Z","shell.execute_reply":"2022-06-22T11:21:31.616451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.618602Z","iopub.execute_input":"2022-06-22T11:21:31.619015Z","iopub.status.idle":"2022-06-22T11:21:31.62939Z","shell.execute_reply.started":"2022-06-22T11:21:31.618983Z","shell.execute_reply":"2022-06-22T11:21:31.628614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_full[\"rank\"].min())\nprint(df_full[\"rank\"].max())","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.630807Z","iopub.execute_input":"2022-06-22T11:21:31.631203Z","iopub.status.idle":"2022-06-22T11:21:31.636865Z","shell.execute_reply.started":"2022-06-22T11:21:31.631167Z","shell.execute_reply":"2022-06-22T11:21:31.63609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_ranks, df_full_raw, ranks","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.638616Z","iopub.execute_input":"2022-06-22T11:21:31.639114Z","iopub.status.idle":"2022-06-22T11:21:31.643405Z","shell.execute_reply.started":"2022-06-22T11:21:31.63903Z","shell.execute_reply":"2022-06-22T11:21:31.642709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[\"rank\"] = df_test.groupby([\"id\", \"cell_type\"]).cumcount()\ndf_test[\"rank\"] = df_test.groupby([\"id\", \"cell_type\"]).apply(lambda x: x[\"rank\"][::-1]).values\ndf_test[\"rank\"] = df_test.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True) * 100","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.644842Z","iopub.execute_input":"2022-06-22T11:21:31.645282Z","iopub.status.idle":"2022-06-22T11:21:31.670346Z","shell.execute_reply.started":"2022-06-22T11:21:31.645243Z","shell.execute_reply":"2022-06-22T11:21:31.669633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.loc[(df_test[\"cell_type\"] == \"markdown\"), \"rank\"] = 0","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.671546Z","iopub.execute_input":"2022-06-22T11:21:31.671981Z","iopub.status.idle":"2022-06-22T11:21:31.677241Z","shell.execute_reply.started":"2022-06-22T11:21:31.671944Z","shell.execute_reply":"2022-06-22T11:21:31.67643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.sort_values([\"id\", \"cell_type\", \"rank\"], ascending=[True, True, False])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.679285Z","iopub.execute_input":"2022-06-22T11:21:31.679778Z","iopub.status.idle":"2022-06-22T11:21:31.691408Z","shell.execute_reply.started":"2022-06-22T11:21:31.679739Z","shell.execute_reply":"2022-06-22T11:21:31.690765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # slightly minus a value from the max value on test datset\n# df_test.loc[df_test.groupby([\"id\"]).idxmax()[\"rank\"], \"rank\"] -= (df_test.loc[df_test[\"cell_type\"] == \"code\", \"rank\"].groupby(\"id\").apply(lambda x: x.nlargest(1)[-1] - x.nlargest(2)[-1]) / 10).values","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.702355Z","iopub.execute_input":"2022-06-22T11:21:31.702779Z","iopub.status.idle":"2022-06-22T11:21:31.70919Z","shell.execute_reply.started":"2022-06-22T11:21:31.702742Z","shell.execute_reply":"2022-06-22T11:21:31.708376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test[[\"rank\", \"cell_type\", \"source\"]]","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.710625Z","iopub.execute_input":"2022-06-22T11:21:31.711124Z","iopub.status.idle":"2022-06-22T11:21:31.721471Z","shell.execute_reply.started":"2022-06-22T11:21:31.711085Z","shell.execute_reply":"2022-06-22T11:21:31.720706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.722835Z","iopub.execute_input":"2022-06-22T11:21:31.723269Z","iopub.status.idle":"2022-06-22T11:21:31.751043Z","shell.execute_reply.started":"2022-06-22T11:21:31.72323Z","shell.execute_reply":"2022-06-22T11:21:31.750396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.75442Z","iopub.execute_input":"2022-06-22T11:21:31.756243Z","iopub.status.idle":"2022-06-22T11:21:31.772087Z","shell.execute_reply.started":"2022-06-22T11:21:31.756205Z","shell.execute_reply":"2022-06-22T11:21:31.771452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.775683Z","iopub.execute_input":"2022-06-22T11:21:31.776506Z","iopub.status.idle":"2022-06-22T11:21:31.785607Z","shell.execute_reply.started":"2022-06-22T11:21:31.776465Z","shell.execute_reply":"2022-06-22T11:21:31.784638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_test[\"rank\"].min())\nprint(df_test[\"rank\"].max())","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.786766Z","iopub.execute_input":"2022-06-22T11:21:31.787141Z","iopub.status.idle":"2022-06-22T11:21:31.79307Z","shell.execute_reply.started":"2022-06-22T11:21:31.787108Z","shell.execute_reply":"2022-06-22T11:21:31.792201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing & Text Cleansing","metadata":{}},{"cell_type":"code","source":"df_full[\"cell_type\"] = df_full[\"cell_type\"].apply(lambda x: 1.0 if x == \"markdown\" else 0.0)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.794942Z","iopub.execute_input":"2022-06-22T11:21:31.79663Z","iopub.status.idle":"2022-06-22T11:21:31.808918Z","shell.execute_reply.started":"2022-06-22T11:21:31.796594Z","shell.execute_reply":"2022-06-22T11:21:31.80803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full[\"cell_type\"] = df_full[\"cell_type\"].astype(\"float32\")\ndf_full[\"rank\"] = df_full[\"rank\"].astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.810241Z","iopub.execute_input":"2022-06-22T11:21:31.810614Z","iopub.status.idle":"2022-06-22T11:21:31.816525Z","shell.execute_reply.started":"2022-06-22T11:21:31.810582Z","shell.execute_reply":"2022-06-22T11:21:31.815718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.817953Z","iopub.execute_input":"2022-06-22T11:21:31.81847Z","iopub.status.idle":"2022-06-22T11:21:31.836713Z","shell.execute_reply.started":"2022-06-22T11:21:31.81843Z","shell.execute_reply":"2022-06-22T11:21:31.83591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[\"cell_type\"] = df_test[\"cell_type\"].apply(lambda x: 1.0 if x == \"markdown\" else 0.0)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.838222Z","iopub.execute_input":"2022-06-22T11:21:31.838892Z","iopub.status.idle":"2022-06-22T11:21:31.844305Z","shell.execute_reply.started":"2022-06-22T11:21:31.838858Z","shell.execute_reply":"2022-06-22T11:21:31.84362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[\"cell_type\"] = df_test[\"cell_type\"].astype(\"float32\")\ndf_test[\"rank\"] = df_test[\"rank\"].astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.845692Z","iopub.execute_input":"2022-06-22T11:21:31.846132Z","iopub.status.idle":"2022-06-22T11:21:31.854291Z","shell.execute_reply.started":"2022-06-22T11:21:31.846095Z","shell.execute_reply":"2022-06-22T11:21:31.853136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.856381Z","iopub.execute_input":"2022-06-22T11:21:31.862351Z","iopub.status.idle":"2022-06-22T11:21:31.879985Z","shell.execute_reply.started":"2022-06-22T11:21:31.862314Z","shell.execute_reply":"2022-06-22T11:21:31.87932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def str_cleansing(x):\n    preproc_lines = []\n    for line in x.split(\"\\n\"):\n        line = line.replace('\\n','')\n        line = line.replace('\\t',' ')\n        \n        # remove alias names\n        tmp = []\n        for idx, value in enumerate(line.split()):\n            if value == \"as\":\n                tmp.append(idx)\n                try:\n                    tmp.append(idx + 1)\n                except:\n                    pass\n        line = \" \".join(series(line.split()).iloc[diff(list(range(len(line.split()))), tmp)].tolist())\n\n        # replace 'import numpy as np' -> 'import numpy' to be easily tokenized\n        # but, below codes are not much effective for codebert tokenizer (codebert seems not to be able to interpret 'import ~') \n        tmp = []\n        skip_flag = True\n        for idx, value in enumerate(line.split()):\n            if skip_flag:\n                if value == \"import\":\n                    try:\n                        tmp.append(\"import \" + line.split()[idx+1])\n                        skip_flag=False\n                    except:\n                        tmp.append(\"import \")\n                elif value == \"from\":\n                    try:\n                        tmp.append(\"from \" + line.split()[idx+1])\n                        skip_flag=False\n                    except:\n                        tmp.append(\"from \") \n                else:\n                    tmp.append(value)\n            else:\n                skip_flag = True\n                continue\n        line = \" \".join(tmp)\n        \n        preproc_lines.append(line)\n        text_finder = re.compile('[^A-Za-z]')\n        preproc_lines[-1] = \" \".join(text_finder.sub(' ', preproc_lines[-1]).split()).lower()\n    preprocessed_script = ' '.join(preproc_lines)\n    preprocessed_script = ' '.join(preprocessed_script.split())\n    return preprocessed_script","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.881111Z","iopub.execute_input":"2022-06-22T11:21:31.881498Z","iopub.status.idle":"2022-06-22T11:21:31.897223Z","shell.execute_reply.started":"2022-06-22T11:21:31.881464Z","shell.execute_reply":"2022-06-22T11:21:31.896255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.groupby(\"id\").size().describe()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.899007Z","iopub.execute_input":"2022-06-22T11:21:31.8997Z","iopub.status.idle":"2022-06-22T11:21:31.917084Z","shell.execute_reply.started":"2022-06-22T11:21:31.899625Z","shell.execute_reply":"2022-06-22T11:21:31.916421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adjust the cell length\nMIN_CELL_LEN = 8\nMAX_CELL_LEN = 256\ndf_full = df_full.loc[df_full.groupby(\"id\").size().index[(df_full.groupby(\"id\").size() >= MIN_CELL_LEN) & (df_full.groupby(\"id\").size() <= MAX_CELL_LEN)]]\n\n# select the train data which has at least 1 cell for both code & markdown\ndf_full = df_full.loc[df_full.index.get_level_values(0).unique()[df_full.groupby(\"id\")[\"cell_type\"].apply(lambda x: True if ((x==0.0).sum() > 1) & ((x==1.0).sum() > 1) else False)]]","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:31.918372Z","iopub.execute_input":"2022-06-22T11:21:31.918858Z","iopub.status.idle":"2022-06-22T11:21:32.000244Z","shell.execute_reply.started":"2022-06-22T11:21:31.91882Z","shell.execute_reply":"2022-06-22T11:21:31.999435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.groupby(\"id\").size().describe()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:32.001648Z","iopub.execute_input":"2022-06-22T11:21:32.002084Z","iopub.status.idle":"2022-06-22T11:21:32.016963Z","shell.execute_reply.started":"2022-06-22T11:21:32.002045Z","shell.execute_reply":"2022-06-22T11:21:32.015451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_full[\"source\"][df_full[\"cell_type\"] == 0.0] = df_full[\"source\"][df_full[\"cell_type\"] == 0.0].apply(lambda x: str_cleansing(x, 0))\n# df_full[\"source\"][df_full[\"cell_type\"] == 1.0] = df_full[\"source\"][df_full[\"cell_type\"] == 1.0].apply(lambda x: str_cleansing(x, 1))\n\ndf_full[\"source\"] = df_full[\"source\"].apply(lambda x: str_cleansing(x))","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:32.018311Z","iopub.execute_input":"2022-06-22T11:21:32.018784Z","iopub.status.idle":"2022-06-22T11:21:39.538141Z","shell.execute_reply.started":"2022-06-22T11:21:32.018733Z","shell.execute_reply":"2022-06-22T11:21:39.537417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:39.542438Z","iopub.execute_input":"2022-06-22T11:21:39.544362Z","iopub.status.idle":"2022-06-22T11:21:39.566541Z","shell.execute_reply.started":"2022-06-22T11:21:39.544324Z","shell.execute_reply":"2022-06-22T11:21:39.565971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test[\"source\"][df_test[\"cell_type\"] == 0.0] = df_test[\"source\"][df_test[\"cell_type\"] == 0.0].apply(lambda x: str_cleansing(x, 0))\n# df_test[\"source\"][df_test[\"cell_type\"] == 1.0] = df_test[\"source\"][df_test[\"cell_type\"] == 1.0].apply(lambda x: str_cleansing(x, 1))\n\ndf_test[\"source\"] = df_test[\"source\"].apply(lambda x: str_cleansing(x))","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:39.570648Z","iopub.execute_input":"2022-06-22T11:21:39.57249Z","iopub.status.idle":"2022-06-22T11:21:39.807722Z","shell.execute_reply.started":"2022-06-22T11:21:39.572454Z","shell.execute_reply":"2022-06-22T11:21:39.807057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:39.808957Z","iopub.execute_input":"2022-06-22T11:21:39.809217Z","iopub.status.idle":"2022-06-22T11:21:39.825489Z","shell.execute_reply.started":"2022-06-22T11:21:39.809181Z","shell.execute_reply":"2022-06-22T11:21:39.824714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training - Distilbert embedding & CatBoost Ranker","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModel\nimport torch\nfrom torch.utils.data import DataLoader\ntorch_device = torch.device('cuda')\n\ncodebert_tokenizer_path = \"../input/ai4code-ms-codebert/microsoft-codebert-base/tokenizer/\"\ncodebert_model_path = \"../input/ai4code-ms-codebert/microsoft-codebert-base/model/\"\n\nbert_tokenizer_path = \"../input/ai4code-bert/bert-base-uncased/tokenizer/\"\nbert_model_path = \"../input/ai4code-bert/bert-base-uncased/model/\"\n\ndistilbert_tokenizer_path = \"../input/ai4code-distilbert/distilbert-base-uncased/tokenizer/\"\ndistilbert_model_path = \"../input/ai4code-distilbert/distilbert-base-uncased/model/\"\n\nMAX_LEN = 256\n\ndef get_fv(tokenizer_path, model_path, x, batch_size=32, device_type=\"gpu\"):\n    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n    if device_type == \"gpu\":\n        model = AutoModel.from_pretrained(model_path).to(torch_device)\n    else:\n        model = AutoModel.from_pretrained(model_path)\n    tmp = []\n    dataloader = DataLoader(x, batch_size, shuffle=False)\n    for idx, value in enumerate(tqdm(dataloader)):  \n        inputs = tokenizer.batch_encode_plus(\n            value,\n            add_special_tokens=True,\n            max_length=MAX_LEN,\n            padding=\"max_length\",\n            return_token_type_ids=False,\n            truncation=True\n        )\n        if device_type == \"gpu\":\n            input_ids = torch.tensor(inputs['input_ids']).to(torch_device)\n        else:\n            input_ids = torch.tensor(inputs['input_ids'])\n        outputs = model(input_ids).last_hidden_state.detach().cpu().numpy().mean(axis=1)\n        tmp.append(outputs)\n    return np.concatenate(tmp, axis=0)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:21:39.826928Z","iopub.execute_input":"2022-06-22T11:21:39.827559Z","iopub.status.idle":"2022-06-22T11:21:46.435479Z","shell.execute_reply.started":"2022-06-22T11:21:39.827487Z","shell.execute_reply":"2022-06-22T11:21:46.434734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_full_fv = np.zeros((df_full.shape[0], 768))\ndf_full_fv = get_fv(distilbert_tokenizer_path, distilbert_model_path, df_full[\"source\"], batch_size=32, device_type=\"gpu\").astype(\"float32\")\ntorch.cuda.empty_cache()\ndf_full.drop([\"source\"], axis=1, inplace=True)\n\ndf_test_fv = np.zeros((df_test.shape[0], 768))\ndf_test_fv = get_fv(distilbert_tokenizer_path, distilbert_model_path, df_test[\"source\"], batch_size=32, device_type=\"gpu\").astype(\"float32\")\ntorch.cuda.empty_cache()\ndf_test.drop([\"source\"], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:21:46.440213Z","iopub.execute_input":"2022-06-22T11:21:46.442053Z","iopub.status.idle":"2022-06-22T11:22:28.465339Z","shell.execute_reply.started":"2022-06-22T11:21:46.442017Z","shell.execute_reply":"2022-06-22T11:22:28.464625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bisect import bisect\n\ndef create_dataset(x, y, batch_size, shuffle=True, sample_weight=None):\n    if sample_weight is not None:\n        dataset = tf.data.Dataset.from_tensor_slices((x, y, sample_weight))\n    else:\n        dataset = tf.data.Dataset.from_tensor_slices((x, y))\n    dataset = dataset.shuffle(int(batch_size * 4), reshuffle_each_iteration=True) if shuffle else dataset\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(4)\n    return dataset\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max\ndef convert_rank_ids(x):\n    # Convert the cell_id index into a column.\n    x = x.reset_index('cell_id')\n    # Group the cell_ids for each notebook into a list.\n    x = x.groupby('id')['cell_id'].apply(list)\n    return x\ndef flatten_list(xss):\n    return [x for xs in xss for x in xs]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:22:28.466529Z","iopub.execute_input":"2022-06-22T11:22:28.467231Z","iopub.status.idle":"2022-06-22T11:22:28.482497Z","shell.execute_reply.started":"2022-06-22T11:22:28.467193Z","shell.execute_reply":"2022-06-22T11:22:28.481857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna function\ndef optuna_objective_function(trial: Trial, fold, train_x, train_y, train_groups, val_x, val_y, val_groups, categoIdx,\n                              model_name, output_container, ntrees=1000, eta=1e-2, best_ntrees=[None]):\n    if model_name == \"LGB_GOSS\":\n        # objective\n        # regession : \"mae\", \"mse\"\n        # classification - binary : \"binary\"\n        # classification - binary : \"multiclass\" (num_class=n)\n        # ranking : \"xe_ndcg_mart\"\n\n        # metric\n        # regession : \"mae\", \"mse\", \"rmse\"\n        # classification - binary : \"binary_logloss\", \"binary_error\", \"auc\"\n        # classification - muticlass : \"multi_logloss\", \"multi_error\"\n        # ranking : \"ndcg\", \"map\"\n\n        tuning_params = {\n            \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [1e-2, 5e-3, 1e-3]),\n            \"num_leaves\": trial.suggest_categorical(\"num_leaves\", [pow(2, i) - 1 for i in [4, 5, 6, 7, 8]]),\n            # goss sampling hyper-parameter replacing the \"sumample\"\n            \"top_rate\": trial.suggest_float(\"top_rate\", 0.2, 0.5, step=0.1),\n            \"other_rate\": trial.suggest_float(\"other_rate\", 0.2, 0.5, step=0.1),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 0.8, step=0.1),\n            \"reg_lambda\": trial.suggest_categorical(\"reg_lambda\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"min_child_weight\": trial.suggest_categorical(\"min_child_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 1, 51, step=2),\n            \"min_gain_to_split\": trial.suggest_categorical(\"min_gain_to_split\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            # for binary\n            \"scale_pos_weight\": trial.suggest_categorical(\"scale_pos_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n        }\n        model = lgb.LGBMClassifier(boosting_type=\"goss\", objective=\"binary\",\n                                   n_estimators=ntrees, device_type=\"gpu\",\n                                   random_state=fold, verbose=-1, **tuning_params)\n        cb_list = [\n            lgb.early_stopping(stopping_rounds=int(ntrees * 0.2), first_metric_only=True, verbose=False, min_delta=0.001),\n        ]\n        model.fit(train_x, train_y, categorical_feature=categoIdx,\n                  eval_set=(val_x,val_y), eval_metric=\"auc\", callbacks=cb_list)\n        best_ntrees[0] = model.best_iteration_\n    elif model_name == \"LGB_GBM\":\n        # objective\n        # regession : \"mae\", \"mse\"\n        # classification - binary : \"binary\"\n        # classification - binary : \"multiclass\" (num_class=n)\n        # ranking : \"xe_ndcg_mart\"\n\n        # metric\n        # regession : \"mae\", \"mse\", \"rmse\"\n        # classification - binary : \"binary_logloss\", \"binary_error\", \"auc\"\n        # classification - muticlass : \"multi_logloss\", \"multi_error\"\n        # ranking : \"ndcg\", \"map\"\n\n        tuning_params = {\n            \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [1e-2, 5e-3, 1e-3]),\n            \"num_leaves\": trial.suggest_categorical(\"num_leaves\", [pow(2, i) - 1 for i in [4, 5, 6, 7, 8]]),\n            \"subsample\": trial.suggest_float(\"subsample\", 0.5, 0.8, step=0.1),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 0.8, step=0.1),\n            \"reg_lambda\": trial.suggest_categorical(\"reg_lambda\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"min_child_weight\": trial.suggest_categorical(\"min_child_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 1, 51, step=2),\n            \"min_gain_to_split\": trial.suggest_categorical(\"min_gain_to_split\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            # for binary\n            \"scale_pos_weight\": trial.suggest_categorical(\"scale_pos_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n        }\n\n        model = lgb.LGBMClassifier(boosting_type=\"gbdt\", objective=\"binary\",\n                                   n_estimators=ntrees, device_type=\"gpu\",\n                                   random_state=fold, verbose=-1, **tuning_params)\n        cb_list = [\n            lgb.early_stopping(stopping_rounds=int(ntrees * 0.2), first_metric_only=True, verbose=False, min_delta=0.001),\n        ]\n        model.fit(train_x, train_y, categorical_feature=categoIdx,\n                  eval_set=(val_x,val_y), eval_metric=\"auc\", callbacks=cb_list)\n        best_ntrees[0] = model.best_iteration_\n    elif model_name == \"XGB_GBT\":\n        # objective\n        # regession : \"reg:absoluteerror\", \"reg:squarederror\"\n        # classification - binary : \"binary:logistic\"\n        # classification - multicalss :\"multi:softmax\" (num_class=n)\n        # ranking : \"rank:ndcg\"\n\n        # metric\n        # regession : \"mae\", \"rmse\"\n        # classification - binary : \"logloss\", \"error@t\" (t=threshold), \"auc\"\n        # classification - multicalss : \"mlogloss\", \"merror\"\n        # ranking : \"ndcg\", \"map\"\n\n        tuning_params = {\n            \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [1e-2, 5e-3, 1e-3]),\n            \"max_depth\": trial.suggest_categorical(\"max_depth\", [4, 5, 6, 7, 8]),\n            \"subsample\": trial.suggest_float(\"subsample\", 0.5, 0.8, step=0.1),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 0.8, step=0.1),\n            \"reg_lambda\": trial.suggest_categorical(\"reg_lambda\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"min_child_weight\": trial.suggest_categorical(\"min_child_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"gamma\": trial.suggest_categorical(\"gamma\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            # for binary\n            \"scale_pos_weight\": trial.suggest_categorical(\"scale_pos_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n        }\n        model = xgb.XGBClassifier(booster=\"gbtree\", objective=\"binary:logistic\",\n                            n_estimators=ntrees, tree_method=\"gpu_hist\",\n                            random_state=fold, verbosity=0, **tuning_params)\n        model.fit(train_x, train_y,\n                  eval_set=[(val_x, val_y)], eval_metric=\"auc\",\n                  early_stopping_rounds=int(ntrees * 0.2), verbose=False)\n        best_ntrees[0] = model.best_iteration\n    elif model_name == \"CAT_GBM\":\n#         CatBoostRanker, Pool, MetricVisualizer\n        # objective\n        # regession : \"MAE\", \"RMSE\"\n        # classification - binary : \"Logloss\"\n        # classification - multicalss :\"MultiClass\"\n        # ranking :\n        # past -> slow, low quality -> high quality\n        # RMSE\n        # QueryRMSE\n        # PairLogit\n        # PairLogitPairwise\n        # YetiRank\n        # YetiRankPairwise\n\n        # metric\n        # regession : \"MAE\", \"RMSE\", \"R2\"\n        # classification - binary : \"Logloss\", \"Accuracy\", \"AUC\", \"F1\"\n        # classification - multicalss : \"MultiClass\", \"Accuracy\", \"TotalF1\" (average=Weighted,Macro,Micro)\n        # ranking : \"PairLogit\", \"YetiRank\", \"NDCG\", \"MAP\"\n\n        tuning_params = {\n            \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [5e-3]),\n            \"max_depth\": trial.suggest_categorical(\"max_depth\", [4, 6, 8]),\n            \"bagging_temperature\": trial.suggest_categorical(\"bagging_temperature\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            # rsm = colsample_bylevel (not supported for GPU)\n            # \"rsm\": trial.suggest_float(\"rsm\", 0.5, 0.8, step=0.1)\n            \"random_strength\": trial.suggest_categorical(\"random_strength\", [0.01, 0.05, 0.1, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0]),\n            \"reg_lambda\": trial.suggest_categorical(\"reg_lambda\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            \"min_child_samples\": trial.suggest_float(\"min_child_samples\", 1, 51, step=2),\n            # for binary\n            # \"scale_pos_weight\": trial.suggest_categorical(\"scale_pos_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n        }\n\n#         model = cat.CatBoostClassifier(boosting_type=\"Plain\", loss_function=\"YetiRank\", eval_metric=\"Logloss\",\n#                                     n_estimators=ntrees, task_type=\"GPU\", bootstrap_type=\"Bayesian\",\n#                                     verbose=False, random_state=fold, **tuning_params)\n        model = cat.CatBoostRanker(boosting_type=\"Plain\", loss_function=\"QueryRMSE\", eval_metric=\"QueryRMSE\",\n                            n_estimators=ntrees, task_type=\"GPU\", bootstrap_type=\"Bayesian\",\n                            verbose=False, random_state=fold, **tuning_params)\n    \n        train_pool = cat.Pool(\n            data=train_x,\n            label=train_y,\n            group_id=train_groups\n        )\n\n        val_pool = cat.Pool(\n            data=val_x,\n            label=val_y,\n            group_id=val_groups\n        )\n\n        model.fit(train_pool, cat_features=categoIdx,\n                eval_set=val_pool, early_stopping_rounds=int(ntrees * 0.2), use_best_model=True,\n                verbose=False)\n        best_ntrees[0] = model.best_iteration_\n    elif model_name == \"CAT_ORD\":\n        # objective\n        # regession : \"MAE\", \"RMSE\"\n        # classification - binary : \"Logloss\"\n        # classification - multicalss :\"MultiClass\"\n        # ranking : \"PairLogit\", \"YetiRank\"\n\n        # metric\n        # regession : \"MAE\", \"RMSE\", \"R2\"\n        # classification - binary : \"Logloss\", \"Accuracy\", \"AUC\", \"F1\"\n        # classification - multicalss : \"MultiClass\", \"Accuracy\", \"TotalF1\" (average=Weighted,Macro,Micro)\n        # ranking : \"PairLogit\", \"YetiRank\", \"NDCG\", \"MAP\"\n\n        tuning_params = {\n            \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [5e-3]),\n            \"max_depth\": trial.suggest_categorical(\"max_depth\", [4, 6, 8]),\n            # \"bagging_temperature\": trial.suggest_categorical(\"bagging_temperature\", list(np.linspace(1e-3, 1.0, num=75, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=25, endpoint=True))),\n            # rsm = colsample_bylevel (not supported for GPU)\n            # \"rsm\": trial.suggest_float(\"rsm\", 0.5, 0.8, step=0.1),\n            \"random_strength\": trial.suggest_categorical(\"random_strength\", [0.01, 0.1, 1.0, 2.0, 3.0]),\n            \"reg_lambda\": trial.suggest_categorical(\"reg_lambda\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n            # \"min_child_samples\": trial.suggest_float(\"min_child_samples\", 1, 51, step=2),\n            # for binary\n            # \"scale_pos_weight\": trial.suggest_categorical(\"scale_pos_weight\", list(np.linspace(1e-3, 1.0, num=150, endpoint=False)) + list(np.linspace(1.0, 1e+2, num=50, endpoint=True))),\n        }\n\n        model = cat.CatBoostClassifier(boosting_type=\"Ordered\", loss_function=\"Logloss\", eval_metric=\"Logloss\",\n                                    n_estimators=ntrees, task_type=\"GPU\", bootstrap_type=\"Bayesian\",\n                                    verbose=False, random_state=fold, **tuning_params)\n        model.fit(train_x, train_y, cat_features=categoIdx,\n                eval_set=[(val_x, val_y)], early_stopping_rounds=int(ntrees * 0.2), use_best_model=True,\n                verbose=False)\n        best_ntrees[0] = model.best_iteration_\n    else:\n        print(\"unknown\")\n        return -1\n    \n    pred = df_full.loc[ids[val_idx]].copy()\n    pred.loc[(pred[\"cell_type\"] == 1.0), \"rank\"] = model.predict(val_pool)[(pred[\"cell_type\"] == 1.0)]\n    optuna_score = kendall_tau(df_orders.loc[ids[val_idx]].sort_index(), convert_rank_ids(pred.sort_values([\"id\", \"rank\"], ascending=[True, False])))\n    \n#     # classification\n#     pred = model.predict_proba(val_x)\n#     optuna_score = metrics.f1_score(val_y.tolist(), [1.0 if i > threshold else 0.0 for i in pred[:,1]])\n    \n#     # ranking\n#     pred = model.predict(val_x)\n#     optuna_score = metrics.mean_squared_error(val_y.tolist(), pred)\n\n    if optuna_score > output_container[\"score\"]:\n        print(\"found the best !\")\n        if best_ntrees[0] is not None:\n            print(\"number of trees :\", best_ntrees)\n        output_container[\"model\"] = model\n        output_container[\"pred\"] = pred\n        output_container[\"score\"] = optuna_score\n\n    # trial.report(optuna_score, trial.number)\n    # if trial.should_prune():\n    #     raise optuna.TrialPruned()\n\n    return optuna_score","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:22:28.49017Z","iopub.execute_input":"2022-06-22T11:22:28.490609Z","iopub.status.idle":"2022-06-22T11:22:28.539247Z","shell.execute_reply.started":"2022-06-22T11:22:28.490569Z","shell.execute_reply":"2022-06-22T11:22:28.538453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_fold_training(fold, train_idx, val_idx, sample_weight=None, finetuning=False):\n    tmp_time = time()\n    print(\"\\n===== Fold\", fold, \"=====\\n\")\n    \n    train_x = df_full.loc[(df_full.index.get_level_values(0).isin(ids[val_idx]) & (df_full[\"cell_type\"] == 1.0).values)]\n    train_groups = train_x.index.get_level_values(0).to_list()\n    train_y = train_x[\"rank\"].copy()\n    train_x = df_full_fv[(df_full.index.get_level_values(0).isin(ids[val_idx]) & (df_full[\"cell_type\"] == 1.0).values)]\n    \n    val_x = df_full.loc[ids[val_idx]]\n    val_groups = val_x.index.get_level_values(0).to_list()\n    val_y = val_x[\"rank\"].copy()\n    val_x = df_full_fv[findIdx(df_full.index.get_level_values(0), ids[val_idx])]\n    \n    output_container = {\"model\": None, \"pred\": None, \"score\": -np.inf}\n    optuna_direction = 'maximize'\n    optuna_trials = 5\n    optuna_timout = int(6 * 3600 / kfolds_spliter.get_n_splits())\n    optuna_study = create_study(direction=optuna_direction, sampler=TPESampler())\n    \n    best_ntrees = [0]\n    optuna_study.optimize(\n        lambda trial: optuna_objective_function(\n            trial, fold, train_x, train_y, train_groups, val_x, val_y, val_groups, categoIdx=None, model_name=\"CAT_GBM\", output_container=output_container,\n            ntrees=ntrees, eta=eta, best_ntrees=best_ntrees\n        ),\n        n_jobs=1, n_trials=optuna_trials, timeout=optuna_timout\n    )\n    \n    model_list.append(output_container[\"model\"])\n    params_list.append(optuna_study.best_params)\n    if best_ntrees[0] is not None:\n        params_list[-1][\"ntrees\"] = best_ntrees[0]\n    print(\"fold\", fold, \"best params :\", params_list[-1])\n    \n    val_pred.loc[ids[val_idx], \"rank\"] = output_container[\"pred\"][\"rank\"]\n    fold_metric.append(output_container[\"score\"])\n    print(\"fold\", fold, \"score :\", fold_metric[-1])\n    \n    test_x = df_test.copy()\n    test_groups = test_x.index.get_level_values(0).to_list()\n    test_y = None\n    test_x = df_test_fv[:]\n    \n    test_pool = cat.Pool(\n        data=test_x,\n        label=test_y,\n        group_id=test_groups\n    )\n    \n    test_pred.loc[(test_pred[\"cell_type\"] == 1.0), \"rank\"] += (model_list[-1].predict(test_pool) / n_folds)[(test_pred[\"cell_type\"] == 1.0)]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-22T11:26:12.15675Z","iopub.execute_input":"2022-06-22T11:26:12.157207Z","iopub.status.idle":"2022-06-22T11:26:12.174475Z","shell.execute_reply.started":"2022-06-22T11:26:12.157167Z","shell.execute_reply":"2022-06-22T11:26:12.173443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learning parameter setting\nntrees = 5000\neta = 5e-3\n\n# ntrees = 100\n# eta = 1e-2\n\n# for group fold split\nids = df_full.index.unique('id')\ndf_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\nancestors = df_ancestors.loc[ids, 'ancestor_id']\n\n# stratified sampling by each length of code blocks\nnb_length_spliter = KBinsDiscretizer(n_bins=10, strategy=\"quantile\", encode=\"ordinal\")\nstrat_y = nb_length_spliter.fit_transform(df_full.groupby('id').size().to_frame()).flatten().astype(\"int32\")","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:22:28.562907Z","iopub.execute_input":"2022-06-22T11:22:28.563196Z","iopub.status.idle":"2022-06-22T11:22:28.780996Z","shell.execute_reply.started":"2022-06-22T11:22:28.563128Z","shell.execute_reply":"2022-06-22T11:22:28.78018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = []\nhistory_list = []\nparams_list = []\nfold_metric = []\n\nval_pred = df_full[[\"cell_type\", \"rank\"]].copy()\nval_pred.loc[val_pred[\"cell_type\"]==1.0, \"rank\"] = 0\ntest_pred = df_test[[\"cell_type\", \"rank\"]].copy()\ntest_pred.loc[test_pred[\"cell_type\"]==1.0, \"rank\"] = 0\n\nn_folds = 5\nkfolds_spliter = StratifiedGroupKFold(n_folds, shuffle=True, random_state=42)\n\nstart_time_training = time()\n# fold training\nfor fold, (train_idx, val_idx) in enumerate(kfolds_spliter.split(range(len(ids)), y=strat_y, groups=ancestors)):\n    start_mem = memory_usage()   \n    do_fold_training(fold, train_idx, val_idx, None, finetuning=False)\n    gc.collect()\n    end_mem = memory_usage()\n    print(\"@Memory leaked :\", end_mem - start_mem, \"\\n\")\nend_time_training = time()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-06-22T11:26:14.26152Z","iopub.execute_input":"2022-06-22T11:26:14.262011Z","iopub.status.idle":"2022-06-22T11:27:08.402201Z","shell.execute_reply.started":"2022-06-22T11:26:14.261971Z","shell.execute_reply":"2022-06-22T11:27:08.401595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, value in enumerate(fold_metric):\n    print(\"Fold\", idx, \"score :\", value)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:27:08.405789Z","iopub.execute_input":"2022-06-22T11:27:08.406578Z","iopub.status.idle":"2022-06-22T11:27:08.414701Z","shell.execute_reply.started":"2022-06-22T11:27:08.406541Z","shell.execute_reply":"2022-06-22T11:27:08.413986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Average score :\", np.mean(fold_metric))","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:27:08.416212Z","iopub.execute_input":"2022-06-22T11:27:08.417382Z","iopub.status.idle":"2022-06-22T11:27:08.423741Z","shell.execute_reply.started":"2022-06-22T11:27:08.417313Z","shell.execute_reply":"2022-06-22T11:27:08.42296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# sample submission on markdown\ntest_pred.loc[(test_pred[\"cell_type\"] == 1.0), \"rank\"].head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:27:08.425608Z","iopub.execute_input":"2022-06-22T11:27:08.425917Z","iopub.status.idle":"2022-06-22T11:27:08.442122Z","shell.execute_reply.started":"2022-06-22T11:27:08.425882Z","shell.execute_reply":"2022-06-22T11:27:08.44154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred.loc[(test_pred[\"cell_type\"] == 1.0), \"rank\"].hist()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T11:27:08.445705Z","iopub.execute_input":"2022-06-22T11:27:08.447546Z","iopub.status.idle":"2022-06-22T11:27:08.755985Z","shell.execute_reply.started":"2022-06-22T11:27:08.447509Z","shell.execute_reply":"2022-06-22T11:27:08.754747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submit = (\n    convert_rank_ids(test_pred.sort_values([\"id\", \"rank\"], ascending=[True, False]))\n    .apply(' '.join)  # list of ids -> string of ids\n    .rename_axis('id')\n    .rename('cell_order')\n)\ny_submit","metadata":{"execution":{"iopub.status.busy":"2022-06-21T13:14:20.971674Z","iopub.execute_input":"2022-06-21T13:14:20.972326Z","iopub.status.idle":"2022-06-21T13:14:20.98838Z","shell.execute_reply.started":"2022-06-21T13:14:20.97229Z","shell.execute_reply":"2022-06-21T13:14:20.987815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submit.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-21T13:14:20.991916Z","iopub.execute_input":"2022-06-21T13:14:20.993621Z","iopub.status.idle":"2022-06-21T13:14:21.001688Z","shell.execute_reply.started":"2022-06-21T13:14:20.993588Z","shell.execute_reply":"2022-06-21T13:14:21.000821Z"},"trusted":true},"execution_count":null,"outputs":[]}]}