{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":48575,"databundleVersionId":5145258,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-26T16:27:20.754791Z","iopub.execute_input":"2024-10-26T16:27:20.755547Z","iopub.status.idle":"2024-10-26T16:27:21.739728Z","shell.execute_reply.started":"2024-10-26T16:27:20.755506Z","shell.execute_reply":"2024-10-26T16:27:21.738757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyarrow.parquet as pq\nfrom sklearn import metrics\nimport scipy.stats\nimport seaborn as sns\nimport matplotlib.pyplot as plt \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom catboost import CatBoostClassifier, Pool, CatBoostRanker\nimport torch\nimport time  # для оценки времени\nimport torch.utils.data as data_utils\nimport tqdm\nfrom typing import List\nimport os\nimport torch\nimport numpy as np\nimport pandas as pd\nimport random\nfrom sklearn.metrics import roc_auc_score\nfrom torchvision.ops import sigmoid_focal_loss\nfrom torch.nn.utils import clip_grad_norm_\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:27:24.204494Z","iopub.execute_input":"2024-10-26T16:27:24.205000Z","iopub.status.idle":"2024-10-26T16:27:32.554154Z","shell.execute_reply.started":"2024-10-26T16:27:24.204969Z","shell.execute_reply":"2024-10-26T16:27:32.553104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvcc --version","metadata":{"execution":{"iopub.status.busy":"2024-07-08T05:55:01.368907Z","iopub.execute_input":"2024-07-08T05:55:01.369314Z","iopub.status.idle":"2024-07-08T05:55:02.429846Z","shell.execute_reply.started":"2024-07-08T05:55:01.369280Z","shell.execute_reply":"2024-07-08T05:55:02.428484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATA_PATH = \"/kaggle/input/alfa-bank-pd-credit-history/data_for_competition/train_data/\"\nTEST_DATA_PATH = \"/kaggle/input/alfa-bank-pd-credit-history/data_for_competition/test_data/\"\nTRAIN_TARGET_PATH = \"/kaggle/input/alfa-bank-pd-credit-history/data_for_competition/train_target.csv\"\nTRAIN_FEATURES_PATH = \"../data/train_features_gb/\"\nTEST_FEATURES_PATH = \"../data/test_features_gb/\"","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:27:32.555867Z","iopub.execute_input":"2024-10-26T16:27:32.556547Z","iopub.status.idle":"2024-10-26T16:27:32.561246Z","shell.execute_reply.started":"2024-10-26T16:27:32.556516Z","shell.execute_reply":"2024-10-26T16:27:32.560232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_dataset = TRAIN_DATA_PATH\ndataset_paths = sorted([os.path.join(path_to_dataset, filename) for filename in os.listdir(path_to_dataset)])\ndataset_paths","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:27:32.562488Z","iopub.execute_input":"2024-10-26T16:27:32.562781Z","iopub.status.idle":"2024-10-26T16:27:32.579153Z","shell.execute_reply.started":"2024-10-26T16:27:32.562758Z","shell.execute_reply":"2024-10-26T16:27:32.578211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport tqdm\ndef read_parquet_dataset_from_local(path_to_dataset: str, start_from: int = 0,\n                                     num_parts_to_read: int = 2, columns=None, verbose=False) -> pd.DataFrame:\n    \"\"\"\n    читает num_parts_to_read партиций, преобразует их к pd.DataFrame и возвращает\n    :param path_to_dataset: путь до директории с партициями\n    :param start_from: номер партиции, с которой начать чтение\n    :param num_parts_to_read: количество партиций, которые требуется прочитать\n    :param columns: список колонок, которые нужно прочитать из партиции\n    :return: pd.DataFrame\n    \"\"\"\n\n    res = []\n    dataset_paths = sorted([os.path.join(path_to_dataset, filename) for filename in os.listdir(path_to_dataset)])\n    \n    start_from = max(0, start_from)\n    chunks = dataset_paths[start_from: start_from + num_parts_to_read]\n    if verbose:\n        print('Reading chunks:\\n')\n        for chunk in chunks:\n            print(chunk)\n    for chunk_path in chunks:\n        chunk = pd.read_parquet(chunk_path,columns=columns)\n        res.append(chunk)\n    return pd.concat(res).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:27:36.039559Z","iopub.execute_input":"2024-10-26T16:27:36.040261Z","iopub.status.idle":"2024-10-26T16:27:36.049305Z","shell.execute_reply.started":"2024-10-26T16:27:36.040230Z","shell.execute_reply":"2024-10-26T16:27:36.048279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OHEAggregator(object):\n    \n    def __init__(self):\n        self.encoded_features = None\n        \n    def __extract_count_aggregations(self, data_frame: pd.DataFrame, mode: str) -> pd.DataFrame:\n        feature_columns = list(data_frame.columns.values)\n        feature_columns.remove(\"id\")\n        feature_columns.remove(\"rn\")\n\n        dummies = pd.get_dummies(data_frame[feature_columns], columns=feature_columns)\n        dummy_features = dummies.columns.values\n        \n        ohe_features = pd.concat([data_frame, dummies], axis=1)\n        ohe_features = ohe_features.drop(columns=feature_columns)\n        return ohe_features\n        \n    def __transform_data(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int=50,\n                                     mode: str = \"fit_transform\", save_to_path=None, verbose: bool=False):\n        assert mode in [\"fit_transform\", \"transform\"], f\"Unrecognized mode: {mode}! Please use one of the following modes: \\\"fit_transform\\\", \\\"transform\\\"\"\n        preprocessed_frames = []\n        for step in tqdm.notebook.tqdm(range(0, num_parts_total, num_parts_to_preprocess_at_once), \n                                       desc=\"Transforming sequential credit data\"):\n            data_frame = read_parquet_dataset_from_local(path_to_dataset, start_from=step, \n                                                         num_parts_to_read=num_parts_to_preprocess_at_once, \n                                                         verbose=verbose)\n            features = self.__extract_count_aggregations(data_frame, mode=mode)\n            if save_to_path:\n                features.to_parquet(os.path.join(save_to_path, f\"processed_chunk_{step}.pq\"))\n            preprocessed_frames.append(features)\n        \n        features = pd.concat(preprocessed_frames)\n        features.fillna(np.uint8(0), inplace=True)\n        dummy_features = list(features.columns.values)\n        dummy_features.remove(\"id\")\n        if mode == \"fit_transform\":\n            self.encoded_features = dummy_features\n        else:\n            assert not self.encoded_features is None, \"Transformer not fitted\"\n            for col in self.encoded_features:\n                if not col in dummy_features:\n                    features[col] = np.uint8(0)\n        return features[[\"id\"]+self.encoded_features]\n    \n    def fit_transform(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int = 50,\n                      save_to_path=None, verbose: bool=False):\n        return self.__transform_data(path_to_dataset=path_to_dataset,\n                                     num_parts_to_preprocess_at_once=num_parts_to_preprocess_at_once,\n                                     num_parts_total=num_parts_total, mode=\"fit_transform\",\n                                     save_to_path=save_to_path, verbose=verbose)\n    def transform(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int=50,\n                  save_to_path=None, verbose: bool=False):\n        return self.__transform_data(path_to_dataset=path_to_dataset,\n                                     num_parts_to_preprocess_at_once=num_parts_to_preprocess_at_once,\n                                     num_parts_total=num_parts_total, mode=\"transform\",\n                                     save_to_path=save_to_path, verbose=verbose)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:27:39.207994Z","iopub.execute_input":"2024-10-26T16:27:39.208385Z","iopub.status.idle":"2024-10-26T16:27:39.223655Z","shell.execute_reply.started":"2024-10-26T16:27:39.208353Z","shell.execute_reply":"2024-10-26T16:27:39.222488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CountAggregator(object):\n    \n    def __init__(self):\n        self.encoded_features = None\n        \n    def __extract_count_aggregations(self, data_frame: pd.DataFrame, mode: str) -> pd.DataFrame:\n        # one-hot-encoding\n        feature_columns = list(data_frame.columns.values)\n        feature_columns.remove(\"id\")\n        feature_columns.remove(\"rn\")\n\n        dummies = pd.get_dummies(data_frame[feature_columns], columns=feature_columns)\n        dummy_features = dummies.columns.values\n        \n        ohe_features = pd.concat([data_frame, dummies], axis=1)\n        ohe_features = ohe_features.drop(columns=feature_columns)\n        \n        # count aggregation\n        ohe_features.groupby(\"id\")\n        features = ohe_features.groupby(\"id\")[dummy_features].sum().reset_index(drop=False)\n        return features\n        \n    def __transform_data(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int=50,\n                                     mode: str = \"fit_transform\", save_to_path=None, verbose: bool=False):\n        assert mode in [\"fit_transform\", \"transform\"], f\"Unrecognized mode: {mode}! Please use one of the following modes: \\\"fit_transform\\\", \\\"transform\\\"\"\n        preprocessed_frames = []\n        for step in tqdm.notebook.tqdm(range(0, num_parts_total, num_parts_to_preprocess_at_once), \n                                       desc=\"Transforming sequential credit data\"):\n            data_frame = read_parquet_dataset_from_local(path_to_dataset, start_from=step, \n                                                         num_parts_to_read=num_parts_to_preprocess_at_once, \n                                                         verbose=verbose)\n            features = self.__extract_count_aggregations(data_frame, mode=mode)\n            if save_to_path:\n                features.to_parquet(os.path.join(save_to_path, f\"processed_chunk_{step}.pq\"))\n            preprocessed_frames.append(features)\n        \n        features = pd.concat(preprocessed_frames)\n        features.fillna(np.uint8(0), inplace=True)\n        dummy_features = list(features.columns.values)\n        dummy_features.remove(\"id\")\n        if mode == \"fit_transform\":\n            self.encoded_features = dummy_features\n        else:\n            assert not self.encoded_features is None, \"Transformer not fitted\"\n            for col in self.encoded_features:\n                if not col in dummy_features:\n                    features[col] = np.uint8(0)\n        return features[[\"id\"]+self.encoded_features]\n    \n    def fit_transform(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int = 50,\n                      save_to_path=None, verbose: bool=False):\n        return self.__transform_data(path_to_dataset=path_to_dataset,\n                                     num_parts_to_preprocess_at_once=num_parts_to_preprocess_at_once,\n                                     num_parts_total=num_parts_total, mode=\"fit_transform\",\n                                     save_to_path=save_to_path, verbose=verbose)\n    def transform(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int=50,\n                  save_to_path=None, verbose: bool=False):\n        return self.__transform_data(path_to_dataset=path_to_dataset,\n                                     num_parts_to_preprocess_at_once=num_parts_to_preprocess_at_once,\n                                     num_parts_total=num_parts_total, mode=\"transform\",\n                                     save_to_path=save_to_path, verbose=verbose)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T14:26:14.334093Z","iopub.execute_input":"2024-10-26T14:26:14.335012Z","iopub.status.idle":"2024-10-26T14:26:14.350514Z","shell.execute_reply.started":"2024-10-26T14:26:14.334975Z","shell.execute_reply":"2024-10-26T14:26:14.349466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WeightMeanAggregator(object):\n    \n    def __init__(self, alpha=1):\n        self.encoded_features = None\n        self.alpha = alpha\n        \n    def __extract_aggregations(self, data_frame: pd.DataFrame, mode: str) -> pd.DataFrame:\n        # one-hot-encoding\n        feature_columns = list(data_frame.columns.values)\n        feature_columns.remove(\"id\")\n        feature_columns.remove(\"rn\")\n\n        dummies = pd.get_dummies(data_frame[feature_columns], columns=feature_columns)\n        dummy_features = dummies.columns.values\n        \n        ohe_features = pd.concat([data_frame, dummies], axis=1)\n        ohe_features = ohe_features.drop(columns=feature_columns)\n        \n        # lenght of history\n        history_lenght = ohe_features.groupby('id')['rn'].max().reset_index(drop=False)\n        history_lenght = history_lenght.rename(columns={'rn': 'history_lenght'})\n        ohe_features = ohe_features.merge(history_lenght, on='id')\n\n        ohe_features['weight'] = (ohe_features['rn'] / ohe_features['history_lenght'])**self.alpha\n        sum_weight = ohe_features.groupby('id')['weight'].sum().reset_index(drop=False).rename(columns={'weight': 'sum_weight'})\n        ohe_features = ohe_features.merge(sum_weight, on='id')\n        \n        generated_features = list(ohe_features.columns)\n        generated_features.remove(\"id\")\n        generated_features.remove(\"rn\")\n        generated_features.remove(\"history_lenght\")\n        generated_features.remove(\"weight\")\n        generated_features.remove(\"sum_weight\")\n        \n        for feature in generated_features:\n            ohe_features[feature] = ohe_features[feature] * ohe_features['weight'] / ohe_features['sum_weight']\n        \n        # weight aggregation\n        ohe_features.groupby(\"id\")\n        features = ohe_features.groupby(\"id\")[generated_features].sum().reset_index(drop=False)\n        features = features.merge(history_lenght, on='id')\n        \n        return features  \n        \n    def __transform_data(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int=50,\n                                     mode: str = \"fit_transform\", save_to_path=None, verbose: bool=False):\n        assert mode in [\"fit_transform\", \"transform\"], f\"Unrecognized mode: {mode}! Please use one of the following modes: \\\"fit_transform\\\", \\\"transform\\\"\"\n        preprocessed_frames = []\n        for step in tqdm.notebook.tqdm(range(0, num_parts_total, num_parts_to_preprocess_at_once), \n                                       desc=\"Transforming sequential credit data\"):\n            data_frame = read_parquet_dataset_from_local(path_to_dataset, start_from=step, \n                                                         num_parts_to_read=num_parts_to_preprocess_at_once, \n                                                         verbose=verbose)\n            features = self.__extract_aggregations(data_frame, mode=mode)\n            if save_to_path:\n                features.to_parquet(os.path.join(save_to_path, f\"processed_chunk_{step}.pq\"))\n            preprocessed_frames.append(features)\n        \n        features = pd.concat(preprocessed_frames)\n        features.fillna(np.uint8(0), inplace=True)\n        dummy_features = list(features.columns.values)\n        dummy_features.remove(\"id\")\n        if mode == \"fit_transform\":\n            self.encoded_features = dummy_features\n        else:\n            assert not self.encoded_features is None, \"Transformer not fitted\"\n            for col in self.encoded_features:\n                if not col in dummy_features:\n                    features[col] = np.uint8(0)\n        return features[[\"id\"]+self.encoded_features]\n    \n    def fit_transform(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int = 50,\n                      save_to_path=None, verbose: bool=False):\n        return self.__transform_data(path_to_dataset=path_to_dataset,\n                                     num_parts_to_preprocess_at_once=num_parts_to_preprocess_at_once,\n                                     num_parts_total=num_parts_total, mode=\"fit_transform\",\n                                     save_to_path=save_to_path, verbose=verbose)\n    def transform(self, path_to_dataset: str, num_parts_to_preprocess_at_once: int = 1, num_parts_total: int=50,\n                  save_to_path=None, verbose: bool=False):\n        return self.__transform_data(path_to_dataset=path_to_dataset,\n                                     num_parts_to_preprocess_at_once=num_parts_to_preprocess_at_once,\n                                     num_parts_total=num_parts_total, mode=\"transform\",\n                                     save_to_path=save_to_path, verbose=verbose)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T14:26:22.292851Z","iopub.execute_input":"2024-10-26T14:26:22.293201Z","iopub.status.idle":"2024-10-26T14:26:22.312890Z","shell.execute_reply.started":"2024-10-26T14:26:22.293175Z","shell.execute_reply":"2024-10-26T14:26:22.311807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregator_count = CountAggregator()\naggregator_weight_mean = WeightMeanAggregator(alpha=1.4)\naggregator_ohe = OHEAggregator()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:29:49.902397Z","iopub.execute_input":"2024-10-26T16:29:49.903395Z","iopub.status.idle":"2024-10-26T16:29:49.907629Z","shell.execute_reply.started":"2024-10-26T16:29:49.903356Z","shell.execute_reply":"2024-10-26T16:29:49.906581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_data_ohe = aggregator_ohe.fit_transform(path_to_dataset = TRAIN_DATA_PATH, num_parts_to_preprocess_at_once=1, num_parts_total=3, \n                                      save_to_path=None, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:29:53.722125Z","iopub.execute_input":"2024-10-26T16:29:53.723051Z","iopub.status.idle":"2024-10-26T16:39:14.070231Z","shell.execute_reply.started":"2024-10-26T16:29:53.723014Z","shell.execute_reply":"2024-10-26T16:39:14.069052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_data_weight_mean = aggregator_weight_mean.fit_transform(TRAIN_DATA_PATH, num_parts_to_preprocess_at_once=1, num_parts_total=3, \n                                      save_to_path=None, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:53:34.316266Z","iopub.execute_input":"2024-10-26T13:53:34.316624Z","iopub.status.idle":"2024-10-26T13:54:34.321344Z","shell.execute_reply.started":"2024-10-26T13:53:34.316596Z","shell.execute_reply":"2024-10-26T13:54:34.320426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data_weight_mean.merge(train_data_count,how='left', on='id')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_weight_mean","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:54:34.322870Z","iopub.execute_input":"2024-10-26T13:54:34.323160Z","iopub.status.idle":"2024-10-26T13:54:34.579551Z","shell.execute_reply.started":"2024-10-26T13:54:34.323135Z","shell.execute_reply":"2024-10-26T13:54:34.578572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_count = aggregator_count.fit_transform(TRAIN_DATA_PATH, num_parts_to_preprocess_at_once=1, num_parts_total=3, \n                                      save_to_path=None, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T14:30:52.934842Z","iopub.execute_input":"2024-10-26T14:30:52.935640Z","iopub.status.idle":"2024-10-26T14:31:40.198137Z","shell.execute_reply.started":"2024-10-26T14:30:52.935606Z","shell.execute_reply":"2024-10-26T14:31:40.197277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target = pd.read_csv('/kaggle/input/alfa-bank-pd-credit-history/data_for_competition/train_target.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:39:14.072426Z","iopub.execute_input":"2024-10-26T16:39:14.072851Z","iopub.status.idle":"2024-10-26T16:39:14.795552Z","shell.execute_reply.started":"2024-10-26T16:39:14.072814Z","shell.execute_reply":"2024-10-26T16:39:14.794654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data_ohe.merge(train_target,how='left', on='id')","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:39:14.796933Z","iopub.execute_input":"2024-10-26T16:39:14.797586Z","iopub.status.idle":"2024-10-26T16:39:20.703138Z","shell.execute_reply.started":"2024-10-26T16:39:14.797548Z","shell.execute_reply":"2024-10-26T16:39:20.702318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:39:20.704970Z","iopub.execute_input":"2024-10-26T16:39:20.705275Z","iopub.status.idle":"2024-10-26T16:39:22.917157Z","shell.execute_reply.started":"2024-10-26T16:39:20.705249Z","shell.execute_reply":"2024-10-26T16:39:22.916220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data_count.merge(train_target,how='left', on='id')\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2024-10-26T14:31:40.677181Z","iopub.execute_input":"2024-10-26T14:31:40.677508Z","iopub.status.idle":"2024-10-26T14:31:42.571492Z","shell.execute_reply.started":"2024-10-26T14:31:40.677482Z","shell.execute_reply":"2024-10-26T14:31:42.570517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data_weight_mean.merge(train_target,how='left', on='id')\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:36:51.097736Z","iopub.execute_input":"2024-07-07T09:36:51.098111Z","iopub.status.idle":"2024-07-07T09:36:58.262714Z","shell.execute_reply.started":"2024-07-07T09:36:51.098080Z","shell.execute_reply":"2024-07-07T09:36:58.261346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"подготовка данных закончилась, начинается обучение","metadata":{}},{"cell_type":"code","source":"data_frame = read_parquet_dataset_from_local(path_to_dataset,num_parts_to_read=3)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:55:31.093647Z","iopub.execute_input":"2024-10-20T14:55:31.093938Z","iopub.status.idle":"2024-10-20T14:55:35.337359Z","shell.execute_reply.started":"2024-10-20T14:55:31.093913Z","shell.execute_reply":"2024-10-20T14:55:35.336384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = list(data_frame.columns.values)\nfeature_cols","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:55:35.338717Z","iopub.execute_input":"2024-10-20T14:55:35.339038Z","iopub.status.idle":"2024-10-20T14:55:35.346677Z","shell.execute_reply.started":"2024-10-20T14:55:35.339011Z","shell.execute_reply":"2024-10-20T14:55:35.345581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = feature_cols\n\ntarget = 'flag'  \ntargets = train_data[\"flag\"].values","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:46:10.865877Z","iopub.execute_input":"2024-10-20T14:46:10.866623Z","iopub.status.idle":"2024-10-20T14:46:10.871260Z","shell.execute_reply.started":"2024-10-20T14:46:10.866582Z","shell.execute_reply":"2024-10-20T14:46:10.870113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(class_weight='balanced')","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:55:35.347902Z","iopub.execute_input":"2024-10-20T14:55:35.348237Z","iopub.status.idle":"2024-10-20T14:55:35.356982Z","shell.execute_reply.started":"2024-10-20T14:55:35.348208Z","shell.execute_reply":"2024-10-20T14:55:35.356061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target\ndata_frame = data_frame.merge(train_target,how = 'left', on = 'id')\ndata_frame","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:55:35.360272Z","iopub.execute_input":"2024-10-20T14:55:35.360738Z","iopub.status.idle":"2024-10-20T14:55:38.343250Z","shell.execute_reply.started":"2024-10-20T14:55:35.360708Z","shell.execute_reply":"2024-10-20T14:55:38.342062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_all = ['id', 'pre_since_opened', 'pre_since_confirmed', 'pre_pterm',\n       'pre_fterm', 'pre_till_pclose', 'pre_till_fclose',\n       'pre_loans_credit_limit', 'pre_loans_next_pay_summ',\n       'pre_loans_outstanding', 'pre_loans_total_overdue',\n       'pre_loans_max_overdue_sum', 'pre_loans_credit_cost_rate', \n       'is_zero_loans5', 'is_zero_loans530', 'is_zero_loans3060',\n       'is_zero_loans6090', 'is_zero_loans90', 'pre_util', 'pre_over2limit',\n       'pre_maxover2limit', 'is_zero_util', 'is_zero_over2limit',\n       'is_zero_maxover2limit', \n       'enc_loans_account_holder_type', 'enc_loans_credit_status',\n       'enc_loans_credit_type', 'enc_loans_account_cur', 'pclose_flag',\n       'fclose_flag']\n\nlr.fit(data_frame[feature_cols], data_frame['flag'])","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:55:38.344638Z","iopub.execute_input":"2024-10-20T14:55:38.344964Z","iopub.status.idle":"2024-10-20T14:55:54.689039Z","shell.execute_reply.started":"2024-10-20T14:55:38.344936Z","shell.execute_reply":"2024-10-20T14:55:54.687644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = lr.predict(data_frame[feature_cols])\nmetrics.roc_auc_score(data_frame['flag'], y)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:55:54.691123Z","iopub.execute_input":"2024-10-20T14:55:54.692035Z","iopub.status.idle":"2024-10-20T14:56:01.127395Z","shell.execute_reply.started":"2024-10-20T14:55:54.691984Z","shell.execute_reply":"2024-10-20T14:56:01.126341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_class = lr.predict(data_frame[feature_cols])\nunique, counts = np.unique(preds_class, return_counts=True)\nprint (unique, counts)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:56:01.128727Z","iopub.execute_input":"2024-10-20T14:56:01.129086Z","iopub.status.idle":"2024-10-20T14:56:04.416573Z","shell.execute_reply.started":"2024-10-20T14:56:01.129059Z","shell.execute_reply":"2024-10-20T14:56:04.415283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique, counts = np.unique( data_frame['flag'], return_counts=True)\nprint (unique, counts)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T14:59:06.972636Z","iopub.execute_input":"2024-10-20T14:59:06.973603Z","iopub.status.idle":"2024-10-20T14:59:07.374314Z","shell.execute_reply.started":"2024-10-20T14:59:06.973543Z","shell.execute_reply":"2024-10-20T14:59:07.373136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = catboost_pool = Pool(data_frame[feature_cols],\n                                 data_frame['flag'])\n\nmodel = CatBoostClassifier(iterations=20,\n                           depth=2,\n                           learning_rate=1,\n                           loss_function='Logloss',\n                           verbose=True)\n# train the model\nmodel.fit(data_frame[feature_cols], data_frame['flag'])\n# make the prediction using the resulting model\npreds_class = model.predict(test_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T15:05:22.737024Z","iopub.execute_input":"2024-10-20T15:05:22.738089Z","iopub.status.idle":"2024-10-20T15:05:45.746614Z","shell.execute_reply.started":"2024-10-20T15:05:22.738046Z","shell.execute_reply":"2024-10-20T15:05:45.745725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique, counts = np.unique(preds_class, return_counts=True)\nprint (unique, counts)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T15:05:54.556220Z","iopub.execute_input":"2024-10-20T15:05:54.556653Z","iopub.status.idle":"2024-10-20T15:05:54.589149Z","shell.execute_reply.started":"2024-10-20T15:05:54.556614Z","shell.execute_reply":"2024-10-20T15:05:54.588139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:32:18.139269Z","iopub.status.idle":"2024-10-26T13:32:18.140109Z","shell.execute_reply.started":"2024-10-26T13:32:18.139843Z","shell.execute_reply":"2024-10-26T13:32:18.139865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install catboost==1.2.5","metadata":{"execution":{"iopub.status.busy":"2024-07-07T13:52:20.255462Z","iopub.execute_input":"2024-07-07T13:52:20.255979Z","iopub.status.idle":"2024-07-07T13:53:07.617698Z","shell.execute_reply.started":"2024-07-07T13:52:20.255944Z","shell.execute_reply":"2024-07-07T13:53:07.615362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = list(train_data.columns.values)\nfeature_cols.remove(\"flag\")\nlen(feature_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:39:22.918471Z","iopub.execute_input":"2024-10-26T16:39:22.918869Z","iopub.status.idle":"2024-10-26T16:39:22.925654Z","shell.execute_reply.started":"2024-10-26T16:39:22.918835Z","shell.execute_reply":"2024-10-26T16:39:22.924712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = feature_cols\ntarget = 'flag'  \ntargets = train_data[\"flag\"].values","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:39:22.926918Z","iopub.execute_input":"2024-10-26T16:39:22.927562Z","iopub.status.idle":"2024-10-26T16:39:22.936485Z","shell.execute_reply.started":"2024-10-26T16:39:22.927534Z","shell.execute_reply":"2024-10-26T16:39:22.935620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedKFold(n_splits=10, random_state=596, shuffle=True)\n\noof = np.zeros(len(train_data))\ntrain_preds = np.zeros(len(train_data))\n\nmodels_catboost_base = []\n\nfor fold_, (train_idx, val_idx) in enumerate(cv.split(train_data, targets), 1):\n    print(f\"Training with fold {fold_} started\")\n    \n    model_catboost = CatBoostClassifier(\n                    verbose=3000,\n                    loss_function='Logloss',\n                    eval_metric='AUC',\n                    early_stopping_rounds=500,\n                    #task_type=\"GPU\",\n                    task_type=\"CPU\",\n                    iterations=50000,\n                    learning_rate=0.01, \n                    auto_class_weights = 'Balanced',\n                    depth=5, \n                    l2_leaf_reg= 1,\n                    random_state=42\n    )\n      \n    train, val = train_data.iloc[train_idx], train_data.iloc[val_idx]\n    \n    model_catboost.fit(\n                  train[feature_names], train[target], \n                  eval_set=(val[feature_names], val[target]),\n                  plot=False\n    )\n    oof[val_idx] = model_catboost.predict_proba(val[feature_cols])[:, 1]\n    train_preds[train_idx] += model_catboost.predict_proba(train[feature_cols])[:, 1] / (cv.n_splits-1)\n    models_catboost_base.append(model_catboost)\n    print(f\"Training with fold {fold_} completed\")","metadata":{"execution":{"iopub.status.busy":"2024-10-26T16:39:22.937617Z","iopub.execute_input":"2024-10-26T16:39:22.937877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#models_catboost_ranker\n#занял на порядок больше времени, примерно с такими же результатами","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['Group_id'] = train_data['id']//187500\ntrain_data['Group_id'] = train_data['Group_id'].apply(lambda x: int(x)) ","metadata":{"execution":{"iopub.status.busy":"2024-09-01T17:01:05.219934Z","iopub.execute_input":"2024-09-01T17:01:05.220749Z","iopub.status.idle":"2024-09-01T17:01:05.556064Z","shell.execute_reply.started":"2024-09-01T17:01:05.220710Z","shell.execute_reply":"2024-09-01T17:01:05.555146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = list(train_data.columns.values)\nfeature_cols.remove(\"flag\")\nlen(feature_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:58:09.888683Z","iopub.execute_input":"2024-10-26T13:58:09.889031Z","iopub.status.idle":"2024-10-26T13:58:09.896099Z","shell.execute_reply.started":"2024-10-26T13:58:09.889004Z","shell.execute_reply":"2024-10-26T13:58:09.894995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target = pd.read_csv('/kaggle/input/alfa-bank-pd-credit-history/data_for_competition/train_target.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:59:09.627081Z","iopub.execute_input":"2024-10-26T13:59:09.627746Z","iopub.status.idle":"2024-10-26T13:59:10.952740Z","shell.execute_reply.started":"2024-10-26T13:59:09.627711Z","shell.execute_reply":"2024-10-26T13:59:10.951950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.merge(train_target,how='left', on='id')\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:59:23.295078Z","iopub.execute_input":"2024-10-26T13:59:23.295793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = feature_cols\ntarget = 'flag'  \ntargets = train_data[\"flag\"].values","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:58:12.201294Z","iopub.execute_input":"2024-10-26T13:58:12.201968Z","iopub.status.idle":"2024-10-26T13:58:12.206028Z","shell.execute_reply.started":"2024-10-26T13:58:12.201933Z","shell.execute_reply":"2024-10-26T13:58:12.205195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedKFold(n_splits=10, random_state=754, shuffle=True)\n\noof = np.zeros(len(train_data))\n\nmodels_catboost_ranker = []\n\nfor fold_, (train_idx, val_idx) in enumerate(cv.split(train_data, targets), 1):\n    print(f\"Training with fold {fold_} started\")\n    \n    model_catboost = CatBoostRanker(\n                    verbose=3000,\n                    loss_function='PairLogit:max_pairs=1000000',\n                    eval_metric='AUC',\n                    early_stopping_rounds=500,\n                    task_type=\"CPU\",\n                    iterations=50000,\n                    learning_rate=0.01,       \n                    depth=7, \n                    l2_leaf_reg= 1,\n                    random_state=42,\n    )\n    \n    train, val = train_data.iloc[train_idx], train_data.iloc[val_idx]\n    \n    train_pool = Pool(\n            data=train[feature_names],\n            label=train[target].values\n            ,group_id=train[\"Group_id\"].values\n            )\n\n    val_pool = Pool(\n           data=val[feature_names],\n           label=val[target].values\n           ,group_id=val[\"Group_id\"].values\n        )\n    \n    \n    model_catboost.fit(\n                  train_pool, \n                  eval_set=val_pool,\n                  plot=False\n    )\n    oof[val_idx] = model_catboost.predict(val[feature_cols])\n\n    models_catboost_ranker.append(model_catboost)\n    print(f\"Training with fold {fold_} completed\")","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:58:14.547230Z","iopub.execute_input":"2024-10-26T13:58:14.547977Z","iopub.status.idle":"2024-10-26T13:58:19.203180Z","shell.execute_reply.started":"2024-10-26T13:58:14.547943Z","shell.execute_reply":"2024-10-26T13:58:19.202037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.describe()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:41:56.138109Z","iopub.execute_input":"2024-05-21T06:41:56.139102Z","iopub.status.idle":"2024-05-21T06:42:40.744590Z","shell.execute_reply.started":"2024-05-21T06:41:56.139058Z","shell.execute_reply":"2024-05-21T06:42:40.743523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T10:50:19.624410Z","iopub.execute_input":"2024-05-18T10:50:19.624753Z","iopub.status.idle":"2024-05-18T10:50:19.649268Z","shell.execute_reply.started":"2024-05-18T10:50:19.624726Z","shell.execute_reply":"2024-05-18T10:50:19.647576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#corr = df1.corr()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T10:51:14.464409Z","iopub.execute_input":"2024-05-18T10:51:14.464787Z","iopub.status.idle":"2024-05-18T10:55:53.612807Z","shell.execute_reply.started":"2024-05-18T10:51:14.464757Z","shell.execute_reply":"2024-05-18T10:55:53.611080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(200,100))\nsns.heatmap(corr,annot=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T10:58:37.124561Z","iopub.execute_input":"2024-05-18T10:58:37.126969Z","iopub.status.idle":"2024-05-18T10:58:55.169733Z","shell.execute_reply.started":"2024-05-18T10:58:37.126888Z","shell.execute_reply":"2024-05-18T10:58:55.167820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subs = df1[['pre_loans5','pre_loans530','pre_loans3060','pre_loans6090','pre_loans90']]\nsubs","metadata":{"execution":{"iopub.status.busy":"2024-05-18T11:03:27.939444Z","iopub.execute_input":"2024-05-18T11:03:27.940387Z","iopub.status.idle":"2024-05-18T11:03:29.378472Z","shell.execute_reply.started":"2024-05-18T11:03:27.940313Z","shell.execute_reply":"2024-05-18T11:03:29.376570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_subs = subs.corr()","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:31:55.899345Z","iopub.execute_input":"2024-05-10T13:31:55.900039Z","iopub.status.idle":"2024-05-10T13:31:58.468167Z","shell.execute_reply.started":"2024-05-10T13:31:55.899999Z","shell.execute_reply":"2024-05-10T13:31:58.467012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nsns.heatmap(corr_subs,annot=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:31:58.469627Z","iopub.execute_input":"2024-05-10T13:31:58.470078Z","iopub.status.idle":"2024-05-10T13:31:58.862776Z","shell.execute_reply.started":"2024-05-10T13:31:58.470029Z","shell.execute_reply":"2024-05-10T13:31:58.861451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subs.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T11:03:34.323531Z","iopub.execute_input":"2024-05-18T11:03:34.323930Z","iopub.status.idle":"2024-05-18T11:03:37.735070Z","shell.execute_reply.started":"2024-05-18T11:03:34.323896Z","shell.execute_reply":"2024-05-18T11:03:37.733673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['pre_loans5'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T11:03:31.054715Z","iopub.execute_input":"2024-05-18T11:03:31.055606Z","iopub.status.idle":"2024-05-18T11:03:31.315874Z","shell.execute_reply.started":"2024-05-18T11:03:31.055566Z","shell.execute_reply":"2024-05-18T11:03:31.314547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['pre_loans530'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T11:03:51.662511Z","iopub.execute_input":"2024-05-18T11:03:51.662986Z","iopub.status.idle":"2024-05-18T11:03:51.926452Z","shell.execute_reply.started":"2024-05-18T11:03:51.662949Z","shell.execute_reply":"2024-05-18T11:03:51.924672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pre_loans не имеет смысла включать в features потому что они все по-разному закодированы, а значение 16 может быть ничем не лучше 6","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:32:02.116746Z","iopub.execute_input":"2024-05-10T13:32:02.117098Z","iopub.status.idle":"2024-05-10T13:32:02.122446Z","shell.execute_reply.started":"2024-05-10T13:32:02.117069Z","shell.execute_reply":"2024-05-10T13:32:02.121391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subs_enc = df1[['enc_paym_0',\n 'enc_paym_1',\n 'enc_paym_2',\n 'enc_paym_3',\n 'enc_paym_4',\n 'enc_paym_5',\n 'enc_paym_6',\n 'enc_paym_7',\n 'enc_paym_8',\n 'enc_paym_9',\n 'enc_paym_10',\n 'enc_paym_11',\n 'enc_paym_12',\n 'enc_paym_13',\n 'enc_paym_14',\n 'enc_paym_15',\n 'enc_paym_16',\n 'enc_paym_17',\n 'enc_paym_18',\n 'enc_paym_19',\n 'enc_paym_20',\n 'enc_paym_21',\n 'enc_paym_22',\n 'enc_paym_23',\n 'enc_paym_24' ]]","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:32:02.123709Z","iopub.execute_input":"2024-05-10T13:32:02.124047Z","iopub.status.idle":"2024-05-10T13:32:03.904049Z","shell.execute_reply.started":"2024-05-10T13:32:02.124021Z","shell.execute_reply":"2024-05-10T13:32:03.902770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#corr_enc = subs_enc.corr()","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:32:03.905701Z","iopub.execute_input":"2024-05-10T13:32:03.906165Z","iopub.status.idle":"2024-05-10T13:32:03.911621Z","shell.execute_reply.started":"2024-05-10T13:32:03.906124Z","shell.execute_reply":"2024-05-10T13:32:03.910402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.figure(figsize=(20,10))\n#sns.heatmap(corr_enc,annot=True)\n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:32:03.913128Z","iopub.execute_input":"2024-05-10T13:32:03.913488Z","iopub.status.idle":"2024-05-10T13:32:03.920359Z","shell.execute_reply.started":"2024-05-10T13:32:03.913458Z","shell.execute_reply":"2024-05-10T13:32:03.919333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_0'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:36.485433Z","iopub.execute_input":"2024-05-19T15:36:36.486339Z","iopub.status.idle":"2024-05-19T15:36:36.741812Z","shell.execute_reply.started":"2024-05-19T15:36:36.486287Z","shell.execute_reply":"2024-05-19T15:36:36.740781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_24'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:37.262952Z","iopub.execute_input":"2024-05-19T15:36:37.264159Z","iopub.status.idle":"2024-05-19T15:36:37.518354Z","shell.execute_reply.started":"2024-05-19T15:36:37.264123Z","shell.execute_reply":"2024-05-19T15:36:37.517142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_1'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:37.520745Z","iopub.execute_input":"2024-05-19T15:36:37.521292Z","iopub.status.idle":"2024-05-19T15:36:37.808945Z","shell.execute_reply.started":"2024-05-19T15:36:37.521249Z","shell.execute_reply":"2024-05-19T15:36:37.807622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_23'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:37.810627Z","iopub.execute_input":"2024-05-19T15:36:37.811042Z","iopub.status.idle":"2024-05-19T15:36:38.067637Z","shell.execute_reply.started":"2024-05-19T15:36:37.811003Z","shell.execute_reply":"2024-05-19T15:36:38.066224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_22'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:38.069168Z","iopub.execute_input":"2024-05-19T15:36:38.069648Z","iopub.status.idle":"2024-05-19T15:36:38.330467Z","shell.execute_reply.started":"2024-05-19T15:36:38.069608Z","shell.execute_reply":"2024-05-19T15:36:38.328977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_21'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:38.332139Z","iopub.execute_input":"2024-05-19T15:36:38.332626Z","iopub.status.idle":"2024-05-19T15:36:38.593214Z","shell.execute_reply.started":"2024-05-19T15:36:38.332590Z","shell.execute_reply":"2024-05-19T15:36:38.591396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_20'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:38.598340Z","iopub.execute_input":"2024-05-19T15:36:38.598815Z","iopub.status.idle":"2024-05-19T15:36:38.852234Z","shell.execute_reply.started":"2024-05-19T15:36:38.598775Z","shell.execute_reply":"2024-05-19T15:36:38.851052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['enc_paym_19'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:39.118234Z","iopub.execute_input":"2024-05-19T15:36:39.118711Z","iopub.status.idle":"2024-05-19T15:36:39.373470Z","shell.execute_reply.started":"2024-05-19T15:36:39.118676Z","shell.execute_reply":"2024-05-19T15:36:39.371884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['is_zero_loans5'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:39.375235Z","iopub.execute_input":"2024-05-19T15:36:39.375738Z","iopub.status.idle":"2024-05-19T15:36:39.631571Z","shell.execute_reply.started":"2024-05-19T15:36:39.375702Z","shell.execute_reply":"2024-05-19T15:36:39.630240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['is_zero_loans530'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T15:36:39.633285Z","iopub.execute_input":"2024-05-19T15:36:39.633635Z","iopub.status.idle":"2024-05-19T15:36:39.889108Z","shell.execute_reply.started":"2024-05-19T15:36:39.633606Z","shell.execute_reply":"2024-05-19T15:36:39.887544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_zero = df1[['is_zero_loans5','is_zero_loans530','is_zero_loans3060','is_zero_loans6090','is_zero_loans90']]\ncorr_zero = corr_zero.corr()\nplt.figure(figsize=(20,10))\nsns.heatmap(corr_zero,annot=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T11:04:39.216218Z","iopub.execute_input":"2024-05-18T11:04:39.216894Z","iopub.status.idle":"2024-05-18T11:04:43.540557Z","shell.execute_reply.started":"2024-05-18T11:04:39.216839Z","shell.execute_reply":"2024-05-18T11:04:43.539120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.columns","metadata":{"execution":{"iopub.status.busy":"2024-05-18T11:04:43.543233Z","iopub.execute_input":"2024-05-18T11:04:43.543692Z","iopub.status.idle":"2024-05-18T11:04:43.553912Z","shell.execute_reply.started":"2024-05-18T11:04:43.543649Z","shell.execute_reply":"2024-05-18T11:04:43.552098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['pre_pterm'].value_counts()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-10T13:33:26.972162Z","iopub.execute_input":"2024-05-10T13:33:26.972618Z","iopub.status.idle":"2024-05-10T13:33:27.229854Z","shell.execute_reply.started":"2024-05-10T13:33:26.972585Z","shell.execute_reply":"2024-05-10T13:33:27.228139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_all = ['id', 'pre_since_opened', 'pre_since_confirmed', 'pre_pterm',\n       'pre_fterm', 'pre_till_pclose', 'pre_till_fclose',\n       'pre_loans_credit_limit', 'pre_loans_next_pay_summ',\n       'pre_loans_outstanding', 'pre_loans_total_overdue',\n       'pre_loans_max_overdue_sum', 'pre_loans_credit_cost_rate', \n       'is_zero_loans5', 'is_zero_loans530', 'is_zero_loans3060',\n       'is_zero_loans6090', 'is_zero_loans90', 'pre_util', 'pre_over2limit',\n       'pre_maxover2limit', 'is_zero_util', 'is_zero_over2limit',\n       'is_zero_maxover2limit', \n       'enc_loans_account_holder_type', 'enc_loans_credit_status',\n       'enc_loans_credit_type', 'enc_loans_account_cur', 'pclose_flag',\n       'fclose_flag']","metadata":{"execution":{"iopub.status.busy":"2024-06-30T12:57:01.659151Z","iopub.execute_input":"2024-06-30T12:57:01.659527Z","iopub.status.idle":"2024-06-30T12:57:01.666498Z","shell.execute_reply.started":"2024-06-30T12:57:01.659495Z","shell.execute_reply":"2024-06-30T12:57:01.664991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['id','rn','pre_till_pclose','pre_loans_credit_limit',\n'pre_loans_next_pay_summ',\n'pre_loans_outstanding',\n'pre_loans_total_overdue',\n'pre_loans_max_overdue_sum',\n'pre_loans_credit_cost_rate','pre_util','enc_loans_account_cur']","metadata":{"execution":{"iopub.status.busy":"2024-06-30T12:57:01.667775Z","iopub.execute_input":"2024-06-30T12:57:01.668164Z","iopub.status.idle":"2024-06-30T12:57:01.684517Z","shell.execute_reply.started":"2024-06-30T12:57:01.668102Z","shell.execute_reply":"2024-06-30T12:57:01.683216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df1[features_all].merge(df_target,how = 'left', on = 'id')\ndf_train","metadata":{"execution":{"iopub.status.busy":"2024-06-30T12:40:15.696315Z","iopub.execute_input":"2024-06-30T12:40:15.697083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FEATURES_PATH = \"../data/train_features_gb/\"\nTEST_FEATURES_PATH = \"../data/test_features_gb/\"","metadata":{"execution":{"iopub.status.busy":"2024-06-30T13:06:19.106843Z","iopub.execute_input":"2024-06-30T13:06:19.107390Z","iopub.status.idle":"2024-06-30T13:06:19.113213Z","shell.execute_reply.started":"2024-06-30T13:06:19.107343Z","shell.execute_reply":"2024-06-30T13:06:19.111877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train2 = df_train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['flag'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:48:46.278718Z","iopub.execute_input":"2024-05-27T15:48:46.278999Z","iopub.status.idle":"2024-05-27T15:48:46.539466Z","shell.execute_reply.started":"2024-05-27T15:48:46.278977Z","shell.execute_reply":"2024-05-27T15:48:46.538362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(class_weight='balanced')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:46:19.943726Z","iopub.execute_input":"2024-05-21T06:46:19.944112Z","iopub.status.idle":"2024-05-21T06:46:19.949157Z","shell.execute_reply.started":"2024-05-21T06:46:19.944083Z","shell.execute_reply":"2024-05-21T06:46:19.948062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr.fit(df1[features], df_train['flag'])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:46:21.765538Z","iopub.execute_input":"2024-05-21T06:46:21.766250Z","iopub.status.idle":"2024-05-21T06:46:40.737285Z","shell.execute_reply.started":"2024-05-21T06:46:21.766218Z","shell.execute_reply":"2024-05-21T06:46:40.736143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = lr.predict(df1[features])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:46:40.739509Z","iopub.execute_input":"2024-05-21T06:46:40.740146Z","iopub.status.idle":"2024-05-21T06:46:42.948391Z","shell.execute_reply.started":"2024-05-21T06:46:40.740109Z","shell.execute_reply":"2024-05-21T06:46:42.947204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique, counts = np.unique(y, return_counts=True)\nprint (unique, counts)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:46:43.607575Z","iopub.execute_input":"2024-05-21T06:46:43.608246Z","iopub.status.idle":"2024-05-21T06:46:44.273939Z","shell.execute_reply.started":"2024-05-21T06:46:43.608210Z","shell.execute_reply":"2024-05-21T06:46:44.272904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train['flag'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:46:44.275242Z","iopub.execute_input":"2024-05-21T06:46:44.276191Z","iopub.status.idle":"2024-05-21T06:46:44.531897Z","shell.execute_reply.started":"2024-05-21T06:46:44.276154Z","shell.execute_reply":"2024-05-21T06:46:44.530725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics.roc_auc_score(df_train['flag'], y)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:46:44.533200Z","iopub.execute_input":"2024-05-21T06:46:44.533575Z","iopub.status.idle":"2024-05-21T06:46:51.219035Z","shell.execute_reply.started":"2024-05-21T06:46:44.533546Z","shell.execute_reply":"2024-05-21T06:46:51.217934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics.classification_report(df_train['flag'], y)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:47:10.377597Z","iopub.execute_input":"2024-05-21T06:47:10.378711Z","iopub.status.idle":"2024-05-21T06:48:06.626182Z","shell.execute_reply.started":"2024-05-21T06:47:10.378660Z","shell.execute_reply":"2024-05-21T06:48:06.625109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr2 = LogisticRegression(class_weight = {0:29,1:1})\nlr2.fit(df1[features], df_train['flag'])\ny2 = lr2.predict(df1[features])\nunique, counts = np.unique(y2, return_counts=True)\nprint (unique, counts)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:54:55.377137Z","iopub.execute_input":"2024-05-21T06:54:55.377600Z","iopub.status.idle":"2024-05-21T06:56:31.256793Z","shell.execute_reply.started":"2024-05-21T06:54:55.377571Z","shell.execute_reply":"2024-05-21T06:56:31.255641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics.classification_report(df_train['flag'], y2)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T06:52:51.375202Z","iopub.execute_input":"2024-05-21T06:52:51.375608Z","iopub.status.idle":"2024-05-21T06:53:48.769096Z","shell.execute_reply.started":"2024-05-21T06:52:51.375571Z","shell.execute_reply":"2024-05-21T06:53:48.767912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#не могу понять как выставить правильно веса. при class_weight='balanced' весь прогноз уходит в дефолт, при задании с помощью словаря {0:29,1:1} весь \n#прогноз уходит в недефолт, а при {0:1,1:29} в получается аналогично balanced","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = catboost_pool = Pool(df1[features],\n                                 df_train['flag'])\n\nmodel = CatBoostClassifier(iterations=5,\n                           depth=2,\n                           learning_rate=1,\n                           loss_function='Logloss',\n                           verbose=True)\n# train the model\nmodel.fit(df1[features], df_train['flag'])\n# make the prediction using the resulting model\npreds_class = model.predict(test_data)\n#preds_proba = model.predict_proba(test_data)\nunique, counts = np.unique(preds_class, return_counts=True)\nprint (unique, counts)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:42:52.809368Z","iopub.execute_input":"2024-05-21T07:42:52.809998Z","iopub.status.idle":"2024-05-21T07:43:17.846057Z","shell.execute_reply.started":"2024-05-21T07:42:52.809952Z","shell.execute_reply":"2024-05-21T07:43:17.845092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.roc_auc_score(df_train['flag'], preds_class))\nmetrics.classification_report(df_train['flag'], preds_class)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:43:17.847579Z","iopub.execute_input":"2024-05-21T07:43:17.848248Z","iopub.status.idle":"2024-05-21T07:44:22.215880Z","shell.execute_reply.started":"2024-05-21T07:43:17.848216Z","shell.execute_reply":"2024-05-21T07:44:22.214625Z"},"trusted":true},"execution_count":null,"outputs":[]}]}