{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install git+https://gitlab.com/nyker510/vivid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntest_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\n                       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for c in train_df.columns:\n    vc = train_df[c].value_counts()\n#     print(vc)\n\n    if c == 'id':\n        continue\n\n    test_set = set(test_df[c].unique())\n    train_set = set(train_df[c].unique())\n    print(train_set - test_set)\n    \n    n_intersection = len(train_set & test_set)\n    print(f'{c} intersection: {n_intersection / len(train_set)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['ord_0'].unique()\n\ntest_df['ord_0'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(train_df[c].unique()) -","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vivid.featureset.encodings import OneHotEncodingAtom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vivid.utils import get_logger, timer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger = get_logger(__name__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold, StratifiedKFold\n\nclass TargetEncodingTransformer:\n    def __init__(self, cv=10):\n        self.cv = cv\n        self.mapping_ = None\n\n    def fit(self, X, y):\n        fold = StratifiedKFold(self.cv)\n        if isinstance(X, pd.Series):\n            X = X.values\n        if isinstance(y, pd.Series):\n            y = y.values\n\n        oof = np.zeros_like(y, dtype=np.float)\n#         logger.info(f'Cardinarity: {len(np.unique(X))}')\n        logger.info(X.shape)\n        s = pd.Series(X)\n        self.mapping_ = pd.DataFrame(y).groupby(X).mean()[0].to_dict()\n#         print(self.mapping_)\n\n        for idx1, idx2 in fold.split(X, y):\n            vc = pd.DataFrame(y[idx1]).groupby(X[idx1]).mean()[0].to_dict()\n            val = s[idx2].map(vc).fillna(np.mean(y[idx1]))\n            oof[idx2] = val\n\n        return oof\n\n    def transform(self, X):\n        retval = pd.Series(X).map(self.mapping_)\n        return retval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vivid.featureset import AbstractAtom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TargetEncodingAtom(AbstractAtom):\n    use_columns = ['nom_0', 'nom_1',\n       'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',\n       'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5', 'day', 'month']\n\n    def __init__(self):\n        self.mapping = {}\n\n    def call(self, input_df: pd.DataFrame, y):\n        output_df = pd.DataFrame()\n        for col in self.use_columns:\n            name = f'{col}'\n            encoder = self.mapping.get(name, None)\n\n            if encoder is not None:\n                x = encoder.transform(input_df[col].values)\n            else:\n                encoder = TargetEncodingTransformer()\n                x = encoder.fit(input_df[col], y)\n                self.mapping[name] = encoder\n            output_df[name] = x\n\n        return output_df.add_prefix('TE_')\n\n    def fit(self, input_df: pd.DataFrame, y) -> pd.DataFrame:\n        self.mapping = {}\n        self.call(input_df, y)\n        return self\n\n    def transform(self, input_df):\n        return self.call(input_df, y=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TargetEncodingAtom().generate(train_df, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\n\nclass CatOneHotEoncodingAtom(OneHotEncodingAtom):\n    use_columns = ['bin_0', 'bin_1', 'bin_2', 'bin_3']\n    \n    def fit(self, input_df: pd.DataFrame, y=None):\n        self.mapping_ = OrderedDict()\n        for c in self.use_columns:\n            cat = input_df[c].dropna().unique()\n            self.mapping_[c] = cat\n        return self","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CatOneHotEoncodingAtom().generate(train_df, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vivid.out_of_fold.boosting import LGBMClassifierOutOfFold, XGBoostClassifierOutOfFold\nfrom vivid.out_of_fold.boosting.block import create_boosting_seed_blocks\nfrom vivid.out_of_fold.linear import LogisticOutOfFold\nfrom vivid.out_of_fold.ensumble import RFClassifierFeatureOutOfFold\nfrom vivid.out_of_fold.base import BaseOutOfFoldFeature\nfrom vivid.core import MergeFeature, EnsembleFeature\nfrom vivid.out_of_fold.base import BaseOutOfFoldFeature\n\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom rgf.sklearn import RGFClassifier\nfrom sklearn.linear_model import LogisticRegression","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class KaggleKernelMixin:\n    pred_ = None\n    def save_best_models(self, best_models):\n        pass\n    \n    def predict(self, input_df):\n        if self.pred_:\n            logger.info(f'{self}: load from cache')\n            return self.pred_\n        self.pred_ = super().predict(input_df)\n        return self.pred_\n    \nclass FillnaMixin:\n    def call(self, df_source, y=None, test=False):\n        if not test:\n            self.mappings_ = df_source.mean()\n        df_source = df_source.fillna(self.mappings_)\n        return super().call(df_source, y, test)\n\nclass CustomLGBM(KaggleKernelMixin, LGBMClassifierOutOfFold):\n    initial_params = {\n        'n_estimators': 10000,\n        'objective': 'binary',\n        'feature_fraction': .9,\n        'learning_rate': .05,\n        'max_depth': 5,\n        'num_leaves': 17\n    }\n    \nclass LogisticOptuna(FillnaMixin, KaggleKernelMixin, LogisticOutOfFold):\n    initial_params = {\n        'input_scaling': 'standard'\n    }\n\nclass XGB(KaggleKernelMixin, XGBoostClassifierOutOfFold):\n    pass\n\nclass SimpleLGBM(KaggleKernelMixin, LGBMClassifierOutOfFold):\n    initial_params = {\n        'n_estimators': 10000,\n        'learning_rate': .05,\n        'reg_lambda': 1.,\n        'reg_alpha': 1.,\n        'feature_fraction': .7,\n        'max_depth': 3,\n    }\n    \nclass RF(FillnaMixin, KaggleKernelMixin, RFClassifierFeatureOutOfFold):\n    initial_params = {'n_estimators': 125, 'max_features': 0.2, 'max_depth': 25, 'min_samples_leaf': 4, 'n_jobs': -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vivid.featureset.molecules import create_molecule, MoleculeFeature","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ExtraTree(FillnaMixin, KaggleKernelMixin, BaseOutOfFoldFeature):\n    model_class = ExtraTreesClassifier\n    initial_params = {'n_estimators': 100, 'max_features': 0.5, 'max_depth': 18, 'min_samples_leaf': 4, 'n_jobs': -1}\n    \nclass RGF(FillnaMixin, KaggleKernelMixin, BaseOutOfFoldFeature):\n    model_class = RGFClassifier\n    initial_params = {'algorithm': 'RGF_Sib', 'loss': 'Log'}\n    \nclass Logistic(FillnaMixin, KaggleKernelMixin, BaseOutOfFoldFeature):\n    model_class = LogisticRegression\n    init_params = { 'input_scaling': 'standard' }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basic_molecule = create_molecule([\n    CatOneHotEoncodingAtom(),\n    TargetEncodingAtom()\n], name='basic')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"entry_point = MoleculeFeature(basic_molecule, root_dir='/kaggle/working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"single_models = [\n    Logistic(parent=entry_point, name='logistic'),\n    CustomLGBM(parent=entry_point, name='lgbm'),\n    XGB(parent=entry_point, name='xgb'),\n    RF(parent=entry_point, name='rf'),\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merge = MergeFeature(single_models[:], name='merged', root_dir=entry_point.root_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stacking_models = [\n    Logistic(parent=merge, name='stack_logistic'),\n    CustomLGBM(parent=merge, name='stack_lgbm'),\n    EnsembleFeature(single_models[:], name='ensumble', root_dir=entry_point.root_dir)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for m in stacking_models:\n    m.fit(train_df, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_model = stacking_models[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = final_model.predict(test_df).values[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('/kaggle/working/submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}