{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pylab as plt\nfrom IPython.display import HTML\nimport warnings\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, StratifiedKFold\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression, PassiveAggressiveClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, ExtraTreesRegressor\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_selection import RFE\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.mixture import GaussianMixture\nfrom category_encoders import  LeaveOneOutEncoder, BinaryEncoder, TargetEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import roc_auc_score\nfrom xgboost import XGBRegressor\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoost\nfrom catboost import Pool\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nimport time\nimport logging\n\n\nsample_submission = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\")\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[['nom_5','nom_6','nom_7','nom_8','nom_9','target']][60:70]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create new features   nom_x by nom_y\ntrain[\"nom_0by1\"]=train.nom_0.str.cat(train['nom_1'])\ntrain[\"nom_0by2\"]=train.nom_0.str.cat(train['nom_2'])\ntrain[\"nom_0by3\"]=train.nom_0.str.cat(train['nom_3'])\ntrain[\"nom_0by4\"]=train.nom_0.str.cat(train['nom_4'])\ntest[\"nom_0by1\"]=test.nom_0.str.cat(test['nom_1'])\ntest[\"nom_0by2\"]=test.nom_0.str.cat(test['nom_2'])\ntest[\"nom_0by3\"]=test.nom_0.str.cat(test['nom_3'])\ntest[\"nom_0by4\"]=test.nom_0.str.cat(test['nom_4'])\n\ntrain[\"nom_1by2\"]=train.nom_1.str.cat(train['nom_2'])\ntrain[\"nom_1by3\"]=train.nom_1.str.cat(train['nom_3'])\ntrain[\"nom_1by4\"]=train.nom_1.str.cat(train['nom_4'])\ntest[\"nom_1by2\"]=test.nom_1.str.cat(test['nom_2'])\ntest[\"nom_1by3\"]=test.nom_1.str.cat(test['nom_3'])\ntest[\"nom_1by4\"]=test.nom_1.str.cat(test['nom_4'])\n\n\ntrain[\"nom_2by3\"]=train.nom_2.str.cat(train['nom_3'])\ntrain[\"nom_2by4\"]=train.nom_2.str.cat(train['nom_4'])\ntest[\"nom_2by3\"]=test.nom_2.str.cat(test['nom_3'])\ntest[\"nom_2by4\"]=test.nom_2.str.cat(test['nom_4'])\n\ntrain[\"nom_3by4\"]=train.nom_3.str.cat(train['nom_4'])\ntest[\"nom_3by4\"]=test.nom_3.str.cat(test['nom_4'])\n\n#create new features   bin_x by nom_y\ntrain[\"bin_3by4\"]=train.bin_3.str.cat(train['bin_4'])\ntest[\"bin_3by4\"]=test.bin_3.str.cat(test['bin_4'])\n\n\ntrain[\"bin_3bynom_0\"]=train.bin_3.str.cat(train['nom_0'])\ntrain[\"bin_3bynom_1\"]=train.bin_3.str.cat(train['nom_1'])\ntrain[\"bin_3bynom_2\"]=train.bin_3.str.cat(train['nom_2'])\ntrain[\"bin_3bynom_3\"]=train.bin_3.str.cat(train['nom_3'])\ntrain[\"bin_3bynom_4\"]=train.bin_3.str.cat(train['nom_4'])\ntest[\"bin_3bynom_0\"]=test.bin_3.str.cat(test['nom_0'])\ntest[\"bin_3bynom_1\"]=test.bin_3.str.cat(test['nom_1'])\ntest[\"bin_3bynom_2\"]=test.bin_3.str.cat(test['nom_2'])\ntest[\"bin_3bynom_3\"]=test.bin_3.str.cat(test['nom_3'])\ntest[\"bin_3bynom_4\"]=test.bin_3.str.cat(test['nom_4'])\n\n\ntrain[\"bin_4bynom_0\"]=train.bin_4.str.cat(train['nom_0'])\ntrain[\"bin_4bynom_1\"]=train.bin_4.str.cat(train['nom_1'])\ntrain[\"bin_4bynom_2\"]=train.bin_4.str.cat(train['nom_2'])\ntrain[\"bin_4bynom_3\"]=train.bin_4.str.cat(train['nom_3'])\ntrain[\"bin_4bynom_4\"]=train.bin_4.str.cat(train['nom_4'])\ntest[\"bin_4bynom_0\"]=test.bin_4.str.cat(test['nom_0'])\ntest[\"bin_4bynom_1\"]=test.bin_4.str.cat(test['nom_1'])\ntest[\"bin_4bynom_2\"]=test.bin_4.str.cat(test['nom_2'])\ntest[\"bin_4bynom_3\"]=test.bin_4.str.cat(test['nom_3'])\ntest[\"bin_4bynom_4\"]=test.bin_4.str.cat(test['nom_4'])\n\ntrain[\"nom_5bynom_8\"]=train.nom_5.str.cat(train['nom_8'])\ntest[\"nom_5bynom_8\"]=test.nom_5.str.cat(test['nom_8'])\n\ntrain[\"nom_6bynom_9\"]=train.nom_6.str.cat(train['nom_9'])\ntest[\"nom_6bynom_9\"]=test.nom_6.str.cat(test['nom_9'])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.nom_0by1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create new features   day as string\ntrain['day_obj']=train.day.astype('str')\ntest['day_obj']=test.day.astype('str')\n\ntrain['month_obj']=train.month.astype('str')\ntest['month_obj']=test.month.astype('str')\n\n#create new features   ord_x as object\ntrain['ord_0_obj']=train.ord_0\ntrain['ord_1_obj']=train.ord_1\ntrain['ord_2_obj']=train.ord_2\ntrain['ord_3_obj']=train.ord_3\ntrain['ord_4_obj']=train.ord_4\ntrain['ord_5_obj']=train.ord_5\ntest['ord_0_obj']=test.ord_0\ntest['ord_1_obj']=test.ord_1\ntest['ord_2_obj']=test.ord_2\ntest['ord_3_obj']=test.ord_3\ntest['ord_4_obj']=test.ord_4\ntest['ord_5_obj']=test.ord_5\n\ntrain['nom_0_obj']=train.nom_0\ntrain['nom_1_obj']=train.nom_1\ntrain['nom_2_obj']=train.nom_2\ntrain['nom_3_obj']=train.nom_3\ntrain['nom_4_obj']=train.nom_4\ntest['nom_0_obj']=test.nom_0\ntest['nom_1_obj']=test.nom_1\ntest['nom_2_obj']=test.nom_2\ntest['nom_3_obj']=test.nom_3\ntest['nom_4_obj']=test.nom_4\n\ntrain['bin_0_obj']=train.bin_0\ntrain['bin_1_obj']=train.bin_1\ntrain['bin_2_obj']=train.bin_2\ntrain['bin_3_obj']=train.bin_3\ntrain['bin_4_obj']=train.bin_4\ntest['bin_0_obj']=test.bin_0\ntest['bin_1_obj']=test.bin_1\ntest['bin_2_obj']=test.bin_2\ntest['bin_3_obj']=test.bin_3\ntest['bin_4_obj']=test.bin_4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create new features   month by day\ntrain[\"daybymonth\"]=train.month_obj.str.cat(train['day_obj'])\ntest[\"daybymonth\"]=test.month_obj.str.cat(test['day_obj'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.ord_1_obj","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Categorical features\n\n#cat_cols=['nom_5','nom_6','nom_7','nom_8','nom_9','day_obj','month_obj','ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj',\n#          'ord_5_obj','nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj','bin_0_obj','bin_1_obj','bin_2_obj','bin_3_obj','bin_4_obj']\n\n#Try new features and select only good effectors\n\ncat_cols=['nom_5','nom_6','nom_7','nom_8','nom_9','day_obj','month_obj','ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj',\n          'ord_5_obj',\n          'nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj',\n          'bin_0_obj','bin_1_obj','bin_2_obj','bin_3_obj','bin_4_obj',\n          \"nom_3by4\",\"bin_4bynom_0\"]\n\n#Remain bad effector\n#\"nom_0by1\"\"nom_0by2\"\"nom_0by3\"\"nom_0by4\"\"nom_1by2\"\"nom_1by3\"\"nom_1by4\"\"nom_2by3\"\"nom_2by4\",\"bin_3by4\",\"bin_3bynom_0\",\"bin_3bynom_1\"\n#,\"bin_3bynom_3\",\"bin_3bynom_2\",\"bin_3bynom_4\",\"bin_4bynom_1\",\"bin_4bynom_2\"\"bin_4bynom_3\",\"bin_4bynom_4\"\n\n\n#Target encorder\nfor c in cat_cols:\n    data_tmp=pd.DataFrame({c:train[c],'target':train.target})\n    target_mean=data_tmp.groupby(c)['target'].mean()\n    test[c]=test[c].map(target_mean)\n    \n    tmp=np.repeat(np.nan, train.shape[0])\n    \n    kf=KFold(n_splits=4, shuffle=True, random_state=72)\n    for idx_1,idx_2 in kf.split(train):\n        target_mean=data_tmp.iloc[idx_1].groupby(c)['target'].mean()\n        tmp[idx_2]=train[c].iloc[idx_2].map(target_mean)\n        \n        \n    train[c]=tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create new features\ntrain['nom_mean']=(train.nom_5+train.nom_6+train.nom_7)/3\ntest['nom_mean']=(test.nom_5+test.nom_6+test.nom_7)/3","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Numerical features\n\ncols_to_use = ['ord_1', 'ord_2', 'ord_3', 'ord_4','ord_5','nom_0','nom_1','nom_2','nom_3','nom_4','bin_3','bin_4']  #cat_columns\n#cols_to_use = ['ord_1', 'ord_2', 'ord_3', 'ord_4','ord_5','nom_2']\n#cols_to_use = ['nom_0','nom_1','nom_2','nom_3','nom_4','ord_5']\n\n#Try new features and select only good effectors\n\n#num_cols_to_use = ['bin_0','bin_1','bin_2','ord_0']\n#num_cols_to_use = ['ord_0','bin_0','bin_1','bin_2','day','month','nom_5','nom_6','nom_7','nom_8','nom_9','day_obj','month_obj',\n#                   'ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj','ord_5_obj',\n#                   'nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj',\n#                   'bin_0_obj','bin_1_obj','bin_2_obj','bin_3_obj','bin_4_obj']\nnum_cols_to_use = ['ord_0','bin_0','bin_1','bin_2','day','month','nom_5','nom_6','nom_7','nom_8','nom_9','day_obj','month_obj',\n                   'ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj','ord_5_obj',\n                   'nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj',\n                   'bin_0_obj','bin_1_obj','bin_2_obj','bin_3_obj','bin_4_obj',\n                   \"nom_3by4\",\"bin_4bynom_0\"\n                   ]\n\n\n#num_cols_to_use = ['day_obj','month_obj','ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj','ord_5_obj',\n#                   'nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj','bin_0_obj','bin_1_obj','bin_2_obj',\n#                   'bin_3_obj','bin_4_obj']\n\n#bad effect\n#\"nom_0by1\"\"nom_0by2\"\"nom_0by3\"\"nom_0by4\"\"nom_1by2\"\"nom_1by3\"\"nom_1by4\"\"nom_2by3\"\"nom_2by4\",\"bin_3by4\",\"bin_3bynom_0\",\"bin_3bynom_1\"\n#,\"bin_3bynom_3\",\"bin_3bynom_2\",\"bin_3bynom_4\",\"bin_4bynom_1\",\"bin_4bynom_2\",\"bin_4bynom_3\",\"bin_4bynom_4\"\ncols_sum = cols_to_use + num_cols_to_use\n\nX = train[cols_sum]\ny = train.target\n\n#One Hot Encoder\n\nfrom sklearn.model_selection import train_test_split\ntrain_X,val_X,train_y,val_y = train_test_split(X,y,random_state = 0)\n\nX_val = test[cols_sum]\n\nnumerical_transformer = SimpleImputer(strategy='mean')\n\ncategorical_transformer = Pipeline(steps=[\n    ('imputer',SimpleImputer(strategy='constant')),\n    ('onehot',OneHotEncoder(handle_unknown='ignore'))\n])\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_transformer, num_cols_to_use),\n        ('cat',categorical_transformer, cols_to_use)\n    ])\n\n#Try some models and select only good result\n\n#params={'objective':'binary:logistic','eta':0.05,'gamma':0.0,'lambda':1.0,'min_child_weight':1,'max_depth':10,'colsample_bytree':0.8,'random_state':71}\n#my_model = XGBRegressor(**params,num_round=5000, early_stopping_rounds=10,n_estimators=500)\n#my_model = XGBRegressor(n_estimators=800, learning_rate=0.2,objective ='binary:logistic')\n#my_model = XGBClassifier(n_estimators=50, learning_rate=0.01, max_depth=5)\nmy_model = CatBoost({'loss_function':'Logloss','num_boost_round':2500,'learning_rate':0.03,'early_stopping_rounds': 10,'depth':2})\n#my_model = CatBoostClassifier(custom_loss=['Accuracy'],random_seed=42,iterations=1800,learning_rate=0.7,early_stopping_rounds=10,depth=10)\n#my_model = LGBMClassifier(n_estimators=1500, learning_rate=0.5, max_depth=10,num_leaves=50,max_bin=500)\n#my_model = RandomForestClassifier(n_estimators=50, random_state=0)\n\nmy_pipeline = Pipeline(steps=[('preprocessor',preprocessor),\n                      ('model',my_model)\n                     ])\n\nmy_pipeline.fit(train_X,train_y)\n\nprediction = my_pipeline.predict(X_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nval_predictions = my_pipeline.predict(val_X)\nprint(roc_auc_score(val_y,val_predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.ord_0_obj","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ss=sample_submission\nss.target = prediction\nss.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}