{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\npd.set_option(\"display.max_columns\",50)\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier\nfrom sklearn.preprocessing import LabelEncoder,OneHotEncoder,OrdinalEncoder,StandardScaler\nfrom sklearn.model_selection import train_test_split,GridSearchCV,KFold,RandomizedSearchCV,StratifiedKFold\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.metrics import roc_auc_score\nimport pandas_profiling","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtypes = {\"day\":\"float32\",\"month\":\"float32\",\"target\":\"uint8\",\"bin_0\":\"float32\",\"bin_1\":\"float32\",\"bin_2\":\"float32\",\"ord_0\":\"float32\"}\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\",dtype=dtypes)\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\",dtype=dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Except id and target variables, all the other have missing values.\n2. Most of the columns are objects and few are floats.\n3. There are different types of categorical variables, Binary, Nominal, Ordinal and cyclical variables (day and month)"},{"metadata":{"trusted":true},"cell_type":"code","source":"test.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop(\"id\",axis=1,inplace=True)\nSubmission = test[['id']]\ntest.drop('id',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train['target'],)\nplt.title(\"Distribution of Dependent Variable\")\nplt.xlabel(\"Target\")\nplt.ylabel(\"Count\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Dependent variable is imbalanced (although the imbalance is not that extreme).\n2. 81% and 19% are the ratios of the values of Dependent variable."},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [col for col in train.columns if col!='target']\nbin_cols = ['bin_0','bin_1','bin_2','bin_3','bin_4']\nord_cols = ['ord_0','ord_1','ord_2','ord_3','ord_4','ord_5']\nnom_cols = ['nom_0','nom_1','nom_2','nom_3','nom_4','nom_5','nom_6','nom_7','nom_8','nom_9']\nprint (\"Dependent Variables are:{}\".format(cols))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in cols:\n    print (\"Unique Values in {} variable in Train data are:{}\".format(col,train[col].nunique()))\n    print (\"Unique Values in {} variable in Test data are:{}\".format(col,test[col].nunique()))\n    print (\"--------------------------------------------------------------------------------\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. There are 5 Binary Categorical Variables. \n2. There are 10 Categorical Variables out of which 5 variables that have large number of unique values. Also these variables do not have any ordering.\n3. There are 6 categorical variables whose values have specific ordering and one of these has large number of unique values. \n4. There are day(day of week) and month variables that have expected number of unique values. "},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in cols:\n    print (\"Percentage of Missing Values in {} variable in Train data are:{}\".format(col,train[col].isna().sum()/len(train)))\n    print (\"Percentage of Missing Values in {} variable in Test data are:{}\".format(col,test[col].isna().sum()/len(test)))\n    print (\"--------------------------------------------------------------------------------\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Most of the variables have same number of missing values in both Train and Test datasets. \n2. Since there are no continuous variables here, the missing values in these variables have to be imputed with mode."},{"metadata":{},"cell_type":"markdown","source":"### Binary Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in bin_cols:\n    train[col].fillna(train[col].value_counts().index[0],inplace=True)\n    test[col].fillna(test[col].value_counts().index[0],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting bin_3 and bin_4 variables in the form of 0's and 1's\nmapping = {\"T\":1,\"F\":0,\"Y\":1,\"N\":0}\ntrain['bin_4'] = train['bin_4'].map(mapping)\ntrain['bin_3'] = train['bin_3'].map(mapping)\n\ntest['bin_4'] = test['bin_4'].map(mapping)\ntest['bin_3'] = test['bin_3'].map(mapping)\n\n# converting the float values to int\nfor col in ['bin_0','bin_1','bin_2']:\n    train[col] = train[col].astype('int')\n    test[col] = test[col].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for ind,col in enumerate(train[bin_cols]):\n    plt.figure(ind)\n    sns.countplot(x=col,data=train,hue='target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in ['bin_0','bin_1','bin_2','bin_3','bin_4']:\n    print (\"Value Count of {} Variable grouped by the target variable:\\n\".format(col),train.groupby(col)['target'].value_counts(normalize=True))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. For each Binary Variable, the distribution of target variable is almost the same (81% to 19%)."},{"metadata":{},"cell_type":"markdown","source":"### Ordinal Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in ord_cols:\n    train[col].fillna(train[col].value_counts().index[0],inplace=True)\n    test[col].fillna(test[col].value_counts().index[0],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for ind,col in enumerate(train[ord_cols]):\n    plt.figure(figsize=(14,6))\n    plt.figure(ind)\n    sns.countplot(x=col,data=train,order=train[col].value_counts().index.values,orient='h')\n    plt.xticks(rotation=90)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\nfor df in [train,test]:\n    df['ord_5'] = le.fit_transform(df['ord_5'])\n\n# Converting ordinal columns ord_0,ord_1,ord_2,ord_3,ord_4 to category data type with the assumed ordering\ntrain['ord_0'] = train['ord_0'].astype('category')\ntrain['ord_0'] = train['ord_0'].cat.set_categories([1.0,2.0,3.0],ordered=True)\ntrain['ord_0'] = train['ord_0'].cat.codes\n\ntrain['ord_1'] = train['ord_1'].astype('category')\ntrain['ord_1'] = train['ord_1'].cat.set_categories([\"Novice\",\"Contributor\",\"Expert\",\"Master\",\"Grandmaster\"],ordered=True)\ntrain['ord_1'] = train['ord_1'].cat.codes\n\ntrain['ord_2'] = train['ord_2'].astype('category')\ntrain['ord_2'] = train['ord_2'].cat.set_categories([\"Freezing\",\"Cold\",\"Warm\",\"Hot\",\"Boiling Hot\",\"Lava Hot\"],ordered=True)\ntrain['ord_2'] = train['ord_2'].cat.codes\n\ntrain['ord_3'] = train['ord_3'].astype('category')\ntrain['ord_3'] = train['ord_3'].cat.set_categories([\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"g\",\"h\",\"i\",\"j\",\"k\",\"l\",\"m\",\"n\",\"o\"],ordered=True)\ntrain['ord_3'] = train['ord_3'].cat.codes\n\ntrain['ord_4'] = train['ord_4'].astype('category')\ntrain['ord_4'] = train['ord_4'].cat.set_categories([\"A\",\"B\",\"C\",\"D\",\"E\",\"F\",\"G\",\"H\",\"I\",\"J\",\"K\",\"L\",\"M\",\"N\",\"O\",\"P\",\"Q\",\"R\",\"S\",\"T\",\"U\",\"V\",\"W\",\"X\",\"Y\",\"Z\"],ordered=True)\ntrain['ord_4'] = train['ord_4'].cat.codes\n\n\n\n# Converting ordinal columns ord_0,ord_1,ord_2,ord_3,ord_4 to category data type with the assumed ordering\ntest['ord_0'] = test['ord_0'].astype('category')\ntest['ord_0'] = test['ord_0'].cat.set_categories([1,2,3],ordered=True)\ntest['ord_0'] = test['ord_0'].cat.codes\n\ntest['ord_1'] = test['ord_1'].astype('category')\ntest['ord_1'] = test['ord_1'].cat.set_categories([\"Novice\",\"Contributor\",\"Expert\",\"Master\",\"Grandmaster\"],ordered=True)\ntest['ord_1'] = test['ord_1'].cat.codes\n\ntest['ord_2'] = test['ord_2'].astype('category')\ntest['ord_2'] = test['ord_2'].cat.set_categories([\"Freezing\",\"Cold\",\"Warm\",\"Hot\",\"Boiling Hot\",\"Lava Hot\"],ordered=True)\ntest['ord_2'] = test['ord_2'].cat.codes\n\ntest['ord_3'] = test['ord_3'].astype('category')\ntest['ord_3'] = test['ord_3'].cat.set_categories([\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"g\",\"h\",\"i\",\"j\",\"k\",\"l\",\"m\",\"n\",\"o\"],ordered=True)\ntest['ord_3'] = test['ord_3'].cat.codes\n\ntest['ord_4'] = test['ord_4'].astype('category')\ntest['ord_4'] = test['ord_4'].cat.set_categories([\"A\",\"B\",\"C\",\"D\",\"E\",\"F\",\"G\",\"H\",\"I\",\"J\",\"K\",\"L\",\"M\",\"N\",\"O\",\"P\",\"Q\",\"R\",\"S\",\"T\",\"U\",\"V\",\"W\",\"X\",\"Y\",\"Z\"],ordered=True)\ntest['ord_4'] = test['ord_4'].cat.codes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Nominal Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Filling missing values in the nominal columns with the mode.\nfor col in nom_cols:\n    train[col].fillna(train[col].value_counts().index[0],inplace=True)\n    test[col].fillna(test[col].value_counts().index[0],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = ['nom_0','nom_1','nom_2','nom_3','nom_4']\nfor ind,col in enumerate(train[cols]):\n    plt.figure(ind)\n    sns.countplot(x=col,data=train,order=train[col].value_counts().index.values,hue='target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dummy variables of nominal variables with low cardinality\n\n# Dummy encoding\nnom_0_dummy = pd.get_dummies(train['nom_0'],prefix=\"nom_0\",)\ntrain = pd.concat([train,nom_0_dummy],axis=1)\ntrain.drop(\"nom_0\",axis=1,inplace=True)\n\nnom_1_dummy = pd.get_dummies(train['nom_1'],prefix=\"nom_1\")\ntrain = pd.concat([train,nom_1_dummy],axis=1)\ntrain.drop(\"nom_1\",axis=1,inplace=True)\n\nnom_2_dummy = pd.get_dummies(train['nom_2'],prefix=\"nom_2\")\ntrain = pd.concat([train,nom_2_dummy],axis=1)\ntrain.drop(\"nom_2\",axis=1,inplace=True)\n\nnom_3_dummy = pd.get_dummies(train['nom_3'],prefix=\"nom_3\")\ntrain = pd.concat([train,nom_3_dummy],axis=1)\ntrain.drop(\"nom_3\",axis=1,inplace=True)\n\nnom_4_dummy = pd.get_dummies(train['nom_4'],prefix=\"nom_4\")\ntrain = pd.concat([train,nom_4_dummy],axis=1)\ntrain.drop(\"nom_4\",axis=1,inplace=True)\n\n# Dummy encoding\nnom_0_dummy = pd.get_dummies(test['nom_0'],prefix=\"nom_0\",)\ntest = pd.concat([test,nom_0_dummy],axis=1)\ntest.drop(\"nom_0\",axis=1,inplace=True)\n\nnom_1_dummy = pd.get_dummies(test['nom_1'],prefix=\"nom_1\")\ntest = pd.concat([test,nom_1_dummy],axis=1)\ntest.drop(\"nom_1\",axis=1,inplace=True)\n\nnom_2_dummy = pd.get_dummies(test['nom_2'],prefix=\"nom_2\")\ntest = pd.concat([test,nom_2_dummy],axis=1)\ntest.drop(\"nom_2\",axis=1,inplace=True)\n\nnom_3_dummy = pd.get_dummies(test['nom_3'],prefix=\"nom_3\")\ntest = pd.concat([test,nom_3_dummy],axis=1)\ntest.drop(\"nom_3\",axis=1,inplace=True)\n\nnom_4_dummy = pd.get_dummies(test['nom_4'],prefix=\"nom_4\")\ntest = pd.concat([test,nom_4_dummy],axis=1)\ntest.drop(\"nom_4\",axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mean encoding the nominal variables that have hign cardinality\nnom_5_target_encoding = np.round(train.groupby('nom_5')['target'].mean(),decimals=2).to_dict()\ntrain['nom_5_target_encoding'] = train['nom_5'].map(nom_5_target_encoding)\n\nnom_6_target_encoding = np.round(train.groupby('nom_6')['target'].mean(),decimals=2).to_dict()\ntrain['nom_6_target_encoding'] = train['nom_6'].map(nom_6_target_encoding)\n\nnom_7_target_encoding = np.round(train.groupby('nom_7')['target'].mean(),decimals=2).to_dict()\ntrain['nom_7_target_encoding'] = train['nom_7'].map(nom_7_target_encoding)\n\nnom_8_target_encoding = np.round(train.groupby('nom_8')['target'].mean(),decimals=2).to_dict()\ntrain['nom_8_target_encoding'] = train['nom_8'].map(nom_8_target_encoding)\n\nnom_9_target_encoding = np.round(train.groupby('nom_9')['target'].mean(),decimals=2).to_dict()\ntrain['nom_9_target_encoding'] = train['nom_9'].map(nom_9_target_encoding)\n\n\ntest['nom_5_target_encoding'] = test['nom_5'].map(nom_5_target_encoding)\ntest['nom_6_target_encoding'] = test['nom_6'].map(nom_6_target_encoding)\ntest['nom_7_target_encoding'] = test['nom_7'].map(nom_7_target_encoding)\ntest['nom_8_target_encoding'] = test['nom_8'].map(nom_8_target_encoding)\ntest['nom_9_target_encoding'] = test['nom_9'].map(nom_9_target_encoding)\n\ntest['nom_6_target_encoding'].fillna(test['nom_6_target_encoding'].mean(),inplace=True)\n\ntrain.drop(['nom_5','nom_6','nom_7','nom_8','nom_9'],axis=1,inplace=True)\ntest.drop(['nom_5','nom_6','nom_7','nom_8','nom_9'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Handling Cyclical Features such as day and month\nfor df in [train,test]:\n    df['day'].fillna(df['day'].value_counts().index[0],inplace=True)\n    df['month'].fillna(df['month'].value_counts().index[0],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sine and Cosine transformation of the cyclical features such as day and month\ndef date_cyc_enc(df, col, max_vals):\n    df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)\n    df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)\n    return df\n\ntrain = date_cyc_enc(train, 'day', 7)\ntest = date_cyc_enc(test, 'day', 7) \n\ntrain = date_cyc_enc(train, 'month', 12)\ntest = date_cyc_enc(test, 'month', 12)\ntrain.drop(['day','month'],axis=1,inplace=True)\ntest.drop(['day','month'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols_to_transform = ['ord_0','ord_1','ord_2','ord_3','ord_4','ord_5']\nscaled_train = train.copy()\nfeatures_train = scaled_train[cols_to_transform]\nscaler = StandardScaler().fit(features_train.values)\nfeatures_train = scaler.transform(features_train.values)\nscaled_train[cols_to_transform] = features_train\n\nscaled_test = test.copy()\nfeatures_test = scaled_test[cols_to_transform]\nscaler = StandardScaler().fit(features_test.values)\nfeatures_test = scaler.transform(features_test.values)\nscaled_test[cols_to_transform] = features_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = scaled_train[[col for col in scaled_train.columns if col!='target']]\ny = scaled_train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_Train,X_Test,y_Train,y_Test = train_test_split(X,y,test_size=0.2,random_state=42,stratify=y)\nprint (X_Train.shape)\nprint (X_Test.shape)\nprint (y_Train.shape)\nprint (y_Test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_1 = lgb.LGBMClassifier(boosting_type='goss',objective='binary',random_state=42,n_jobs=-1,verbose=1,class_weight='balanced')\nparams = {\"max_depth\":[3,4,5,6,7,-1],\n          \"learning_rate\":[0.01,0.05,0.1,0.3],\n          \"subsample\":[0.5,0.6,0.7,0.8,0.9],\n          \"colsample_bytree\":[0.5,0.6,0.7,0.8,0.9],\n          \"reg_alpha\":[0.5,1,2,5,10],\n          \"reg_lambda\":[0.5,1,2,5,10],\n          \"num_leaves\":[7,15,31,63,127],\n          \"n_estimators\":list(range(50,500,50)),\n          \"min_data_in_leaf\":[1,3,5,10,15,25]}\nrandom_search_1 = RandomizedSearchCV(estimator=clf_1,param_distributions=params,cv=10,scoring='roc_auc')\nrandom_search_1.fit(X_Train,y_Train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_search_1.best_estimator_,random_search_1.best_score_,random_search_1.best_params_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ser = pd.Series(random_search_1.best_estimator_.feature_importances_,X_Train.columns).sort_values()\nser.plot(kind='bar',figsize=(10,6))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"Submission['target']=random_search_1.predict_proba(scaled_test)[:,1]\nSubmission.to_csv(\"Latest.csv\",index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Submission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}