{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is derived from Kaggle Grandmaster Bojan Tunguz's NB\nhttps://www.kaggle.com/tunguz/elo-with-h2o-automl This notebook merely builds on his fantastic feature engineering (194 features!) and tries to select the best features using the new python feature selection library: featurewiz\n# FeatureWiz\n<a href=\"https://github.com/AutoViML/featurewiz\"><img src=\"https://i.ibb.co/ZLdZMZg/featurewiz-logos.png\" alt=\"featurewiz-logos\" border=\"0\"></a>\n","metadata":{}},{"cell_type":"markdown","source":"### We are first going to download some libraries","metadata":{}},{"cell_type":"code","source":"!pip install lazytransform --ignore-installed --no-deps","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-23T00:26:23.715383Z","iopub.execute_input":"2022-05-23T00:26:23.715876Z","iopub.status.idle":"2022-05-23T00:26:27.402654Z","shell.execute_reply.started":"2022-05-23T00:26:23.715738Z","shell.execute_reply":"2022-05-23T00:26:27.401641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install featurewiz --ignore-installed --no-deps\n!pip install xlrd --ignore-installed --no-deps","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-23T00:26:31.224298Z","iopub.execute_input":"2022-05-23T00:26:31.224631Z","iopub.status.idle":"2022-05-23T00:26:37.340966Z","shell.execute_reply.started":"2022-05-23T00:26:31.224597Z","shell.execute_reply":"2022-05-23T00:26:37.340059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### You need to install this on Kaggle notebooks since Kaggle has an incompatible version of Pillow ##\n!pip install Pillow==9.0.0","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-23T00:26:40.973123Z","iopub.execute_input":"2022-05-23T00:26:40.973479Z","iopub.status.idle":"2022-05-23T00:26:55.004528Z","shell.execute_reply.started":"2022-05-23T00:26:40.973446Z","shell.execute_reply":"2022-05-23T00:26:55.003210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### use only these 3 datasets ##\ntrainfile = '/kaggle/input/mitsuru-features-only-train-and-test/train_df_noindex.csv'\ntestfile = '../input/mitsuru-features-only-train-and-test/test_df_noindex.csv'\nsubfile = '/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv'","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:26:57.372722Z","iopub.execute_input":"2022-05-23T00:26:57.373111Z","iopub.status.idle":"2022-05-23T00:26:57.379004Z","shell.execute_reply.started":"2022-05-23T00:26:57.373067Z","shell.execute_reply":"2022-05-23T00:26:57.377813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:26:58.192225Z","iopub.execute_input":"2022-05-23T00:26:58.192584Z","iopub.status.idle":"2022-05-23T00:26:59.403931Z","shell.execute_reply.started":"2022-05-23T00:26:58.192550Z","shell.execute_reply":"2022-05-23T00:26:59.402765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(trainfile)\ntest = pd.read_csv(testfile)\nsub = pd.read_csv(subfile)\nprint(train.shape, test.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:26:59.931324Z","iopub.execute_input":"2022-05-23T00:26:59.931675Z","iopub.status.idle":"2022-05-23T00:27:15.212564Z","shell.execute_reply.started":"2022-05-23T00:26:59.931638Z","shell.execute_reply":"2022-05-23T00:27:15.211861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = ['target']\nidvars = []\nmodeltype = 'Regression' ### This is needed to make it simpler for us to evaluate results","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:15.214085Z","iopub.execute_input":"2022-05-23T00:27:15.214518Z","iopub.status.idle":"2022-05-23T00:27:15.219681Z","shell.execute_reply.started":"2022-05-23T00:27:15.214479Z","shell.execute_reply":"2022-05-23T00:27:15.218611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_target = False\nimbalanced = False","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:15.221266Z","iopub.execute_input":"2022-05-23T00:27:15.222435Z","iopub.status.idle":"2022-05-23T00:27:15.239587Z","shell.execute_reply.started":"2022-05-23T00:27:15.222391Z","shell.execute_reply":"2022-05-23T00:27:15.238572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df[target].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:17.296902Z","iopub.execute_input":"2022-05-23T00:27:17.297411Z","iopub.status.idle":"2022-05-23T00:27:17.302110Z","shell.execute_reply.started":"2022-05-23T00:27:17.297366Z","shell.execute_reply":"2022-05-23T00:27:17.301473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make sure you set the environment preferences to \"latest\" in the Kaggle menu options to the right -> see here","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:19.041897Z","iopub.execute_input":"2022-05-23T00:27:19.042895Z","iopub.status.idle":"2022-05-23T00:27:19.047899Z","shell.execute_reply.started":"2022-05-23T00:27:19.042849Z","shell.execute_reply":"2022-05-23T00:27:19.046966Z"}}},{"cell_type":"code","source":"#!pip install scikit-learn --upgrade","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:23.290416Z","iopub.execute_input":"2022-05-23T00:27:23.291019Z","iopub.status.idle":"2022-05-23T00:27:23.296564Z","shell.execute_reply.started":"2022-05-23T00:27:23.290963Z","shell.execute_reply":"2022-05-23T00:27:23.295358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import the required libraries","metadata":{}},{"cell_type":"code","source":"from lazytransform import LazyTransformer","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:26.810387Z","iopub.execute_input":"2022-05-23T00:27:26.810720Z","iopub.status.idle":"2022-05-23T00:27:27.650886Z","shell.execute_reply.started":"2022-05-23T00:27:26.810684Z","shell.execute_reply":"2022-05-23T00:27:27.649475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import featurewiz as FW","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:27:30.808456Z","iopub.execute_input":"2022-05-23T00:27:30.808779Z","iopub.status.idle":"2022-05-23T00:27:32.252210Z","shell.execute_reply.started":"2022-05-23T00:27:30.808749Z","shell.execute_reply":"2022-05-23T00:27:32.251162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We can try different types of category encoders to see which one works best for this dataset. We will start with a simple encoder and a Robust Scaler since we have many outliers in the dataset","metadata":{}},{"cell_type":"code","source":"encoders = 'auto' ### auto, label, woe encoders are all good ones to start with \nscalers = 'std' ### MinMax is also a good scaler to try ###","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:28:35.044758Z","iopub.execute_input":"2022-05-23T00:28:35.045987Z","iopub.status.idle":"2022-05-23T00:28:35.051066Z","shell.execute_reply.started":"2022-05-23T00:28:35.045921Z","shell.execute_reply":"2022-05-23T00:28:35.049959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We must always check the data types before applying any transforms","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:28:37.210448Z","iopub.execute_input":"2022-05-23T00:28:37.211551Z","iopub.status.idle":"2022-05-23T00:28:37.407288Z","shell.execute_reply.started":"2022-05-23T00:28:37.211493Z","shell.execute_reply":"2022-05-23T00:28:37.406419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if isinstance(target, str):\n    preds = [x for x in list(train) if x not in [target]+idvars]\nelse:\n    preds = [x for x in list(train) if x not in target+idvars]\nlen(preds)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:28:46.982852Z","iopub.execute_input":"2022-05-23T00:28:46.983479Z","iopub.status.idle":"2022-05-23T00:28:46.991779Z","shell.execute_reply.started":"2022-05-23T00:28:46.983437Z","shell.execute_reply":"2022-05-23T00:28:46.991073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check if there are any mixed data types in data - are numeric features numeric? are string features object? These kind of questions should be answered and fixed immediately","metadata":{}},{"cell_type":"code","source":"#train[preds].dtypes","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:28:54.671638Z","iopub.execute_input":"2022-05-23T00:28:54.672306Z","iopub.status.idle":"2022-05-23T00:28:54.677212Z","shell.execute_reply.started":"2022-05-23T00:28:54.672237Z","shell.execute_reply":"2022-05-23T00:28:54.676183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train[''].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:29:00.745759Z","iopub.execute_input":"2022-05-23T00:29:00.746063Z","iopub.status.idle":"2022-05-23T00:29:00.750462Z","shell.execute_reply.started":"2022-05-23T00:29:00.746032Z","shell.execute_reply":"2022-05-23T00:29:00.749226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We need to split data before doing any feature engg or feature transformations.\nOtherwise there will be data leakage which is a major mistake newbies make","metadata":{}},{"cell_type":"code","source":"## we are going to use a random_state for now and see ########\nfrom sklearn.model_selection import train_test_split\ntrainx, testx = train_test_split(train, train_size=0.80, random_state=99)\nprint(trainx.shape, testx.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:29:08.638047Z","iopub.execute_input":"2022-05-23T00:29:08.638677Z","iopub.status.idle":"2022-05-23T00:29:09.119243Z","shell.execute_reply.started":"2022-05-23T00:29:08.638625Z","shell.execute_reply":"2022-05-23T00:29:09.118071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### If there are mixed datatypes in object column. We need to fix them using lazytransform which has a handy function to convert mixed data types to numeric ###","metadata":{}},{"cell_type":"code","source":"from lazytransform import convert_all_object_columns_to_numeric\ntrainx, testx = convert_all_object_columns_to_numeric(trainx, testx)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:29:13.313341Z","iopub.execute_input":"2022-05-23T00:29:13.314464Z","iopub.status.idle":"2022-05-23T00:29:15.021822Z","shell.execute_reply.started":"2022-05-23T00:29:13.314410Z","shell.execute_reply":"2022-05-23T00:29:15.020643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's build a simple model with all the features to set a baseline","metadata":{}},{"cell_type":"code","source":"infcols=[]\ninfcols = FW.EDA_find_remove_columns_with_infinity(trainx)\nprint(len(infcols))\ntrainx.drop(infcols, axis=1, inplace=True)\ntestx.drop(infcols, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:30:38.516603Z","iopub.execute_input":"2022-05-23T00:30:38.517298Z","iopub.status.idle":"2022-05-23T00:30:39.153839Z","shell.execute_reply.started":"2022-05-23T00:30:38.517240Z","shell.execute_reply":"2022-05-23T00:30:39.152731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if isinstance(target, str):\n    preds = [x for x in list(trainx) if x not in [target]+idvars]\nelse:\n    preds = [x for x in list(trainx) if x not in target+idvars]\nlen(preds)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:31:07.091986Z","iopub.execute_input":"2022-05-23T00:31:07.092675Z","iopub.status.idle":"2022-05-23T00:31:07.101555Z","shell.execute_reply.started":"2022-05-23T00:31:07.092632Z","shell.execute_reply":"2022-05-23T00:31:07.100464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = FW.complex_XGBoost_model(X_train=trainx[preds], y_train=trainx[target], \n                        X_test=testx[preds], log_y=False, \n                        GPU_flag=False, scaler='', enc_method='label', n_splits=5, verbose=-1)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:31:08.328766Z","iopub.execute_input":"2022-05-23T00:31:08.329127Z","iopub.status.idle":"2022-05-23T00:31:40.588036Z","shell.execute_reply.started":"2022-05-23T00:31:08.329094Z","shell.execute_reply":"2022-05-23T00:31:40.587112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if modeltype != 'Regression':\n    #y_preds = lazy.yformer.inverse_transform(y_preds)\n    y_preds = outputs[0]\nelse:\n    y_preds = outputs[0]\ny_preds[:4]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:31:43.797657Z","iopub.execute_input":"2022-05-23T00:31:43.798026Z","iopub.status.idle":"2022-05-23T00:31:43.806685Z","shell.execute_reply.started":"2022-05-23T00:31:43.797989Z","shell.execute_reply":"2022-05-23T00:31:43.805824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if modeltype == 'Regression':\n    from sklearn.metrics import r2_score, mean_squared_error\n    print('R-Squared = %0.0f%%' %(100*r2_score(testx[target],y_preds)))\n    print('RMSE = %0.2f' %np.sqrt(mean_squared_error(testx[target],y_preds)))\n    #plot_scatter(test[target],testm[target+'_XGBoost_predictions'])\nelse:\n    from sklearn.metrics import balanced_accuracy_score, classification_report\n    if isinstance(target, str): \n        print('Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target],y_preds)))\n        print(classification_report(testx[target],y_preds))\n    elif len(target) == 1:\n            print('Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target],y_preds)))\n            print(classification_report(testx[target],y_preds))\n    else:\n        for each_i, target_name in enumerate(target):\n            print('For %s:' %target_name)\n            print('    Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target].values[:,each_i],y_preds[:,each_i])))\n            print(classification_report(testx[target].values[:,each_i],y_preds[:,each_i]))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:31:44.766443Z","iopub.execute_input":"2022-05-23T00:31:44.767268Z","iopub.status.idle":"2022-05-23T00:31:44.792696Z","shell.execute_reply.started":"2022-05-23T00:31:44.767206Z","shell.execute_reply":"2022-05-23T00:31:44.791552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let us do some feature engg to see if we can add groupby features","metadata":{}},{"cell_type":"code","source":"catcols = trainx.select_dtypes(include='object').columns.tolist()\ncatcols = [x for x in catcols if x not in [target]]\nlen(catcols)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:31:53.508695Z","iopub.execute_input":"2022-05-23T00:31:53.509168Z","iopub.status.idle":"2022-05-23T00:31:53.517473Z","shell.execute_reply.started":"2022-05-23T00:31:53.509125Z","shell.execute_reply":"2022-05-23T00:31:53.516775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols = trainx.select_dtypes(include='number').columns.tolist()\nif isinstance(target, str):\n    numcols = [x for x in numcols if x not in [target]+idvars]\nelse:\n    numcols = [x for x in numcols if x not in target+idvars]\nlen(numcols)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:31:54.358602Z","iopub.execute_input":"2022-05-23T00:31:54.359049Z","iopub.status.idle":"2022-05-23T00:31:54.511635Z","shell.execute_reply.started":"2022-05-23T00:31:54.359018Z","shell.execute_reply":"2022-05-23T00:31:54.510405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"### This is what we do for adding groupby features ###\nrcc = FW.Groupby_Aggregator(categoricals=catcols,aggregates=['count','mean','min','max'], \n                            numerics=numcols)\nrcc","metadata":{}},{"cell_type":"code","source":"#train = rcc.fit_transform(trainx)\n#test = rcc.transform(testx)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:32:13.657779Z","iopub.execute_input":"2022-05-23T00:32:13.658537Z","iopub.status.idle":"2022-05-23T00:32:13.663020Z","shell.execute_reply.started":"2022-05-23T00:32:13.658491Z","shell.execute_reply":"2022-05-23T00:32:13.662039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if isinstance(target, str):\n    train = trainx[preds+[target]]\nelse:\n    train = trainx[preds+target]\ntest = testx[preds]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:32:14.199430Z","iopub.execute_input":"2022-05-23T00:32:14.200522Z","iopub.status.idle":"2022-05-23T00:32:14.358913Z","shell.execute_reply.started":"2022-05-23T00:32:14.200466Z","shell.execute_reply":"2022-05-23T00:32:14.357749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## check if the features look correct before and after ##\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:32:15.875049Z","iopub.execute_input":"2022-05-23T00:32:15.875755Z","iopub.status.idle":"2022-05-23T00:32:15.904312Z","shell.execute_reply.started":"2022-05-23T00:32:15.875710Z","shell.execute_reply":"2022-05-23T00:32:15.903345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape, test.shape)\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:32:18.343628Z","iopub.execute_input":"2022-05-23T00:32:18.344083Z","iopub.status.idle":"2022-05-23T00:32:18.370026Z","shell.execute_reply.started":"2022-05-23T00:32:18.344050Z","shell.execute_reply":"2022-05-23T00:32:18.369276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let us do some feature selection since it has too many correlated vars","metadata":{}},{"cell_type":"code","source":"#### Make sure you set correlation limit and other defaults as needed ###\ntrainm, testm = FW.featurewiz(train, target, corr_limit=0.70, verbose=2, sep=',', \n\t\theader=0, test_data=test,feature_engg='', category_encoders='',\n\t\tdask_xgboost_flag=False, nrows=None)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:32:20.578200Z","iopub.execute_input":"2022-05-23T00:32:20.578578Z","iopub.status.idle":"2022-05-23T00:41:48.368976Z","shell.execute_reply.started":"2022-05-23T00:32:20.578542Z","shell.execute_reply":"2022-05-23T00:41:48.367885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(testm.shape)\ntestm.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:41:56.265239Z","iopub.execute_input":"2022-05-23T00:41:56.265644Z","iopub.status.idle":"2022-05-23T00:41:56.281932Z","shell.execute_reply.started":"2022-05-23T00:41:56.265600Z","shell.execute_reply":"2022-05-23T00:41:56.280711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(trainm.shape)\ntrainm.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:41:57.154057Z","iopub.execute_input":"2022-05-23T00:41:57.155223Z","iopub.status.idle":"2022-05-23T00:41:57.181053Z","shell.execute_reply.started":"2022-05-23T00:41:57.155169Z","shell.execute_reply":"2022-05-23T00:41:57.180179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train.columns)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:04.855098Z","iopub.execute_input":"2022-05-23T00:42:04.855441Z","iopub.status.idle":"2022-05-23T00:42:04.862867Z","shell.execute_reply.started":"2022-05-23T00:42:04.855405Z","shell.execute_reply":"2022-05-23T00:42:04.861628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We will now use the same featurewiz model to see if selected features are making the model better","metadata":{"execution":{"iopub.execute_input":"2022-04-04T19:07:39.675933Z","iopub.status.busy":"2022-04-04T19:07:39.675683Z","iopub.status.idle":"2022-04-04T19:07:39.700584Z","shell.execute_reply":"2022-04-04T19:07:39.699253Z","shell.execute_reply.started":"2022-04-04T19:07:39.675905Z"}}},{"cell_type":"code","source":"select = testm.columns.tolist()\n#select = oldselect\nlen(select)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:08.824670Z","iopub.execute_input":"2022-05-23T00:42:08.825188Z","iopub.status.idle":"2022-05-23T00:42:08.832112Z","shell.execute_reply.started":"2022-05-23T00:42:08.825137Z","shell.execute_reply":"2022-05-23T00:42:08.831221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs2 = FW.complex_XGBoost_model(trainm[select], trainm[target], \n                        testm[select],  log_y=False, \n                        GPU_flag=False, scaler='', \n                        enc_method='label', n_splits=5, verbose=-1)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:10.726552Z","iopub.execute_input":"2022-05-23T00:42:10.727604Z","iopub.status.idle":"2022-05-23T00:42:18.021933Z","shell.execute_reply.started":"2022-05-23T00:42:10.727553Z","shell.execute_reply":"2022-05-23T00:42:18.020967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dicto = {1:'f',0:'t'}\n#dicto ={}","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:23.006395Z","iopub.execute_input":"2022-05-23T00:42:23.006737Z","iopub.status.idle":"2022-05-23T00:42:23.011823Z","shell.execute_reply.started":"2022-05-23T00:42:23.006703Z","shell.execute_reply":"2022-05-23T00:42:23.010629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if modeltype != 'Regression':\n    y_preds = outputs2[0]\n    #y_preds = pd.Series(y_preds).map(dicto)\nelse:\n    y_preds = outputs2[0]\ny_preds[:4]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:23.503962Z","iopub.execute_input":"2022-05-23T00:42:23.505215Z","iopub.status.idle":"2022-05-23T00:42:23.513634Z","shell.execute_reply.started":"2022-05-23T00:42:23.505152Z","shell.execute_reply":"2022-05-23T00:42:23.512713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if modeltype == 'Regression':\n    from sklearn.metrics import r2_score, mean_squared_error\n    print('R-Squared = %0.0f%%' %(100*r2_score(testx[target],y_preds)))\n    print('RMSE = %0.2f' %np.sqrt(mean_squared_error(testx[target],y_preds)))\n    #plot_scatter(test[target],testm[target+'_XGBoost_predictions'])\nelse:\n    from sklearn.metrics import balanced_accuracy_score, classification_report\n    if isinstance(target, str): \n        print('Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target],y_preds)))\n        print(classification_report(testx[target],y_preds))\n    elif len(target) == 1:\n            print('Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target],y_preds)))\n            print(classification_report(testx[target],y_preds))\n    else:\n        for each_i, target_name in enumerate(target):\n            print('For %s:' %target_name)\n            print('    Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target].values[:,each_i],y_preds[:,each_i])))\n            print(classification_report(testx[target].values[:,each_i],y_preds[:,each_i]))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:24.741687Z","iopub.execute_input":"2022-05-23T00:42:24.742008Z","iopub.status.idle":"2022-05-23T00:42:24.761688Z","shell.execute_reply.started":"2022-05-23T00:42:24.741976Z","shell.execute_reply":"2022-05-23T00:42:24.760500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let us use LazyTransform to build a better pipeline ","metadata":{}},{"cell_type":"code","source":"x_train = trainx[select]\ny_train = trainx[target]\nx_test = testx[select]\ny_test = testx[target].values","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:27.341067Z","iopub.execute_input":"2022-05-23T00:42:27.341990Z","iopub.status.idle":"2022-05-23T00:42:27.372882Z","shell.execute_reply.started":"2022-05-23T00:42:27.341932Z","shell.execute_reply":"2022-05-23T00:42:27.371731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### If this appears to be an Imbalanced dataset. Let us set Imbalanced to True in LazyTransform\n#pd.DataFrame(y_train.value_counts(1))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:28.161170Z","iopub.execute_input":"2022-05-23T00:42:28.161510Z","iopub.status.idle":"2022-05-23T00:42:28.166710Z","shell.execute_reply.started":"2022-05-23T00:42:28.161476Z","shell.execute_reply":"2022-05-23T00:42:28.165455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now we will try a simple model first in the lazytransform pipeline\n1. We can set Imbalanced flag to be True and LazyTransform will automatically apply SMOTE to balance the data sets classes.\n2. By consistently applying different encoders we were able to select the best encoder. That helps","metadata":{}},{"cell_type":"markdown","source":"## # let us try a model with lazytransform and see if it is better\nA stacking regressor or classifier typically gives better results than an individual estimator since it combines predictions from multiple estimators.","metadata":{}},{"cell_type":"code","source":"### Let us use a simple model ##\nfrom sklearn.linear_model import LogisticRegression\nfrom lightgbm import LGBMClassifier, LGBMRegressor\nfrom sklearn.multioutput import MultiOutputRegressor, MultiOutputClassifier\nfrom xgboost import XGBClassifier, XGBRegressor\nif modeltype =='Regression':\n    #lgb = LGBMRegressor(n_estimators=150, random_state=0)\n    lgb = XGBRegressor(n_estimators=150, random_state=0, \n                        base_score=None, booster='gbtree')\n    #lgb = FW.Stacking_Regressor()\n    #lgb = FW.Blending_Regressor()\n    if isinstance(target, list):\n        if len(target) > 1:\n            lgb = MultiOutputRegressor(lgb)\nelse:\n    #lgb = LGBMClassifier(n_estimators=50, random_state=0)\n    lgb = XGBClassifier(n_estimators=150, random_state=0, \n                        base_score=None, booster='gbtree')\n    #lgb = FW.Stacking_Classifier()\n    if isinstance(target, list):\n        if len(target) > 1:\n            lgb = MultiOutputClassifier(lgb)\nlgb","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:32.348790Z","iopub.execute_input":"2022-05-23T00:42:32.349469Z","iopub.status.idle":"2022-05-23T00:42:32.368709Z","shell.execute_reply.started":"2022-05-23T00:42:32.349416Z","shell.execute_reply":"2022-05-23T00:42:32.366112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train[:3]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:33.900001Z","iopub.execute_input":"2022-05-23T00:42:33.901320Z","iopub.status.idle":"2022-05-23T00:42:33.912648Z","shell.execute_reply.started":"2022-05-23T00:42:33.901242Z","shell.execute_reply":"2022-05-23T00:42:33.911514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### We need to make sure that we transform the target since HeartDisease is categorical\nlazy = LazyTransformer(model=lgb, encoders=encoders, scalers=scalers, \n        date_to_string=False, transform_target=transform_target, imbalanced=imbalanced)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:35.176015Z","iopub.execute_input":"2022-05-23T00:42:35.176359Z","iopub.status.idle":"2022-05-23T00:42:35.180619Z","shell.execute_reply.started":"2022-05-23T00:42:35.176326Z","shell.execute_reply":"2022-05-23T00:42:35.179937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lazy.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:42:36.575421Z","iopub.execute_input":"2022-05-23T00:42:36.576299Z","iopub.status.idle":"2022-05-23T00:43:18.795107Z","shell.execute_reply.started":"2022-05-23T00:42:36.576220Z","shell.execute_reply":"2022-05-23T00:43:18.794435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if modeltype != 'Regression' and transform_target:\n    y_preds = lazy.predict(x_test)\n    #y_preds = lazy.yformer.inverse_transform(y_preds)\nelse:\n    y_preds = lazy.predict(x_test)\ny_preds[:4]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:43:22.753187Z","iopub.execute_input":"2022-05-23T00:43:22.753545Z","iopub.status.idle":"2022-05-23T00:43:22.896588Z","shell.execute_reply.started":"2022-05-23T00:43:22.753510Z","shell.execute_reply":"2022-05-23T00:43:22.895657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if modeltype == 'Regression':\n    from sklearn.metrics import r2_score, mean_squared_error\n    print('R-Squared = %0.0f%%' %(100*r2_score(testx[target],y_preds)))\n    print('RMSE = %0.2f' %np.sqrt(mean_squared_error(testx[target],y_preds)))\n    #plot_scatter(test[target],testm[target+'_XGBoost_predictions'])\nelse:\n    from sklearn.metrics import balanced_accuracy_score, classification_report\n    if isinstance(target, str): \n        print('Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target],y_preds)))\n        print(classification_report(testx[target],y_preds))\n    elif len(target) == 1:\n            print('Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target],y_preds)))\n            print(classification_report(testx[target],y_preds))\n    else:\n        for each_i, target_name in enumerate(target):\n            print('For %s:' %target_name)\n            print('    Bal accu %0.0f%%' %(100*balanced_accuracy_score(testx[target].values[:,each_i],y_preds[:,each_i])))\n            print(classification_report(testx[target].values[:,each_i],y_preds[:,each_i]))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T00:43:24.051382Z","iopub.execute_input":"2022-05-23T00:43:24.052643Z","iopub.status.idle":"2022-05-23T00:43:24.073408Z","shell.execute_reply.started":"2022-05-23T00:43:24.052565Z","shell.execute_reply":"2022-05-23T00:43:24.071984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary: Performing feature engg and feature selection + a lazy pipeline will yield very good results.\nAdditional: Remember to apply Imbalanced when using lazytransform for imbalanced datasets and also send in a more complex model such as Stacking models when dealing with difficult datasets.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}