{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\npd.set_option(\"display.max_columns\", 500)\npd.set_option(\"display.max_rows\", 500)\nfrom scipy.stats.mstats import gmean\nfrom scipy.sparse import hstack\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-22T14:37:51.459929Z","iopub.execute_input":"2022-04-22T14:37:51.460427Z","iopub.status.idle":"2022-04-22T14:37:52.477481Z","shell.execute_reply.started":"2022-04-22T14:37:51.460304Z","shell.execute_reply":"2022-04-22T14:37:52.476690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nfrom math import sqrt\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.metrics import roc_auc_score, mean_absolute_error, mean_squared_error, log_loss\nfrom sklearn.model_selection import KFold, StratifiedKFold, TimeSeriesSplit\n\nimport scipy.stats as ss\nfrom scipy.stats import mode\nimport math as mt\nfrom math import *\nimport matplotlib as mlp\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport seaborn as sns\nfrom tqdm import tqdm\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import cross_val_predict\nimport numpy as np\nimport pandas as pd\nimport time\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom statsmodels.graphics.mosaicplot import mosaic\nimport itertools\nimport random","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:52.478824Z","iopub.execute_input":"2022-04-22T14:37:52.479025Z","iopub.status.idle":"2022-04-22T14:37:53.652310Z","shell.execute_reply.started":"2022-04-22T14:37:52.479001Z","shell.execute_reply":"2022-04-22T14:37:53.651585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(mtype):\n    \n\n    train1 = pd.read_csv(\"../input/football-match-prediction-feature-engineering/train_home_team.csv\")\n    test1 = pd.read_csv(\"../input/football-match-prediction-feature-engineering/test_home_team.csv\")\n    train2 = pd.read_csv(\"../input/football-match-prediction-feature-engineering/train_away_team.csv\")\n    test2 = pd.read_csv(\"../input/football-match-prediction-feature-engineering/test_away_team.csv\")\n    train = pd.concat([train1, train2.drop(['id','match_date','league_name','is_cup'], axis=1)], axis=1)\n    test = pd.concat([test1, test2.drop(['id','match_date','league_name','is_cup'], axis=1)], axis=1)\n\n    del train1, train2, test1, test2\n        \n    tt_and_scores = pd.read_csv(\"../input/football-match-probability-prediction/train_target_and_scores.csv\")\n    tt_and_scores = tt_and_scores.rename(columns={'target':'final_score'})\n    target2int = {'away': 0, 'draw': 1, 'home': 2}\n    tt_and_scores['final_score'] = tt_and_scores['final_score'].map(target2int)\n    \n    train = train.drop('match_date', axis=1)\n    test = test.drop('match_date', axis=1)\n    \n    target = mtype+'_score'\n    train = train.merge(tt_and_scores[['id', target]], on='id', how='inner')\n    train[target] = train[target].astype(int)\n    \n    return train, test, target","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.653392Z","iopub.execute_input":"2022-04-22T14:37:53.653590Z","iopub.status.idle":"2022-04-22T14:37:53.662010Z","shell.execute_reply.started":"2022-04-22T14:37:53.653566Z","shell.execute_reply":"2022-04-22T14:37:53.661140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_list(mtype, df_trn):\n    \n    train_df = df_trn.copy()\n    \n    cat_cols = (['league_name','is_cup','home_team_coach_id', 'away_team_coach_id']+[col for col in train_df.columns if '_ihm' in col]+\n                [col for col in train_df.columns if '_icp' in col]+[col for col in train_df.columns if '_cch' in col]+\n                [col for col in train_df.columns if '_lid' in col])\n        \n    features = [col for col in train_df.columns if col not in cat_cols + ['id', target]]\n    return cat_cols, features","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.664093Z","iopub.execute_input":"2022-04-22T14:37:53.664748Z","iopub.status.idle":"2022-04-22T14:37:53.678172Z","shell.execute_reply.started":"2022-04-22T14:37:53.664716Z","shell.execute_reply":"2022-04-22T14:37:53.677431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scoring_regression(y_true, y_hat):\n    return mean_squared_error(y_true, y_hat)\n\ndef scoring_auc(y_true, y_hat):\n    return roc_auc_score(y_true, y_hat)\n\ndef scoring_logloss(y_true, y_hat):\n    return log_loss(y_true, y_hat)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.679132Z","iopub.execute_input":"2022-04-22T14:37:53.679668Z","iopub.status.idle":"2022-04-22T14:37:53.689190Z","shell.execute_reply.started":"2022-04-22T14:37:53.679632Z","shell.execute_reply":"2022-04-22T14:37:53.688424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prep_train_test(cat_list, other_list, y, df_train, df_test):\n    \n    X_train = df_train[cat_list + other_list + [y]].reset_index(drop=True)\n    X_test = df_test[cat_list + other_list]\n    y_train = X_train[y]\n    \n    for cc in tqdm(cat_list, disable=True):\n        \n        fillna_const = 'MISSING_VALUE'\n        X_train[cc] = X_train[cc].astype('str').fillna(fillna_const)\n        X_test[cc] = X_test[cc].astype('str').fillna(fillna_const)\n        X_train[cc] = X_train[cc].astype('category')\n        X_test[cc] = X_test[cc].astype('category')\n    \n    return X_train, X_test, y_train","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.690230Z","iopub.execute_input":"2022-04-22T14:37:53.690453Z","iopub.status.idle":"2022-04-22T14:37:53.704093Z","shell.execute_reply.started":"2022-04-22T14:37:53.690428Z","shell.execute_reply":"2022-04-22T14:37:53.703333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    \"objective\":\"multiclass\", \n    \"boosting_type\":\"gbdt\", \n    'num_class': 3, \n    'metric': \"multi_logloss\",\n    'learning_rate': 0.00868,        \n    'lambda_l1': 1.90,\n    'lambda_l2': 6.50,\n    'num_leaves': 555,\n    'min_sum_hessian_in_leaf': 20,\n    'feature_fraction': 0.872,\n    'feature_fraction_bynode': 0.5,\n    'bagging_fraction': 0.6,\n    'bagging_freq': 9,\n    'min_data_in_leaf': 512,\n    'max_depth': 7,\n    'seed': 42,\n    'verbosity': -1\n}","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.705259Z","iopub.execute_input":"2022-04-22T14:37:53.705506Z","iopub.status.idle":"2022-04-22T14:37:53.715344Z","shell.execute_reply.started":"2022-04-22T14:37:53.705478Z","shell.execute_reply":"2022-04-22T14:37:53.714455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(cat_cols, features, target, train, test):\n    \n    scores = []\n    p_valid = []\n    yp = []\n    \n    NFOLD = 5\n    kf = StratifiedKFold(n_splits = NFOLD, shuffle=True, random_state = 1998)\n    kf_lr = StratifiedKFold(n_splits = 5, shuffle=True, random_state = 42)\n    \n    X_train, X_test, y_train = prep_train_test(cat_cols, features, target, train, test)\n    \n    for n_fold, (train_index, test_index) in tqdm(enumerate(kf.split(X_train, y_train))):\n\n        # replace cat cols \n        X_train_temp = X_train.iloc[train_index].reset_index(drop=True)\n        X_val_temp = X_train.iloc[test_index].reset_index(drop=True)\n        X_test_temp = X_test.reset_index(drop=True)\n\n        y_train_temp = X_train_temp[target]\n        X_train_temp = X_train_temp.drop([target], axis=1)\n        y_val_temp = X_val_temp[target]\n        X_val_temp = X_val_temp.drop([target], axis=1)\n\n        for cc in tqdm(cat_cols, disable=True):\n\n            le = LabelEncoder()\n            le.fit(list(set(X_test[cc]) | set(X_train[cc])))\n            X_train_temp[cc] = le.transform(X_train_temp[[cc]])\n            X_val_temp[cc] = le.transform(X_val_temp[[cc]])\n            X_test_temp[cc] = le.transform(X_test_temp[[cc]])\n\n        print('X_train shape:', X_train_temp.shape)\n        print('X_valid shape:', X_val_temp.shape)\n\n        lgb_data_train = lgb.Dataset(X_train_temp, \n                                     label=y_train_temp,\n                                     free_raw_data=False)\n\n        lgb_data_valid = lgb.Dataset(X_val_temp, \n                                     label=y_val_temp,\n                                     free_raw_data=False,\n                                     reference=lgb_data_train)\n\n        lgb_model = lgb.train(params, \n                          lgb_data_train, \n                          valid_sets = [lgb_data_train, lgb_data_valid],\n                          verbose_eval = 250,\n                          early_stopping_rounds = 150,\n                          num_boost_round = 6000)\n\n        y_hat_valid_lgb = lgb_model.predict(X_val_temp)\n        y_hat_test_lgb = lgb_model.predict(X_test_temp)\n\n        p_valid.append(y_hat_valid_lgb.squeeze())\n        yp.append(y_hat_test_lgb.squeeze())\n        score_valid = scoring_logloss(y_train[test_index].values, y_hat_valid_lgb)\n        print(f'score_lgb : {score_valid}')            \n            \n        scores.append(score_valid)    \n        lgb.plot_importance(lgb_model, max_num_features=35, importance_type='gain', figsize=(12,10))\n    \n    return yp, scores","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.716716Z","iopub.execute_input":"2022-04-22T14:37:53.717156Z","iopub.status.idle":"2022-04-22T14:37:53.731121Z","shell.execute_reply.started":"2022-04-22T14:37:53.717117Z","shell.execute_reply":"2022-04-22T14:37:53.730392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_TYPE = 'final'\ntrain, test, target = read_data(MODEL_TYPE)\ncat_cols, features = feat_list(MODEL_TYPE, train)\nyp, scores = run_training(cat_cols, features, target, train, test)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T14:37:53.732397Z","iopub.execute_input":"2022-04-22T14:37:53.732718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NFOLDS = 5\nprint(sum(scores)/NFOLDS)\nscores","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = pd.DataFrame()\nsubm['id'] = test['id']\nsubm[['away', 'draw', 'home']] = pd.DataFrame(sum(yp)/NFOLDS, columns=['away', 'draw', 'home'])\nsubm[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}