{"cells":[{"metadata":{},"cell_type":"markdown","source":"Train various models on the training set created in [this kernel](http://https://www.kaggle.com/marginalreturns/create-training-set-with-four-factor-stats) and then remember which model works best to go to [this kernel](https://www.kaggle.com/marginalreturns/ncaam20-stage-1-submission?scriptVersionId=29287789) and submit predictions."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import log_loss\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename)) \n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"training_set = pd.read_csv('/kaggle/input/create-training-set-with-four-factor-stats/training_set.csv')\nrecord = pd.read_csv('/kaggle/input/create-training-set-with-four-factor-stats/record.csv')\nsubmission = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_set.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To test models, we will remove each season, build a model on the other seasons, and test the model on the removed season. I would like to print out accuracy (how many games correct?) and logloss. We will reset the model for each season to avoid data leakage. "},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn.base\n\ndef model_test(model, model_params):\n    model_name = type(model).__name__\n    print(model_name)\n    model_results[model_name + ' Accuracy'] = 0\n    model_results[model_name + ' LogLoss'] = 0    \n\n    cvresults = pd.DataFrame()\n    gs = GridSearchCV(model, model_params, scoring='neg_log_loss', n_jobs=-1, verbose = 3, cv = 3)\n    x_train = training_set[x_vars]\n    y_train = training_set['Result']\n\n    cv = gs.fit(x_train, y_train)\n    params = cv.best_params_\n    results = cv.cv_results_\n    cvresults = cvresults.append(results, ignore_index=True)\n\n    clf = model.set_params(**params)\n           \n    for season in seasons:\n        print('Processing ', season)\n        x_train = training_set[training_set['Season'] != season][x_vars]        \n        x_test = training_set[training_set['Season'] == season][x_vars]\n        y_train = training_set[training_set['Season'] != season]['Result']\n        y_test = training_set[training_set['Season'] == season]['Result']\n        \n        model = clf.fit(x_train, y_train)\n        y_proba = model.predict_proba(x_test)[:,1]\n        \n        \n        model_results.loc[model_results['Season']==season,model_name + ' Accuracy'] = model.score(x_test,y_test)\n        model_results.loc[model_results['Season']==season,model_name + ' LogLoss'] = log_loss(y_test, y_proba)\n        \n    return model_results, cvresults","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.ensemble import RandomForestClassifier, VotingClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\n\nx_vars = ['deltaSeed','deltaRPI','deltaPace','deltaAdjORtng','deltaAdjDRtng','deltaOeFG','deltaTOP','deltaOR%','deltaFTR']\n\nrfc = RandomForestClassifier()\nrfcparams = {'n_estimators' : [100,200,300],\n             'criterion' : ['gini','entropy'],\n             'min_samples_split' : [2,4,],\n             'min_samples_leaf' : [2,4],\n             'random_state' : [0],}\n\nmlp = MLPClassifier()\nmlpparams = {'solver' : ['lbfgs'],\n             'max_iter': [1000, 2500],\n             'alpha' : 10.0 ** -np.arange(1, 5),\n             'hidden_layer_sizes' : [10, 25, 50],\n             'random_state' : [0],}\n\nknn = KNeighborsClassifier()\nknnparams = {'weights' : ['uniform','distance']}\n\nsvc = SVC()\nsvcparams = {'probability' : [True],\n             'random_state' : [0]}\n\nlr = LogisticRegression()\nlrparams = {'C':[0.5,1.0,2.0,3.0],\n            'random_state' : [0],\n            'max_iter' : [100,200,500]}\n\neclf = VotingClassifier(estimators = [('rfc',rfc),('mlp',mlp),('svc',svc),('lr',lr)], voting='soft')\n\nmodels = {rfc : rfcparams,\n          mlp : mlpparams,\n          knn : knnparams,\n          svc : svcparams,\n          lr : lrparams,\n          }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_results = pd.DataFrame()\nseasons = list(training_set['Season'].unique())\nmodel_results['Season'] = seasons\nfor model in models:\n    model_results, cvresults = model_test(model, model_params = models[model])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_results.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's try using an ensemble voting method by itself and compare it"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nparams = {'rfc__n_estimators' : [100,200,300], \n          'rfc__criterion' : ['gini'],\n          'rfc__min_samples_split' : [2,4,],\n          'rfc__min_samples_leaf' : [2,4],\n          'mlp__solver' : ['lbfgs'],\n          'mlp__max_iter': [1000, 2500],\n          'mlp__alpha' : 10.0 ** -np.arange(1, 5),\n          'mlp__hidden_layer_sizes' : [10, 25, 50],\n          'mlp__random_state' : [0],\n          'mlp__verbose' : [False],\n          'lr__C':[0.5,1.0,2.0,3.0],\n          'lr__random_state' : [0],\n          'lr__max_iter' : [100,200,500],\n          'svc__probability':[True]}\n\neclf = VotingClassifier(estimators = [('rfc',rfc),('mlp',mlp),('svc',svc),('lr',lr)], voting='soft')\n        \nclf = GridSearchCV(estimator=eclf, param_grid=params, cv=3)\n\ntrain_x, test_x, train_y, test_y = train_test_split(training_set[x_vars], training_set['Result'], test_size = 0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = clf.fit(train_x, train_y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Once we pick a model, we will need to build our submission file, which is done in the [other kernel.](https://www.kaggle.com/marginalreturns/ncaam20-stage-1-submission?scriptVersionId=29287789)"},{"metadata":{"trusted":true},"cell_type":"code","source":"cvresults","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}