{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.ensemble import AdaBoostRegressor\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:28:28.010406Z","iopub.execute_input":"2021-11-01T07:28:28.011381Z","iopub.status.idle":"2021-11-01T07:28:29.331892Z","shell.execute_reply.started":"2021-11-01T07:28:28.011268Z","shell.execute_reply":"2021-11-01T07:28:29.331048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install xgboost\n! pip install lightgbm\n! pip install tensorflow","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:28:29.33323Z","iopub.execute_input":"2021-11-01T07:28:29.333439Z","iopub.status.idle":"2021-11-01T07:29:06.789053Z","shell.execute_reply.started":"2021-11-01T07:28:29.333413Z","shell.execute_reply":"2021-11-01T07:29:06.788108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/beyond-analysis/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:06.791116Z","iopub.execute_input":"2021-11-01T07:29:06.791378Z","iopub.status.idle":"2021-11-01T07:29:11.25418Z","shell.execute_reply.started":"2021-11-01T07:29:06.791347Z","shell.execute_reply":"2021-11-01T07:29:11.253364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\ncate = ['CATEGORY_1', 'CATEGORY_2']\ndf[cate]= df[cate].apply(le.fit_transform)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:11.25596Z","iopub.execute_input":"2021-11-01T07:29:11.256204Z","iopub.status.idle":"2021-11-01T07:29:11.683062Z","shell.execute_reply.started":"2021-11-01T07:29:11.256174Z","shell.execute_reply":"2021-11-01T07:29:11.682114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df.groupby('UNIQUE_IDENTIFIER').mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:11.68432Z","iopub.execute_input":"2021-11-01T07:29:11.684566Z","iopub.status.idle":"2021-11-01T07:29:11.987187Z","shell.execute_reply.started":"2021-11-01T07:29:11.684539Z","shell.execute_reply":"2021-11-01T07:29:11.986193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:11.98885Z","iopub.execute_input":"2021-11-01T07:29:11.989182Z","iopub.status.idle":"2021-11-01T07:29:11.998782Z","shell.execute_reply.started":"2021-11-01T07:29:11.989139Z","shell.execute_reply":"2021-11-01T07:29:11.997697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf1.drop(['PRACTICE_ENTRY_NUMBER', 'PRACTICE_ENTRY', 'ENTRY_NUMBER','WINNINGS_2', 'ENTRY', 'UNIQUE_IDENTIFIER', 'DEPOSIT_2'], inplace = True, axis= 1)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:12.000154Z","iopub.execute_input":"2021-11-01T07:29:12.000394Z","iopub.status.idle":"2021-11-01T07:29:12.014584Z","shell.execute_reply.started":"2021-11-01T07:29:12.000365Z","shell.execute_reply":"2021-11-01T07:29:12.013531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:12.016492Z","iopub.execute_input":"2021-11-01T07:29:12.01679Z","iopub.status.idle":"2021-11-01T07:29:12.028494Z","shell.execute_reply.started":"2021-11-01T07:29:12.016755Z","shell.execute_reply":"2021-11-01T07:29:12.027568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df1.drop(['Y1', 'Y2'], axis=1)\ny = df1[['Y1', 'Y2']]","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:12.029972Z","iopub.execute_input":"2021-11-01T07:29:12.030212Z","iopub.status.idle":"2021-11-01T07:29:12.044032Z","shell.execute_reply.started":"2021-11-01T07:29:12.030183Z","shell.execute_reply":"2021-11-01T07:29:12.043319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, train_size = 0.9)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:12.047363Z","iopub.execute_input":"2021-11-01T07:29:12.048226Z","iopub.status.idle":"2021-11-01T07:29:12.075324Z","shell.execute_reply.started":"2021-11-01T07:29:12.048188Z","shell.execute_reply":"2021-11-01T07:29:12.074344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBRegressor\nimport lightgbm as lgb\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:12.076678Z","iopub.execute_input":"2021-11-01T07:29:12.07701Z","iopub.status.idle":"2021-11-01T07:29:17.779327Z","shell.execute_reply.started":"2021-11-01T07:29:12.076979Z","shell.execute_reply":"2021-11-01T07:29:17.778371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:17.780495Z","iopub.execute_input":"2021-11-01T07:29:17.782314Z","iopub.status.idle":"2021-11-01T07:29:17.849079Z","shell.execute_reply.started":"2021-11-01T07:29:17.782275Z","shell.execute_reply":"2021-11-01T07:29:17.848152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.add(Dense(128, input_dim=16, activation='relu'))\nmodel.add(Dense(64,input_dim=128,  activation='relu'))\nmodel.add(Dense(16,input_dim=64, activation='relu'))\nmodel.add(Dense(2,input_dim=16,activation='relu'))","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:17.850639Z","iopub.execute_input":"2021-11-01T07:29:17.851269Z","iopub.status.idle":"2021-11-01T07:29:17.923295Z","shell.execute_reply.started":"2021-11-01T07:29:17.851219Z","shell.execute_reply":"2021-11-01T07:29:17.922405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='mean_squared_error', optimizer='adam', metrics=[tf.keras.metrics.RootMeanSquaredError()])","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:17.925094Z","iopub.execute_input":"2021-11-01T07:29:17.925405Z","iopub.status.idle":"2021-11-01T07:29:17.943033Z","shell.execute_reply.started":"2021-11-01T07:29:17.925363Z","shell.execute_reply":"2021-11-01T07:29:17.942333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:17.944187Z","iopub.execute_input":"2021-11-01T07:29:17.944472Z","iopub.status.idle":"2021-11-01T07:29:17.952642Z","shell.execute_reply.started":"2021-11-01T07:29:17.944423Z","shell.execute_reply":"2021-11-01T07:29:17.951792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ba = model.fit(X_train, y_train, epochs= 100 )","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:29:17.954128Z","iopub.execute_input":"2021-11-01T07:29:17.954616Z","iopub.status.idle":"2021-11-01T07:39:08.985776Z","shell.execute_reply.started":"2021-11-01T07:29:17.954573Z","shell.execute_reply":"2021-11-01T07:39:08.984838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict3 = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:08.987564Z","iopub.execute_input":"2021-11-01T07:39:08.987793Z","iopub.status.idle":"2021-11-01T07:39:09.410066Z","shell.execute_reply.started":"2021-11-01T07:39:08.987764Z","shell.execute_reply":"2021-11-01T07:39:09.409265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = lgb.LGBMRegressor()\nmodel1 = XGBRegressor()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:09.41144Z","iopub.execute_input":"2021-11-01T07:39:09.411688Z","iopub.status.idle":"2021-11-01T07:39:09.416238Z","shell.execute_reply.started":"2021-11-01T07:39:09.41166Z","shell.execute_reply":"2021-11-01T07:39:09.415499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf1 = MultiOutputRegressor(XGBRegressor(random_state = 0)).fit(X_train, y_train)\nclf2 = MultiOutputRegressor(lgb.LGBMRegressor(random_state = 0)).fit(X_train, y_train)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:09.417898Z","iopub.execute_input":"2021-11-01T07:39:09.418518Z","iopub.status.idle":"2021-11-01T07:39:22.597891Z","shell.execute_reply.started":"2021-11-01T07:39:09.418453Z","shell.execute_reply":"2021-11-01T07:39:22.597175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict1 = clf1.predict(X_test)\ny_predict2 = clf2.predict(X_test)\ny_predict3 = model.predict(X_test)\nfinal_predict = (y_predict1+y_predict2+y_predict3)/3","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:22.599624Z","iopub.execute_input":"2021-11-01T07:39:22.600212Z","iopub.status.idle":"2021-11-01T07:39:23.015697Z","shell.execute_reply.started":"2021-11-01T07:39:22.600168Z","shell.execute_reply":"2021-11-01T07:39:23.015065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score =  mean_squared_error(y_test, final_predict, squared=False)\nscore","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:23.017332Z","iopub.execute_input":"2021-11-01T07:39:23.018012Z","iopub.status.idle":"2021-11-01T07:39:23.027968Z","shell.execute_reply.started":"2021-11-01T07:39:23.017967Z","shell.execute_reply":"2021-11-01T07:39:23.027071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading test data","metadata":{}},{"cell_type":"code","source":"tdf = pd.read_csv('../input/beyond-analysis/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:23.029799Z","iopub.execute_input":"2021-11-01T07:39:23.030482Z","iopub.status.idle":"2021-11-01T07:39:25.526489Z","shell.execute_reply.started":"2021-11-01T07:39:23.030413Z","shell.execute_reply":"2021-11-01T07:39:25.52564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdf.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.527659Z","iopub.execute_input":"2021-11-01T07:39:25.527869Z","iopub.status.idle":"2021-11-01T07:39:25.533694Z","shell.execute_reply.started":"2021-11-01T07:39:25.527843Z","shell.execute_reply":"2021-11-01T07:39:25.532908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\ncate = ['CATEGORY_1', 'CATEGORY_2']\ntdf[cate]= df[cate].apply(le.fit_transform)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.535057Z","iopub.execute_input":"2021-11-01T07:39:25.535259Z","iopub.status.idle":"2021-11-01T07:39:25.732467Z","shell.execute_reply.started":"2021-11-01T07:39:25.535234Z","shell.execute_reply":"2021-11-01T07:39:25.731587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdf1 = tdf.groupby('UNIQUE_IDENTIFIER').mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.73396Z","iopub.execute_input":"2021-11-01T07:39:25.734175Z","iopub.status.idle":"2021-11-01T07:39:25.952366Z","shell.execute_reply.started":"2021-11-01T07:39:25.73415Z","shell.execute_reply":"2021-11-01T07:39:25.951602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdf2 = tdf1.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.953657Z","iopub.execute_input":"2021-11-01T07:39:25.953938Z","iopub.status.idle":"2021-11-01T07:39:25.960097Z","shell.execute_reply.started":"2021-11-01T07:39:25.953904Z","shell.execute_reply":"2021-11-01T07:39:25.959204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdf1.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.961364Z","iopub.execute_input":"2021-11-01T07:39:25.961588Z","iopub.status.idle":"2021-11-01T07:39:25.972938Z","shell.execute_reply.started":"2021-11-01T07:39:25.961556Z","shell.execute_reply":"2021-11-01T07:39:25.971883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdf1.drop(['PRACTICE_ENTRY_NUMBER', 'PRACTICE_ENTRY', 'ENTRY_NUMBER','WINNINGS_2', 'ENTRY', 'UNIQUE_IDENTIFIER', 'DEPOSIT_2'], inplace = True, axis= 1)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.978614Z","iopub.execute_input":"2021-11-01T07:39:25.979787Z","iopub.status.idle":"2021-11-01T07:39:25.98888Z","shell.execute_reply.started":"2021-11-01T07:39:25.979736Z","shell.execute_reply":"2021-11-01T07:39:25.988146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict1 = clf1.predict(tdf1)\ny_predict2 = clf2.predict(tdf1)\ny_predict3 = model.predict(tdf1)\n\nfinal_predict = (y_predict1+y_predict2+y_predict3)/3","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:25.997012Z","iopub.execute_input":"2021-11-01T07:39:25.997483Z","iopub.status.idle":"2021-11-01T07:39:28.341361Z","shell.execute_reply.started":"2021-11-01T07:39:25.997426Z","shell.execute_reply":"2021-11-01T07:39:28.340574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame(final_predict, columns = ['Y1', 'Y2'])\nresult.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:28.343233Z","iopub.execute_input":"2021-11-01T07:39:28.34403Z","iopub.status.idle":"2021-11-01T07:39:28.358439Z","shell.execute_reply.started":"2021-11-01T07:39:28.343979Z","shell.execute_reply":"2021-11-01T07:39:28.357607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['UNIQUE_IDENTIFIER'] = tdf2['UNIQUE_IDENTIFIER']\nresult","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:28.360296Z","iopub.execute_input":"2021-11-01T07:39:28.361079Z","iopub.status.idle":"2021-11-01T07:39:28.377118Z","shell.execute_reply.started":"2021-11-01T07:39:28.36103Z","shell.execute_reply":"2021-11-01T07:39:28.376273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = result[['UNIQUE_IDENTIFIER', 'Y1', 'Y2']]","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:28.378732Z","iopub.execute_input":"2021-11-01T07:39:28.379979Z","iopub.status.idle":"2021-11-01T07:39:28.386017Z","shell.execute_reply.started":"2021-11-01T07:39:28.379847Z","shell.execute_reply":"2021-11-01T07:39:28.385143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_csv('prediction9.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T07:39:28.387084Z","iopub.execute_input":"2021-11-01T07:39:28.387285Z","iopub.status.idle":"2021-11-01T07:39:28.649059Z","shell.execute_reply.started":"2021-11-01T07:39:28.387261Z","shell.execute_reply":"2021-11-01T07:39:28.64791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}