{"cells":[{"metadata":{},"cell_type":"markdown","source":"Thank you for sharing submission and your way.\n\nI appreciate all of you whose kernal is used by me."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\n\nfrom sklearn.metrics import mean_squared_error","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#reduce the memory usage\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading & cleaning data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# using original data\nroot = Path('../input/ashrae-feather-format-for-fast-loading')\n# train_df = pd.read_feather(root/'train.feather')\ntest_df = pd.read_feather(root/'test.feather')\nbuilding_meta_df = pd.read_feather(root/\"building_metadata.feather\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using LK data\nleak_df = pd.read_feather('../input/ashrae-leak-data-station/leak.feather')\nleak_df.fillna(0, inplace=True)\nleak_df = leak_df[(leak_df.timestamp.dt.year > 2016) & (leak_df.timestamp.dt.year < 2019)]\nleak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0 # remove large negative values\nleak_df = leak_df[leak_df.building_id!=245]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(leak_df.duplicated().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(leak_df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation for models using leak data"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kfold_submission = pd.read_csv('../input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)\nhalfhalf_submission = pd.read_csv('../input/ashrae-half-and-half/submission.csv', index_col=0)\nhighway_submission = pd.read_csv('../input/ashrae-highway-kernel-route4/submission.csv', index_col=0)\ngep3_submission = pd.read_csv(\"../input/ashrae-great-energy-predictor-iii-eda-model/Normal.csv\", index_col=0)\nsimple_submission = pd.read_csv(\"../input/ashrae-simple-data-cleanup-lb-1-08-no-leaks/submission.csv\", index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"pred1\"] = kfold_submission.meter_reading\ntest_df[\"pred2\"] = halfhalf_submission.meter_reading\ntest_df[\"pred3\"] = highway_submission.meter_reading\ntest_df[\"pred4\"] = gep3_submission.meter_reading\ntest_df[\"pred5\"] = simple_submission.meter_reading\n\ndel kfold_submission, halfhalf_submission, highway_submission, gep3_submission\ndel simple_submission\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.loc[test_df.pred3 < 0, \"pred3\"] = 0\ntest_df.loc[test_df.pred4 < 0, \"pred4\"] = 0\ntest_df.loc[test_df.pred5 < 0, \"pred5\"] = 0\ntest_df = reduce_mem_usage(test_df)\nleak_df = reduce_mem_usage(leak_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'pred4', 'pred5', 'row_id']],\n                        left_on = ['building_id','meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'],\n                        how = \"left\")\nleak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"leak_df[\"pred1_l1p\"] = np.log1p(leak_df.pred1)\nleak_df[\"pred2_l1p\"] = np.log1p(leak_df.pred2)\nleak_df[\"pred3_l1p\"] = np.log1p(leak_df.pred3)\nleak_df[\"pred4_l1p\"] = np.log1p(leak_df.pred4)\nleak_df[\"pred5_l1p\"] = np.log1p(leak_df.pred5)\nleak_df[\"meter_reading_l1p\"] = np.log1p(leak_df.meter_reading)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"leak_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"leak_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(leak_df[leak_df.pred1_l1p.isnull()])\nprint(leak_df[leak_df.pred4_l1p.isnull()])\nprint(leak_df[leak_df.pred5_l1p.isnull()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del building_meta_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"# kfold\nsns.distplot(leak_df.pred1_l1p)\nsns.distplot(leak_df.meter_reading_l1p)\n\nleak_score = np.sqrt(mean_squared_error(leak_df.pred1_l1p, leak_df.meter_reading_l1p))\nprint ('score1=', leak_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# half and half\nsns.distplot(leak_df.pred2_l1p)\nsns.distplot(leak_df.meter_reading_l1p)\n\nleak_score = np.sqrt(mean_squared_error(leak_df.pred2_l1p, leak_df.meter_reading_l1p))\nprint ('score2=', leak_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#highway route4\nsns.distplot(leak_df.pred3_l1p)\nsns.distplot(leak_df.meter_reading_l1p)\n\nleak_score = np.sqrt(mean_squared_error(leak_df.pred3_l1p, leak_df.meter_reading_l1p))\nprint ('score3=', leak_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#gep3\nsns.distplot(leak_df.pred4_l1p)\nsns.distplot(leak_df.meter_reading_l1p)\n\nleak_score = np.sqrt(mean_squared_error(leak_df.pred4_l1p, leak_df.meter_reading_l1p))\nprint ('score4=', leak_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#simple_clean\nsns.distplot(leak_df.pred5_l1p)\nsns.distplot(leak_df.meter_reading_l1p)\n\nleak_score = np.sqrt(mean_squared_error(leak_df.pred5_l1p, leak_df.meter_reading_l1p))\nprint ('score5=', leak_score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Find Best Weight"},{"metadata":{"trusted":true},"cell_type":"code","source":"all_combinations = list(np.linspace(0.15,0.4,15))\nall_combinations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"l = [all_combinations, all_combinations, all_combinations, all_combinations, all_combinations]\n# all_l = list(itertools.product(*l)) + list(itertools.product(*reversed(l)))\nall_l = list(itertools.product(*l))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### filtering sumation of weight"},{"metadata":{"trusted":true},"cell_type":"code","source":"filtered_combis = [l for l in all_l if l[0] + l[1] + l[2] + l[3] + l[4] > 0.95 and l[0] + l[1] + l[2] + l[3] + l[4] < 1.03]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(filtered_combis))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_combi = [] # of the form (i, score)\nfor i, combi in enumerate(filtered_combis):\n    #print(\"Now at: \" + str(i) + \" out of \" + str(len(filtered_combis))) # uncomment to view iterations\n    score1 = combi[0]\n    score2 = combi[1]\n    score3 = combi[2]\n    score4 = combi[3]\n    score5 = combi[4]\n    v = score1 * leak_df['pred1'].values + score2 * leak_df['pred3'].values + score3 * leak_df['pred2'].values + score4 * leak_df['pred4'] + score5 * leak_df['pred5']\n    vl1p = np.log1p(v)\n    curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))\n    \n    if best_combi:\n        prev_score = best_combi[0][1]\n        if curr_score < prev_score:\n            best_combi[:] = []\n            best_combi += [(i, curr_score)]\n    else:\n        best_combi += [(i, curr_score)]\n            \nscore = best_combi[0][1]\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_df = pd.read_feather(root/'test.feather')\n# kfold_submission = pd.read_csv('../input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)\n# halfhalf_submission = pd.read_csv('../input/ashrae-half-and-half/submission.csv', index_col=0)\n# highway_submission = pd.read_csv('../input/ashrae-highway-kernel-route4/submission.csv', index_col=0)\n\n# test_df[\"pred1\"] = kfold_submission.meter_reading\n# test_df[\"pred2\"] = halfhalf_submission.meter_reading\n# test_df[\"pred3\"] = highway_submission.meter_reading\n\n# del kfold_submission, halfhalf_submission, highway_submission\n# gc.collect()\n\n# test_df.loc[test_df.pred3 < 0, \"pred3\"] = 0\n# test_df = reduce_mem_usage(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_form = pd.read_feather(os.path.join(root, 'sample_submission.feather'))\nfinal_combi = filtered_combis[best_combi[0][0]]\nw1 = final_combi[0]\nw2 = final_combi[1]\nw3 = final_combi[2]\nw4 = final_combi[3]\nw5 = final_combi[4]\nprint(\"Best weight is w1(kfold):{}, w2(halfhalf):{}, w3(highway):{}, w4(gep3):{}, w5(simple):{}\".format(w1, w2, w3, w4, w5))\n\nsubmission_form['meter_reading'] = w1 * test_df.pred1 +  w2 * test_df.pred3  + w3 * test_df.pred2 + w4 * test_df.pred4 + w5 * test_df.pred5\nsubmission_form.loc[submission_form.meter_reading < 0, 'meter_reading'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(np.log1p(submission_form.meter_reading))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id').dropna()\nprint(len(leak_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_form.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(np.log1p(submission_form.meter_reading))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_form.to_csv('submission.csv', index=False, float_format='%.4f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}