{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np\nimport pandas as pd\n\nimport warnings\nwarnings.simplefilter('ignore')\n\n\nimport os, gc, sys, warnings, random, math, psutil, pickle\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### Helpers ######\n\n\n## Seeder\n# :seed to make all processes deterministic     # type: int\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    \n## Simple \"Memory profilers\" to see memory usage\ndef get_memory_usage():\n    return np.round(psutil.Process(os.getpid()).memory_info()[0]/2.**30, 2) \n        \ndef sizeof_fmt(num, suffix='B'):\n    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n        if abs(num) < 1024.0:\n            return \"%3.1f%s%s\" % (num, unit, suffix)\n        num /= 1024.0\n    return \"%.1f%s%s\" % (num, 'Yi', suffix)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"####### Vars ######\n\nSEED = 42\nLOCAl_TEST = False\nseed_everything(SEED)\nTARGET = 'meter_reading'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_pickle('/kaggle/input/data-minification-ashrae/train.pkl')\ntest_df = pd.read_pickle('/kaggle/input/data-minification-ashrae/test.pkl')\n\nbuilding_df = pd.read_pickle('/kaggle/input/data-minification-ashrae/building_metadata_metadata.pkl')\n\ntrain_weather_df = pd.read_pickle('/kaggle/input/data-minification-ashrae/weather_train.pkl')\ntest_weather_df = pd.read_pickle('/kaggle/input/data-minification-ashrae/weather_test.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"################ Building DF merge through concat ########################\n# Benefits of concat:\n## Faster for huge datasets (columns number)\n## No dtype change for dataset\n## Consume less memmory \n\ntemp_df = train_df[['building_id']]\ntemp_df = temp_df.merge(building_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ntemp_df = test_df[['building_id']]\ntemp_df = temp_df.merge(building_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntest_df = pd.concat([test_df, temp_df], axis=1)\n\ndel building_df, temp_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"################ Weather DF merge over concat (to not lose type) ###########################\n# Benefits of concat:\n## Faster for huge datasets (columns number)\n## No dtype change for dataset\n## Consume less memmory \n\ntemp_df = train_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(train_weather_df, on=['site_id','timestamp'], how='left')\ndel temp_df['site_id'], temp_df['timestamp']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ntemp_df = test_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(test_weather_df, on=['site_id','timestamp'], how='left')\ndel temp_df['site_id'], temp_df['timestamp']\ntest_df = pd.concat([test_df, temp_df], axis=1)\n\ndel train_weather_df, test_weather_df, temp_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"############# Trick to use kernel hdd to store results #############\n\n# You can save just test_df or both if have sufficient space\ntrain_df.to_pickle('train_df.pkl')\ntest_df.to_pickle('test_df.pkl')\n   \ndel train_df, test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Check memory usage #################################\n\nfor name, size in sorted(((name, sys.getsizeof(value)) for name,value in locals().items()),\n                         key= lambda x: -x[1])[:10]:\n    print(\"{:>30}: {:>8}\".format(name,sizeof_fmt(size)))\nprint('Memory in Gb', get_memory_usage())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model params\n\nimport lightgbm as lgb\nlgb_params = {\n                    'objective':'regression',\n                    'boosting_type':'gbdt',\n                    'metric':'rmse',\n                    'n_jobs':-1,\n                    'learning_rate':0.05,\n                    'num_leaves': 2**8,\n                    'max_depth':-1,\n                    'tree_learner':'serial',\n                    'colsample_bytree': 0.7,\n                    'subsample_freq':1,\n                    'subsample':0.7,\n                    'n_estimators':800,\n                    'max_bin':255,\n                    'verbose':-1,\n                    'seed': SEED,\n                    'early_stopping_rounds':100, \n                } ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Using LightGBM Method for create Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Models saving\nmodel_filename = 'lgbm'\nmodels = []\n\n# Load train_df from hdd\ntrain_df = pd.read_pickle('train_df.pkl')\n\nremove_columns = ['timestamp',TARGET]\nfeatures_columns = [col for col in list(train_df) if col not in remove_columns]\n\nif LOCAl_TEST:\n    tr_data = lgb.Dataset(train_df.iloc[:15000000][features_columns], label=np.log1p(train_df.iloc[:15000000][TARGET]))\n    vl_data = lgb.Dataset(train_df.iloc[15000000:][features_columns], label=np.log1p(train_df.iloc[15000000:][TARGET]))\n    eval_sets = [tr_data,vl_data]\nelse:\n    tr_data = lgb.Dataset(train_df[features_columns], label=np.log1p(train_df[TARGET]))\n    eval_sets = [tr_data]\n\n# Remove train_df from hdd\nos.system('rm train_df.pkl')\n\n# Lets make 5 seeds mix model\nfor cur_seed in [42,43,44,45,46]:\n    \n    # Seed everything\n    seed_everything(cur_seed)\n    lgb_params['seed'] = cur_seed\n    \n    estimator = lgb.train(\n                lgb_params,\n                tr_data,\n                valid_sets = eval_sets,\n                verbose_eval = 100,\n            )\n\n    # For CV you may add fold number\n    # pickle.dump(estimator, open(model_filename + '__fold_' + str(i) + '.bin', \"wb\"))\n    pickle.dump(estimator, open(model_filename + '__seed_' + str(cur_seed)  + '.bin', 'wb'))\n    models.append(model_filename + '__seed_' + str(cur_seed)  + '.bin')\n\nif not LOCAl_TEST:\n    del tr_data, train_df\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"######### Predict #############\n\nif not LOCAl_TEST:\n    \n    # Load test_df from hdd\n    test_df = pd.read_pickle('test_df.pkl')\n    \n    # Remove unused columns\n    test_df = test_df[features_columns]\n    \n    # Remove test_df from hdd\n    os.system('rm test_df.pkl')\n    \n    # Read submission file\n    submission = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv')\n\n    # Remove row_id for a while\n    del submission['row_id']\n    \n    for model_path in models:\n        print('Predictions for', model_path)\n        estimator = pickle.load(open(model_path, 'rb'))\n\n        predictions = []\n        batch_size = 2000000\n        for batch in range(int(len(test_df)/batch_size)+1):\n            print('Predicting batch:', batch)\n            predictions += list(np.expm1(estimator.predict(test_df[features_columns].iloc[batch*batch_size:(batch+1)*batch_size])))\n            \n        submission['meter_reading'] += predictions\n        \n    # Average over models\n    submission['meter_reading'] /= len(models)\n    \n    # Delete test_df\n    del test_df\n     \n    # Fix negative values\n    submission['meter_reading'] = submission['meter_reading'].clip(0,None)\n\n    # Restore row_id\n    submission['row_id'] = submission.index\n    \n    \n    \n    #### Check ####\n    \n    print(submission.iloc[:20])\n    print(submission['meter_reading'].describe())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}