{"cells":[{"metadata":{},"cell_type":"markdown","source":"## TL; DR\n### ・ We have over 40,000,000  test records in this competition\n### ・ I'll share some tips to handle such a large dataset"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nDATA_DIR = \"../input/ashrae-energy-prediction/\"\nSUB_PATH = os.path.join(DATA_DIR, \"sample_submission.csv\")\n\ntarget_col = \"meter_reading\"\ndemo = True\nsample = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ===============================\n# test_preds is your predictions\n# ===============================\n\ntest_preds = np.random.randn(41697600).reshape(-1, 1)\nprint(test_preds.shape)\ntest_preds[:sample]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if demo:\n    chunks = 4000000\n    j = 0\n    for i, sub in enumerate(pd.read_csv(SUB_PATH, chunksize=chunks, iterator=True)):\n        print(i, \":\", sub.shape)\n        sub[target_col] = test_preds[j:j+chunks]\n        j += chunks\n        \n        if i == 0:\n            sub.to_csv('submission.csv', header=True, mode='a', index=False)\n        else: \n            sub.to_csv('submission.csv', header=False, mode='a', index=False)\n            \n        del sub\n        gc.collect()        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nsub.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}