{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d4573267-dd00-4a94-effd-5217d5d2fc14"},"outputs":[],"source":"from IPython.display import display, HTML\nimport time; start_time = time.time()\nimport numpy as np\nimport pandas as pd\nimport resource\n\nchunksize_ = 50000\nf = open('stations.csv','w')\nf.write('Id,Line,Station,Feature,Value,Response\\n')\ntrainl = sum(1 for line in open('../input/train_numeric.csv')) - 1\ntrain1 = pd.read_csv('../input/train_numeric.csv', iterator=True, low_memory=False, chunksize=chunksize_)\nfor i in range(int(trainl / train1.chunksize) + 1):\n    df = train1.get_chunk(train1.chunksize)\n    stations = []\n    for c in df.columns:\n        if c not in ['Id','Response']:\n            df_c = df[['Id',c,'Response']]\n            df_c = df_c.dropna()\n            df_c = df_c.reset_index(drop=True)\n            for j in range(len(df_c)):\n                c1, c2, c3 = c.split('_')\n                stations.append([df_c['Id'][j], c1[1:], c2[1:], c3[1:], df_c[c][j], df_c['Response'][j]])\n    for r in range(len(stations)):\n        f.write(','.join(map(str, stations[r])) + \"\\n\")\n    #print(i, round(((time.time() - start_time)/60),2), resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)\n    if len(df)<1:\n        break\n    break\nf.close()\ns = pd.read_csv('stations.csv')\ns.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a6b73b91-90f8-b099-f2ae-2b058406b984"},"outputs":[],"source":"s.describe()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ed35dd36-ab1f-612f-49bf-042f61cc3458"},"outputs":[],"source":"s[s['Response']==1].describe()"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}