{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#credits to original: https://www.kaggle.com/scirpus/to-overcome-the-terrible-data\n#Please thank him above ^ as this was much simpler thanks to Panda's json_normalize\n#key changes to his version: removed joins, inplace drop, and made it more verbose\nimport pandas as pd\nimport json,os,gc,time\ndef currenttime(): return time.time()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def Load(file, max_rows=None, DROP=[], with_prefix=True):\n    from pandas.io.json import json_normalize\n    t0 = currenttime()\n    columns = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    df = pd.read_csv(file, \n                     converters={column: json.loads for column in columns}, \n                     dtype={'fullVisitorId': 'str'}, \n                     nrows=max_rows)\n    print(df.shape)\n    lst = [df]\n    for c in columns:\n        tmp = json_normalize(df[c])\n        if with_prefix: tmp.columns = [f\"{c}.{subcolumn}\" for subcolumn in tmp.columns]\n        lst.append(tmp)     \n        print('%20s  +%s'%(c,tmp.shape))\n    df = pd.concat(lst, axis=1)\n    print('->',df.shape)\n    DROP = [c for c in columns+DROP if c in df.columns]\n    df.drop(DROP, axis=1, inplace=True)\n    print('DROP %s ->'%len(DROP),df.shape)\n    print('Loaded %s in %s sec'%(file,currenttime()-t0))\n    return df\n\n#feel free to check for your self (set it to [])\nuseless=['device.browserSize', 'device.browserVersion', 'device.flashVersion', 'device.language', 'device.mobileDeviceBranding', 'device.mobileDeviceInfo', 'device.mobileDeviceMarketingName', 'device.mobileDeviceModel', 'device.mobileInputSelector', 'device.operatingSystemVersion', 'device.screenColors', 'device.screenResolution', 'geoNetwork.cityId', 'geoNetwork.latitude', 'geoNetwork.longitude', 'geoNetwork.networkLocation', 'socialEngagementType', 'totals.visits', 'trafficSource.adwordsClickInfo.criteriaParameters','trafficSource.campaignCode']\n\n#full data size will be (1708337, 36)\nLoad('../input/train.csv',max_rows=5000,DROP=useless).sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57ed0244cb8ab2b4be9e359784e08b8bf455efd7"},"cell_type":"code","source":"def GetData():\n    R = Load('../input/train.csv',DROP=useless)\n    E = Load('../input/test.csv',DROP=useless)\n    R['is_train'] = 1\n    E['is_train'] = 0\n    print('Train/test dimensions:',R.shape,E.shape)\n    return R.append(E, sort=True)\n\ndata = GetData()\ngc.enable()\ngc.collect()\nprint()\nS = pd.read_csv('../input/sample_submission.csv')\nprint()\nprint(sorted(data.columns),'\\n')\nprint(sorted(S.columns),'\\n')\ndata.shape,S.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"712df2b63386dad0ff8bd93af7526d710260d046"},"cell_type":"code","source":"def H5DF_load(fileName):\n    t0 = currenttime()\n    if not os.path.exists(fileName):\n        raise ValueError(\"Error: file does not exits: {}\".format(fileName))\n    try:\n        print(\"Reading %20s\" % fileName, end='', flush=True)\n        data = pd.read_hdf(fileName,'table')\n        print(\" %15s  %.3f seconds\" % (data.shape, currenttime()-t0))\n        return data\n    except:\n        raise ValueError(\"file = {}\".format(fileName))\n\n#Expects a dataframe\ndef H5DF_save(data,fileName, append=False):\n    t0 = currenttime()\n    print(\"Saving %20s\" % fileName, end='', flush=True)\n    data.to_hdf(fileName,'table',append=append)\n    print(\" %15s  %.3f seconds\" % (data.shape, currenttime()-t0))\n\n#if you plan to reload this often    \nH5DF_save(data,'data.h5df')\ndata = H5DF_load('data.h5df')\n\n# The warning given is because of irregular data types. It results to a more complicated serialization process. To resolve, you need to set the data type appropriately. It will be much faster!","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3577842ff67ca7742982f49a75b4ab7ab939e80"},"cell_type":"code","source":"data.sample(100)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}