{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nprint(os.listdir(\"../input\"))\nimport gc\ngc.enable()\nfeatures = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\\\n       'visitNumber', 'visitStartTime', 'device.browser',\\\n       'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',\\\n       'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country',\\\n       'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region',\\\n       'geoNetwork.subContinent', 'totals.bounces', 'totals.hits',\\\n       'totals.newVisits', 'totals.pageviews', 'totals.transactionRevenue',\\\n       'trafficSource.adContent', 'trafficSource.campaign',\\\n       'trafficSource.isTrueDirect', 'trafficSource.keyword',\\\n       'trafficSource.medium', 'trafficSource.referralPath',\\\n       'trafficSource.source']\ndef load_df(csv_path='../input/train_v2.csv'):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    ans = pd.DataFrame()\n    dfs = pd.read_csv(csv_path, sep=',',\n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                    chunksize = 100000)\n    for df in dfs:\n        df.reset_index(drop = True,inplace = True)\n        for column in JSON_COLUMNS:\n            column_as_df = json_normalize(df[column])\n            column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n            df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n        print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n        use_df = df[features]\n        del df\n        gc.collect()\n        ans = pd.concat([ans, use_df], axis = 0).reset_index(drop = True)\n        print(ans.shape)\n    return ans\n\ntrain = load_df()\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6b0e0787f24b182d312b727e4560e68a7a22e24"},"cell_type":"code","source":"test = load_df(\"../input/test_v2.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train.to_csv(\"train.csv\", index=False)\ntest.to_csv(\"test.csv\", index = False)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dff2d280b8ee115f4cf11076dc08b53a58160a6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}