{"cells":[{"metadata":{"_uuid":"006380ad6101e0a7a92e54f806d5c118f5a2e372"},"cell_type":"markdown","source":"# 1. Quick start: read csv and flatten json fields\n\nHi! This notebook just loads the data and flattens the json fields. \nI have put the code in one function so you can copy it easily.\n\nAlso, I dumped the processed dataframes to disk so they are easily importable from within another kernel."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"%%time\ndf_small = load_df(nrows=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc88f0dda2ce6a73df522428ab6e3195542d7603","collapsed":true},"cell_type":"code","source":"%%time\ndf_train = load_df()\ndf_test = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31ebbc9d59d9e945a58f8918c0cfe0fccc30585c","collapsed":true},"cell_type":"code","source":"%%time\ndf_train.to_csv(\"train-flattened.csv\", index=False)\ndf_test.to_csv(\"test-flattened.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dfcc21666893751cbd3139d3af83ebab7d527ee","collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}