{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nimport gc\nimport sys\nimport math\n\nfrom pandas.io.json import json_normalize\nfrom datetime import datetime\n\nimport os\nprint(os.listdir(\"../input\"))\n['sample_submission_v2.csv', 'test_v2.csv', 'train_v2.csv']\ngc.enable()\n\nfeatures = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\\\n       'visitNumber', 'visitStartTime', 'device.browser',\\\n       'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',\\\n       'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country',\\\n       'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region',\\\n       'geoNetwork.subContinent', 'totals.bounces', 'totals.hits',\\\n       'totals.newVisits', 'totals.pageviews', 'totals.transactionRevenue',\\\n       'trafficSource.adContent', 'trafficSource.campaign',\\\n       'trafficSource.isTrueDirect', 'trafficSource.keyword',\\\n       'trafficSource.medium', 'trafficSource.referralPath',\\\n       'trafficSource.source', 'customDimensions']\n\ndef load_df(csv_path):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    ans = pd.DataFrame()\n    dfs = pd.read_csv(csv_path, sep=',',\n            converters={column: json.loads for column in JSON_COLUMNS}, \n            dtype={'fullVisitorId': 'str'}, # Important!!\n            chunksize=100000,nrows=50000)\n    for df in dfs:\n        df.reset_index(drop=True, inplace=True)\n        for column in JSON_COLUMNS:\n            column_as_df = json_normalize(df[column])\n            column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n            df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n        #print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n        use_df = df[features]\n        del df\n        gc.collect()\n        ans = pd.concat([ans, use_df], axis=0).reset_index(drop=True)\n        #print(ans.shape)\n    return ans\n\ntrain = load_df('../input/train_v2.csv')\ntest = load_df('../input/test_v2.csv')\ntrain.head()\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":false},"cell_type":"code","source":"for c in train.columns.values:\n    if c not in test.columns.values: print(c)\n\n        \ntrain['totals.transactionRevenue'].fillna(0, inplace=True)\ntrain['totals.transactionRevenue'] = np.log1p(train['totals.transactionRevenue'].astype(float))\nprint(train['totals.transactionRevenue'].describe()) \n\ntest['totals.transactionRevenue'] = np.nan\nall_data = train.append(test, sort=False).reset_index(drop=True)\nprint(all_data.info())\nall_data.head(20)\n\n\n#null_cnt = train.isnull().sum().sort_values()\n#print(null_cnt[null_cnt > 0])\n   \n  \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"68b53de6219a72b6f7583b401e97ce4637a570c9"},"cell_type":"code","source":"null_cnt = train.isnull().sum().sort_values()\n# print(null_cnt[null_cnt > 0])\n# fillna object feature\nfor col in ['trafficSource.keyword',\n            'trafficSource.referralPath',\n            'trafficSource.adContent']:\n    all_data[col].fillna('unknown', inplace=True)\n\n# fillna numeric feature\nall_data['totals.pageviews'].fillna(1, inplace=True)\nall_data['totals.newVisits'].fillna(0, inplace=True)\nall_data['totals.bounces'].fillna(0, inplace=True)\nall_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int)\nall_data['totals.newVisits'] = all_data['totals.newVisits'].astype(int)\nall_data['totals.bounces'] = all_data['totals.bounces'].astype(int)\n\n# fillna boolean feature\nall_data['trafficSource.isTrueDirect'].fillna(False, inplace=True)\n# drop constant column\nconstant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]\n#for c in constant_column:\n#    print(c + ':', train[c].unique())\n\nprint('drop columns:', constant_column)\nall_data.drop(constant_column, axis=1, inplace=True)\n\n# pickup any visitor\nall_data[all_data['fullVisitorId'] == '7813149961404844386'].sort_values(by='visitNumber')[\n    ['date','visitId','visitNumber','totals.hits','totals.pageviews']]\ntrain_rev = train[train['totals.transactionRevenue'] > 0].copy()\nprint(len(train_rev))\n# train_rev.head()\n\n\nvals = train_rev['customDimensions'].value_counts()\nvals.head()\n\n\n\n\n\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7546e382aac011ce2e847eb46f10ac142e7ab7c"},"cell_type":"code","source":"subA=all_data.loc[all_ data['customDimensions'].isin(vals.index.values),'customDimensions']\nsubA.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}