{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport matplotlib.pyplot as plt\nimport json\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv', parse_dates=['date'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c26f4f267fe23064072a3ad26865c276267feace"},"cell_type":"code","source":"# Create new dataframes from JSON fields\n\ndevice_df = df.device.apply(json.loads).values.tolist()\ndevice_df = pd.DataFrame.from_records(device_df)\n\ngeoNetwork_df = df.geoNetwork.apply(json.loads).values.tolist()\ngeoNetwork_df = pd.DataFrame.from_records(geoNetwork_df)\n\ntotals_df = df.totals.apply(json.loads).values.tolist()\ntotals_df = pd.DataFrame.from_records(totals_df)\n\ntrafficSource_df = df.trafficSource.apply(json.loads).values.tolist()\ntrafficSource_df = pd.DataFrame.from_records(trafficSource_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fe65c7c8c56e9b2420479dc276b11302dd90365"},"cell_type":"code","source":"# Merge with the original DataFrame\ndf.drop(['device', 'geoNetwork', 'totals', 'trafficSource'], axis=1, inplace=True)\ndf = pd.concat([df, geoNetwork_df, device_df, totals_df, trafficSource_df], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b26661b1ded98645e23e5ac65c220cbebf3e30c9"},"cell_type":"code","source":"# DataFrame with new fields\ndf.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1058fe709e92c19b661783e55fb2c50d1dc57cc"},"cell_type":"code","source":"# There are several of this columns with only one value, so they can be discarded.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8419f1beb9ee8f500eb6f40930a6685d88f48415"},"cell_type":"code","source":"df.channelGrouping.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"Channel Grouping\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d312b2811a660f82cdf5eaf7d0158559cc36c3b"},"cell_type":"code","source":"df.date.dt.year.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"Year\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f740d6afc866ce0f2f91c1cf1a0a94e36783e4a6"},"cell_type":"code","source":"df.date.dt.month.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"Month\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac008fd1879a957a50151ada68c4f88b6d6c5aa7"},"cell_type":"code","source":"# Can be discarded\ndf.socialEngagementType.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"socialEngagementType\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d00bc8f049f286a5f584a794a036877878f4735"},"cell_type":"code","source":"df.visitNumber.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"visitNumber\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93d33578548a4b10866fc0538f9c6b22ff310e74"},"cell_type":"code","source":"df.city.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"city\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24d571e54bc1c5a02ae8aa5113ca851ac1b3c679"},"cell_type":"code","source":"# Can be discarded\ndf.cityId.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"cityId\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07d9d02909e0b3b5f58cb02870b55d9d1360fc1f"},"cell_type":"code","source":"df.continent.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"continent\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfd9d8105188a7102672936eb611ad0a1dae553c"},"cell_type":"code","source":"df.country.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"country\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0f43152fe243a2e08ad8c39644c89f65f974ef6"},"cell_type":"code","source":"# Can be discarded\ndf.latitude.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"latitude\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26707e13e5651f301a09a44e4a166cabc5ad3cb7"},"cell_type":"code","source":"# Can be discarded\ndf.longitude.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"longitude\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f049fa92da83d221978db7f6b352507913ba9b2"},"cell_type":"code","source":"# Maybe can merge \"not available in demo dataset\" with \"(not set)?\"\ndf.metro.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"metro\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f321ede08fc303e2ec1bf2464d34780c70d672ea"},"cell_type":"code","source":"df.networkDomain.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"networkDomain\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efb6273bf769dea0679827d08d348b07e7f8f17a"},"cell_type":"code","source":"# Can be discarded\ndf.networkLocation.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"networkLocation\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"716d5438122616a558f1a66a54141bcb74677db5"},"cell_type":"code","source":"df.region.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"region\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e495dafff37a88ac55350e67a220dfb12eb650d6"},"cell_type":"code","source":"df.subContinent.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"subContinent\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab96c69f9e269ff3032460773ef6e3d3a5281d0a"},"cell_type":"code","source":"df.browser.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"browser\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eb8f686736837ba9dcf9a779b3f960fa6bee988"},"cell_type":"code","source":"# Can be discarded\ndf.browserSize.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"browserSize\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91574d5dfcb07f6237b296e43310978f52969f03"},"cell_type":"code","source":"# Can be discarded\ndf.browserVersion.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"browserVersion\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05c4402ec31ef27890397cf2d568dffdf8d6af44"},"cell_type":"code","source":"df.deviceCategory.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"deviceCategory\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3b2197a733969da085a5f4e4fa3ffb6a4aabd58"},"cell_type":"code","source":"# Can be discarded\ndf.flashVersion.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"flashVersion\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af45bef41a2c54f3295a81e2a6a58f716c0c212e"},"cell_type":"code","source":"# Warn: Falses are set as NaN's\ndf['isMobile'] = df['isMobile'].fillna(False)\ndf.isMobile.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"isMobile\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc3361ddd57935ea92c0a7748a921636ab3f7b6e"},"cell_type":"code","source":"# All this can be discarded\nprint(df.language.value_counts())\nprint(df.mobileDeviceBranding.value_counts())\nprint(df.mobileDeviceInfo.value_counts())\nprint(df.mobileDeviceMarketingName.value_counts())\nprint(df.mobileDeviceModel.value_counts())\nprint(df.mobileInputSelector.value_counts())\nprint(df.operatingSystemVersion.value_counts())\nprint(df.screenColors.value_counts())\nprint(df.screenResolution.value_counts())\nprint(df.bounces.value_counts())\nprint(df.newVisits.value_counts())\nprint(df.visits.value_counts())\nprint(df.campaignCode.value_counts())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b4942c900bc57340774101692d0eb5170354656"},"cell_type":"code","source":"df.operatingSystem.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"operatingSystem\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38424c78bb80a3f9e0065ce1aaef8f5fe071e852"},"cell_type":"code","source":"df.hits.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"hits\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b56bc3ebcabe854179a1f2bf7d999d46f72a1258"},"cell_type":"code","source":"df.pageviews.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"pageviews\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f47b2d8e6174c348cb3d8544e193a3604b8b51ba"},"cell_type":"code","source":"df.adContent.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"adContent\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1397be2bef99d05efeba0842500bfaebb344632f"},"cell_type":"code","source":"df.campaign.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"campaign\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16b345aaafca7da28677ad4065d8ae0638b04613"},"cell_type":"code","source":"df.keyword.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"keyword\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c0f2eb3440c658c84882d4a638e3c5c0ac514bf"},"cell_type":"code","source":"df.medium.value_counts().plot(kind='bar', figsize=(15, 5), rot=70, title=\"medium\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d3bf0b80d6dd34ef43a3dcad3424120b3c43476"},"cell_type":"code","source":"df.referralPath.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"referralPath\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d78ef9eaeae379edba81b6eb65864e8a95c5eb1a"},"cell_type":"code","source":"df.source.value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"source\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e159593ce718742d1012e670c41e98810ea9eb39"},"cell_type":"code","source":"adwordsClickInfo = df['adwordsClickInfo'].apply(pd.Series)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f588f96a3961ab9d9d0f9feb2215fa1913f049b"},"cell_type":"code","source":"adwordsClickInfo.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95288668cdb95aca34ffb1cb8f1959a07cbf932d"},"cell_type":"code","source":"# Maybe some of this can be useful, not sure yet.\nprint(adwordsClickInfo['criteriaParameters'].value_counts())\nprint(adwordsClickInfo['page'].value_counts())\nprint(adwordsClickInfo['slot'].value_counts())\nprint(adwordsClickInfo['gclId'].value_counts())\nprint(adwordsClickInfo['adNetworkType'].value_counts())\nprint(adwordsClickInfo['isVideoAd'].value_counts())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a000cff5bdd402755f8243700a3da03b24efcebe"},"cell_type":"code","source":"# Mostly Nan's, can be discarded\nadwordsClickInfo['targetingCriteria'].isna().sum() / len(adwordsClickInfo)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0190e320c57af7b785a022bbaa5f1b6e497a83cc"},"cell_type":"code","source":"# Target variable\n# Warn: Contains mostly NaN's\ndf['transactionRevenue'].isna().sum() / len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57266bea60ea2d9148f08d50e1c04f568799a7b4"},"cell_type":"code","source":"# I interpret NaN as if there was no transaction.\ndf['transactionRevenue'] = df.transactionRevenue.fillna(0)\n\ndf['transactionRevenue'].value_counts()[:20].plot(kind='bar', figsize=(15, 5), rot=70, title=\"transactionRevenue\");","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}