{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0820d47b7cb1039358c05ae253258d8c71bf1027"},"cell_type":"code","source":"#import the data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train.head()\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02bebea3f07714d631b9a13e77879e35db894268"},"cell_type":"code","source":"# dataset discovery\ntrain.info\ntest.info\ntrain.columns\ntrain.dtypes\ntest.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4449643867c84255e672d014430eebb08a1e2c0d"},"cell_type":"code","source":"# create new dataframes\ntrain_df = train\ntest_df = test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14c1562ac05b3f9b253b4bde738d372ac7a31400"},"cell_type":"code","source":"# parse the json objects in the train dataframe\nimport json\nfrom pandas.io.json import json_normalize\n#train_df = train_df.join(json_normalize(train_df[\"totals\"].tolist()).add_prefix(\"totals.\")).drop([\"totals\"], axis=1)\ntrain_df = train_df.join(json_normalize(train_df['trafficSource'].map(json.loads).tolist()).add_prefix('trafficSource.'))\\\n    .drop(['trafficSource'], axis=1)\ntrain_df = train_df.join(json_normalize(train_df['device'].map(json.loads).tolist()).add_prefix('device.'))\\\n    .drop(['device'], axis=1)\ntrain_df = train_df.join(json_normalize(train_df['geoNetwork'].map(json.loads).tolist()).add_prefix('geoNetwork.'))\\\n    .drop(['geoNetwork'], axis=1)\ntrain_df = train_df.join(json_normalize(train_df['totals'].map(json.loads).tolist()).add_prefix('totals.'))\\\n    .drop(['totals'], axis=1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab23423020e6a966f50299d6fcb2629a91a16c2b"},"cell_type":"code","source":"# parse the json objects in the test dataframe\n\ntest_df = test_df.join(json_normalize(test_df['trafficSource'].map(json.loads).tolist()).add_prefix('trafficSource.'))\\\n    .drop(['trafficSource'], axis=1)\ntest_df = test_df.join(json_normalize(test_df['device'].map(json.loads).tolist()).add_prefix('device.'))\\\n    .drop(['device'], axis=1)\ntest_df = test_df.join(json_normalize(test_df['geoNetwork'].map(json.loads).tolist()).add_prefix('geoNetwork.'))\\\n    .drop(['geoNetwork'], axis=1)\ntest_df = test_df.join(json_normalize(test_df['totals'].map(json.loads).tolist()).add_prefix('totals.'))\\\n    .drop(['totals'], axis=1)\ntest_df.head()\n#test_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"763bdad2b04010feea3511b0b3eaf24eed615fb1"},"cell_type":"code","source":"#train_df['totals.transactionRevenue'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4171d8c0e464f24485990f3814d35364d2d7934"},"cell_type":"code","source":"#transform the date colummns - conver integer to date \ntrain_df['visitStartTime'] = pd.to_datetime(train_df['visitStartTime'],unit='s')\ntrain_df['date'] = pd.to_datetime(train_df['date'].astype(str),format='%Y%m%d')\n\ntest_df['visitStartTime'] = pd.to_datetime(test_df['visitStartTime'],unit='s')\ntest_df['date'] = pd.to_datetime(test_df['date'].astype(str),format='%Y%m%d')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76e0876a6143afadd8c91844f4acfeb85f4009f8"},"cell_type":"code","source":"# import required libraries\nimport xgboost as xgb\nfrom sklearn.preprocessing import StandardScaler,Imputer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline,FeatureUnion\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn_pandas import DataFrameMapper, CategoricalImputer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"950a1b2383f0bba43e394f83655869f936c24969"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c805f8ff7ef60d5c66d88afb5da589e6f3ff12e"},"cell_type":"code","source":"# find the index of columns of interest\ncols = [\"channelGrouping\",\"socialEngagementType\",\"visitNumber\",\"trafficSource.adContent\",\"trafficSource.adwordsClickInfo.adNetworkType\",\"trafficSource.adwordsClickInfo.criteriaParameters\",\"trafficSource.adwordsClickInfo.gclId\"\n        ,\"trafficSource.adwordsClickInfo.isVideoAd\",\"trafficSource.adwordsClickInfo.page\",\"trafficSource.adwordsClickInfo.slot\",\"trafficSource.campaign\",\"trafficSource.isTrueDirect\",\"trafficSource.keyword\",\"trafficSource.medium\"\n       ,\"trafficSource.referralPath\",\"trafficSource.source\",\"device.deviceCategory\",\"geoNetwork.city\",\"geoNetwork.continent\",\"geoNetwork.country\",\"geoNetwork.metro\",\"geoNetwork.networkDomain\",\"geoNetwork.region\",\"geoNetwork.subContinent\",\"totals.bounces\",\"totals.hits\",\"totals.newVisits\",\"totals.pageviews\",\"totals.visits\",\"totals.transactionRevenue\"]\n#train_df.head()\n[train_df.columns.get_loc(c) for c in train_df.columns if c in cols]\n[test_df.columns.get_loc(c) for c in test_df.columns if c in cols]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfc145c94f96ec1540d0b3f759f7bb6436c9a29e"},"cell_type":"code","source":"# split the train data into a matrix of predictor variables and a vector of target variable\nX_train, y_train = train_df.iloc[:,[0,4,6,8,9,10,12,13,14,15,17,18,19,20,21,22,25,38,40,41,46,47,48,49,50,51,52,54]],train_df.iloc[:,53]\nX_test = test_df.iloc[:,[0,4,6,8,9,10,12,13,14,15,16,17,18,19,20,21,24,37,39,40,45,46,47,48,49,50,51,52]]\nX_train.head()\ny_train.head()\nX_test.head()\nX_train.info()\nX_test.info()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b4d5346e151c5c25f68f52c99666c8ee6f9c120"},"cell_type":"markdown","source":"## Clustering for dataset exploration"},{"metadata":{"_uuid":"8c2dc428f3d94c79540b170ac44c76194b08f288"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}