{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport ast\nimport json\nimport matplotlib.mlab as mlab\nimport matplotlib.pyplot as plt\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"test = pd.read_csv('../input/test.csv', dtype=object)\ntrain = pd.read_csv('../input/train.csv', dtype=object)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b084a999dc74cfabd29d14603ffed96853f5f21d"},"cell_type":"code","source":"test.head(1)\nids = test['fullVisitorId'].values\nidss = ids.astype('str')\nlen(ids), type(ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c299c0e526486efd528f6c915a7584403db3d150"},"cell_type":"code","source":"uniq_ids = np.unique(idss)\nnum_ids = len(uniq_ids) #Unique IDs in the test.csv\nlen(uniq_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7d7e2cdb3076828c77b7fd26c1a7f54bcb76e95"},"cell_type":"code","source":"#Create a dummy but valid submission\n#d = {'fullVisitorId':uniq_ids,'PredictedLogRevenue':np.zeros(num_ids)}\n#df = pd.DataFrame(d)\n#df.to_csv('./submission_all_zeros.csv', index=False)\n#print(os.listdir(\".\"))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_uuid":"d43db6e7cea3d381f30ac58b5bcd8163c3b464a2"},"cell_type":"markdown","source":"**Check Data for the First Time**"},{"metadata":{"trusted":true,"_uuid":"8d842dc3e094e0d10d7a75edc2eb09b5a2295121"},"cell_type":"code","source":"idx = 5\ns = train.iloc[idx]\nprint(train.columns)\nfor c in train:\n    print(train[c][:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37dfe190175c0ce01a5fccf4f80186c35fa87e74"},"cell_type":"code","source":"totals = train['device'][1]\ntotals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bb4d49646f07e5d1a6ed3676af683fef6f7934e"},"cell_type":"code","source":"#TEST\ns = set()\ns.update([1,2,2])\nprint(s, len(s))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"330a04dcf66e359be7bb79dbe4acc3cb313ea55e"},"cell_type":"code","source":"print(type(totals))\ns = json.loads(totals)\nprint(type(s), s)\n#ast.literal_eval(totals)\n#df.apply (dict_to_cols,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba7601fde4839c1a758d4a80e73bdf952c056ed6"},"cell_type":"code","source":"def expand_tree_column(col_data):\n    keys = set()\n    ds = []\n    for cd in col_data:\n        d = json.loads(cd)\n        ds.append(d)\n        keys.update(list(d.keys()))\n    #print('unique keys: %s', keys)\n    res = {k:[] for k in keys}\n    for d in ds:\n        for k in res.keys():\n            res[k].append(d.get(k, None))\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3d17ba4353c095dfe4b92da72ac62be2a4abecb"},"cell_type":"code","source":"def expand_col(df, col_name):\n    col_data = np.array(df[col_name])\n    res = expand_tree_column(col_data)\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb9b5efc4dd334e82bfc39e48006d026b81596ad"},"cell_type":"code","source":"#Expand dictionary-like columns in train.csv\ndict_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']\ntraine = train.copy()\nfor dict_col in dict_cols:\n    print('processing column: {}'.format(dict_col))\n    res = expand_col(train, dict_col)\n    traine = traine.drop(dict_col, axis=1)\n    for k, vec in res.items():\n        traine[k] = vec\ntraine.to_csv('./train_expanded.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3222e247f5716092473bfe3437d9ec41b23ade5"},"cell_type":"code","source":"#Expand dictionary-like columns in test.csv\nteste = test.copy()\nfor dict_col in dict_cols:\n    print('processing column: {}'.format(dict_col))\n    res = expand_col(test, dict_col)\n    teste = teste.drop(dict_col, axis=1)\n    for k, vec in res.items():\n        teste[k] = vec\nteste.to_csv('./test_expanded.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7c2c31794ce2e7413bbccdcc7bee8a22e1ca8ea","_kg_hide-output":false},"cell_type":"code","source":"#TEST\ndd = {'a':1, 'b':2, 'c':3}\nfor d,v in dd.items():\n    print(d,v)\n\nprint(len(traine), len(train), len(teste), len(test))\nprint(os.listdir(\"./\"))\ntraine.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"628ba2572b2e76d89444e798e27479912e034589"},"cell_type":"code","source":"teste = pd.read_csv('./test_expanded.csv', dtype=object)\ntraine = pd.read_csv('./train_expanded.csv', dtype=object)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"118a37bceffc69a4ce6502fd522702efd52ccbc0"},"cell_type":"code","source":"trs = np.array(traine['transactionRevenue'])\ntrs = [0 if np.isnan(float(tr)) else float(tr) for tr in trs]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"129ec68a28ece276d23a5879ec553c13e28bc849"},"cell_type":"code","source":"trsn = [np.log10(tr) for tr in trs if tr > 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25496be161c024280682c1b82ef7714cdc69e8c4"},"cell_type":"code","source":"trsnn = [tr for tr in trsn if tr >=7 and tr < 9]\nlen(trsnn), len(trsn)\nlen(trsnn)/len(trsn)/2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74f74f52b8cb24428710006a760635c4fe40c96b"},"cell_type":"code","source":"a,b,c= plt.hist(trsn, density=1, bins=[4,5,6,7,9,10,11])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeddab5d524a5b2e35d73bd90562e7cd9887758a"},"cell_type":"code","source":"a, b, c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef71fad64afb2b277447a45c5bffaedc88c51f07"},"cell_type":"code","source":"c[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40b3a2e03b0e1d3c336304dcf6d6399a3e1cf4ab"},"cell_type":"code","source":"10**8,10**7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cfa92956ef0fc5355e642918a2bd8a9ba50ff7b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}