{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nimport gc\n\nimport json\nfrom pandas.io.json import json_normalize\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom tqdm import trange,tqdm\nfrom collections import Counter\n\npd.options.display.max_rows = 20\npd.options.display.max_columns = 999\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_pickle('../input/gs-get-src-data/train_df', compression='gzip')\nprint(train_df.shape)\ntest_df = pd.read_pickle('../input/gs-get-src-data/test_df', compression='gzip')\nprint(test_df.shape)\ntrain_df_u = pd.read_pickle('../input/gs-get-src-data/train_df_u', compression='gzip')\nprint(train_df_u.shape)\ntest_df_u = pd.read_pickle('../input/gs-get-src-data/test_df_u', compression='gzip')\nprint(test_df_u.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7c1bdea01b08acd1daef436257e52a3afe95ca6"},"cell_type":"code","source":"id_col = 'fullVisitorId'\ncnt = train_df[id_col].value_counts()\nids = cnt.loc[cnt>1].index.values\nprint(ids.shape)\ndf = train_df.loc[train_df[id_col].isin(ids)]\nprint(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f18ad8c4a7f267440bbd93fa972fe50f34dfeb92"},"cell_type":"code","source":"gdf = df.groupby(id_col)['target'].sum()\ntids = gdf.loc[gdf>0].index.values\nprint(tids.shape)\ntdf = df.loc[df[id_col].isin(tids)]\nprint(tdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2bd8ab73c0453f890f22ae09636c1e83a652306"},"cell_type":"code","source":"gdf = tdf.loc[:,[id_col,'target']]\ngdf['target'] = gdf['target']>0\ngdf = gdf.groupby(id_col)['target'].sum()\ntids2 = gdf.loc[gdf>1].index.values\nprint(tids2.shape)\ntdf2 = tdf.loc[tdf[id_col].isin(tids2)]\nprint(tdf2.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0ff7df3cfa2e2d66ba9fe14b2d954b72d134663","scrolled":false},"cell_type":"code","source":"cols = list(tdf2.columns)\ncols = [col for col in cols if col != id_col] + [id_col]\ntdf2[cols].loc[tdf2[id_col]==tids2[3]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e45b394072b1fd68d3cd98b0d1edb11e163cfb8"},"cell_type":"code","source":"tdf2.groupby([id_col,'visit_year'])['target'].sum().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7212446426c065a3fd52feedf95b9d2c2d1cf45"},"cell_type":"code","source":"cids = np.intersect1d(train_df.fullVisitorId, test_df.fullVisitorId)\nprint(len(cids))\nprint(train_df.loc[train_df.fullVisitorId.isin(cids)].shape)\nprint(test_df.loc[test_df.fullVisitorId.isin(cids)].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdca3710be4067e512e9d9dc17718e3a779a9bac"},"cell_type":"code","source":"cdf = tdf.loc[tdf[id_col].isin(cids)].groupby(id_col)['target'].sum()\nprint(cdf.shape)\ncdf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67d51c42d1eb5ae28c0077a536e45849acbd16cc"},"cell_type":"code","source":"sub = pd.read_csv('../input/gs-lgb-uv-ensemble-1-1/lgb_uv_ensemble_1_1.csv')\nprint(sub.shape)\nsub1 = sub.join(cdf, on=id_col)\nind = sub1.target.notnull()\nsub1.loc[ind, 'PredictedLogRevenue'] = (sub1.loc[ind, 'PredictedLogRevenue']+np.log1p(sub1.loc[ind, 'target']))/2\nsubmission = sub1.drop('target', axis=1)\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fcfcf73e49469c633b97ab5463371767f6ea1c2"},"cell_type":"code","source":"submission.to_csv('test_all.csv', index=False, float_format='%.5f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}