{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0806a9660c15452a52b5c3aee5efcfc5f986f4b8"},"cell_type":"markdown","source":"### 0. Intro\n\nSorry for commenting alomost all code, but kaggle servers can't calculate all at commit.\n\nIt's simple for report in university course"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"### 1. Convert data to json format"},{"metadata":{"_uuid":"daf854db0a47eb86f8933b5d264088d3a1ba24f8"},"cell_type":"markdown","source":"**Use this kernel https://www.kaggle.com/ravann/1-step-by-step-format-data-to-columnar-format/notebook\n**"},{"metadata":{"_uuid":"d8ec82fe00d73bda8acd7aef030eee7f22f858ba"},"cell_type":"markdown","source":"### 2. Data exploring and filtering. Part 1"},{"metadata":{"trusted":true,"_uuid":"0e2b6e3ea45f39fc2a9fdedf2bb6b6801d45264a"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport datetime as dt\nimport seaborn as sns\nimport pickle\nfrom collections import defaultdict\nfrom mpl_toolkits.mplot3d import Axes3D\nimport gc\n\nsns.set()\ngc.enable()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb75eb5b9c040c18f19ed8e5c5ef623a23a53959"},"cell_type":"markdown","source":"**Load dataset**"},{"metadata":{"trusted":true,"_uuid":"f70235edbfc8bb5be25ef7a42048c4a62f5fdac8"},"cell_type":"code","source":"#df_train = pd.read_csv(\"../input/prepaired-data-of-customer-revenue-prediction/train_flat.csv\", converters={'fullVisitorId': str})\n#df_test = pd.read_csv(\"../input/prepaired-data-of-customer-revenue-prediction/test_flat.csv\", converters={'fullVisitorId': str})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34e372451c34336d809b87c702c3617bb6d444e2"},"cell_type":"markdown","source":"**View dataset**"},{"metadata":{"trusted":true,"_uuid":"4b49dd083fd18d84345945d2620ad451c42b2349"},"cell_type":"code","source":"#df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2c4de29fba234ddf6cba98aaf084d84f6c382f6"},"cell_type":"code","source":"#df_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af4e67389fe1825fdc378d563cd5d5c46c4f302b"},"cell_type":"markdown","source":"**Fill missing values in totals_transactionRevenues by zeros**"},{"metadata":{"trusted":true,"_uuid":"cab13b2356f3455186b10ade5eb4eff01c1cb984"},"cell_type":"code","source":"#df_train[\"totals_transactionRevenue\"] = df_train[\"totals_transactionRevenue\"].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edfa3e8b397572b1a630add79551f372658b43ff"},"cell_type":"markdown","source":"**Check whether columns in train and in test are equal**"},{"metadata":{"trusted":true,"_uuid":"f9b91bf8844758c2da4abed592457c4bd0c6d386"},"cell_type":"code","source":"#train_col = np.array(df_train.columns)\n#test_col = np.array(df_test.columns)\n#print(set(train_col) - set(test_col))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e995d9bca7737cfd5274d59085f2ea0318feb653"},"cell_type":"markdown","source":"**Remove strange column trafficSource_campaignCode from test**"},{"metadata":{"trusted":true,"_uuid":"af7a17b47731fd6814f9c09e59dd5c00cf60fef3"},"cell_type":"code","source":"#df_train = df_train.drop(columns=[\"trafficSource_campaignCode\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6cbb75d696d72c35bbf234e03f14f7f6a33e1c58"},"cell_type":"markdown","source":"**View at \"socialEngagementType\"**"},{"metadata":{"trusted":true,"_uuid":"098a222361cd5c48f8e8a4646f6239df8e08d007"},"cell_type":"code","source":"#print(np.unique(df_train[\"socialEngagementType\"], return_counts=True))\n#print(np.unique(df_test[\"socialEngagementType\"], return_counts=True))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86077d3797b1ecf64d6944ae329a038c56d19518"},"cell_type":"markdown","source":"**Remove this \"usefull\" feature from train and test**"},{"metadata":{"trusted":true,"_uuid":"3658efabb807067abcc0f1235379cc3c4f53e322"},"cell_type":"code","source":"#df_train = df_train.drop(columns=[\"socialEngagementType\"])\n#df_test = df_test.drop(columns=[\"socialEngagementType\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9cc8c0ec2ea6d02fb9eba3ccd6cfd5bde84c8f6"},"cell_type":"markdown","source":"**Explore feature values in train**"},{"metadata":{"trusted":true,"_uuid":"55d33daf52c17a016145e75c4efaf58d53be7929"},"cell_type":"code","source":"#df_train_eq_nan = df_train.fillna(-1543)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7e2180a316385194dac11f89e1c487c31370d9d"},"cell_type":"code","source":"\"\"\"for col_name in np.array(df_train_eq_nan.columns):\n    print(col_name)\n    try:\n        print(np.unique(np.array(df_train_eq_nan[col_name]), return_counts=True))\n    except Exception:\n        print(np.unique(np.array(df_train_eq_nan[col_name]).astype(str), return_counts=True))\n    print(\"-\" * 43)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34cecdc5850a42ba056e9159907c86f7b7bd1b08"},"cell_type":"markdown","source":"**After watching this values I aggreagated the following groups of featues**"},{"metadata":{"trusted":true,"_uuid":"b8758ae98ec85f12edeced1df9132a3429ac78ae"},"cell_type":"code","source":"# numerical = [\"visitNumber\", \"visitStartTime\", \"totals_bounces\", \"totals_hits\", \"totals_newVisits\", \"totals_pageviews\", \n#             \"trafficSource_adwordsClickInfo.isVideoAd\", \"trafficSource_adwordsClickInfo.page\", \n#             \"trafficSource_isTrueDirect\", \"device_isMobile\"]\n\n# categorial = [\"channelGrouping\", \"date\", \"device_browser\", \"device_deviceCategory\", \"device_operatingSystem\",\n#                \"geoNetwork_city\", \"geoNetwork_continent\",\"geoNetwork_metro\", \"geoNetwork_country\",\n#               \"geoNetwork_networkDomain\", \"geoNetwork_region\",\n#                \"geoNetwork_subContinent\", \"trafficSource_adContent\", \"trafficSource_adwordsClickInfo.adNetworkType\",\n#                \"trafficSource_adwordsClickInfo.slot\", \"trafficSource_campaign\", \"trafficSource_keyword\", \"trafficSource_medium\",\n#                \"trafficSource_source\"]\n\n# saved_trash = [\"fullVisitorId\"]\n\n# trash_trash = [\"totals_visits\", \"device_browserSize\", \"device_browserVersion\",\n#                 \"device_flashVersion\", \"device_language\", \"device_mobileDeviceBranding\", \"device_mobileDeviceInfo\",\n#                 \"device_mobileDeviceMarketingName\", \"device_mobileDeviceModel\", \"device_mobileInputSelector\",\n#                 \"device_screenColors\", \"device_screenResolution\", \"geoNetwork_cityId\", \"geoNetwork_latitude\",\n#                 \"geoNetwork_longitude\", \"geoNetwork_networkLocation\", \"trafficSource_adwordsClickInfo.criteriaParameters\",\n#                 \"device_operatingSystemVersion\"]\n                \n# wanted_to_trash = [\"sessionId\", \"visitId\", \"trafficSource_adwordsClickInfo.gclId\", \"trafficSource_referralPath\"]\n\n# in_future_wanted_to_trash = [\"visitStartTime\", \"date\", \"device_isMobile\", \"geoNetwork_city\", \"geoNetwork_metro\",\n#                 \"geoNetwork_networkDomain\", \"trafficSource_adContent\", \"trafficSource_keyword\"]\n\n# to_heal = [\"totals_bounces\", \"totals_newVisits\", \"totals_pageviews\", \"trafficSource_adwordsClickInfo.adNetworkType\",\n#             \"device_browser\", \"trafficSource_adwordsClickInfo.isVideoAd\", \"trafficSource_adwordsClickInfo.slot\", \n#             \"trafficSource_adwordsClickInfo.page\", \"trafficSource_campaign\", \"trafficSource_isTrueDirect\",\n#             \"trafficSource_medium\", \"trafficSource_source\", \"device_operatingSystem\", \"geoNetwork_city\",\n#             \"geoNetwork_region\",\n#             \"geoNetwork_continent\", \"geoNetwork_country\", \"geoNetwork_metro\", \"geoNetwork_networkDomain\",\n#             \"geoNetwork_subContinent\", \"trafficSource_adContent\", \"trafficSource_keyword\"]\n\n# answer_feature = [\"totals_transactionRevenue\"]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44149ecec96a8b7f79bf48c213e9c7534357a58b"},"cell_type":"markdown","source":"* numerical -- _numerical features_\n* categorial -- _categorial features_\n* saved_trash -- _actually fullVisitorId_\n* trash_trash -- _unusefull features. Has a lot of nans or the same value for all objects_\n* wanted_to_trash -- _explore this featuers now but wants to delete them (e.g. all values are different for all objects)_\n* in_future_wanted_to_trash -- <i>features that can be dublicated by others (e.g. divice_isMobile may be duplicated by One-Hot-encoding of device_deviceCategory) or there is not any dependence between them and answer. But now don't remove them</i>\n* to_heal -- _fill nans by other value_\n* answer_feature -- _feature with answer_"},{"metadata":{"_uuid":"d5f32c6d9bb132654caf501ef07a305f673f7455"},"cell_type":"markdown","source":"**Remove features from trash_trash**"},{"metadata":{"trusted":true,"_uuid":"b91704df427cf2092cb1c9663191c5079d01354f"},"cell_type":"code","source":"# df_train = df_train.drop(columns=trash_trash)\n# df_test = df_test.drop(columns=trash_trash)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50037ed2a10f80d316591039ebd49a592609e6ac"},"cell_type":"markdown","source":"**Explore features from wanted_to_trash**"},{"metadata":{"trusted":true,"_uuid":"ef6be980f035372a5d3d8c4f4d1bad0fcbbeaca0"},"cell_type":"code","source":"# df_train_eq_nan = df_train.fillna(-1543)\n# for col_name in wanted_to_trash:\n#     print(col_name)\n#     un = None\n#     try:\n#         un = np.unique(np.array(df_train_eq_nan[col_name]), return_counts=True)\n#     except Exception:\n#         un = np.unique(np.array(df_train_eq_nan[col_name]).astype(str), return_counts=True)\n#     print(un)\n#     print(\"DIFFERENT COUNT: \", un[0].shape[0])\n#     print(\"-\" * 43)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1bd5ea75767f13c390df6423b476a517c3d78617"},"cell_type":"markdown","source":"**Seems that SessionId and visitId should be removed. Let see at them:**"},{"metadata":{"_uuid":"4e25e7155b19d672d56b7bbe536f455c37ff3081"},"cell_type":"markdown","source":"**Explore the objects with same SessionId**"},{"metadata":{"trusted":true,"_uuid":"ed2ad64eb280101c372663319c5c0aebfc6f98c6"},"cell_type":"code","source":"# un = np.unique(df_train[\"sessionId\"], return_counts=True)\n# idx = np.where(un[1] != 1)[0]\n# repeat_session_id = un[0][idx]\n\n# repeated_df = df_train[df_train[\"sessionId\"].isin(repeat_session_id)].sort_values([\"sessionId\"]).iloc[:, :20]\n# repeated_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e92fc3b5efb699ca701d3f435d52be6b48655790"},"cell_type":"markdown","source":"**Let's see on features date and SessionId. Obviously if SessionIds are the same then fullVisitodIds are also the same. It seems that difference between dates for equal SessinId is 1 day. Perhaps the session was started at one day and ended at the next day, but there are two records about this one session.  So we can not only delete feature SessionId but also delete duplicated objects with the same SessionId (delete more earlier objects because there is much more information about later objects).**"},{"metadata":{"_uuid":"ebc6a0c37ebeff517510711a80aeee913000216a"},"cell_type":"markdown","source":"**Check if is that true that there are only two record for such \"bad\" equal SessionId:**"},{"metadata":{"trusted":true,"_uuid":"a63f4a626f27d20965296b200404b1b13954ea78"},"cell_type":"code","source":"# np.all(\n#     df_train[df_train[\"sessionId\"].isin(repeat_session_id)][[\"date\", \"sessionId\"]].groupby(\"sessionId\").count() == 2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"773c71762f46d93b98e1fa2504e315e888ff4a5e"},"cell_type":"markdown","source":"**So delete objects and SessionId:**"},{"metadata":{"trusted":true,"_uuid":"47b76ecf8f97cdfd4e0d0c904fed9a214a930e2d"},"cell_type":"code","source":"# def parse_datetime(strdate):\n#     year, month, day = list(map(lambda x: int(x), [strdate[:4], strdate[4:6], strdate[6:8]]))\n#     return dt.datetime(year=year, month=month, day=day)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed66bd4f366e0509443d5dcf285e13701fe947c7"},"cell_type":"code","source":"# bad_idxs = []\n# for session_id in repeat_session_id:\n#     part_df = repeated_df[repeated_df[\"sessionId\"] == session_id]\n#     dt1, dt2 = parse_datetime(str(part_df[\"date\"].iloc[0])), parse_datetime(str(part_df[\"date\"].iloc[1]))\n#     if dt1 < dt2:\n#         bad_idxs.append(part_df.iloc[0].name)\n#     else:\n#         bad_idxs.append(part_df.iloc[1].name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e60bd8ecd09af77748faa6af6fadc57a3b6dd74a"},"cell_type":"code","source":"# len(bad_idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0594b1283310d540a4827b99d9b6437068241bb"},"cell_type":"code","source":"# df_train = df_train.drop(bad_idxs)\n# df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a20bad328a554ecf706f47a1c4b7bb141be0f68b"},"cell_type":"code","source":"# df_train.index = np.arange(df_train.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0390cc4d919f4369edbb4ac7b55df8e6dbd79878"},"cell_type":"code","source":"# df_train_eq_nan = df_train.fillna(-1543)\n# for col_name in wanted_to_trash:\n#     print(col_name)\n#     un = None\n#     try:\n#         un = np.unique(np.array(df_train_eq_nan[col_name]), return_counts=True)\n#     except Exception:\n#         un = np.unique(np.array(df_train_eq_nan[col_name]).astype(str), return_counts=True)\n#     print(un)\n#     print(\"DIFFERENT COUNT: \", un[0].shape[0])\n#     print(\"-\" * 43)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e26a9f7cf45348293c0a18eb9ef322c5274f3b2"},"cell_type":"code","source":"# df_train = df_train.drop(columns=[\"sessionId\"])\n# df_test = df_test.drop(columns=[\"sessionId\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c150af245885349f97dc4ca603e9c5aa6cf448e"},"cell_type":"markdown","source":"**Let's see on VisitId:**"},{"metadata":{"trusted":true,"_uuid":"65de2ee4de63c1dabdcf059ea15a296540e0987c"},"cell_type":"code","source":"# un = np.unique(df_train[\"visitId\"], return_counts=True)\n# idx = np.where(un[1] > 1)[0]\n# repeat_session_id = un[0][idx]\n# repeated_df = df_train[df_train[\"visitId\"].isin(repeat_session_id)].sort_values([\"visitId\"]).iloc[:, :20]\n# repeated_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2461853817cd2615fe66e99b7959299cb844b96b"},"cell_type":"code","source":"# len(repeat_session_id)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6d674e52571d8308e468824da88395d6ad373e7"},"cell_type":"markdown","source":"**We can see that records with same visitId has different fullVisitorId but the same visitStartTime. Because of simultaneous sessions data was received concurrently and this sessions has the same visidId. So we can't remove records with same visitId but can remove feature visitId**"},{"metadata":{"trusted":true,"_uuid":"9df1748ce7751eb453e777252ac73f3f15220e95"},"cell_type":"code","source":"# df_train = df_train.drop(columns=[\"visitId\"])\n# df_test = df_test.drop(columns=[\"visitId\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a6acc0e399ce5b48dd1a9571026f02adf01d096"},"cell_type":"markdown","source":"**Think about feature trafficSource_adwordsClickInfo.gclId: it has a lot of nans**"},{"metadata":{"trusted":true,"_uuid":"3be9eb85427e9f50f5f9b2f8d12dd42a213e5433"},"cell_type":"code","source":"# df_train_eq_nan = df_train.fillna(-1543)\n# un = np.unique(np.array(df_train_eq_nan[\"trafficSource_adwordsClickInfo.gclId\"]).astype(str), return_counts=True)\n# idx = np.where(un[0] != \"-1543\")[0]\n# not_null_CI_train = un[0][idx]\n# not_null_CI_train, not_null_CI_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28f392aa52423b5a079eb1fed771853a19455308"},"cell_type":"code","source":"# df_test_eq_nan = df_test.fillna(-1543)\n# un = np.unique(np.array(df_test_eq_nan[\"trafficSource_adwordsClickInfo.gclId\"]).astype(str), return_counts=True)\n# idx = np.where(un[0] != \"-1543\")[0]\n# not_null_CI_test = un[0][idx]\n# not_null_CI_test, not_null_CI_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c12a7e5ba2b094000a82d3aadaa97483857aa75"},"cell_type":"code","source":"# CI_intersect = set(not_null_CI_train) & set(not_null_CI_test)\n# len(CI_intersect)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c80e1801fb4c114d03511c65a31b2efcdb620cf3"},"cell_type":"markdown","source":"**There is a little intersect and it seems to be such king of information coding. I suppose we should remove this feature**"},{"metadata":{"trusted":true,"_uuid":"1e293639987bf21a837d06d9f4cc264ec8b8bd1e"},"cell_type":"code","source":"# df_train = df_train.drop(columns=[\"trafficSource_adwordsClickInfo.gclId\"])\n# df_test = df_test.drop(columns=[\"trafficSource_adwordsClickInfo.gclId\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f4e433c3ba58d4b18c184346e2e1a54aab97dc5"},"cell_type":"markdown","source":"**Think about feature trafficSource_referralPath:**"},{"metadata":{"trusted":true,"_uuid":"48c8e7ab66a09899df670811bff66b24861b9a22"},"cell_type":"code","source":"# df_train_eq_nan = df_train.fillna(-1543)\n# un = np.unique(np.array(df_train_eq_nan[\"trafficSource_referralPath\"]).astype(str), return_counts=True)\n# idx = np.where(un[0] != \"-1543\")[0]\n# not_null_RP_train = un[0][idx]\n# not_null_RP_train, not_null_RP_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebcc51b631da4efbcfa84d0f358ba2b4efcae48c"},"cell_type":"code","source":"# df_test_eq_nan = df_test.fillna(-1543)\n# un = np.unique(np.array(df_test_eq_nan[\"trafficSource_referralPath\"]).astype(str), return_counts=True)\n# idx = np.where(un[0] != \"-1543\")[0]\n# not_null_RP_test = un[0][idx]\n# not_null_RP_test, not_null_RP_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3954149f22726f0a68ebabd7f4275337c1366285"},"cell_type":"code","source":"# del df_train_eq_nan, df_test_eq_nan\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70ad098ddd4f13224e0efc9b02feb588230038a3"},"cell_type":"markdown","source":"**Let explore how many objects has revenue when trafficSource_referralPath isn't a nan**"},{"metadata":{"trusted":true,"_uuid":"555c65aa55d955d74b1559c2bec77ff899f34232"},"cell_type":"code","source":"# revenues = df_train[df_train[\"trafficSource_referralPath\"].isin(not_null_RP_train)][\"totals_transactionRevenue\"]\n# revenues[revenues != 0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33d48c9575d53618891f5be1ebb860847dcfbc3b"},"cell_type":"code","source":"# df_train[df_train[\"totals_transactionRevenue\"] != 0].shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"783a9036eccb63b2a9c2cc8c9e50b2fc04de4dbb"},"cell_type":"markdown","source":"**So it seems that this feature is important**"},{"metadata":{"trusted":true,"_uuid":"deaffcebb0289a9d585301ece7b9ca7456166453"},"cell_type":"code","source":"# categorial.append(\"trafficSource_referralPath\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"09762787d1039c53ded02d472976e837366f944d"},"cell_type":"markdown","source":"**Fill nans:**"},{"metadata":{"trusted":true,"_uuid":"3e5abdfe06f89c5272983b918cf7da0ae94977fc"},"cell_type":"code","source":"# zero_filling = [\"totals_bounces\", \"totals_newVisits\", \"totals_pageviews\", \"trafficSource_adwordsClickInfo.isVideoAd\",\n#                 \"trafficSource_adwordsClickInfo.page\", \"trafficSource_isTrueDirect\"]\n# empty_filling = [\"device_browser\", \"trafficSource_adwordsClickInfo.adNetworkType\",\n#                  \"trafficSource_adwordsClickInfo.slot\", \"geoNetwork_city\", \"geoNetwork_continent\", \n#                  \"geoNetwork_country\", \"geoNetwork_metro\", \"geoNetwork_networkDomain\", \"geoNetwork_region\",\n#                  \"geoNetwork_subContinent\", \"trafficSource_adContent\", \"trafficSource_keyword\",\n#                  \"trafficSource_campaign\", \"trafficSource_medium\", \"trafficSource_source\",\n#                  \"device_operatingSystem\", \"trafficSource_referralPath\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7df0da3230939089ff1cfeb490de77c1655e1c1"},"cell_type":"code","source":"# df_train[zero_filling] = df_train[zero_filling].fillna(0)\n# df_train[\"trafficSource_adwordsClickInfo.isVideoAd\"] = df_train[\"trafficSource_adwordsClickInfo.isVideoAd\"].apply(lambda x: 1 if x == 0 else 0)\n# df_train[\"trafficSource_isTrueDirect\"] = df_train[\"trafficSource_isTrueDirect\"].apply(lambda x: 1 if x else 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9ec7e8a5c5d5d4b0d804ac40a44c638d55ea853"},"cell_type":"code","source":"# df_test[zero_filling] = df_test[zero_filling].fillna(0)\n# df_test[\"trafficSource_adwordsClickInfo.isVideoAd\"] = df_test[\"trafficSource_adwordsClickInfo.isVideoAd\"].apply(lambda x: 1 if x == 0 else 0)\n# df_test[\"trafficSource_isTrueDirect\"] = df_test[\"trafficSource_isTrueDirect\"].apply(lambda x: 1 if x else 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"baa7010bfa3bc33c42d37bc2f4420db91ff34654"},"cell_type":"code","source":"# df_train[empty_filling] = df_train[empty_filling].fillna(\"@\")\n# df_test[empty_filling] = df_test[empty_filling].fillna(\"@\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7350166db4fb437ff98a72fd86d28cf10cf590a0"},"cell_type":"code","source":"# df_train[\"device_isMobile\"] = df_train[\"device_isMobile\"].apply(lambda x: 1 if x else 0)\n# df_test[\"device_isMobile\"] = df_test[\"device_isMobile\"].apply(lambda x: 1 if x else 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed31eb7e79a0a78476c939574664110a7993dbf1"},"cell_type":"code","source":"# set(df_train.columns) - (set(numerical) | set(categorial))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5023b5ea8be2202701f267542f55808402c7b6da"},"cell_type":"code","source":"# len(numerical) + len(categorial), df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e17fa9f68cd7be0dee1e5a68af2136a7c632ecfa"},"cell_type":"markdown","source":"**Save result**"},{"metadata":{"trusted":true,"_uuid":"a30c20c91ad45def281f646f2da8a7b78bf335d1"},"cell_type":"code","source":"#df_train.to_csv(\"./kernel/data/train_filtered.csv\", sep=\",\", index=False)\n#df_test.to_csv(\"./kernel/data/test_filtered.csv\", sep=\",\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"967c693559d78600cba1325f8e239acc3c82773f"},"cell_type":"code","source":"#df_train = pd.read_csv(\"../input/prepaired-data-of-customer-revenue-prediction/train_filtered.csv\", converters={'fullVisitorId': str}, sep=\",\")\n#df_test = pd.read_csv(\"../input/prepaired-data-of-customer-revenue-prediction/test_filtered.csv\", converters={'fullVisitorId': str}, sep=\",\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e0d01f4f0c4f5db6bbeb89bc2763464fbfe2c6b"},"cell_type":"markdown","source":"### 3. Data exploring and filtering. Part 2"},{"metadata":{"_uuid":"6919fb1018bfa1e21db8ea27a333d92fd3e277b9"},"cell_type":"markdown","source":"**Add some new OHE features from categorial features**"},{"metadata":{"trusted":true,"_uuid":"800344d61821325cd29e8fb25262981d30b0123f"},"cell_type":"code","source":"# for col_name in categorial:\n#     print(col_name)\n#     un_train = np.unique(np.array(df_train[col_name]).astype(str), return_counts=True)\n#     un_test = np.unique(np.array(df_test[col_name]).astype(str), return_counts=True)\n#     print(\"TRAIN: \", un_train[0])\n#     print(\"TEST: \", un_test[0])\n#     print(\"DIFFERENT TRAIN COUNT: \", un_train[0].shape[0])\n#     print(\"DIFFERENT TEST COUNT: \", un_test[0].shape[0])\n#     print(\"-\" * 43)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d99aa2916dc324b9ba414e682b4c895eb0a3368"},"cell_type":"markdown","source":"**We can select the following groups of categorial features:**"},{"metadata":{"trusted":true,"_uuid":"21e447cd9c1b3b14f11cd59d715149f8fbef62d6"},"cell_type":"code","source":"# easy_OHE = [\"channelGrouping\", \"device_deviceCategory\", \"geoNetwork_continent\", \"geoNetwork_subContinent\", \"trafficSource_medium\"]\n# easy_OHE_but_prepare = [\"trafficSource_adwordsClickInfo.adNetworkType\", \"trafficSource_adwordsClickInfo.slot\"]\n\n# bad_categorial = [\"device_browser\", \"date\", \"device_operatingSystem\", \"geoNetwork_city\", \"geoNetwork_metro\", \"geoNetwork_country\",\n#                   \"geoNetwork_networkDomain\", \"geoNetwork_region\", \"trafficSource_adContent\",\n#                   \"trafficSource_campaign\", \"trafficSource_keyword\", \"trafficSource_source\",\n#                  \"trafficSource_referralPath\"]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50c6faae8127d699b372e504cc24b37102f596c9"},"cell_type":"markdown","source":"* easy_OHE -- _easy to do One-Hot-Encoding_\n* easy_OHE_but_prepare -- _easy to do One-Hot-Encoding, but feature values in train and in test are not the same_\n* bad_categorial -- _other categorial features_"},{"metadata":{"trusted":true,"_uuid":"55f5005f26d4df0b49d3d3581c335c9911949e8f"},"cell_type":"code","source":"# from sklearn.preprocessing import OneHotEncoder as OHE\n# from sklearn.preprocessing import LabelEncoder as OE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddc0e8fc3df09a85aa7a6932dafea1a11eb0d775"},"cell_type":"markdown","source":"**Assumption: since we will encode the device_deviceCategory, the isMobile feature can be thrown out, as it will be a duplicate. So we will check it**"},{"metadata":{"trusted":true,"_uuid":"76551bbe7d1dda930d404105eb834b99b5ad91d7"},"cell_type":"code","source":"# y_train = np.array(df_train[answer_feature])\n# id_numeration_train = np.array(df_train[saved_trash])\n# X_train_numerical = np.array(df_train[numerical])\n\n# id_numeration_test = np.array(df_test[saved_trash])\n# X_test_numerical = np.array(df_test[numerical])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd023c8ec4bdda5a2f9ef1aeda02c17c55df4d1a"},"cell_type":"code","source":"# train_easy_OHE = np.array(df_train[easy_OHE])\n# test_easy_OHE = np.array(df_test[easy_OHE])\n\n# easy_OEs = [OE() for i in range(train_easy_OHE.shape[1])]\n# for f in range(train_easy_OHE.shape[1]):\n#     easy_OEs[f].fit(train_easy_OHE[:, f])\n#     train_easy_OHE[:, f] = easy_OEs[f].transform(train_easy_OHE[:, f])\n#     test_easy_OHE[:, f] = easy_OEs[f].transform(test_easy_OHE[:, f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e68f5fa1ddbaf1e7ddb844b9c5b634e3570968a7"},"cell_type":"code","source":"# easy_enc = OHE(sparse=False)\n# easy_enc.fit(train_easy_OHE)\n# train_easy_OHE_conv = easy_enc.transform(train_easy_OHE)\n# test_easy_OHE_conv = easy_enc.transform(test_easy_OHE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"52d88b3fab1f9dbaade90c8ef0c864055fdc85eb"},"cell_type":"markdown","source":"**Suppose that isMobile means that device_deviceCategory equals \"mobile\":**"},{"metadata":{"trusted":true,"_uuid":"cd30dd572a16e6034becf3b04d9ead1f508be1ea"},"cell_type":"code","source":"# errors = np.where(\n#     np.array(train_easy_OHE_conv[:, 9]).astype(bool)\n#     != np.array(df_train[\"device_isMobile\"]))[0]\n# errors.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"faf26c398b548ed4ebafe78bfd718b671d973a56"},"cell_type":"markdown","source":"**No, it's not a truth. Suppose that isMobile means that device_deviceCategory equals \"mobile\" or \"tablet\":**"},{"metadata":{"trusted":true,"_uuid":"1fc19119900d63f73fc87289e4e3bf3d067fdf40"},"cell_type":"code","source":"# errors = np.where(\n#     ((np.array(train_easy_OHE_conv[:, 10]).astype(bool)) | (np.array(train_easy_OHE_conv[:, 9]).astype(bool)))\n#     != np.array(df_train[\"device_isMobile\"]))[0]\n# errors.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40cab329e99c8d5a05440a8682bf1fcb5f8a7440"},"cell_type":"markdown","source":"**It's truth, but there is some errors that are needed to be fixed**"},{"metadata":{"trusted":true,"_uuid":"c8edf5948ab95df077362744541b946b0e98ebbb"},"cell_type":"code","source":"# display(df_train.iloc[errors, 10:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d908dfec6acc4a6cd33b81d457a8b9fb28185a91"},"cell_type":"code","source":"# deviceCategory_idx = 11\n# is_mobile_idx = 12","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"248f676e4aeaa1fa1b5fbd051fdb81e571f4766d"},"cell_type":"markdown","source":"**Apparently, there are errors in deviceCategory rather than in isMobie. Moreover, in erroneous examples in favor of the error says feature OS. Let's make this correction: where isMobile in erroneous examples = 0 -- set desktop, otherwise -- mobile.**"},{"metadata":{"trusted":true,"_uuid":"790cd20d8c6616e3e0d2a38a9ec2f96a9678ca96"},"cell_type":"code","source":"# df_train.iloc[errors, deviceCategory_idx] = (\n#     df_train.iloc[errors, is_mobile_idx].apply(lambda x: \"desktop\" if x == 0 else \"mobile\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a800d60a7091651149c74613f0fb45281d3f24"},"cell_type":"code","source":"# errors_te = np.where(\n#     ((np.array(test_easy_OHE_conv[:, 10]).astype(bool)) | (np.array(test_easy_OHE_conv[:, 9]).astype(bool)))\n#     != np.array(df_test[\"device_isMobile\"]))[0]\n# errors_te.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"268a3889eb2ff1961561da945f8699d81c1f1c1d"},"cell_type":"code","source":"# df_test.iloc[errors_te, deviceCategory_idx - 1] = (\n#     df_test.iloc[errors_te, is_mobile_idx - 1].apply(lambda x: \"desktop\" if x == 0 else \"mobile\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3595d1b56fe087254cdbf0cddd577aebacd6f3e"},"cell_type":"code","source":"# train_easy_OHE = np.array(df_train[easy_OHE])\n# test_easy_OHE = np.array(df_test[easy_OHE])\n# easy_OEs = [OE() for i in range(train_easy_OHE.shape[1])]\n# for f in range(train_easy_OHE.shape[1]):\n#     easy_OEs[f].fit(train_easy_OHE[:, f])\n#     train_easy_OHE[:, f] = easy_OEs[f].transform(train_easy_OHE[:, f])\n#     test_easy_OHE[:, f] = easy_OEs[f].transform(test_easy_OHE[:, f])\n# easy_enc = OHE(sparse=False)\n# easy_enc.fit(train_easy_OHE)\n# train_easy_OHE_conv = easy_enc.transform(train_easy_OHE)\n# test_easy_OHE_conv = easy_enc.transform(test_easy_OHE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f28aa165b8f0b2fe221519f275ad62151ec491a"},"cell_type":"code","source":"# errors2 = np.where(\n#     ((np.array(train_easy_OHE_conv[:, 10]).astype(bool)) | (np.array(train_easy_OHE_conv[:, 9]).astype(bool)))\n#     != np.array(df_train[\"device_isMobile\"]))[0]\n# print(errors2.shape)\n# errors_te2 = np.where(\n#     ((np.array(test_easy_OHE_conv[:, 10]).astype(bool)) | (np.array(test_easy_OHE_conv[:, 9]).astype(bool)))\n#     != np.array(df_test[\"device_isMobile\"]))[0]\n# print(errors_te2.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"85493c1659331c26316fcd3bdf37bc208c5edae9"},"cell_type":"markdown","source":"**There is no errors!**"},{"metadata":{"trusted":true,"_uuid":"3f7bb74fae37d41bb74276ef0140346e3b2c942b"},"cell_type":"code","source":"# easy_OHE_names = ['channelGrouping_(Other)',\n#  'channelGrouping_Affiliates',\n#  'channelGrouping_Direct',\n#  'channelGrouping_Display',\n#  'channelGrouping_Organic Search',\n#  'channelGrouping_Paid Search',\n#  'channelGrouping_Referral',\n#  'channelGrouping_Social',\n#  'device_deviceCategory_desktop',\n#  'device_deviceCategory_mobile',\n#  'device_deviceCategory_tablet',\n#  'geoNetwork_continent_@',\n#  'geoNetwork_continent_Africa',\n#  'geoNetwork_continent_Americas',\n#  'geoNetwork_continent_Asia',\n#  'geoNetwork_continent_Europe',\n#  'geoNetwork_continent_Oceania',\n#  'geoNetwork_subContinent_@',\n#  'geoNetwork_subContinent_Australasia',\n#  'geoNetwork_subContinent_Caribbean',\n#  'geoNetwork_subContinent_Central America',\n#  'geoNetwork_subContinent_Central Asia',\n#  'geoNetwork_subContinent_Eastern Africa',\n#  'geoNetwork_subContinent_Eastern Asia',\n#  'geoNetwork_subContinent_Eastern Europe',\n#  'geoNetwork_subContinent_Melanesia',\n#  'geoNetwork_subContinent_Micronesian Region',\n#  'geoNetwork_subContinent_Middle Africa',\n#  'geoNetwork_subContinent_Northern Africa',\n#  'geoNetwork_subContinent_Northern America',\n#  'geoNetwork_subContinent_Northern Europe',\n#  'geoNetwork_subContinent_Polynesia',\n#  'geoNetwork_subContinent_South America',\n#  'geoNetwork_subContinent_Southeast Asia',\n#  'geoNetwork_subContinent_Southern Africa',\n#  'geoNetwork_subContinent_Southern Asia',\n#  'geoNetwork_subContinent_Southern Europe',\n#  'geoNetwork_subContinent_Western Africa',\n#  'geoNetwork_subContinent_Western Asia',\n#  'geoNetwork_subContinent_Western Europe',\n#  'trafficSource_medium_@',\n#  'trafficSource_medium_affiliate',\n#  'trafficSource_medium_cpc',\n#  'trafficSource_medium_cpm',\n#  'trafficSource_medium_organic',\n#  'trafficSource_medium_referral']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a266939139436bf07343df09dfcaac160c47e3b"},"cell_type":"markdown","source":"**Let's encode easy_OHE_but_prepare:**"},{"metadata":{"trusted":true,"_uuid":"4e0625595764445c45dc689181b87fc683e38cd6"},"cell_type":"code","source":"# df_test[\"trafficSource_adwordsClickInfo.adNetworkType\"] = (\n#     df_test[\"trafficSource_adwordsClickInfo.adNetworkType\"].apply(lambda x: \"@\" if x == 'Content' else x))\n# df_test[\"trafficSource_adwordsClickInfo.slot\"] = (\n#     df_test[\"trafficSource_adwordsClickInfo.slot\"].apply(lambda x: \"@\" if x == 'Google Display Network' else x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf1c9542a1e2b1601bf979095159ec88196a3768"},"cell_type":"code","source":"# for col_name in easy_OHE_but_prepare:\n#     print(col_name)\n#     un_train = np.unique(np.array(df_train[col_name]).astype(str), return_counts=True)\n#     un_test = np.unique(np.array(df_test[col_name]).astype(str), return_counts=True)\n#     print(\"TRAIN: \", un_train[0])\n#     print(\"TEST: \", un_test[0])\n#     print(\"DIFFERENT TRAIN COUNT: \", un_train[0].shape[0])\n#     print(\"DIFFERENT TEST COUNT: \", un_test[0].shape[0])\n#     print(\"-\" * 43)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71e6c0f3092a40ab363830dea25b5b83ef8982f1"},"cell_type":"code","source":"# train_easy_OHE_prep = np.array(df_train[easy_OHE_but_prepare])\n# test_easy_OHE_prep = np.array(df_test[easy_OHE_but_prepare])\n# easy_OEs_prep = [OE() for i in range(train_easy_OHE_prep.shape[1])]\n# for f in range(train_easy_OHE_prep.shape[1]):\n#     easy_OEs_prep[f].fit(train_easy_OHE_prep[:, f])\n#     train_easy_OHE_prep[:, f] = easy_OEs_prep[f].transform(train_easy_OHE_prep[:, f])\n#     test_easy_OHE_prep[:, f] = easy_OEs_prep[f].transform(test_easy_OHE_prep[:, f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9f6739d6102fe9127c9e4590928a46b42d78e23"},"cell_type":"code","source":"# easy_enc_prep = OHE(sparse=False)\n# easy_enc_prep.fit(train_easy_OHE_prep)\n# train_easy_OHE_prep_conv = easy_enc_prep.transform(train_easy_OHE_prep)\n# test_easy_OHE_prep_conv = easy_enc_prep.transform(test_easy_OHE_prep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"092f57c6b7cf65ca185b818efa7e4f4e328676b1"},"cell_type":"code","source":"# easy_OHE_prep_names = ['trafficSource_adwordsClickInfo.adNetworkType_@',\n#  'trafficSource_adwordsClickInfo.adNetworkType_Google Search',\n#  'trafficSource_adwordsClickInfo.adNetworkType_Search partners',\n#  'trafficSource_adwordsClickInfo.slot_@',\n#  'trafficSource_adwordsClickInfo.slot_RHS',\n#  'trafficSource_adwordsClickInfo.slot_Top']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"026b0879406ac5289314531317f77eea1d114fb9"},"cell_type":"markdown","source":"**Let's factorize all bad_categorial features**"},{"metadata":{"trusted":true,"_uuid":"ecd23532734ebd7c4bf68492fa71903fd65cfc54"},"cell_type":"code","source":"# train_bad_cat = df_train[bad_categorial].copy()\n# test_bad_cat = df_test[bad_categorial].copy()\n# for f in bad_categorial:\n#     train_bad_cat[f], indexer = pd.factorize(train_bad_cat[f])\n#     test_bad_cat[f] = indexer.get_indexer(test_bad_cat[f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"674f6d0b61d932554385df90c32f800e9489a817"},"cell_type":"code","source":"# y_train_clf = (y_train.ravel() > 0).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"472933403dce0d2d5c4d009939c2eef54835cbb4"},"cell_type":"markdown","source":"**But some of values of features let's convert via OHE. Explore how feature values are distributed for record with some revenue and with no revenue:**"},{"metadata":{"trusted":true,"_uuid":"38aa2cd5695a48654ad62d8c1148297f8585d557"},"cell_type":"code","source":"# with_rev = df_train.iloc[y_train_clf == True, :][bad_categorial]\n# no_rev = df_train.iloc[y_train_clf == False, :][bad_categorial]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d305ec00750aab30f99d97d4421a0432c70664f9"},"cell_type":"code","source":"# for feature in bad_categorial:\n#     vals = with_rev[feature]\n#     un, cnt = np.unique(vals, return_counts=True)\n#     data = np.hstack((un.reshape((-1, 1)), cnt.reshape((-1, 1))))\n\n#     top = 20\n#     df = pd.DataFrame(data, columns=[\"feature\", \"count features\"])\n#     plt.figure(figsize=(15, 8))\n#     sns.barplot(x=\"count features\", y='feature', data=df.sort_values(\"count features\", ascending=False).iloc[:top])\n#     plt.title(\"FEATURE {} HAVE REVENUE\".format(feature), fontsize=18)\n#     plt.show()\n    \n#     vals = no_rev[feature]\n#     un, cnt = np.unique(vals, return_counts=True)\n#     data = np.hstack((un.reshape((-1, 1)), cnt.reshape((-1, 1))))\n#     top = 20\n#     df = pd.DataFrame(data, columns=[\"feature\", \"count features\"])\n#     plt.figure(figsize=(15, 8))\n#     sns.barplot(x=\"count features\", y='feature', data=df.sort_values(\"count features\", ascending=False).iloc[:top])\n#     plt.title(\"FEATURE {} HAVEN'T REVENUE\".format(feature), fontsize=18)\n#     plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ea7b01fe4c3545799726f4af8b30225ed534c31"},"cell_type":"markdown","source":"**Let's make a top of values for each feature:**"},{"metadata":{"trusted":true,"_uuid":"a8c5ae52bc769ce67d04d824c6e401e5188d0297"},"cell_type":"code","source":"# top_rev =    [5, 0, 6, 20, 6, 4, 7, 10, 2, 5, 5, 16, 5]\n# top_no_rev = [11, 0, 6, 10, 5, 10, 7, 4, 3, 4, 5, 10, 7] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1366deb918766c2eafe54a8a855098e67fd2fca4"},"cell_type":"code","source":"# bad_cat_to_OHE = defaultdict(set)\n\n# for feature, t_r, t_n_r in zip(bad_categorial, top_rev, top_no_rev):\n#     vals = with_rev[feature]\n#     un, cnt = np.unique(vals, return_counts=True)\n#     sortidx = np.argsort(cnt)\n#     bad_cat_to_OHE[feature] |= set(un[sortidx][::-1][:t_r])\n    \n#     vals = no_rev[feature]\n#     un, cnt = np.unique(vals, return_counts=True)\n#     sortidx = np.argsort(cnt)\n#     bad_cat_to_OHE[feature] |= set(un[sortidx][::-1][:t_n_r])\n#     if \"@\" in bad_cat_to_OHE[feature]:\n#         bad_cat_to_OHE[feature].remove(\"@\")\n        \n#     test_vals = set(df_test[feature])\n#     bad_cat_to_OHE[feature] &= test_vals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ec1f75a188a18dd3fb1a9b0d0f5928a0fb13178"},"cell_type":"code","source":"# ordered_dict = dict()\n# for (name, value) in bad_cat_to_OHE.items():\n#     ordered_dict[name] = list(value)\n    \n# bad_cat_features_OHE_names = []\n# for (name, value) in ordered_dict.items():\n#     if len(value) > 0:\n#         for val in value:\n#             bad_cat_features_OHE_names.append(\"{}_{}\".format(name, val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e77590a9fa38ca5149efbbf8c3ec059d8863b4ae"},"cell_type":"code","source":"# add_len = len(bad_cat_features_OHE_names)\n# add_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"273feff4a0b46d93e5811c8a9aaacb24967af44a"},"cell_type":"code","source":"# train_bad_cat_to_OHE = np.zeros((df_train.shape[0], add_len))\n# test_bad_cat_to_OHE = np.zeros((df_test.shape[0], add_len))\n# for num, feature_value in enumerate(bad_cat_features_OHE_names):\n#     idx = feature_value.rfind(\"_\")\n#     feature, value = feature_value[:idx], feature_value[idx + 1:]\n#     train_bad_cat_to_OHE[:, num] = np.array(df_train[feature] == value).astype(int)\n#     test_bad_cat_to_OHE[:, num] = np.array(df_test[feature] == value).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b36ce682ef909baae5a9b633c50bcfc15bb6b74"},"cell_type":"markdown","source":"**Add to categorial features feature weekday:**"},{"metadata":{"trusted":true,"_uuid":"ec7809d0018205cc918bd6f667448ad1ab50621b"},"cell_type":"code","source":"# bad_categorial += [\"weekday\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f28d8e3d5ad92023852a645fa80947ad685bfbe"},"cell_type":"code","source":"# tr_wd = np.array(pd.to_datetime(df_train[\"date\"], format=\"%Y%m%d\").apply(lambda x: x.weekday())).reshape((-1, 1))\n# te_wd = np.array(pd.to_datetime(df_test[\"date\"], format=\"%Y%m%d\").apply(lambda x: x.weekday())).reshape((-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6107017d2d96213b40adf0384d87d41a95f642ad"},"cell_type":"markdown","source":"**Collect all together:**"},{"metadata":{"trusted":true,"_uuid":"6029e47aaf44cbc8944a66c5dc1265877efa86a7"},"cell_type":"code","source":"# X_train = np.hstack((X_train_numerical, train_easy_OHE_conv,\n#                       train_easy_OHE_prep_conv, train_bad_cat_to_OHE, train_bad_cat, tr_wd))\n# X_test = np.hstack((X_test_numerical, test_easy_OHE_conv,\n#                      test_easy_OHE_prep_conv, test_bad_cat_to_OHE, test_bad_cat, te_wd))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ac0be6ea1f3e147d678503cbc1b82524e4abb14"},"cell_type":"code","source":"# feature_names = numerical + easy_OHE_names + easy_OHE_prep_names + bad_cat_features_OHE_names + bad_categorial\n# len(feature_names), X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4b526e604e6301b3c22e61241c9cb190258e7ef"},"cell_type":"markdown","source":"**Let's visualize train and test:**"},{"metadata":{"trusted":true,"_uuid":"74efe862c7564af88e71f2967882d2239dd73a6b"},"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD, PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.preprocessing import StandardScaler\nimport scipy\nimport pickle\nimport re\nimport time\nfrom sklearn.utils import shuffle as skshuffle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e065385ff4723aab321d68cc5d222f038dc6228"},"cell_type":"code","source":"# def visualize_data(data_train, y_train, data_test, amount_train=None, amount_test=None,\n#                    scale=True, threeD=False, shuffle=False, alpha=1.0):\n    \n#     def visualize_2d(data_train, y_train, data_test):\n#         tsvd = PCA(n_components=2)\n#         tsvd.fit(data_train)\n#         Z_train = tsvd.fit_transform(data_train)\n#         Z_test = tsvd.transform(data_test)\n\n#         classes_amount = np.unique(y_train).shape[0]\n#         cm = plt.get_cmap('jet')\n#         plt.figure(figsize=(18, 15))\n#         plt.scatter(Z_train[:, 0], Z_train[:, 1],\n#                     c=y_train.ravel(), cmap='RdYlGn_r', alpha=alpha) # green - 1 blue 0\n#         plt.title(\"Train data\")\n#         plt.figure(figsize=(18, 15))\n#         plt.scatter(Z_test[:, 0], Z_test[:, 1],\n#                     c=[\"plum\"] * Z_test.shape[0], alpha=alpha)\n#         plt.title(\"Test data\")\n#         plt.show()\n        \n#     def visualize_3d(data_train, y_train, data_test):\n#         tsvd = PCA(n_components=3)\n#         tsvd.fit(data_train)\n#         Z_train = tsvd.fit_transform(data_train)\n#         Z_test = tsvd.transform(data_test)\n\n#         classes_amount = np.unique(y_train).shape[0]\n#         cm = plt.get_cmap('jet')\n        \n#         fig = plt.figure(figsize=(18, 15))\n#         ax = fig.add_subplot(111, projection='3d')\n#         sc = ax.scatter(Z_train[:, 0], Z_train[:, 1], Z_train[:, 2],\n#                 c=y_train.ravel(), cmap='RdYlGn_r', alpha=alpha)\n#         plt.title(\"Train data\")\n#         fig = plt.figure(figsize=(18, 15))\n#         ax = fig.add_subplot(111, projection='3d')\n#         sc = ax.scatter(Z_test[:, 0], Z_test[:, 1], Z_test[:, 2],\n#                 c=[\"plum\"] * Z_test.shape[0], alpha=alpha)\n#         plt.title(\"Test data\")\n#         plt.show()\n        \n#     if amount_train is None:\n#         amount_train = data_train.shape[0]\n#     else:\n#         amount_train = int(amount_train * data_train.shape[0])\n        \n#     if amount_test is None:\n#         amount_test = data_test.shape[0]\n#     else:\n#         amount_test = int(amount_test * data_test.shape[0])\n        \n#     if shuffle:\n#         data_train, y_train = skshuffle(data_train, y_train)\n#         data_test = skshuffle(data_test)\n    \n#     if scale:\n#         scaler = StandardScaler(copy=True)\n#         scaler.fit(data_train)\n#         X_train = scaler.transform(data_train)\n#         X_test = scaler.transform(data_test)\n#     else:\n#         X_train = data_train\n#         X_test = data_test\n        \n#     start = time.time()\n#     if threeD:\n#         visualize_3d(X_train[:amount_train, :], y_train[:amount_train], X_test[:amount_train, :])\n#     else:\n#         visualize_2d(X_train[:amount_train, :], y_train[:amount_train], X_test[:amount_train, :])\n#     print(time.time() - start)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d015b4a26f08fad596f4cd3546fc0da67ff3372d"},"cell_type":"code","source":"# y_train_clf = np.copy(y_train)\n# y_train_clf[y_train_clf > 0] = 1\n# y_train_clf[y_train_clf == 0] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0363e17018f483f5d55d0ad03bf9af13d1bd9c5"},"cell_type":"code","source":"# visualize_data(X_train, y_train_clf, X_test, amount_train=0.2,\n#                amount_test=0.2, threeD=False, alpha=0.5, scale=True, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7efec08f6f8dd9e49b61080213353a50bb7534df"},"cell_type":"markdown","source":"**Red points -- records with some revenue**"},{"metadata":{"trusted":true,"_uuid":"00a4a2fd5a1ab6d6e84541df96e3c1eaeba4cbb4"},"cell_type":"code","source":"# visualize_data(X_train, y_train_clf, X_test, amount_train=0.2,\n#                amount_test=0.2, threeD=True, alpha=0.5, scale=True, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9befbc1242a037e9230ee53c168fa6606291d5c9"},"cell_type":"markdown","source":"### 4. Train and predict"},{"metadata":{"trusted":true,"_uuid":"2c84579bf990f3f0e553e4ff920aa8e96536013a"},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import KFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error, roc_auc_score, log_loss\nfrom sklearn.preprocessing import StandardScaler as SS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdcf74de4e338cbf2097b6058f6bfd305456c423"},"cell_type":"code","source":"def tune_threshold(base_cv, preds, y_true):\n    diffs = []\n    MIN_VALS = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1]\n    for MIN_VAL in MIN_VALS:\n        print(\"TH: \", MIN_VAL)\n        print(\"RMSE CV {}\".format(base_cv))\n        y2 = np.expm1(preds)\n        y2[y2 < MIN_VAL] = 0\n        neo = mean_squared_error(np.log1p(y_true), np.log1p(y2)) ** .5\n        print(\"RMSE NEW CV: \", neo)\n        print(\"DIFF: \", base_cv - neo)\n        diffs.append(base_cv - neo)\n        print(\"-\" * 43)\n    idx = np.argmax(np.array(diffs))\n    return MIN_VALS[idx]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18c24d8e3742fef881a769ba0b77042d824208a9"},"cell_type":"markdown","source":"**next cell based on https://www.kaggle.com/nikitpatel/lightgbm-with-best-hyperparameter-tuning/notebook**"},{"metadata":{"trusted":true,"_uuid":"3a21943af8303781c84a7a2f6504477a157fa481"},"cell_type":"code","source":"def get_folds(visitor_ids_in_train, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(np.unique(visitor_ids_in_train)))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(visitor_ids_in_train.shape[0])\n    df = pd.DataFrame()\n    df[\"fullVisitorId\"] = visitor_ids_in_train\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb12575965109754d69f2086db3b4311c6261d63"},"cell_type":"markdown","source":"**Based on some several attempts to fit lgbm I selected the following groups of meaningless features:**"},{"metadata":{"trusted":true,"_uuid":"4346f66fa28163725e3b30aac4cfe74ab310baa3"},"cell_type":"code","source":"# bad_features = [\n#        'trafficSource_adwordsClickInfo.isVideoAd',\n#        'channelGrouping_(Other)', 'geoNetwork_subContinent_Australasia',\n#        'geoNetwork_subContinent_Central Asia',\n#        'geoNetwork_subContinent_Eastern Africa',\n#        'geoNetwork_subContinent_Eastern Europe',\n#        'geoNetwork_subContinent_Melanesia',\n#        'geoNetwork_subContinent_Micronesian Region',\n#        'geoNetwork_subContinent_Middle Africa',\n#        'geoNetwork_subContinent_Northern Africa',\n#        'geoNetwork_subContinent_Polynesia',\n#        'geoNetwork_subContinent_Southern Africa',\n#        'geoNetwork_subContinent_Southern Asia',\n#        'geoNetwork_subContinent_Western Africa',\n#        'trafficSource_adwordsClickInfo.adNetworkType_Search partners',\n#        'trafficSource_adwordsClickInfo.slot_@',\n#        'trafficSource_adwordsClickInfo.slot_RHS'\n# ]\n\n# bad_features2 = [\n#        'device_browser_Android Webview', 'device_browser_Opera',\n#        'device_browser_Opera Mini', 'device_browser_Safari (in-app)',\n#        'device_browser_UC Browser', 'device_browser_YaBrowser',\n#        'geoNetwork_city_Bangkok', 'geoNetwork_city_Ho Chi Minh City',\n#        'geoNetwork_city_London', 'geoNetwork_continent_Africa',\n#        'geoNetwork_continent_Oceania', 'geoNetwork_country_Germany',\n#        'geoNetwork_country_India', 'geoNetwork_country_Japan',\n#        'geoNetwork_country_Mexico', 'geoNetwork_country_Thailand',\n#        'geoNetwork_country_Turkey', 'geoNetwork_country_United Kingdom',\n#        'geoNetwork_country_Vietnam', 'geoNetwork_metro_London',\n#        'geoNetwork_networkDomain_hinet.net',\n#        'geoNetwork_networkDomain_ttnet.com.tr',\n#        'geoNetwork_region_England',\n#        'geoNetwork_subContinent_Southeast Asia',\n#        'geoNetwork_subContinent_Southern Europe',\n#        'geoNetwork_subContinent_Western Europe',\n#        'trafficSource_adContent_Google Online Store',\n#        'trafficSource_adwordsClickInfo.adNetworkType_Google Search',\n#        'trafficSource_campaign_Data Share Promo',\n#        'trafficSource_medium_affiliate', 'trafficSource_medium_cpc',\n#        'trafficSource_source_Partners', 'trafficSource_source_baidu',\n#        'trafficSource_source_facebook.com',\n#        'trafficSource_source_gdeals.googleplex.com',\n#        'trafficSource_source_google.com',\n#        'trafficSource_source_m.facebook.com'\n# ]\n\n# bad_features2 += [\"visitStartTime\", \"date\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab4b8000a7a01f7f8aa8992d45214d2f988862ba"},"cell_type":"code","source":"# len(feature_names)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90dc4abc22f265cbadf44e9c8f9abf18d5e607e4"},"cell_type":"markdown","source":"**Remove meaningless features:**"},{"metadata":{"trusted":true,"_uuid":"1236b73c3524a7ef656d1b3c2f1377d3431371d8"},"cell_type":"code","source":"# categorial = bad_categorial\n# names_indices = dict()\n# for idx, name in enumerate(feature_names):\n#     names_indices[name] = idx\n    \n# good_indices = []\n# new_feature_names = []\n# for (name, idx) in names_indices.items():\n#     if name not in bad_features and name not in bad_features2:\n#         good_indices.append(idx)\n#         new_feature_names.append(name)\n\n# X_train = X_train[:, good_indices]\n# X_test = X_test[:, good_indices]\n\n# idx = numerical.index(\"visitStartTime\")\n# numerical = numerical[:idx] + numerical[idx + 1:]\n# idx = categorial.index(\"date\")\n# categorial = categorial[:idx] + categorial[idx + 1:]\n\n# cat_indices = np.arange(X_train.shape[1] - len(categorial), X_train.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"223cfc7849efaa182518ce90fdb71c0438b483d0"},"cell_type":"code","source":"# y_train = y_train.ravel()\n# X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78c508f3a78518ad664cb0ebdd9329415fe5d7e8"},"cell_type":"markdown","source":"**Save data:**"},{"metadata":{"trusted":true,"_uuid":"9f777684b7a1079c98a0b45590b2d6c114f3ba8a"},"cell_type":"code","source":"\"\"\"data_pkl = {\n    \"X_train\" : X_train,\n    \"y_train\" : y_train,\n    \"X_test\" : X_test,\n    \"feature_names\" : new_feature_names,\n    \"categorial\" : categorial,\n    \"numerical\" : numerical,\n    \"id_numeration_train\" : id_numeration_train,\n    \"id_numeration_test\" : id_numeration_test,\n}\nwith open(\"./kernel/prepaired_data/data_all_features.pkl\", \"wb\") as fout:\n    pickle.dump(data_pkl, fout)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a863a5101aa40e82e5e96894504861d19ac0f13"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2806948bfa9ceef0ae8f67932f2b4bbae7ca9074"},"cell_type":"code","source":"with open(\"../input/prepaired-data-of-customer-revenue-prediction/data_all_features.pkl\", \"rb\") as fin:\n    data = pickle.load(fin)\nX_train, y_train = data[\"X_train\"], data[\"y_train\"]\nX_test = data[\"X_test\"]\nid_numeration_train = data[\"id_numeration_train\"]\nid_numeration_test = data[\"id_numeration_test\"]\nfeature_names = data[\"feature_names\"]\nnumerical = data[\"numerical\"]\ncategorial = data[\"categorial\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37fbfc81b2e80a42d40faed9e45e87b9310ba227"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf0706dd4537a75b26ab544f95e82cd396cb6899"},"cell_type":"markdown","source":"### First part of train-prediction logic"},{"metadata":{"_uuid":"c6784d88a484411de9c1f9091fe5a20c6c01c47d"},"cell_type":"markdown","source":"**based on ideas of https://www.kaggle.com/ogrellier/using-classification-for-predictions/notebook and\nhttps://www.kaggle.com/nikitpatel/lightgbm-with-best-hyperparameter-tuning/notebook**"},{"metadata":{"_uuid":"4a9dc4f65b006d12762c472b9cf9b7b63ddb8bf2"},"cell_type":"markdown","source":"**Scale numerical features:**"},{"metadata":{"trusted":true,"_uuid":"c94505aabebdb84adb76e4339e2b0a27b8e46c17"},"cell_type":"code","source":"scaler = SS()\nscaler.fit(X_train[:, :len(numerical)])\nX_train_scaled = np.copy(X_train)\nX_test_scaled = np.copy(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0de68f9624e7d5d152472f08dd3e3feb13c010d"},"cell_type":"code","source":"X_train_scaled[:, :len(numerical)] = scaler.transform(X_train[:, :len(numerical)])\nX_test_scaled[:, :len(numerical)] = scaler.transform(X_test[:, :len(numerical)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e54aebb5224fc15081cf5133a9c64d5495d3dd2"},"cell_type":"code","source":"cat_indices = np.arange(X_train.shape[1] - len(categorial), X_train.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5311df62d62b0878de72a52b005e3f3f340bdca2"},"cell_type":"code","source":"cat_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb2e15291ca7b255b0e2ebe0d2f7dc4044080822"},"cell_type":"code","source":"X_train_new = np.hstack((X_train_scaled[:, cat_indices], X_train_scaled[:, :cat_indices[0]]))\nX_test_new = np.hstack((X_test_scaled[:, cat_indices], X_test_scaled[:, :cat_indices[0]]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad33e7617987a9f3b0b25e361a4eceb4a98aa54b"},"cell_type":"code","source":"new_feature_names = feature_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"249630e65eddb42e4990643b5133f6aea29942b9"},"cell_type":"code","source":"cat_ind_new = [int(x) for x in np.arange(len(cat_indices)).astype(np.int32)]\nnew_new_feature_names = new_feature_names[-len(cat_ind_new):] + new_feature_names[:-len(cat_ind_new)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2968fc7498d3513f39d1f231a65fe6957bb3037"},"cell_type":"code","source":"n_splits = 5\n\nfolds = get_folds(id_numeration_train.ravel(), n_splits=n_splits)\n\noof_reg_preds = np.zeros(X_train.shape[0])\nsub_reg_preds_cv = np.zeros(X_test.shape[0])\nimportances = pd.DataFrame()\n\nnames = new_new_feature_names\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = X_train_new[trn_], y_train[trn_]\n    val_x, val_y = X_train_new[val_], y_train[val_]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1543 + 1543,\n        subsample=.9,\n        max_depth=15,\n        colsample_bytree=.9,\n        random_state=1543\n    )\n    start = time.time()\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        verbose=50,\n        eval_metric=\"rmse\",\n        categorical_feature=cat_ind_new\n    )\n    print(\"REGR TIME: \", time.time() - start)\n    imp_df = pd.DataFrame()\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    imp_df['feature'] = names\n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(X_test_new, num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds_cv += _preds / n_splits # maybe fix\n    \nprint(\"CV RMSE: \", mean_squared_error(np.log1p(y_train), oof_reg_preds) ** .5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdddb4e730ebb5e45625a133fa94989e41bdb5d5"},"cell_type":"code","source":"importances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(10, 30))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c938e9f17dd1b2718708efcb97f16cdab46bbe01"},"cell_type":"code","source":"base = mean_squared_error(np.log1p(y_train), oof_reg_preds) ** .5\nMIN_VAL = tune_threshold(base, oof_reg_preds, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a692bf0c0688a362301b8e198402a4e514e16e"},"cell_type":"code","source":"MIN_VAL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df4c35c6c64f9a864e6df93676491409e04376ef"},"cell_type":"code","source":"y_p1 = np.expm1(sub_reg_preds_cv)\ny_p1[y_p1 < MIN_VAL] = 0\nsub_reg_preds_cv_correted = np.log1p(y_p1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df73a3a8a717c2eb2f82221ece928423da46b7fa"},"cell_type":"code","source":"del X_train_scaled, X_test_scaled, X_train_new, X_test_new\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e67a40e9e53688301e8344b7110ccdb3050e178c"},"cell_type":"markdown","source":"### Second part of train-prediction logic"},{"metadata":{"_uuid":"bf91eb6788c224bfa55c42a4922152ecc44f6e3b"},"cell_type":"markdown","source":"**Process train data:**"},{"metadata":{"_uuid":"7fc80789d98b3bf6350ef3feaf34ae8d84e5f586"},"cell_type":"markdown","source":"**Add statistic features for _numerical_ features**"},{"metadata":{"trusted":true,"_uuid":"1779500c9fa4e497235c1148e7b33ed34f5be5de"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d3e194ba261cfa5c542965a316ef4b889b4b77c"},"cell_type":"code","source":"train_num = pd.DataFrame(X_train[:, :-len(categorial)], columns=new_feature_names[:-len(categorial)], index=None)\ntrain_num[\"fullVisitorId\"] = id_numeration_train\ntrn_data_num = train_num.groupby('fullVisitorId').mean()\n#trn_data_num1.columns = [\"{}_mean\".format(x) for x in trn_data_num1.columns]\n#trn_data_num2 = train_num.groupby(\"fullVisitorId\").std().fillna(0)\n#trn_data_num2.columns = [\"{}_std\".format(x) for x in trn_data_num2.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ffc4ff201ea798ecc369dc223dc8d37edd6217d"},"cell_type":"code","source":"#trn_data_num = pd.concat([trn_data_num1, trn_data_num2], axis=1)\n#trn_data_num.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b96f4edfa004507a6af3fb5596a9eec9889d206"},"cell_type":"code","source":"#del trn_data_num1, trn_data_num2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c81e9076d005ec6ae94b98ada78b6f677f46d394"},"cell_type":"markdown","source":"**Add moda for _categorial_ features**"},{"metadata":{"trusted":true,"_uuid":"4b92692d515af6acbceab23b1208dace9a730bf8"},"cell_type":"code","source":"# it's works about 2 hours\n\"\"\"\ntrain_cat = pd.DataFrame(X_train[:, -len(categorial):], columns=new_feature_names[-len(categorial):], index=None)\ntrain_cat[\"fullVisitorId\"] = id_numeration_train\ntrn_data_cat = train_cat.groupby(\"fullVisitorId\").aggregate(lambda x: x.value_counts().index[0])\ntrn_data_cat.to_csv(\"./kernel/prepaired_data/train_categorial_features_moda.csv\", sep=\",\")\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83ff3d06e1b7eb034c58a3cbbf9320596c37e1f1"},"cell_type":"code","source":"trn_data_cat = pd.read_csv(\"../input/prepaired-data-of-customer-revenue-prediction/train_categorial_features_moda.csv\", sep=\",\",\n                           converters={\"fullVisitorId\" : str}, index_col=\"fullVisitorId\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d9252a31b3913bd3999c88bfd712f94c82be4ad"},"cell_type":"code","source":"np.all(trn_data_num.index == trn_data_cat.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad2a457a2a57825d37c905455dbf25856abe8c1"},"cell_type":"code","source":"trn_data = pd.concat([trn_data_num, trn_data_cat], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aed91d57a8914dab905ef5120290639019dd230c"},"cell_type":"code","source":"train_num[\"pred\"] = np.expm1(oof_reg_preds)\ntrn_pred_list = train_num[['fullVisitorId', 'pred']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.pred))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})\ntrn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)\ntrn_feats = trn_all_predictions.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1d097ebaafd24eb5bdd6031f27a2c1e9d54cc66"},"cell_type":"code","source":"del train_num, trn_data_num, trn_data_cat\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a840517a04146a2d20104a15b5ca84ef9ed52538"},"cell_type":"code","source":"trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))\ntrn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))\ntrn_all_predictions['t_std'] = np.log1p(trn_all_predictions[trn_feats].std(axis=1).fillna(0))\ntrn_all_predictions['t_min'] = np.log1p(trn_all_predictions[trn_feats].min(axis=1))\ntrn_all_predictions['t_var_coef'] = np.log1p(trn_all_predictions[trn_feats].std(axis=1).fillna(0) / (trn_all_predictions[trn_feats].mean(axis=1) + 1e-6))\ntrn_all_predictions['t_max'] = np.log1p(trn_all_predictions[trn_feats].max(axis=1))\ntrn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats]).sum(axis=1)\ntrn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0).sum(axis=1))\ntrn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfda7075ac33d49213d047efa0aef2eb4eeaea72"},"cell_type":"code","source":"trn_all_predictions['t_std'] = trn_all_predictions['t_std'].fillna(0)\ntrn_all_predictions['t_var_coef'] = trn_all_predictions['t_var_coef'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aff7ce60930454704c798925d808ef4f1dd6e065"},"cell_type":"code","source":"full_data = pd.concat([trn_data, trn_all_predictions], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68a5875630f209bde3b507b04da2218485e6ad1a"},"cell_type":"code","source":"del trn_data, trn_all_predictions\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9766f8a1b68750dacd4ebfe3671d5fb7668415f8"},"cell_type":"markdown","source":"**Process test data:**"},{"metadata":{"trusted":true,"_uuid":"f6383ea80ff797fbf35d92a77b068ce49f7de687"},"cell_type":"code","source":"test_num = pd.DataFrame(X_test[:, :-len(categorial)], columns=new_feature_names[:-len(categorial)], index=None)\ntest_num[\"fullVisitorId\"] = id_numeration_test\nsub_data_num = test_num.groupby('fullVisitorId').mean()\n#sub_data_num1.columns = [\"{}_mean\".format(x) for x in sub_data_num1.columns]\n#sub_data_num2 = test_num.groupby('fullVisitorId').std().fillna(0)\n#sub_data_num2.columns = [\"{}_std\".format(x) for x in sub_data_num2.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ddcbd93dbc49add15f776d5944ea0c2c3f7b3d3"},"cell_type":"code","source":"#sub_data_num = pd.concat([sub_data_num1, sub_data_num2], axis=1)\n#sub_data_num.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3c41ceb690191a4ff513ae054ae101635099cbf"},"cell_type":"code","source":"#del sub_data_num1, sub_data_num2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c5b26cbb67d05d22be72aa892f3ce2e110ac302"},"cell_type":"code","source":"# it's works about 2 hours\n\"\"\"\ntest_cat = pd.DataFrame(X_test[:, -len(categorial):], columns=new_feature_names[-len(categorial):], index=None)\ntest_cat[\"fullVisitorId\"] = id_numeration_test\ntest_data_cat = test_cat.groupby(\"fullVisitorId\").aggregate(lambda x: x.value_counts().index[0])\ntest_data_cat.to_csv(\"./kernel/prepaired_data/test_categorial_features_moda.csv\", sep=\",\")\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"211707ad9a5088da998d93ad87e53ec3481b9933"},"cell_type":"code","source":"test_data_cat = pd.read_csv(\"../input/prepaired-data-of-customer-revenue-prediction/test_categorial_features_moda.csv\", sep=\",\",\n                           converters={\"fullVisitorId\" : str}, index_col=\"fullVisitorId\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"547a5695e46b984c65af890b0e6054754be66263"},"cell_type":"code","source":"np.all(sub_data_num.index == test_data_cat.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af62872b00c77b3b3c7ebf42e63dff1afdb473dc"},"cell_type":"code","source":"sub_data = pd.concat([sub_data_num, test_data_cat], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3c8180b65b997d9ce624834a64f27f9d164137a"},"cell_type":"code","source":"del sub_data_num, test_data_cat\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39b08df99e110cb39d45c573d20a276dbcbdd9ae"},"cell_type":"code","source":"test_num[\"pred\"] = np.expm1(sub_reg_preds_cv_correted)\nsub_pred_list = test_num[['fullVisitorId', 'pred']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.pred))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93eb0aa1f257a4e482500033b84fc7b90641e3a7"},"cell_type":"code","source":"del test_num\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"add0cf533c7e676489ca7570a36d120a74075719"},"cell_type":"code","source":"sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)\nfor f in trn_feats:\n    if f not in sub_all_predictions.columns:\n        sub_all_predictions[f] = np.nan\nsub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))\nsub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))\nsub_all_predictions['t_std'] = np.log1p(sub_all_predictions[trn_feats].std(axis=1).fillna(0))\nsub_all_predictions['t_min'] = np.log1p(sub_all_predictions[trn_feats].min(axis=1))\nsub_all_predictions['t_var_coef'] = np.log1p(sub_all_predictions[trn_feats].std(axis=1).fillna(0) / (sub_all_predictions[trn_feats].mean(axis=1) + 1e-6))\nsub_all_predictions['t_max'] = np.log1p(sub_all_predictions[trn_feats].max(axis=1))\nsub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats]).sum(axis=1)\nsub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0).sum(axis=1))\nsub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73edf46f17736dd2a2a3a7736c3af38310bedbd1"},"cell_type":"code","source":"sub_all_predictions['t_std'] = sub_all_predictions['t_std'].fillna(0)\nsub_all_predictions['t_var_coef'] = sub_all_predictions['t_var_coef'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33ba713279339b68b47dee2fd70a7a5926b7562c"},"cell_type":"code","source":"sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76b62ebeddee3bd2142a04b7104541d9d75b9e08"},"cell_type":"code","source":"del sub_data, sub_all_predictions\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"437257c4aa4dd058053597fa13554f822f58f491"},"cell_type":"code","source":"full_data.shape, sub_full_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28fec89f1f3235596663380410c25c1cc9341e7b"},"cell_type":"markdown","source":"**Scale features and train-predict:**"},{"metadata":{"trusted":true,"_uuid":"0a12677e4ae3f5c4144ed1d630cfae078187ba2f"},"cell_type":"code","source":"tmp = pd.DataFrame()\ntmp[\"fullVisitorId\"] = id_numeration_train.ravel()\ntmp['target'] = y_train\ntrn_user_target = tmp[['fullVisitorId', 'target']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5d0afde592e8b54e87bd646bd0402a2d6698cc2"},"cell_type":"code","source":"feat = categorial + list(set(full_data.columns) - set(categorial))\ncat_ind = [int(x) for x in np.arange(len(categorial))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc196e7c4acb76f1dbdbe68401413a51d33ae5ea"},"cell_type":"code","source":"full_data = full_data[feat]\nsub_full_data = sub_full_data[feat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04427f21247914e4862426d59dea976dd364bdf5"},"cell_type":"code","source":"features_to_scale = list(set(full_data.columns) - set(categorial) - set(trn_feats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb91732711435623f9d594174f5039779ad76f86"},"cell_type":"code","source":"scaler2 = SS()\nscaler2.fit(np.array(full_data[features_to_scale]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b21e4f881f7049a8ccb92eef31e78259d6e9a113"},"cell_type":"code","source":"full_data[features_to_scale] = scaler2.transform(np.array(full_data[features_to_scale]))\nsub_full_data[features_to_scale] = scaler2.transform(np.array(sub_full_data[features_to_scale]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dffd271b11e0de5f74bace5dcc56e486f1a06987"},"cell_type":"code","source":"params={'learning_rate': 0.05,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 200,\n        'verbose': 1,\n        \"subsample\": 0.99,\n        \"colsample_bytree\": 0.99,\n        \"random_state\":1543,\n        'max_depth': 14,\n        'lambda_l2': 0.02085548700474218,\n        'lambda_l1': 0.004107624022751344,\n        'bagging_fraction': 0.7934712636944741,\n        'feature_fraction': 0.686612409641711,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea4b1b929f713cc21a64ba00d630452b6cc9661e"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fddd32bf9ecd22c180a740a268be6603af5f2b4"},"cell_type":"code","source":"n_splits=5\nfolds2 = get_folds(np.array(trn_user_target.index), n_splits=n_splits)\n\noof_preds2 = np.zeros(full_data.shape[0])\nsub_preds2 = np.zeros(sub_full_data.shape[0])\nvis_importances = pd.DataFrame()\n\nfeat = categorial + list(set(full_data.columns) - set(categorial))\n\nfor fold_, (trn_, val_) in enumerate(folds2):\n    trn_x, trn_y = np.array(full_data.iloc[trn_][feat]), np.array(trn_user_target['target'].iloc[trn_])\n    val_x, val_y = np.array(full_data.iloc[val_][feat]), np.array(trn_user_target['target'].iloc[val_])\n    \n    reg = lgb.LGBMRegressor(**params,\n        #num_leaves=31,\n        # learning_rate=0.03,\n        n_estimators=1543,\n        # subsample=.93,\n        # colsample_bytree=.94,\n        # random_state=1543\n    )\n    start = time.time()\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=50,\n        categorical_feature=cat_ind\n    )\n    print(\"FIT TIME: {}\".format(time.time() - start))\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = feat\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)\n    \n    oof_preds2[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_preds2[oof_preds2 < 0] = 0\n    \n    # Make sure features are in the same order\n    _preds2 = reg.predict(np.array(sub_full_data[feat]), num_iteration=reg.best_iteration_)\n    _preds2[_preds2 < 0] = 0\n    sub_preds2 += _preds2 / n_splits # ensembling\n    \nprint(\"RMSE CV: \", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds2) ** .5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6711a7c4e96ae8da32cf0b6f8a53dcaa825f728"},"cell_type":"code","source":"oof_preds2.max(), _preds2.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e14259b4fab84f8982870d6bb10bc2e6780a8d3"},"cell_type":"code","source":"MIN_VAL2 = tune_threshold(\n    mean_squared_error(np.log1p(trn_user_target['target']), oof_preds2) ** .5,\n    oof_preds2, trn_user_target['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d09c88b0343a48d2158c94bf52bf274d634d530b"},"cell_type":"code","source":"vis_importances['gain_log'] = np.log1p(vis_importances['gain'])\nmean_gain = vis_importances[['gain', 'feature']].groupby('feature').mean()\nvis_importances['mean_gain'] = vis_importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 200))\nsns.barplot(x='gain_log', y='feature', data=vis_importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"835d2cdcbeb71042977144cef5da5936c454e4c8"},"cell_type":"code","source":"y_pred = np.expm1(sub_preds2)\ny_pred[y_pred < MIN_VAL2] = 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58c4710db78291a5338cd335a9c61fe494dc03d5"},"cell_type":"markdown","source":"**Submit prediction**"},{"metadata":{"trusted":true,"_uuid":"dac3fa8faafbcaf7474fa032c03fae0c05e8023d","_kg_hide-output":false},"cell_type":"code","source":"sub_full_data['PredictedLogRevenue'] = np.log1p(y_pred)\nsub_full_data[['PredictedLogRevenue']].to_csv('kernel_submit_1.csv', index=True) ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}