{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-30T19:41:35.540435Z","iopub.execute_input":"2021-11-30T19:41:35.540721Z","iopub.status.idle":"2021-11-30T19:41:36.563204Z","shell.execute_reply.started":"2021-11-30T19:41:35.540693Z","shell.execute_reply":"2021-11-30T19:41:36.562282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    start_mem_usg = df.memory_usage(deep=True).sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in df.select_dtypes(exclude=['object', 'datetime64[ns]']).columns:      \n        if not df[col].isna().sum():\n            # Print current column type\n            print(\"******************************\")\n            print(\"Column: \",col)\n            print(\"dtype before: \",df[col].dtype)\n\n            # make variables for Int, max and min\n            IsInt = False\n            mx = df[col].max()\n            mn = df[col].min()\n\n            # test if column can be converted to an integer\n            asint = df[col].fillna(0).astype(np.int64)\n            result = (df[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True\n\n\n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        df[col] = df[col].astype(np.uint8)\n                    elif mx < 65535:\n                        df[col] = df[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        df[col] = df[col].astype(np.uint32)\n                    else:\n                        df[col] = df[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)    \n\n            # Make float datatypes 32 bit\n            else:\n                df[col] = df[col].astype(np.float32)\n\n            # Print new column type\n            print(\"dtype after: \",df[col].dtype)\n            print(\"******************************\")\n\n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = df.memory_usage(deep=True).sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return df\n\ndef get_fresh_dataset():\n    return pd.read_csv('/kaggle/input/ga-customer-revenue-prediction/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:41:36.565099Z","iopub.execute_input":"2021-11-30T19:41:36.565581Z","iopub.status.idle":"2021-11-30T19:41:36.584689Z","shell.execute_reply.started":"2021-11-30T19:41:36.565537Z","shell.execute_reply":"2021-11-30T19:41:36.583868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = get_fresh_dataset()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:41:36.585872Z","iopub.execute_input":"2021-11-30T19:41:36.586231Z","iopub.status.idle":"2021-11-30T19:42:00.787904Z","shell.execute_reply.started":"2021-11-30T19:41:36.586203Z","shell.execute_reply":"2021-11-30T19:42:00.785582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.memory_usage(deep=True).sum() / 1024**2","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:00.790395Z","iopub.execute_input":"2021-11-30T19:42:00.791153Z","iopub.status.idle":"2021-11-30T19:42:02.236624Z","shell.execute_reply.started":"2021-11-30T19:42:00.791106Z","shell.execute_reply":"2021-11-30T19:42:02.235981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:02.237628Z","iopub.execute_input":"2021-11-30T19:42:02.238179Z","iopub.status.idle":"2021-11-30T19:42:02.242098Z","shell.execute_reply.started":"2021-11-30T19:42:02.238143Z","shell.execute_reply":"2021-11-30T19:42:02.241486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device_columns = ['device_' + i for i in json.loads(df_train.device.iloc[0]).keys()]\n# geo_columns = ['geo_' + i for i in json.loads(df_train.geoNetwork.iloc[0]).keys()]\n# total_columns = ['total_' + i for i in json.loads(df_train.totals.iloc[0]).keys()]\n# traffic_columns = ['traffic_' + i for i in json.loads(df_train.trafficSource.iloc[0]).keys()]","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:50:25.18601Z","iopub.execute_input":"2021-11-29T20:50:25.186393Z","iopub.status.idle":"2021-11-29T20:50:25.196965Z","shell.execute_reply.started":"2021-11-29T20:50:25.186341Z","shell.execute_reply":"2021-11-29T20:50:25.196003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(device_columns, len(device_columns))\n# print(geo_columns, len(geo_columns))\n# print(total_columns, len(total_columns))\n# print(traffic_columns, len(traffic_columns))","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:50:25.198675Z","iopub.execute_input":"2021-11-29T20:50:25.199019Z","iopub.status.idle":"2021-11-29T20:50:25.210193Z","shell.execute_reply.started":"2021-11-29T20:50:25.198953Z","shell.execute_reply":"2021-11-29T20:50:25.209444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# type(df_train.device.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:50:25.211738Z","iopub.execute_input":"2021-11-29T20:50:25.211982Z","iopub.status.idle":"2021-11-29T20:50:25.222565Z","shell.execute_reply.started":"2021-11-29T20:50:25.211951Z","shell.execute_reply":"2021-11-29T20:50:25.221721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def parse_json(x):\n#     return [value for key, value in json.loads(x).items()]\n\ndevice_json = [json.loads(x) for x in df_train.device]\n\ndf_device = pd.DataFrame.from_records(device_json)\n# df_train[device_columns] = df_train['device'].apply(lambda x: parse_json(x))","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:02.243358Z","iopub.execute_input":"2021-11-30T19:42:02.243604Z","iopub.status.idle":"2021-11-30T19:42:16.47828Z","shell.execute_reply.started":"2021-11-30T19:42:02.243577Z","shell.execute_reply":"2021-11-30T19:42:16.477351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_device.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:16.479802Z","iopub.execute_input":"2021-11-30T19:42:16.480123Z","iopub.status.idle":"2021-11-30T19:42:16.483981Z","shell.execute_reply.started":"2021-11-30T19:42:16.480082Z","shell.execute_reply":"2021-11-30T19:42:16.483121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_device.memory_usage(deep=True).sum() / 1024**2","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:16.485829Z","iopub.execute_input":"2021-11-30T19:42:16.486181Z","iopub.status.idle":"2021-11-30T19:42:20.171644Z","shell.execute_reply.started":"2021-11-30T19:42:16.486134Z","shell.execute_reply":"2021-11-30T19:42:20.170699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"geo_json = [json.loads(x) for x in df_train.geoNetwork]\ndf_geo = pd.DataFrame.from_records(geo_json)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:20.173369Z","iopub.execute_input":"2021-11-30T19:42:20.173717Z","iopub.status.idle":"2021-11-30T19:42:30.918534Z","shell.execute_reply.started":"2021-11-30T19:42:20.173676Z","shell.execute_reply":"2021-11-30T19:42:30.917776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_geo.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:30.921182Z","iopub.execute_input":"2021-11-30T19:42:30.921673Z","iopub.status.idle":"2021-11-30T19:42:32.503751Z","shell.execute_reply.started":"2021-11-30T19:42:30.921624Z","shell.execute_reply":"2021-11-30T19:42:32.503081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_geo.memory_usage(deep=True).sum() / 1024 ** 2","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:32.50495Z","iopub.execute_input":"2021-11-30T19:42:32.505547Z","iopub.status.idle":"2021-11-30T19:42:35.397998Z","shell.execute_reply.started":"2021-11-30T19:42:32.505514Z","shell.execute_reply":"2021-11-30T19:42:35.397351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"totals_json = [json.loads(x) for x in df_train.totals]\ndf_totals = pd.DataFrame.from_records(totals_json)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:35.398865Z","iopub.execute_input":"2021-11-30T19:42:35.399098Z","iopub.status.idle":"2021-11-30T19:42:40.72143Z","shell.execute_reply.started":"2021-11-30T19:42:35.39907Z","shell.execute_reply":"2021-11-30T19:42:40.720464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traffic_json = [json.loads(x) for x in df_train.trafficSource]\ndf_traffic = pd.DataFrame.from_records(traffic_json)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:40.722759Z","iopub.execute_input":"2021-11-30T19:42:40.723017Z","iopub.status.idle":"2021-11-30T19:42:51.110675Z","shell.execute_reply.started":"2021-11-30T19:42:40.722975Z","shell.execute_reply":"2021-11-30T19:42:51.110024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_totals.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:51.111688Z","iopub.execute_input":"2021-11-30T19:42:51.112275Z","iopub.status.idle":"2021-11-30T19:42:51.61882Z","shell.execute_reply.started":"2021-11-30T19:42:51.112241Z","shell.execute_reply":"2021-11-30T19:42:51.618275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.drop(columns=JSON_COLUMNS, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:56.112533Z","iopub.execute_input":"2021-11-30T19:42:56.112948Z","iopub.status.idle":"2021-11-30T19:42:56.220501Z","shell.execute_reply.started":"2021-11-30T19:42:56.112917Z","shell.execute_reply":"2021-11-30T19:42:56.219476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.concat([df_train, df_totals, df_traffic, df_geo, df_device], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:42:59.628953Z","iopub.execute_input":"2021-11-30T19:42:59.62927Z","iopub.status.idle":"2021-11-30T19:43:00.801051Z","shell.execute_reply.started":"2021-11-30T19:42:59.629238Z","shell.execute_reply":"2021-11-30T19:43:00.800056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.memory_usage(deep=True).sum() / 1024 ** 2","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:10.428155Z","iopub.execute_input":"2021-11-30T19:43:10.428439Z","iopub.status.idle":"2021-11-30T19:43:20.459907Z","shell.execute_reply.started":"2021-11-30T19:43:10.428405Z","shell.execute_reply":"2021-11-30T19:43:20.458903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_geo, df_totals, df_traffic, df_device","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:20.461551Z","iopub.execute_input":"2021-11-30T19:43:20.461802Z","iopub.status.idle":"2021-11-30T19:43:20.989406Z","shell.execute_reply.started":"2021-11-30T19:43:20.461775Z","shell.execute_reply":"2021-11-30T19:43:20.988432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.tail()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:20.991342Z","iopub.execute_input":"2021-11-30T19:43:20.992013Z","iopub.status.idle":"2021-11-30T19:43:21.027033Z","shell.execute_reply.started":"2021-11-30T19:43:20.991949Z","shell.execute_reply":"2021-11-30T19:43:21.026076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(df_train.dtypes)\n# print(df_train.isna().sum())\n\n# import datetime\n\n## Parsing date\n# df_train['date'] = df_train['date'].apply(lambda x: datetime.datetime(x[:4], ))\n\ndf_train = df_train[~df_train.date.isna()]","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:21.029239Z","iopub.execute_input":"2021-11-30T19:43:21.029753Z","iopub.status.idle":"2021-11-30T19:43:25.296185Z","shell.execute_reply.started":"2021-11-30T19:43:21.029706Z","shell.execute_reply":"2021-11-30T19:43:25.295298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum() * 100 / len(df_train)\n\n'''\nCampaign code has 99% nulls. Drop it.\nImpute transaction revenue with 0. \n'''\n\ndf_train = df_train[df_train.columns.difference(['campaignCode', 'adContent'])]\ndf_train.transactionRevenue.fillna(0, inplace=True)\ndf_train.isna().sum() * 100 / len(df_train)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:25.297928Z","iopub.execute_input":"2021-11-30T19:43:25.298245Z","iopub.status.idle":"2021-11-30T19:43:37.513264Z","shell.execute_reply.started":"2021-11-30T19:43:25.298205Z","shell.execute_reply":"2021-11-30T19:43:37.512285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Impute bounces with 0\n## Impute isTrueDirect with False\n## Impute newVisits with False\ndf_train.bounces.fillna(0, inplace=True)\ndf_train.isTrueDirect.fillna(False, inplace=True)\ndf_train.newVisits.fillna(False, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:37.515452Z","iopub.execute_input":"2021-11-30T19:43:37.515778Z","iopub.status.idle":"2021-11-30T19:43:38.778462Z","shell.execute_reply.started":"2021-11-30T19:43:37.515734Z","shell.execute_reply":"2021-11-30T19:43:38.777486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[df_train.columns.difference(['keyword', 'referralPath'])]","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:38.779727Z","iopub.execute_input":"2021-11-30T19:43:38.779974Z","iopub.status.idle":"2021-11-30T19:43:40.728852Z","shell.execute_reply.started":"2021-11-30T19:43:38.779943Z","shell.execute_reply":"2021-11-30T19:43:40.728156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dropping the 100 nulls in pageviews\ndf_train = df_train[df_train['pageviews'] != 'NaN']\ndf_train = df_train[~df_train['pageviews'].isna()]","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:43:40.730664Z","iopub.execute_input":"2021-11-30T19:43:40.730888Z","iopub.status.idle":"2021-11-30T19:43:44.025531Z","shell.execute_reply.started":"2021-11-30T19:43:40.730862Z","shell.execute_reply":"2021-11-30T19:43:44.024559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## No nulls\ndf_train.isna().sum() * 100 / len(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:51:17.86847Z","iopub.execute_input":"2021-11-30T19:51:17.869235Z","iopub.status.idle":"2021-11-30T19:51:23.247036Z","shell.execute_reply.started":"2021-11-30T19:51:17.869174Z","shell.execute_reply":"2021-11-30T19:51:23.246085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train[df_train.transactionRevenue != 0]) * 100 / len(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:52:06.521815Z","iopub.execute_input":"2021-11-30T19:52:06.522111Z","iopub.status.idle":"2021-11-30T19:52:06.69735Z","shell.execute_reply.started":"2021-11-30T19:52:06.522081Z","shell.execute_reply":"2021-11-30T19:52:06.696463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop adwordsClickInfo\ndf_train = df_train[df_train.columns.difference(['adwordsClickInfo'])]","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:11.442977Z","iopub.execute_input":"2021-11-29T20:52:11.443391Z","iopub.status.idle":"2021-11-29T20:52:13.343498Z","shell.execute_reply.started":"2021-11-29T20:52:11.443345Z","shell.execute_reply":"2021-11-29T20:52:13.342338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns\n## For timeseries\n\n# There will be filter by country, continent, and city. visitStartTime, transactionRevenue, week, month, year\n\ndf_train['hour'] = df_train.visitStartTime.dt.hour\ndf_train['minutes'] = df_train.visitStartTime.dt.minute\ndf_train['seconds'] = df_train.visitStartTime.dt.second\ndf_train['month_name'] = df_train.visitStartTime.dt.month_name()\ndf_train['day_of_week'] = df_train.visitStartTime.dt.day_name()\ndf_train['year'] = df_train.visitStartTime.dt.year\n\ndf_timeseries = df_train[['continent', 'country', 'city', 'visitStartTime', 'day_of_week', 'month_name', 'year', 'transactionRevenue']]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.347573Z","iopub.execute_input":"2021-11-29T20:52:13.347871Z","iopub.status.idle":"2021-11-29T20:52:13.41658Z","shell.execute_reply.started":"2021-11-29T20:52:13.347837Z","shell.execute_reply":"2021-11-29T20:52:13.415083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_timeseries.to_csv('timeseries_dataset.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.418001Z","iopub.status.idle":"2021-11-29T20:52:13.418372Z","shell.execute_reply.started":"2021-11-29T20:52:13.418191Z","shell.execute_reply":"2021-11-29T20:52:13.418209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"['bounces', 'browser', 'browserSize', 'browserVersion', 'campaign',\n       'channelGrouping', 'city', 'cityId', 'continent', 'country', 'date',\n       'deviceCategory', 'flashVersion', 'fullVisitorId', 'hits', 'isMobile',\n       'isTrueDirect', 'language', 'latitude', 'longitude', 'medium', 'metro',\n       'mobileDeviceBranding', 'mobileDeviceInfo', 'mobileDeviceMarketingName',\n       'mobileDeviceModel', 'mobileInputSelector', 'networkDomain',\n       'networkLocation', 'newVisits', 'operatingSystem',\n       'operatingSystemVersion', 'pageviews', 'region', 'screenColors',\n       'screenResolution', 'sessionId', 'socialEngagementType', 'source',\n       'subContinent', 'transactionRevenue', 'visitId', 'visitNumber',\n       'visitStartTime', 'visits']","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.4193Z","iopub.status.idle":"2021-11-29T20:52:13.419611Z","shell.execute_reply.started":"2021-11-29T20:52:13.419442Z","shell.execute_reply":"2021-11-29T20:52:13.419459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nQuick thoughts on plots. \nContinents, country, city, long, and lat are given plot them on maps using plotly or the library(starts with f or something) taught in class. \nCheck seasonality, trend, using date\nCheck when do people most visit, least visit using visitStartTime\nCheck 80/20 rule. 80% revenue comes from 20% users. \nRevenue distribution(histogram) of these 20% users. \nFrom here on focuse on 20% users and plot the following: \n1. What devices they use(mobile or desktop)? (plot for both 80% and 20%)\n2. What OS they use?(plot for both 80% and 20%)\n3. What browser they use? (plot for both 80% and 20%)\n4. Revenue per browser? \n5. What device type has highest conversion rate(customer that produce revenue / total number of customers)\n6. Where do the 20% and 80% live(continent, country, subcontinent, region bar plot)\n7. What network domain do 20% and 80% use(see if there is any difference)?\n8. What is the traffic source? \n9. Check distribution of pageviews and Total hits for 20% and 80%. \n\nPlot following timeseries plot: \n1. Time vs revenue(over the years, monthly, weekly, daily plot workday and weekend seperatly for weekly plot\nand check the device type, if it changes and also plot trend line)\n\nPlot following map plot:\n1. Plot revenue % on map\n2. Plot device type on map\n3. Highlight top 3 revenue generating states\n4. Highlight top 3 states with most users(20% and overall)\n5. If possible add a time slider to see how the revenue changes over years, months, weekday, weekend(Try with plotly). \n'''\nTEMPORAL_COLUMNS = [\n    'visitStartTime',\n    'date'\n]\nSPACIAL_COLUMNS = [\n    'city',\n    'continent', \n    'country', \n    'latitude',\n    'longitude',\n    'region',\n    'subContinent'\n]\n\n## parsing date\ndf_train['date'] = df_train['date'].astype('str')\ndf_train['date'] = pd.to_datetime([f'{x[:4]}-{x[4:6]}-{x[6:]}' for x in df_train['date']])","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.420469Z","iopub.status.idle":"2021-11-29T20:52:13.42078Z","shell.execute_reply.started":"2021-11-29T20:52:13.420621Z","shell.execute_reply":"2021-11-29T20:52:13.420637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['date'] = df_train['date'].dt.date","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.422524Z","iopub.status.idle":"2021-11-29T20:52:13.422839Z","shell.execute_reply.started":"2021-11-29T20:52:13.422676Z","shell.execute_reply":"2021-11-29T20:52:13.422691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['date'] = pd.to_datetime(df_train['date'])","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.423765Z","iopub.status.idle":"2021-11-29T20:52:13.424066Z","shell.execute_reply.started":"2021-11-29T20:52:13.423908Z","shell.execute_reply":"2021-11-29T20:52:13.423924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Parsing visitstarttime\n\ndf_train[['visitStartTime', 'date']]\ndf_train.visitStartTime.dtype\ndf_train.visitStartTime = pd.to_datetime(df_train.visitStartTime, unit='s')","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.425343Z","iopub.status.idle":"2021-11-29T20:52:13.425668Z","shell.execute_reply.started":"2021-11-29T20:52:13.425502Z","shell.execute_reply":"2021-11-29T20:52:13.425519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['hour'] = df_train.visitStartTime.dt.hour\ndf_train['minutes'] = df_train.visitStartTime.dt.minute\ndf_train['seconds'] = df_train.visitStartTime.dt.second\ndf_train['month_name'] = df_train.visitStartTime.dt.month_name()\ndf_train['day_of_week'] = df_train.visitStartTime.dt.day_name()\ndf_train['year'] = df_train.visitStartTime.dt.year","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.426823Z","iopub.status.idle":"2021-11-29T20:52:13.427231Z","shell.execute_reply.started":"2021-11-29T20:52:13.426966Z","shell.execute_reply":"2021-11-29T20:52:13.426981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Drop date as we already have visitStartTime\ndf_train = df_train[df_train.columns.difference(['date'])]","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.428489Z","iopub.status.idle":"2021-11-29T20:52:13.428803Z","shell.execute_reply.started":"2021-11-29T20:52:13.428637Z","shell.execute_reply":"2021-11-29T20:52:13.428654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Drop IDs to save space\ndf_train.columns","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.43097Z","iopub.status.idle":"2021-11-29T20:52:13.431328Z","shell.execute_reply.started":"2021-11-29T20:52:13.431157Z","shell.execute_reply":"2021-11-29T20:52:13.431174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip3 install plotly","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.432937Z","iopub.status.idle":"2021-11-29T20:52:13.433427Z","shell.execute_reply.started":"2021-11-29T20:52:13.433179Z","shell.execute_reply":"2021-11-29T20:52:13.433205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nQuick thoughts on plots. \nContinents, country, city, long, and lat are given plot them on maps using plotly or the library(starts with f or something) taught in class. \nCheck seasonality, trend, using date\nCheck when do people most visit, least visit using visitStartTime\nCheck 80/20 rule. 80% revenue comes from 20% users. \nRevenue distribution(histogram) of these 20% users. \nFrom here on focuse on 20% users and plot the following: \n1. What devices they use(mobile or desktop)? (plot for both 80% and 20%)\n2. What OS they use?(plot for both 80% and 20%)\n3. What browser they use? (plot for both 80% and 20%)\n4. Revenue per browser? \n5. What device type has highest conversion rate(customer that produce revenue / total number of customers)\n6. Where do the 20% and 80% live(continent, country, subcontinent, region bar plot)\n7. What network domain do 20% and 80% use(see if there is any difference)?\n8. What is the traffic source? \n9. Check distribution of pageviews and Total hits for 20% and 80%. \n\nPlot following timeseries plot: \n1. Time vs revenue(over the years, monthly, weekly, daily plot workday and weekend seperatly for weekly plot\nand check the device type, if it changes and also plot trend line)\n\nPlot following map plot:\n1. Plot revenue % on map\n2. Plot device type on map\n3. Highlight top 3 revenue generating states\n4. Highlight top 3 states with most users(20% and overall)\n5. If possible add a time slider to see how the revenue changes over years, months, weekday, weekend(Try with plotly). \n'''\n\n# transactionRevenue is object. Converting it to float32\ndf_train.transactionRevenue = df_train.transactionRevenue.astype(np.float32)\n## Plotting timeseries\n# df_train['year'] = df_train.visitStartTime.dt.year\n# df_train['month'] = df_train.visitStartTime.dt.month\n# df_train.set_index('visitStartTime', inplace=True)\ndf_time_of_day = df_train.copy(deep=True)\ndf_time_of_day['hour'] = df_time_of_day.visitStartTime.dt.hour\ndf_time_of_day = df_time_of_day[['hour', 'transactionRevenue']]\ndf_month = df_train.groupby(pd.Grouper(key='visitStartTime', freq='M')).agg('sum').reset_index()[['visitStartTime', 'transactionRevenue']]\ndf_year = df_train.groupby(pd.Grouper(key='visitStartTime', freq='Y')).agg('sum').reset_index()[['visitStartTime', 'transactionRevenue']]\ndf_week = df_train.groupby(pd.Grouper(key='visitStartTime', freq='W')).agg('sum').reset_index()[['visitStartTime', 'transactionRevenue']]\ndf_day = df_train.groupby(pd.Grouper(key='visitStartTime', freq='D')).agg('sum').reset_index()[['visitStartTime', 'transactionRevenue']]\ndf_time_of_day = df_time_of_day.groupby('hour').agg('sum').reset_index()[['hour', 'transactionRevenue']]\ndf_day_of_week = df_train.groupby('day_of_week').agg('sum').reset_index()[['day_of_week', 'transactionRevenue']]\n# df_year = df_year[['year', 'month', 'transactionRevenue']]\n","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.435299Z","iopub.status.idle":"2021-11-29T20:52:13.435807Z","shell.execute_reply.started":"2021-11-29T20:52:13.435542Z","shell.execute_reply":"2021-11-29T20:52:13.435569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.439465Z","iopub.status.idle":"2021-11-29T20:52:13.439869Z","shell.execute_reply.started":"2021-11-29T20:52:13.439683Z","shell.execute_reply":"2021-11-29T20:52:13.439702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure()\n\nyear_subplot=fig.add_subplot(3, 2, 1)\nmonthly_subplot = fig.add_subplot(3, 2, 2)\nweekly_subplot=fig.add_subplot(3, 2, 3)\ndaily_subplot = fig.add_subplot(3, 2, 4)\n# store_nbr_subplot = fig.add_subplot(3, 2)\n\nfigure_size = (25, 25)\n\n# xticks = df_family.index\n\ndf_year[['visitStartTime', 'transactionRevenue']].plot(kind='line', figsize=figure_size, ax=year_subplot)\n# family_subplot.set_title(\"Sales trend by family\")\n# family_subplot.set_ylabel(\"Average sales\")\n# family_subplot.set_xlabel(\"Years\")\n# family_subplot.set_xticks(df_family.index)\n# family_subplot.set_xticklabels(df_family.index, rotation=90)\n\ndf_month[['visitStartTime', 'transactionRevenue']].plot(kind='line', figsize=figure_size, ax=monthly_subplot)\n# city_subplot.set_title(\"Sales trend by city\")\n# city_subplot.set_ylabel(\"Average sales\")\n# city_subplot.set_xlabel(\"Years\")\n\ndf_week[['visitStartTime', 'transactionRevenue']].plot(kind='line', figsize=figure_size, ax=weekly_subplot)\n# locale_subplot.set_title(\"Sales trend by locale\")\n# locale_subplot.set_ylabel(\"Average sales\")\n# locale_subplot.set_xlabel(\"Years\")\n\ndf_day[['visitStartTime', 'transactionRevenue']].plot(kind='line', figsize=figure_size, ax=daily_subplot)\n# state_subplot.set_title(\"Sales trend by state\")\n# state_subplot.set_ylabel(\"Average sales\")\n# state_subplot.set_xlabel(\"Years\")\n\nplt.subplot(2, 2, 1)\nplt.plot(list(df_year['visitStartTime']), list(df_year['transactionRevenue']))\nplt.subplot(2, 2, 2)\nplt.plot(list(df_month['visitStartTime']), list(df_month['transactionRevenue']))\nplt.subplot(2, 2, 3)\nplt.plot(list(df_week['visitStartTime']), list(df_week['transactionRevenue']))\nplt.subplot(2, 2, 4)\nplt.plot(list(df_day['visitStartTime']), list(df_day['transactionRevenue']))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.44215Z","iopub.status.idle":"2021-11-29T20:52:13.442509Z","shell.execute_reply.started":"2021-11-29T20:52:13.442333Z","shell.execute_reply":"2021-11-29T20:52:13.442357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('whitegrid')\nfigure_size = (25, 25)\nsns.set(rc={'figure.figsize':(25,25)})\n\ndf_day_ma = df_day.copy(deep=True)\ndf_day_ma['rolling_average'] = df_day_ma.transactionRevenue.rolling(7).mean().shift(-3)\ndf_day_ma = df_day_ma[['visitStartTime', 'rolling_average']]\n\ndf_week_ma = df_week.copy(deep=True)\ndf_week_ma['rolling_average'] = df_week_ma.transactionRevenue.rolling(7).mean().shift(-3)\ndf_week_ma = df_week_ma[['visitStartTime', 'rolling_average']]\n\n\n\n\nplt.subplot(2, 3, 1)\nsns.lineplot(x='visitStartTime', y='transactionRevenue', data=df_day)\nsns.lineplot(x='visitStartTime', y='rolling_average', data=df_day_ma)\nplt.subplot(2, 3, 2)\nsns.lineplot(x='visitStartTime', y='transactionRevenue', data=df_week)\nsns.lineplot(x='visitStartTime', y='rolling_average', data=df_week_ma)\nplt.subplot(2, 3, 3)\nsns.lineplot(x='visitStartTime', y='transactionRevenue', data=df_month)\nplt.subplot(2, 3, 4)\nsns.lineplot(x='hour', y='transactionRevenue', data=df_time_of_day)\nplt.subplot(2, 3, 5)\nsns.lineplot(x='day_of_week', y='transactionRevenue', data=df_day_of_week)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T20:52:13.44363Z","iopub.status.idle":"2021-11-29T20:52:13.44395Z","shell.execute_reply.started":"2021-11-29T20:52:13.443787Z","shell.execute_reply":"2021-11-29T20:52:13.443803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install dash==0.29.0","metadata":{"execution":{"iopub.status.busy":"2021-11-29T01:43:49.19003Z","iopub.execute_input":"2021-11-29T01:43:49.190351Z","iopub.status.idle":"2021-11-29T01:44:04.67493Z","shell.execute_reply.started":"2021-11-29T01:43:49.190319Z","shell.execute_reply":"2021-11-29T01:44:04.673658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dash\nimport dash.dash.dash_html_components as html\nfrom dash.dependencies import Input, Output\nfrom dash.exceptions import PreventUpdate\n\nexternal_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']\n\napp = dash.Dash(__name__, external_stylesheets=external_stylesheets)\n\napp.layout = html.Div([\n    html.Button('Click here to see the content', id='show-secret'),\n    html.Div(id='body-div')\n])\n\n@app.callback(\n    Output(component_id='body-div', component_property='children'),\n    Input(component_id='show-secret', component_property='n_clicks')\n)\ndef update_output(n_clicks):\n    if n_clicks is None:\n        raise PreventUpdate\n    else:\n        return \"Elephants are the only animal that can't jump\"\n\nif __name__ == '__main__':\n    app.run_server(debug=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-29T01:44:04.677068Z","iopub.execute_input":"2021-11-29T01:44:04.677443Z","iopub.status.idle":"2021-11-29T01:44:05.071598Z","shell.execute_reply.started":"2021-11-29T01:44:04.677398Z","shell.execute_reply":"2021-11-29T01:44:05.070295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nfrom IPython.display import HTML\n\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv()\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n\n# create a link to download the dataframe\ncreate_download_link(df_timeseries)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T04:55:52.673586Z","iopub.execute_input":"2021-11-29T04:55:52.673874Z","iopub.status.idle":"2021-11-29T04:55:59.877438Z","shell.execute_reply.started":"2021-11-29T04:55:52.673842Z","shell.execute_reply":"2021-11-29T04:55:59.876742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_timeseries = pd.read_csv('./kaggle/input/timseries_dataset.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-29T05:24:07.697336Z","iopub.execute_input":"2021-11-29T05:24:07.698105Z","iopub.status.idle":"2021-11-29T05:24:07.725167Z","shell.execute_reply.started":"2021-11-29T05:24:07.698068Z","shell.execute_reply":"2021-11-29T05:24:07.724074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import the modules we'll need\nfrom IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport base64\n\n# function that takes in a dataframe and creates a text link to  \n# download it (will only work for files < 2MB or so)\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv()\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n# create a random sample dataframe\n# df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))\n\n# create a link to download the dataframe\ncreate_download_link(df_timeseries)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_filters = df_train[['continent', 'country', 'city']]","metadata":{},"execution_count":null,"outputs":[]}]}