{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Necessary librarys\nimport os  # it's a operational system library, to set some informations\nimport random  # random is to generate random values\nfrom ast import literal_eval\nimport pandas as pd  # to manipulate data frames\nimport numpy as np  # to work with matrix\nfrom scipy.stats import kurtosis, skew  # it's to explore some statistics of numerical values\n\nimport matplotlib.pyplot as plt  # to graphics plot\nimport seaborn as sns  # a good library to graphic plots\n\nimport squarify  # to better understand proportion of categorys - it's a treemap layout algorithm\n\n# Importing librarys to use on interactive graphs\nfrom plotly.offline import init_notebook_mode, iplot, plot\nimport plotly.graph_objs as go\n\nimport json  # to convert json in df\nfrom pandas.io.json import json_normalize  # to normalize the json file\nfrom datetime import datetime\n\n# to set a style to all graphs\nplt.style.use('fivethirtyeight')\ninit_notebook_mode(connected=True)\npd.set_option('display.max_columns', 500)\n\n\ndef NumericalColumns(df):  # fillna numeric feature\n    df['totals.pageviews'].fillna(1, inplace=True)  # filling NA's with 1\n    df['totals.newVisits'].fillna(0, inplace=True)  # filling NA's with 0\n    df['totals.bounces'].fillna(0, inplace=True)  # filling NA's with 0\n    df['trafficSource.isTrueDirect'].fillna(False, inplace=True)  # filling boolean with False\n    df['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)  # filling boolean with True\n    df[\"totals.transactionRevenue\"] = df[\"totals.transactionRevenue\"].fillna(0.0).astype(float)  # filling NA with zero\n    df['totals.pageviews'] = df['totals.pageviews'].astype(int)  # setting numerical column as integer\n    df['totals.newVisits'] = df['totals.newVisits'].astype(int)  # setting numerical column as integer\n    df['totals.bounces'] = df['totals.bounces'].astype(int)  # setting numerical column as integer\n    df[\"totals.hits\"] = df[\"totals.hits\"].astype(float)  # setting numerical to float\n    df['totals.visits'] = df['totals.visits'].astype(int)  # seting as int\n    df['totals.totalTransactionRevenue'] = df[\"totals.totalTransactionRevenue\"].fillna(0.0).astype(float)\n    df['trafficSource.adwordsClickInfo.page'] = df['trafficSource.adwordsClickInfo.page'].fillna(0)\n    return df  # return the transformed dataframe\n\n\ndef delete_constant(df_train):\n    discovering_consts = [col for col in df_train.columns if df_train[col].nunique() == 1]\n    # drop constant columns and hits\n    df_train.drop(discovering_consts, axis=1, inplace=True)\n\n    return (df_train)\n\n\ndef process_custom(df_train):\n    df_train['customDimensions'] = df_train['customDimensions'].replace(to_replace=r'\\'', value='\\\"', regex=True)\n\n    df_train['customDimensions'] = df_train['customDimensions'].apply(literal_eval)\n    df_train['customDimensions'] = df_train['customDimensions'].map(lambda a: 0 if (a == []) else a[0]['value'])\n\n    df_train['totals.transactionRevenuelog'] = df_train['totals.transactionRevenue'].apply(lambda x: np.log1p(x))\n    return (df_train)\n\n\n# This function is to extract date features\ndef date_process(df):\n    df[\"date\"] = pd.to_datetime(df[\"date\"], format=\"%Y%m%d\")  # seting the column as pandas datetime\n    df[\"_weekday\"] = df['date'].dt.weekday  # extracting week day\n    df[\"_day\"] = df['date'].dt.day  # extracting day\n    df[\"_month\"] = df['date'].dt.month  # extracting day\n    df[\"_year\"] = df['date'].dt.year  # extracting day\n    df['_visitHour'] = (df['visitStartTime'].apply(lambda x: str(datetime.fromtimestamp(x).hour))).astype(int)\n    df['on_weekend'] = df['date'].apply(lambda x: x.dayofweek >= 5).astype('bool')\n    df['at_night'] = df['date'].apply(lambda x: x.hour <= 5 or x.hour >= 21).astype('bool')\n\n    return df  # returning the df after the transformation\n\n\ndef na_values(df_train):\n    # values to convert to NA\n    na_vals = ['unknown.unknown', '(not set)', 'not available in demo dataset',\n               '(not provided)', '(none)', '<NA>']\n\n    df_train = df_train.replace(na_vals, np.nan, regex=True)\n    return (df_train)\n\n\ndef categorical_data(df_train):\n    cat_cols = ['channelGrouping', 'device.operatingSystem', 'geoNetwork.region', 'geoNetwork.metro', 'geoNetwork.city',\n                'trafficSource.source',\n                '_day', '_month', '_weekday', '_year', 'totals.newVisits', 'device.browser', 'device.deviceCategory',\n                'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'geoNetwork.networkDomain',\n                'geoNetwork.region', 'geoNetwork.subContinent', 'trafficSource.keyword', 'trafficSource.medium',\n                'trafficSource.referralPath', 'trafficSource.source', ]\n    for column in cat_cols:\n        df_train[column] = df_train[column].astype('category')\n\n    return (df_train)\n\n\n\n\n# Code to transform the json format columns in table\ndef json_read(data_frame):\n    cols = list(pd.read_csv(data_frame, nrows=1))\n\n    columns = ['device', 'geoNetwork', 'totals', 'trafficSource']  # Columns that have json format\n    for df in pd.read_csv(data_frame, converters={column: json.loads for column in columns},\n                          dtype={'fullVisitorId': 'str'}, usecols =[i for i in cols if i != 'hits'], chunksize=100000):\n        print(df.head(2))\n        # Importing the dataset\n        df = df.replace(to_replace=r'\\'', value='\\\"', regex=True)\n        columns = ['device', 'geoNetwork', 'totals', 'trafficSource']  # Columns that have json format\n\n        for column in columns:  # loop to finally transform the columns in data frame\n            # It will normalize and set the json to a table\n            column_as_df = json_normalize(df[column])\n            # here will be set the name using the category and subcategory of json columns\n            column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n            column_as_df=column_as_df.set_index(df.index)\n            # after extracting the values, let drop the original columns\n            df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n        df = na_values(df)\n\n        df = NumericalColumns(df)\n        df = process_custom(df)\n        df = date_process(df)\n        df=delete_constant(df)\n        df.to_csv(\"New_train_v2.csv\", index=False, header=True, mode='a')\n        \n\nif __name__ == \"__main__\":\n\n# We will import the data using the name and extension that will be concatenated with dir_path\n      json_read('../input/train_v2.csv')\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd  # to manipulate data frames\ntestCSV=pd.read_csv('../input/test_v2.csv',sep=',')\ntestCSV.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e407456b4271698e804480f128cfe49628205d9e"},"cell_type":"code","source":"import pandas as pd\ntrainCSV = pd.read_csv('New_train_v2.csv', sep=',', encoding='utf-8', low_memory=False)\ntrainCSV.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff3aa0bbb51612d933b77d15a16444c0c379917c"},"cell_type":"code","source":"# the categorical variables for train  data\ncategorical_features_train = trainCSV.select_dtypes(include=[np.object])\ncategorical_features_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"035fa3779c745e78754cdad25ead346170e89cc7"},"cell_type":"code","source":"from datetime import timedelta\ncool_columns = ['fullVisitorId', 'date', 'geoNetwork.city', 'geoNetwork.country', 'device.browser',\n                'visitNumber',  'totals.totalTransactionRevenue','_weekday','_day','_month','_year','_visitHour','on_weekend']\n\ntrainCSV2 = trainCSV[cool_columns]\ntrainCSV2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c0ed449bd12a97340eaa29d5663e5b43142d78f"},"cell_type":"code","source":"trainCSV2.loc[:,['totals.totalTransactionRevenue']].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a7c05089b7bc9a6d548cc84c9ab43b2b59bf2e5"},"cell_type":"code","source":"#exemple*\nimport plotly.graph_objs as go\nfrom plotly.offline import init_notebook_mode, iplot\ntmp = trainCSV['date'].value_counts().to_frame().reset_index().sort_values('index')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c07b9f0bbb23f2e82ce3e40fd71d927654b24738"},"cell_type":"code","source":"#Visualization for Visits by date\nimport plotly.graph_objs as go\nfrom plotly.offline import init_notebook_mode, iplot\ntmp = trainCSV2['date'].value_counts().to_frame().reset_index().sort_values('index')\ntmp = tmp.rename(columns = {\"index\" : \"dateX\", \"date\" : \"visits\"})\n\ntr = go.Scatter(mode=\"lines\", x = tmp[\"dateX\"].astype(str), y = tmp[\"visits\"])\nlayout = go.Layout(title=\"Visits by Date\", height=400)\nfig = go.Figure(data = [tr], layout = layout)\niplot(fig)\n# Visualization for Visits by monthly revenue\ntmp = trainCSV2.groupby(\"date\").agg({\"totals.totalTransactionRevenue\" : \"mean\"}).reset_index()\ntmp = tmp.rename(columns = {\"date\" : \"dateX\", \"totals.totalTransactionRevenue\" : \"mean_revenue\"})\ntr = go.Scatter(mode=\"lines\", x = tmp[\"dateX\"].astype(str), y = tmp[\"mean_revenue\"])\nlayout = go.Layout(title=\"Monthly Revenue by Date\", height=400)\nfig = go.Figure(data = [tr], layout = layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a744e27133a66f8b6862cc7fe4e1706227ece68c"},"cell_type":"code","source":"#Create a train and validation sets \nfrom sklearn.model_selection import train_test_split\nfeatures = [c for c in trainCSV2.columns]\nfeatures.remove(\"totals.totalTransactionRevenue\")\ntrainCSV2[\"totals.totalTransactionRevenue\" ] = np.log1p(trainCSV2[\"totals.totalTransactionRevenue\"].astype(float))\ntrain_x, valid_x, train_y, valid_y = train_test_split(trainCSV2[features], \n                                                      train[\"totals.totalTransactionRevenue\"], test_size=0.25, random_state=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c943778153a66c757c8069d9def9aebc4631d7bd"},"cell_type":"code","source":"#Create LGBM model and train it !\nimport lightgbm as lgb \n\nlgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\",\n              \"num_leaves\" : 50, \"learning_rate\" : 0.02, \n              \"bagging_fraction\" : 0.75, \"feature_fraction\" : 0.8, \"bagging_frequency\" : 9}\n    \nlgb_train = lgb.Dataset(train_x, label=train_y)\nlgb_val = lgb.Dataset(valid_x, label=valid_y)\nmodel = lgb.train(lgb_params, lgb_train, 700, valid_sets=[lgb_val], early_stopping_rounds=150, verbose_eval=20)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aecd1be57f174cc3fac9af4cb2254513ecefb963"},"cell_type":"code","source":"clf = RandomForestClassifier(class_weight='balanced')\nparam_grid = {\n    'min_samples_leaf' : [2, 5, 10, 20], # Best:\n    'max_depth': [2, 10, 20], # Best:\n    'n_estimators': [5, 20, 100, 200] # Best:      \n}\n\nsearch = GridSearchCV(clf, param_grid)\nsearch.fit(train_x, train_y)\n\ny_predicted_probability = search.predict_proba(valid_x)[:,1]\nplot_roc_curve(valid_y, y_predicted_probability, title=\"ROC in test set RF\")\nplot_precision_recall_curve(valid_y, y_predicted_probability)\n\nall_features =pd.DataFrame({'feature': train_x.columns, 'importance': search.best_estimator_.feature_importances_})\nall_features = all_features.sort_values(by=['importance'], ascending=False).set_index('feature')\nall_features[all_features.importance>0].plot.bar()\n\nprint(\"Best parameter (CV score=%0.3f):\" % search.best_score_)\nprint(search.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"216c2f4a8869b0795c4c952d11aa6edcb837e2e6"},"cell_type":"code","source":"clf = XGBClassifier()\nparam_grid = {\n    'max_depth':[8],    \n    'n_estimators': [3], \n    'learning_rate' : [0.1], \n    'min_child_weight' : [100], \n    'reg_lambda': [30] \n}\n\nsearch = GridSearchCV(clf, param_grid)\nsearch.fit(train_x, train_y)\n\ny_predicted_probability = search.predict_proba(valid_x)[:,1]\nplot_roc_curve(valid_y, y_predicted_probability, title=\"ROC in test set RF\")\nplot_precision_recall_curve(valid_y, y_predicted_probability)\n\nall_features =pd.DataFrame({'feature': train_y.columns, 'importance': search.best_estimator_.feature_importances_})\nall_features = all_features.sort_values(by=['importance'], ascending=False).set_index('feature')\nall_features[all_features.importance>0].plot.bar()\n\nprint(\"Best parameter (CV score=%0.3f):\" % search.best_score_)\nprint(search.best_params_)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}