{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-18T13:00:46.83961Z","iopub.execute_input":"2021-08-18T13:00:46.839978Z","iopub.status.idle":"2021-08-18T13:00:46.850986Z","shell.execute_reply.started":"2021-08-18T13:00:46.839943Z","shell.execute_reply":"2021-08-18T13:00:46.850261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The aim of this project is to define an easy framework for regression problems**","metadata":{}},{"cell_type":"markdown","source":"**Define a mesurable objectif :**\n\n* Predict the transaction value (continue value)\n* Metrics :  - Root Mean Squared Error (RMSE): mean_squared_error(y_true, y_pred, squared=False) #give more importance to errors\n* Objectif : RMSE < 2","metadata":{}},{"cell_type":"markdown","source":"**We start by reading the data : Some columns are in Json format, we will use a code that almost all kernels are using and for this work we will use only 10% of the initial data to experiment many machine learning algorithm**","metadata":{}},{"cell_type":"code","source":"# Necessary librarys\nimport random # random is to generate random values\n\n# library of datetime\nfrom datetime import datetime\n\nimport matplotlib.pyplot as plt # to graphics plot\nimport seaborn as sns # a good library to graphic plots\n\nimport json # to convert json in df\nfrom pandas import json_normalize # to normalize the json file","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:00:46.85211Z","iopub.execute_input":"2021-08-18T13:00:46.852522Z","iopub.status.idle":"2021-08-18T13:00:46.862648Z","shell.execute_reply.started":"2021-08-18T13:00:46.852483Z","shell.execute_reply":"2021-08-18T13:00:46.861789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code to transform the json format columns in table\ndef json_read(df):\n\n    columns = ['device', 'geoNetwork', 'totals', 'trafficSource'] # Columns that have json format\n\n    # p is a fractional number to skiprows and read just a random sample of the our dataset.\n    p = 0.1 # *** In this case we will use 10% of data set *** #\n\n    #joining the [ path + df received]\n    data_frame = df\n\n    #Importing the dataset\n    random.seed(0)\n    df = pd.read_csv(data_frame,\n                     converters={column: json.loads for column in columns}, # loading the json columns properly\n                     dtype={'fullVisitorId': 'str'}, # transforming this column to string\n                     skiprows=lambda i: i>0 and random.random() > p # Number of rows that will be imported randomly\n                     )\n\n    for column in columns: #loop to finally transform the columns in data frame\n        #It will normalize and set the json to a table\n        column_as_df = json_normalize(df[column])\n        # here will be set the name using the category and subcategory of json columns\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        # after extracting the values, let drop the original columns\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n    # Printing the shape of dataframes that was imported\n    print(f\"Loaded {os.path.basename(data_frame)}. Shape: {df.shape}\")\n    return df # returning the df after importing and transforming\n\n\n# This function is to extract date features\ndef date_process(df):\n    df[\"date\"] = pd.to_datetime(df[\"date\"], format=\"%Y%m%d\") # seting the column as pandas datetime\n    df[\"_weekday\"] = df['date'].dt.weekday #extracting week day\n    df[\"_day\"] = df['date'].dt.day # extracting day\n    df[\"_month\"] = df['date'].dt.month # extracting day\n    df[\"_year\"] = df['date'].dt.year # extracting day\n    df['_visitHour'] = (df['visitStartTime'].apply(lambda x: str(datetime.fromtimestamp(x).hour))).astype(int)\n\n    return df #returning the df after the transformations\n\ndef Read_data():\n\n    # We will import the data using the name and extension that will be concatenated with dir_path\n    data = json_read(\"/kaggle/input/ga-customer-revenue-prediction/train.csv\")\n    data = date_process(data)\n\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:00:46.879041Z","iopub.execute_input":"2021-08-18T13:00:46.879558Z","iopub.status.idle":"2021-08-18T13:00:46.890004Z","shell.execute_reply.started":"2021-08-18T13:00:46.879501Z","shell.execute_reply":"2021-08-18T13:00:46.889267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = Read_data()\ndata.head()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:00:46.891008Z","iopub.execute_input":"2021-08-18T13:00:46.891375Z","iopub.status.idle":"2021-08-18T13:01:06.555177Z","shell.execute_reply.started":"2021-08-18T13:00:46.891349Z","shell.execute_reply":"2021-08-18T13:01:06.55457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA (Exploratory Data Analysis)**","metadata":{}},{"cell_type":"code","source":"def details_missing_columns(df):\n    '''\n    This function will help us to define the details of the missing values\n    '''\n    total = df.isna().sum().sort_values(ascending = False) # getting the sum of null values and ordering\n    percent = (df.isna().sum() / df.isna().count() * 100 ).sort_values(ascending = False) #getting the percent and order of null\n    state_missing_value = pd.concat([total, percent], axis=1, keys=['Total', 'Percent']) # Concatenating the total and percent\n    print(\"\\nTotal missing value columns: \")\n    print(state_missing_value) # Returning values of nulls different of 0\n\ndef missing_values_columns(df):\n    '''\n    This function will help us to detect the columns with more than 50% missing rate\n    '''\n    hight_missing_value = [col for col in df.columns if (df[col].isna().sum() / df[col].isna().count() * 100) > 50] #missing column more than 50%\n    print('\\nNumber of high missing columns: ', len(hight_missing_value))\n    print('high missing columns rate: ', hight_missing_value)\n\ndef missing_values_rows(df):\n    '''\n    This function will help us to detect rows with more than 50% of missing rate\n    '''\n    retain_row = []\n    for index, row in df.iterrows():\n        #rows with more than 50% of missing values\n        if (df.loc[index, :].isna().sum())/df.shape[1] >= 0.5:\n            retain_row.append(index)\n\n    print('\\nThe number of hight missing rows: ', len(retain_row))\n    print('high missing rows rate: ', retain_row)\n\ndef unique_columns(df):\n\n    # all columns where we have a unique value (constants)\n    # It is useful because this columns give us none information\n    discovering_consts = [col for col in df.columns if df[col].nunique() == 1]\n\n    # printing the total of columns dropped and the name of columns\n    print(\"\\nNumber of columns with just one value: \", len(discovering_consts), \"columns\")\n    print(\"Name of constant columns: \\n\", discovering_consts)\n\n\ndef Shape_size_columns(df):\n\n    # all columns where we have the same size than shape like id\n    # It is useful because this columns give us none information\n    discovering_shape = [col for col in df.columns if df[col].nunique() == df.shape[0]]\n\n    # printing the total of columns dropped and the name of columns\n    print(\"\\nNumber of columns with shape size: \", len(discovering_shape), \"columns\")\n    print(\"Name of constant columns: \\n\", discovering_shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:01:06.556389Z","iopub.execute_input":"2021-08-18T13:01:06.556861Z","iopub.status.idle":"2021-08-18T13:01:06.565888Z","shell.execute_reply.started":"2021-08-18T13:01:06.556831Z","shell.execute_reply":"2021-08-18T13:01:06.565239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Form_analysis(data):\n\n    df = data.copy()\n\n    print(f'Number of ligne : {df.shape[0]} / number of columns : {df.shape[1]}')\n\n    print('\\n types of variables :\\n')\n    print(df.dtypes.value_counts())\n\n    #unique columns\n    unique_columns(df)\n\n    #shape shape size columns\n    Shape_size_columns(df)\n\n    #missing value on columns\n    missing_values_columns(df)\n\n    #missing values on rows\n    missing_values_rows(df)\n    \n\n    details_missing_columns(df)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:01:06.566959Z","iopub.execute_input":"2021-08-18T13:01:06.567314Z","iopub.status.idle":"2021-08-18T13:01:06.585307Z","shell.execute_reply.started":"2021-08-18T13:01:06.567288Z","shell.execute_reply":"2021-08-18T13:01:06.584387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Form_analysis(data)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:01:06.586258Z","iopub.execute_input":"2021-08-18T13:01:06.586561Z","iopub.status.idle":"2021-08-18T13:02:05.063526Z","shell.execute_reply.started":"2021-08-18T13:01:06.586502Z","shell.execute_reply":"2021-08-18T13:02:05.062501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Based on this first analysis we can delete the colomns with hight missing rate or with no information add","metadata":{}},{"cell_type":"code","source":"    #1. Drop unique columns\n    to_drop_unique_columns = ['socialEngagementType', 'device.browserVersion', 'device.browserSize', 'device.operatingSystemVersion',\n     'device.mobileDeviceBranding', 'device.mobileDeviceModel', 'device.mobileInputSelector', 'device.mobileDeviceInfo',\n     'device.mobileDeviceMarketingName', 'device.flashVersion', 'device.language', 'device.screenColors', 'device.screenResolution',\n     'geoNetwork.cityId', 'geoNetwork.latitude', 'geoNetwork.longitude', 'geoNetwork.networkLocation', 'totals.visits', 'totals.bounces',\n     'totals.newVisits', 'trafficSource.adwordsClickInfo.criteriaParameters', 'trafficSource.isTrueDirect',\n     'trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.isVideoAd']\n\n    data.drop(to_drop_unique_columns, axis=1, inplace=True)\n    print(\"Total unique features dropped: \", len(to_drop_unique_columns))\n\n    #2. Drop columns\n    to_drop_columns = ['trafficSource.keyword', 'trafficSource.referralPath', 'trafficSource.adwordsClickInfo.page',\n    'trafficSource.adwordsClickInfo.slot', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adContent', 'fullVisitorId',\n    'sessionId', 'visitId', 'visitStartTime', 'date', 'trafficSource.campaign', 'geoNetwork.metro', 'geoNetwork.region',\n    'geoNetwork.networkDomain', 'geoNetwork.city', 'trafficSource.medium'  ]\n\n    data.drop(to_drop_columns, axis=1, inplace=True)\n    print(\"Total features dropped: \", len(to_drop_columns))\n\n    print('\\n')\n\n    print(\"Shape after dropping: \", data.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:02:05.064821Z","iopub.execute_input":"2021-08-18T13:02:05.06511Z","iopub.status.idle":"2021-08-18T13:02:05.523164Z","shell.execute_reply.started":"2021-08-18T13:02:05.065081Z","shell.execute_reply":"2021-08-18T13:02:05.522134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's analyze one more time our data :","metadata":{}},{"cell_type":"code","source":"Form_analysis(data)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:02:05.525082Z","iopub.execute_input":"2021-08-18T13:02:05.525342Z","iopub.status.idle":"2021-08-18T13:02:47.265473Z","shell.execute_reply.started":"2021-08-18T13:02:05.525315Z","shell.execute_reply":"2021-08-18T13:02:47.264626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We keep 'totals.transactionRevenue' columns because it is our target, it is normal that this column has more than 98% of missing values because the conversion rate on E-commerce is very low. We will deal with the missing value after.\n\nAfter this analysis we take a look in our data unique values to understand the data :","metadata":{}},{"cell_type":"code","source":"def knowningData(df, data_type=object, limit=10): #seting the function with df,\n    n = df.select_dtypes(include=data_type) #selecting the desired data type\n    for column in n.columns: #initializing the loop\n        print(\"##############################################\")\n        print(\"Name of column \", column, ': \\n', \"Uniques: \", df[column].unique()[:limit], \"\\n\",\n              \" | ## Total nulls: \", (round(df[column].isnull().sum() / len(df[column]) * 100,2)),\n              \" | ## Total unique values: \", df.nunique()[column]) #print the data and % of nulls)\n        print(\"#############################################\")","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:02:47.268148Z","iopub.execute_input":"2021-08-18T13:02:47.268592Z","iopub.status.idle":"2021-08-18T13:02:47.274975Z","shell.execute_reply.started":"2021-08-18T13:02:47.268546Z","shell.execute_reply":"2021-08-18T13:02:47.273912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n Object --------\")\nknowningData(data, data_type= object)\nprint(\"\\n int --------\")\nknowningData(data, data_type= int)\nprint(\"\\n float --------\")\nknowningData(data, data_type= float)\nprint(\"\\n bool --------\")\nknowningData(data, data_type= bool)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:02:47.276595Z","iopub.execute_input":"2021-08-18T13:02:47.276979Z","iopub.status.idle":"2021-08-18T13:02:54.127272Z","shell.execute_reply.started":"2021-08-18T13:02:47.276938Z","shell.execute_reply":"2021-08-18T13:02:54.126178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Some numerical values should be integer like page views and hits, the nan transaction value will be set to 0 (nan mean no conversion : no transaction).\n\nSome objectif feature get values like 'not set'\n\nLet's clean and replace some values to get a better view on our data :","metadata":{}},{"cell_type":"code","source":"def Filling_Replacing_Values(df):\n    # fillna numeric feature\n    df['totals.pageviews'].fillna(df['totals.pageviews'].value_counts().index[0], inplace=True) #filling NA's with the most Frequent value\n    df[\"totals.pageviews\"] = df[\"totals.pageviews\"].astype(int) # setting numerical to int\n    df[\"totals.hits\"] = df[\"totals.hits\"].astype(int) # setting numerical to int\n    df[\"totals.transactionRevenue\"] = df[\"totals.transactionRevenue\"].fillna(0.0).astype(float)\n\n    #object feature\n    # --> Replace unknown values\n    df.loc[df['channelGrouping'] == '(Other)', 'channelGrouping'] = np.nan\n    df.loc[df['device.operatingSystem'] == '(not set)', 'device.operatingSystem'] = np.nan\n    df.loc[df['geoNetwork.continent'] == '(not set)', 'geoNetwork.continent'] = np.nan\n  \n    return df #return the transformed dataframe","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:02:54.128512Z","iopub.execute_input":"2021-08-18T13:02:54.128777Z","iopub.status.idle":"2021-08-18T13:02:54.134856Z","shell.execute_reply.started":"2021-08-18T13:02:54.128751Z","shell.execute_reply":"2021-08-18T13:02:54.133844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    data = Filling_Replacing_Values(data)\n    Form_analysis(data)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:02:54.136047Z","iopub.execute_input":"2021-08-18T13:02:54.136298Z","iopub.status.idle":"2021-08-18T13:03:36.705576Z","shell.execute_reply.started":"2021-08-18T13:02:54.136273Z","shell.execute_reply":"2021-08-18T13:03:36.703984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to clean the data we drop nan rows\ndata.dropna(axis = 0, inplace=True)\nprint(\"Shape after dropping: \", data.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:36.706948Z","iopub.execute_input":"2021-08-18T13:03:36.707514Z","iopub.status.idle":"2021-08-18T13:03:36.869099Z","shell.execute_reply.started":"2021-08-18T13:03:36.70742Z","shell.execute_reply":"2021-08-18T13:03:36.868043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Printing some statistics of our target\nprint(\"Target Min Value: \", data[\"totals.transactionRevenue\"].min()) # printing the min value\nprint(\"Target Mean Value: \", data[\"totals.transactionRevenue\"].mean()) # mean value\nprint(\"Target Median Value: \", data[\"totals.transactionRevenue\"].median()) # median value\nprint(\"Target Max Value: \", data[\"totals.transactionRevenue\"].max()) # the max value","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:36.870606Z","iopub.execute_input":"2021-08-18T13:03:36.870997Z","iopub.status.idle":"2021-08-18T13:03:36.879745Z","shell.execute_reply.started":"2021-08-18T13:03:36.870954Z","shell.execute_reply":"2021-08-18T13:03:36.8786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n#. Creation target - features\ny = data['totals.transactionRevenue']\nX = data.drop('totals.transactionRevenue', axis=1)\n\n# train/test set\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:36.88139Z","iopub.execute_input":"2021-08-18T13:03:36.881849Z","iopub.status.idle":"2021-08-18T13:03:36.99752Z","shell.execute_reply.started":"2021-08-18T13:03:36.881803Z","shell.execute_reply":"2021-08-18T13:03:36.996605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now that we create our test and train set with 80% trainset and 20% testset we create our evaluation function :\n-We use mainly RMSE evaluation where y hat is the natural log of the predicted revenue for a customer and y is the natural log of the actual summed revenue value plus one.\n-RMSE is an appropriate metric hear because less than 2% of the custumers has transaction, so we want to amplify the error to not miss transactions","metadata":{}},{"cell_type":"code","source":"def evaluation(model, X_train, X_test, y_train, y_test):\n    from sklearn.metrics import mean_squared_error\n    from sklearn.model_selection import learning_curve\n    \n    print('\\n ----- RMSE Evaluation : \\n')\n\n    y_test = np.log1p(y_test)\n    y_train = np.log1p(y_train)\n\n    model.fit(X_train, y_train)\n\n    y_predict_test = model.predict(X_test)\n    y_predict_train = model.predict(X_train)\n\n    train_score = mean_squared_error(y_train, y_predict_train, squared=False)\n    print('train score =', train_score)\n\n    test_score = mean_squared_error(y_test, y_predict_test, squared=False)\n    print('test score =', test_score)\n\n    '''\n    learning curve visualisation\n    '''\n    \n    N, train_score, val_score = learning_curve(model, X_train, y_train, train_sizes = np.linspace(0.1, 1.0, 10), cv=4, scoring='neg_root_mean_squared_error')\n\n    print(N)\n    plt.plot(N,train_score.mean(axis=1), label='train')\n    plt.plot(N,val_score.mean(axis=1), label='validation')\n    plt.xlabel('train_sizes')\n    plt.legend()\n    plt.show()\n\n    print('-----End Evaluation-----\\n')\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:36.998683Z","iopub.execute_input":"2021-08-18T13:03:36.998927Z","iopub.status.idle":"2021-08-18T13:03:37.006803Z","shell.execute_reply.started":"2021-08-18T13:03:36.998902Z","shell.execute_reply":"2021-08-18T13:03:37.005632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's run our first model to understand more the features (We use StandarScaler and OrdinalEncoder to preprare the data) :","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeRegressor\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import OrdinalEncoder\n\n\n#. Encoding\nX_copy = X_train\nX_copy_test = X_test\nencoder = OrdinalEncoder()\nencoder.fit(X)\n\nX_train = encoder.transform(X_train)\nX_test = encoder.transform(X_test)\n\n\npreprocessor = make_pipeline(StandardScaler())\nDTR = make_pipeline(preprocessor, DecisionTreeRegressor(random_state = 0))\n\nevaluation(DTR, X_train, X_test, y_train, y_test)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:37.008083Z","iopub.execute_input":"2021-08-18T13:03:37.008427Z","iopub.status.idle":"2021-08-18T13:03:41.956542Z","shell.execute_reply.started":"2021-08-18T13:03:37.00836Z","shell.execute_reply":"2021-08-18T13:03:41.955626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that we have an overfiting but this step will show us the most interesting features to use on our final training :","metadata":{}},{"cell_type":"code","source":"'''\nfeatures Selection\n'''\n\nIFe = DTR.named_steps[\"decisiontreeregressor\"].feature_importances_\nprint(pd.DataFrame(IFe, index=X_copy.columns))\npd.DataFrame(IFe, index=X_copy.columns).plot.bar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:41.958522Z","iopub.execute_input":"2021-08-18T13:03:41.958938Z","iopub.status.idle":"2021-08-18T13:03:42.310941Z","shell.execute_reply.started":"2021-08-18T13:03:41.958891Z","shell.execute_reply":"2021-08-18T13:03:42.309719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can observe that there s some not importante features than we can delete to train the model :","metadata":{}},{"cell_type":"code","source":"    #select feature with less than 0.01 importance\n    selection = X.columns[IFe<0.01]\n    print(selection)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:03:42.312232Z","iopub.execute_input":"2021-08-18T13:03:42.312564Z","iopub.status.idle":"2021-08-18T13:03:42.318856Z","shell.execute_reply.started":"2021-08-18T13:03:42.312532Z","shell.execute_reply":"2021-08-18T13:03:42.317837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's drop these features and have beagin the training :","metadata":{}},{"cell_type":"code","source":"to_drop = ['device.browser', 'device.deviceCategory', 'geoNetwork.continent',\n       'geoNetwork.subContinent', '_year']\n\nX_train = pd.DataFrame(data = X_train, index = X_copy.index, columns = X_copy.columns)\nX_test = pd.DataFrame(data = X_test, index = X_copy_test.index, columns = X_copy_test.columns)\n\nX_train.drop(to_drop, axis=1, inplace=True)\nX_test.drop(to_drop, axis=1, inplace=True)\n\nfrom sklearn.linear_model import Ridge\nfrom sklearn.svm import LinearSVR\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\n\nRidge_algo = make_pipeline(preprocessor, Ridge(random_state = 0, alpha=.5))\nSVR = make_pipeline(preprocessor, LinearSVR(random_state = 0))\nNKR = make_pipeline(preprocessor, KNeighborsRegressor(n_jobs = -1))\nBOOST = make_pipeline(preprocessor, GradientBoostingRegressor(random_state = 0))\n\ndict_of_models = {'Ridge':Ridge_algo,\n                  'SVR':SVR,\n                  'NKR':NKR,\n                  'DTR':DTR,\n                  'BOOST':BOOST}\n    \nfor name, model in dict_of_models.items():\n    print('Evaluation of -----',name)\n    evaluation(model, X_train, X_test, y_train, y_test)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T13:07:40.217665Z","iopub.execute_input":"2021-08-18T13:07:40.218015Z","iopub.status.idle":"2021-08-18T13:13:57.422659Z","shell.execute_reply.started":"2021-08-18T13:07:40.217984Z","shell.execute_reply":"2021-08-18T13:13:57.421561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GradientBoostingRegressor seem to be the most promising algorithm, let's focus on him :","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import mean_squared_error\n\n#define our own mse and set greater_is_better=True (to get the opposite of neg_mse)\nmse = make_scorer(mean_squared_error, greater_is_better=True)\n\ndef FocusOn_BOOST(BOOST, X_train, X_test, y_train, y_test):\n    hyper_params = {'gradientboostingregressor__random_state': [0],\n                    'gradientboostingregressor__loss': ['ls', 'lad', 'huber', 'quantile'],\n                    'gradientboostingregressor__learning_rate': [0.05, 0.1, 0.2],\n                    'gradientboostingregressor__n_estimators': [100, 200]}\n\n    grid = GridSearchCV(BOOST, hyper_params, scoring=mse, cv=4, n_jobs = -1)\n\n    grid.fit(X_train, y_train)\n\n    #best parameter :\n    print(grid.best_params_)\n    print(grid.best_score_)\n\n    model = grid.best_estimator_\n\n    evaluation(model, X_train, X_test, y_train, y_test)\n\n#call the function\nFocusOn_BOOST(BOOST, X_train, X_test, y_train, y_test)\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-08-18T14:13:29.900268Z","iopub.execute_input":"2021-08-18T14:13:29.900628Z","iopub.status.idle":"2021-08-18T14:21:56.711063Z","shell.execute_reply.started":"2021-08-18T14:13:29.900594Z","shell.execute_reply":"2021-08-18T14:21:56.709801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**CC:**\n\nWe get a small improvement on our train set and same results on testset.\nOur objectif is ok with an RMSE < 2\n\nThe learning curve shows us that we can improve the results with more data.","metadata":{}}]}