{"cells":[{"metadata":{"trusted":true,"_uuid":"c85168e455b1a9db48a5c4943f1efcf5714d23f0"},"cell_type":"code","source":"\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\n\nimport pandas as pd\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\nimport warnings\nimport feather as fe\nwarnings.filterwarnings('ignore')\n\nfrom pandas.io.json import json_normalize\n\nimport json\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use('fivethirtyeight')\n\nfrom sklearn.preprocessing import Imputer\n\nfrom sklearn import preprocessing\n\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43001c900ed14bc49ab0170a5c6b9f35dd654977"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/ga-customer-revenue-prediction/train.csv\",sep=',')\n\ntest_df = pd.read_csv(\"../input/ga-customer-revenue-prediction/test.csv\",sep=',')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"## This method of flattening the JSON columns is a very popular approach obtained from the Kaggle discussion forums\njson_columns = ['device', 'geoNetwork','totals', 'trafficSource']\ndef load_df(filename):\n path = \"../input/ga-customer-revenue-prediction/\" + filename\n df = pd.read_csv(path, converters={column: json.loads for column in json_columns}, \n dtype={'fullVisitorId': 'str'})\n \n for column in json_columns:\n column_as_df = json_normalize(df[column])\n column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b91740a21c179c2cb81c6cef2810b65b096470d"},"cell_type":"code","source":"train_df = load_df(\"train.csv\")\ntest_df = load_df(\"test.csv\")\n\ntrain_df.to_feather('train.feather')\ntest_df.to_feather('test.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b074e3160e73eeaecfcea04b570c01349e3c24c"},"cell_type":"code","source":"train_df = pd.read_feather('train.feather')\ntest_df = pd.read_feather('test.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93d08ba8d0845c6daf2751e1f5a494c788239895"},"cell_type":"code","source":"count_row = train_df.shape[0] \ncount_col = train_df.shape[1]\nprint(\"For Train : \")\nprint(count_row , count_col)\n\ncount_row = test_df.shape[0] \ncount_col = test_df.shape[1]\nprint(\"For Test : \")\nprint(count_row , count_col)\n\nprint(\"*****************************\")\n\n# train_df=train_df.dropna()\n\n# print(\"After removing NaN\")\n# count_row = train_df.shape[0] \n# count_col = train_df.shape[1]\n# print(count_row , count_col)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2991989cee2198d9b1ccdca30f687b681ccb2c0"},"cell_type":"markdown","source":"**Task 2 : Interesting Visualisations****"},{"metadata":{"_uuid":"1c413a5a5258a0858479e112cac025e250e49ff6"},"cell_type":"markdown","source":"****1. No. of Hits Based on different Countries - Heat Map"},{"metadata":{"trusted":true,"_uuid":"cdc0fa0e957bdd3346391da7809ce80c3125dcc9","scrolled":false},"cell_type":"code","source":"\nimport plotly.graph_objs as go\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\ninit_notebook_mode(connected=True)\n\n# colorscale=[[0.0, 'rgb(165,0,38)'], [0.005, 'rgb(215,48,39)'], [0.01, 'rgb(244,109,67)'], [0.02, 'rgb(253,174,97)'], [0.04, 'rgb(254,224,144)'], [0.05, 'rgb(224,243,248)'], [0.1, 'rgb(171,217,233)'], [0.25, 'rgb(116,173,209)'], [0.5, 'rgb(69,117,180)'], [1.0, 'rgb(49,54,149)']]\ncolorscale = [[0, 'rgb(102,194,165)'], [0.0007, 'rgb(102,194,165)'], \n [0.004, 'rgb(132,200,165)'],[0.009, 'rgb(192,200,165)'],\n [0.01, 'rgb(171,221,164)'], [0.02, 'rgb(230,245,152)'], \n [0.04, 'rgb(255,255,191)'], [0.05, 'rgb(254,224,139)'], \n [0.10, 'rgb(253,174,97)'], [0.25, 'rgb(213,62,79)'], [1.0, 'rgb(158,1,66)']]\ndata = dict(type = 'choropleth', \n locations = train_df[\"geoNetwork_country\"].value_counts().index,\n locationmode = 'country names',\n colorscale = colorscale,\n z = train_df['totals_hits'].value_counts().values, \n text = train_df[\"geoNetwork_country\"].value_counts().index,\n colorbar = {'title':'Total Hits '})\nlayout = dict(title = 'Hits', \n height = 1000,\n geo = dict(showframe = False, \n projection = {'type': 'mercator'})\n )\nchoromap3 = go.Figure(data = [data], layout=layout)\niplot(choromap3)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e64266a052a9506c0819dc70a3659922edc43a0"},"cell_type":"markdown","source":" Correlation b/w different numeric parameters Represented as both a Seaborn heatmap and a Tabular Color Map"},{"metadata":{"trusted":true,"_uuid":"bb54dda000d5e015a16abe83c804a7d3506f4dc0"},"cell_type":"code","source":"train_corr = train_df.copy()\nplt.figure(figsize=(15,15))\nsns.heatmap(train_corr.corr(method=\"pearson\"), annot=True, cmap=\"YlGnBu\")\ntrain_corr.corr(method='pearson').style.format(\"{:.2}\").background_gradient(cmap=plt.get_cmap('Accent'), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6742d6d8c00d2954619dcc168c58e75d42165364"},"cell_type":"markdown","source":"**Plot 2 : Number of transactions Across Time : \n**"},{"metadata":{"trusted":true,"_uuid":"c14c6dd2165fdf3691cbab4154e81238d9cbe3b2","scrolled":false},"cell_type":"code","source":"import datetime\n\ndef scatter_plot(cnt_srs, color):\n trace = go.Scatter(\n x=cnt_srs.index[::-1],\n y=cnt_srs.values[::-1],\n showlegend=False,\n marker=dict(\n color=color,\n ),\n )\n return trace\n\ndef add_date_features(df):\n df['date'] = df['date'].astype(str)\n df[\"date\"] = df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n df[\"date\"] = pd.to_datetime(df[\"date\"])\n \n df[\"month\"] = df['date'].dt.month\n df[\"day\"] = df['date'].dt.day\n df[\"weekday\"] = df['date'].dt.weekday\n return df \n\n# train_df['date'] = train_df['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\ntrain_df = add_date_features(train_df)\ncnt_srs = train_df.groupby('date')['totals_transactionRevenue'].agg(['size', 'count'])\ncnt_srs.columns = [\"count\", \"count of non-zero revenue\"]\ncnt_srs = cnt_srs.sort_index()\n#cnt_srs.index = cnt_srs.index.astype('str')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae1011fb0f339b6497937ebbdc5e547c095618ad","scrolled":true},"cell_type":"code","source":"import plotly.offline as py\nfrom plotly import tools\ntrace = scatter_plot(cnt_srs[\"count\"], 'black')\npy.init_notebook_mode(connected=True)\nfig = tools.make_subplots(rows=1, cols=1, vertical_spacing=0.08,subplot_titles=[\"Date - Count\"])\nfig.append_trace(trace, 1, 1)\npy.iplot(fig, filename='date-plots')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"372357e8bac479d7ae583674a57e0ee26cefca28"},"cell_type":"markdown","source":"**Plot 3 : Channel Grouping**"},{"metadata":{"trusted":true,"_uuid":"b70beb4702731a6193d422b24a52a72ce9c7ecee"},"cell_type":"code","source":"t = train_df['channelGrouping'].value_counts()\nvalues1 = t.values \nindex1 = t.index\ndomain1 = {'x': [0.2, 0.50], 'y': [0.0, 0.33]}\nfig = {\n \"data\": [\n {\n \"values\": values1,\n \"labels\": index1,\n \"domain\": {\"x\": [0, .48]},\n \"marker\" : dict(colors=[\"#y88b3c\" ,'#cb27fb', '#b1b1b2']),\n \"name\": \"Channel Grouping\",\n \"hoverinfo\":\"label+percent+name\",\n \"hole\": .4,\n \"type\": \"pie\"\n }\n ],\n \"layout\": {\"title\":\"Channel Grouping\",\n \"annotations\": [\n {\n \"font\": {\n \"size\": 20\n },\n \"showarrow\": False,\n \"text\": \" \",\n \"x\": 0.11,\n \"y\": 0.5\n }\n ]\n }\n}\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fbc9d0924483e648e8f9679eeb5e0b13ecc4f13"},"cell_type":"markdown","source":"**Task 3 : Clustering Based on geographical data****"},{"metadata":{"_uuid":"7743f490ffa31852b071372468060afee3a2e57c"},"cell_type":"markdown","source":"**Continent**"},{"metadata":{"trusted":true,"_uuid":"2e27992268afccd138ecd849cfbae228e9c47920"},"cell_type":"code","source":"train_df[\"totals_transactionRevenue\"] = train_df[\"totals_transactionRevenue\"].astype('float')\ncontinent = train_df.groupby('geoNetwork_continent')['totals_transactionRevenue'].agg(['size', 'count', 'sum'])\ncontinent.columns = [\"total count\", \"count of non-zero revenue\", \"sum\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"632ade8b9266e1e5858b842fb8a9756dc2ee2454"},"cell_type":"code","source":"continent.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f309fc4188d530d4b2b7bc6f0c3c9dc8d077bbf"},"cell_type":"code","source":"print(continent['total count'])\ncontinent['total count'] = np.log(continent['total count'])\ncontinent.groupby('geoNetwork_continent')['total count'].mean().sort_index().plot.bar(color = 'b');\n\nplt.title('Log(Total Count) vs Continent');\nplt.ylabel('Log(Total Count');\nplt.xlabel('Continent')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38536be178ee9cb503e113b4aef5df9f828bf6fc"},"cell_type":"code","source":"print (continent['count of non-zero revenue'])\n# continent['count of non-zero revenue'] = np.log(continent['count of non-zero revenue'])\ncontinent['count of non-zero revenue'] = np.log(continent['count of non-zero revenue'])\ncontinent.groupby('geoNetwork_continent')['count of non-zero revenue'].mean().plot.bar(color = 'b');\nplt.title('Log(Total Non-Zero Revenue Count) vs Continent');\nplt.ylabel('Log(Non Zero Revenue Count)');\nplt.xlabel('Continent')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed381061c0eb0188499f9387795d19c5c2bf071c"},"cell_type":"code","source":"print(continent['sum'])\ncontinent['sum'] = np.log(continent['sum'])\ncontinent.groupby('geoNetwork_continent')['sum'].mean().sort_index().plot.bar(color = 'b')\nplt.title('Log(Total Revenue) vs Continent')\nplt.ylabel('Revenue')\nplt.xlabel('Continent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ff128a637a52af19ab98b88166c8638f158c3eb"},"cell_type":"markdown","source":"**Sub Continent**"},{"metadata":{"trusted":true,"_uuid":"0b2083037d9424d504db50784dc96f14c96f98b1"},"cell_type":"code","source":"train_df[\"totals_transactionRevenue\"] = train_df[\"totals_transactionRevenue\"].astype('float')\nsubCont = train_df.groupby('geoNetwork_subContinent')['totals_transactionRevenue'].agg(['size', 'count', 'sum'])\nsubCont.columns = [\"total count\", \"count of non-zero revenue\", \"sum\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1259ff157f2003a3956c1ee64f24b0de95032fb"},"cell_type":"code","source":"print (subCont)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d814acac7efebda4c47a5c22a0bc40534a763480"},"cell_type":"code","source":"print(subCont['total count'])\nsubCont['total count'] = np.log(subCont['total count'])\nsubCont.groupby('geoNetwork_subContinent')['total count'].mean().sort_index().plot.bar(color = 'b')\nplt.title('Log( Total Count ) vs Sub-Continent')\nplt.ylabel('Total Count')\nplt.xlabel('Sub-Continent')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e7d63c8e5b95dcf4f4cb6c32e7d8c38b0212cbc"},"cell_type":"code","source":"subCont['count of non-zero revenue'] = np.log(subCont['count of non-zero revenue'])\nprint (subCont['count of non-zero revenue'])\nsubCont.groupby('geoNetwork_subContinent')['count of non-zero revenue'].mean().plot.bar(color = 'b')\nplt.title('Log(Total Non-Zero Revenue Count) vs Sub-Continent')\nplt.ylabel('Log(Total Non-Zero Revenue Count)')\nplt.xlabel('Sub-Continent')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9af18502ec50db73959af3ad489aa110824ccb6"},"cell_type":"code","source":"print(subCont['sum'])\nsubCont['sum'] = np.log(subCont['sum'])\nsubCont.groupby('geoNetwork_subContinent')['sum'].mean().sort_index().plot.bar(color = 'b')\nplt.title('Log(Total Revenue) vs Sub-Continent')\nplt.ylabel('Revenue')\nplt.xlabel('Sub-Continent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2c72d3a1aa0217ad5242387cbfb5b97fbf66012"},"cell_type":"markdown","source":"Clustering - geography and Vsiits"},{"metadata":{"trusted":true,"_uuid":"bbe0fae947ecc171374c64d28d31c97412e4eba1"},"cell_type":"code","source":"geo_cols = [\"geoNetwork_city\", \"geoNetwork_country\", \"geoNetwork_subContinent\", \"geoNetwork_continent\"]\ncolors = [\"#008080\"]\ntraces = []\nfor i, col in enumerate(geo_cols):\n t = train_df[col].value_counts()\n traces.append(go.Bar(marker=dict(color=colors[0]),orientation=\"h\", y = t.index[:15], x = t.values[:15]))\n\nfig = tools.make_subplots(rows=2, cols=2, subplot_titles=[\"Visits: City\", \"Visits: Country\",\"Visits: Sub Continent\",\"Visits: Continent\"], print_grid=False)\nfig.append_trace(traces[0], 1, 1)\nfig.append_trace(traces[1], 1, 2)\nfig.append_trace(traces[2], 2, 1)\nfig.append_trace(traces[3], 2, 2)\nfig['layout'].update(height=600,width=1000, showlegend=False)\niplot(fig)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18718be19cbeefb49724434b10995f0ed3bbb6f4"},"cell_type":"markdown","source":"***Task 4 :: Probability ***"},{"metadata":{"trusted":true,"_uuid":"dcff01eb0da631c0d0632b2065677c2d452c4c42"},"cell_type":"code","source":"train_df[\"totals_transactionRevenue\"] = train_df[\"totals_transactionRevenue\"].astype('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"021a87639de9e55929ab8ff13831f89cfb4711fe"},"cell_type":"code","source":"userCount_df = train_df.groupby('fullVisitorId')['fullVisitorId'].agg(['size'])\nuserCount_df=userCount_df.sort_values(by=['size'],ascending=False)\nprint(userCount_df.head(10))\n\nuserAmount_df = train_df.groupby('fullVisitorId')['totals_transactionRevenue'].agg(['count','mean'])\nuserAmount_df=userAmount_df.sort_values(by=['count'],ascending=False)\nprint(userAmount_df.head(10))\n\ntotalTransaction = userCount_df['size'].sum()\nprint(totalTransaction)\n\nuserCount_df['buying probability'] = userCount_df['size']/totalTransaction\nuserCount_df=userCount_df.sort_values(by=['buying probability'],ascending=False)\nprint(userCount_df.head(10))\n\nprobSum = userCount_df['buying probability'].sum()\nprint(probSum)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90c908bc0cfd9d15cb1098a719b952556164450b"},"cell_type":"markdown","source":"**TASK 1 : Data Processing and Cleanig**\nHere, we are transforming the data trypes of the columns."},{"metadata":{"trusted":true,"_uuid":"95d19a48409b589d60555cb488b77a42f94d9d9c"},"cell_type":"code","source":"train_df = train_df.drop('trafficSource_campaignCode',1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a1033049b0078e83d26edcd687000465b9f32e7"},"cell_type":"code","source":"const_cols = []\nfor col in train_df.columns:\n if len(train_df[col].value_counts()) == 1:\n const_cols.append(col)\n\n## non relevant columns\nnon_relevant = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \"visitId\", \"visitStartTime\"]\ntest_df = add_date_features(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"372634da8b13c760d68cdad52571e7109717416c"},"cell_type":"code","source":"### The label encoding part is inspired from another Kernel :\n\nfrom sklearn.preprocessing import LabelEncoder\n\ncategorical_columns = [c for c in train_df.columns if not c.startswith(\"total\")]\ncategorical_columns = [c for c in categorical_columns if c not in const_cols + non_relevant]\nfor c in categorical_columns:\n\n le = LabelEncoder()\n train_vals = list(train_df[c].values.astype(str))\n test_vals = list(test_df[c].values.astype(str))\n \n le.fit(train_vals + test_vals)\n \n train_df[c] = le.transform(train_vals)\n test_df[c] = le.transform(test_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6060974ceb1b25670ca64f4c5b7b4e7509e6cb9"},"cell_type":"code","source":"def normalize_numerical_columns(df, isTrain = True):\n df[\"totals_hits\"] = df[\"totals_hits\"].astype(float)\n df[\"totals_hits\"] = (df[\"totals_hits\"] - min(df[\"totals_hits\"])) / (max(df[\"totals_hits\"]) - min(df[\"totals_hits\"]))\n\n df[\"totals_pageviews\"] = df[\"totals_pageviews\"].astype(float)\n df[\"totals_pageviews\"] = (df[\"totals_pageviews\"] - min(df[\"totals_pageviews\"])) / (max(df[\"totals_pageviews\"]) - min(df[\"totals_pageviews\"]))\n \n if isTrain:\n df[\"totals_transactionRevenue\"] = df[\"totals_transactionRevenue\"].fillna(0.0)\n return df ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d441f08c58f38e03f0e68e6ce2ab2265c023bf8d"},"cell_type":"code","source":"train_df = normalize_numerical_columns(train_df)\ntest_df = normalize_numerical_columns(test_df, isTrain = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0834b628aa7593e5ec61250e6b81f830d1660cca"},"cell_type":"markdown","source":"**Task 6 : Submission --> LIGHT GBM Model**"},{"metadata":{"trusted":true,"_uuid":"048862d23440e2e2e83de65ebe6ca6efd5881673","scrolled":false},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport lightgbm as lgb \nfeatures = [c for c in train_df.columns if c not in const_cols + non_relevant]\nfeatures.remove(\"totals_transactionRevenue\")\n\ntrain_df_new[\"totals_transactionRevenue\"] = np.log1p(train_df[\"totals_transactionRevenue\"].astype(float))\ntrain_x, valid_x, train_y, valid_y = train_test_split(train_df[features], train_df_new[\"totals_transactionRevenue\"], test_size=0.2, random_state=20)\n\n\nlgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\",\n \"num_leaves\" : 50, \"learning_rate\" : 0.02, \n \"bagging_fraction\" : 0.75, \"feature_fraction\" : 0.8, \"bagging_frequency\" : 9}\n \nlgb_train = lgb.Dataset(train_x, label=train_y)\nlgb_val = lgb.Dataset(valid_x, label=valid_y)\nmodel = lgb.train(lgb_params, lgb_train, 200, valid_sets=[lgb_val], early_stopping_rounds=150, verbose_eval=20)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5281c9217447cce31cb441565fd4ea7fbc0f984c"},"cell_type":"code","source":"preds = model.predict(test_df[features], num_iteration=model.best_iteration)\ntest_df[\"PredictedLogRevenue\"] = np.expm1(preds)\nsubmission = test_df.groupby(\"fullVisitorId\").agg({\"PredictedLogRevenue\" : \"sum\"}).reset_index()\nsubmission[\"PredictedLogRevenue\"] = np.log1p(submission[\"PredictedLogRevenue\"])\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].fillna(0)\nsubmission.to_csv(\"Output_7.csv\", index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd21f7ef8aef892300a71dba48c9e930afcbdcf9"},"cell_type":"markdown","source":"Feature Importance ::"},{"metadata":{"trusted":true,"_uuid":"8eb6f00bf350648cc9cf7d4c1519a1fad28cc05d"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,18))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e4bd479d9f7f19d7062e3335148dcf3db45eddf"},"cell_type":"markdown","source":"**TASK 7 : P-Test**"},{"metadata":{"trusted":true,"_uuid":"5deabba57b157424ea52978b1c3f8eb93cd3d2c2"},"cell_type":"code","source":"train_df_copy = train_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bcb0b7152ee04bb5ac0752cae1e70f91b7f6077","scrolled":false},"cell_type":"code","source":"from sklearn import metrics\ncount = 0\nbestValue = 1.65553\ndef ptest(train_df_copy):\n \n train_df_new = pd.DataFrame()\n train_df_new[\"totals_transactionRevenue\"] = np.log1p(train_df_copy[\"totals_transactionRevenue\"].astype(float))\n train_x, valid_x, train_y, valid_y = train_test_split(train_df_copy[features], train_df_new[\"totals_transactionRevenue\"], test_size=0.2, random_state=20)\n\n\n lgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\",\n \"num_leaves\" : 50, \"learning_rate\" : 0.02, \n \"bagging_fraction\" : 0.75, \"feature_fraction\" : 0.8, \"bagging_frequency\" : 9}\n\n\n lgb_train = lgb.Dataset(train_x, label=train_y)\n lgb_val = lgb.Dataset(valid_x, label=valid_y)\n model = lgb.train(lgb_params, lgb_train, 200, valid_sets=[lgb_val], early_stopping_rounds=100, verbose_eval=200)\n \n pred1 = model.predict(valid_x, num_iteration=model.best_iteration)\n rmse = np.sqrt(metrics.mean_squared_error(pred1, valid_y)) \n return rmse\n\ntesting_cols = ['trafficSource_campaign' , 'totals_hits' , 'totals_pageviews']\nfor cols in testing_cols:\n \n count = 0\n avg = 0 \n for i in range (1,100):\n print(i)\n train_df_copy[cols] = np.random.permutation(train_df_copy[cols])\n rmse = ptest(train_df_copy) \n if rmse