{"cells":[{"metadata":{"_uuid":"4a9094ddd740f86c38d892842f67dc07d1a9f23c"},"cell_type":"markdown","source":"## Deep Learning with Keras - Google Analytics Customer Revenue Prediction\n* Note: this is just a starting point, there's a lot of work to be done.\n* A begginer version with [LGBM](https://www.kaggle.com/dimitreoliveira/lgbm-google-store-revenue-prediction)"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2ed662be3aa5d066387243fcf310e812a1cbec95"},"cell_type":"markdown","source":"### Dependencies"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import os\nimport json\nimport datetime\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom pandas.io.json import json_normalize\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, BatchNormalization, Activation, Dropout\nfrom keras import optimizers\n\n%matplotlib inline\npd.options.display.max_columns = 999","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36892733d801ade05ec57a731ea4054a9d23fb9b"},"cell_type":"markdown","source":"### Auxiliar functions"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"1d3ed22fe6c41863c14c77d95e2d8e7bc3fce432"},"cell_type":"code","source":"def add_time_features(df):\n    df['date'] = pd.to_datetime(df['date'], format='%Y%m%d', errors='ignore')\n    df['year'] = df['date'].apply(lambda x: x.year)\n    df['month'] = df['date'].apply(lambda x: x.month)\n    df['day'] = df['date'].apply(lambda x: x.day)\n    df['weekday'] = df['date'].apply(lambda x: x.weekday())\n    \n    return df\n\n\ndef plot_metrics(loss, val_loss):\n    fig, (ax1) = plt.subplots(1, 1, sharex='col', figsize=(20,7))\n    ax1.plot(loss, label='Train loss')\n    ax1.plot(val_loss, label='Validation loss')\n    ax1.legend(loc='best')\n    ax1.set_title('Loss')\n    plt.xlabel('Epochs')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fcac3f636f8ea4c14ac51791269189a372490b8f"},"cell_type":"markdown","source":"Function to load and convert files borrowed from this [kernel](https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields/notebook), thanks!"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\n    df = pd.read_csv(csv_path, dtype={'fullVisitorId': 'str'}, nrows=nrows)\n\n    for column in JSON_COLUMNS:\n        df = df.join(pd.DataFrame(df.pop(column).apply(pd.io.json.loads).values.tolist(), index=df.index))\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42961d5e360501d9a04c9ee92acd731e403bd873","_kg_hide-input":true},"cell_type":"code","source":"train = load_df(\"../input/train.csv\")\ntest = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc7e985fb9c9d0120ef419de2a5f9ced5e174273"},"cell_type":"markdown","source":"### About the train data"},{"metadata":{"trusted":true,"_uuid":"37e4904b3a2a51aa384e648f1729c809aa63feea","_kg_hide-input":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f88e69f0f6a6614277690c80a7d2107579781ef"},"cell_type":"markdown","source":"### This is how our data looks like"},{"metadata":{"trusted":true,"_uuid":"8e430d6c391669e056c163e1c07b82e8e97de3d2","scrolled":true,"_kg_hide-input":true},"cell_type":"code","source":"print('TRAIN SET')\nprint('Rows: %s' % train.shape[0])\nprint('Columns: %s' % train.shape[1])\nprint('Features: %s' % train.columns.values)\nprint()\nprint('TEST SET')\nprint('Rows: %s' % test.shape[0])\nprint('Columns: %s' % test.shape[1])\nprint('Features: %s' % test.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"992f5f440849702ec3b3edc38e48f6771780f50f"},"cell_type":"markdown","source":"### Feature engineering"},{"metadata":{"trusted":true,"_uuid":"3f37d338e4e2d27345c01ce60eb0fefc52aa011d"},"cell_type":"code","source":"train = add_time_features(train)\ntest = add_time_features(test)\n# Convert target feature to 'float' type.\ntrain[\"transactionRevenue\"] = train[\"transactionRevenue\"].astype('float')\ntrain['hits'] = train['hits'].astype(float)\ntest['hits'] = test['hits'].astype(float)\ntrain['pageviews'] = train['pageviews'].astype(float)\ntest['pageviews'] = test['pageviews'].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f38addade64ef2cbc0123985bb81662a86416a99"},"cell_type":"markdown","source":"### Agregated features"},{"metadata":{"trusted":true,"_uuid":"34d2cbb80f23295033b528b7f22b981d49b7b05b"},"cell_type":"code","source":"# Train\ngp_fullVisitorId_train = train.groupby(['fullVisitorId']).agg('sum')\ngp_fullVisitorId_train['fullVisitorId'] = gp_fullVisitorId_train.index\ngp_fullVisitorId_train['mean_hits_per_day'] = gp_fullVisitorId_train.groupby(['day'])['hits'].transform('mean')\ngp_fullVisitorId_train['mean_pageviews_per_day'] = gp_fullVisitorId_train.groupby(['day'])['pageviews'].transform('mean')\ngp_fullVisitorId_train['sum_hits_per_day'] = gp_fullVisitorId_train.groupby(['day'])['hits'].transform('sum')\ngp_fullVisitorId_train['sum_pageviews_per_day'] = gp_fullVisitorId_train.groupby(['day'])['pageviews'].transform('sum')\ngp_fullVisitorId_train = gp_fullVisitorId_train[['fullVisitorId', 'mean_hits_per_day', 'mean_pageviews_per_day', 'sum_hits_per_day', 'sum_pageviews_per_day']]\ntrain = train.join(gp_fullVisitorId_train, on='fullVisitorId', how='inner', rsuffix='_')\ntrain.drop(['fullVisitorId_'], axis=1, inplace=True)\n\n# Test\ngp_fullVisitorId_test = test.groupby(['fullVisitorId']).agg('sum')\ngp_fullVisitorId_test['fullVisitorId'] = gp_fullVisitorId_test.index\ngp_fullVisitorId_test['mean_hits_per_day'] = gp_fullVisitorId_test.groupby(['day'])['hits'].transform('mean')\ngp_fullVisitorId_test['mean_pageviews_per_day'] = gp_fullVisitorId_test.groupby(['day'])['pageviews'].transform('mean')\ngp_fullVisitorId_test['sum_hits_per_day'] = gp_fullVisitorId_test.groupby(['day'])['hits'].transform('sum')\ngp_fullVisitorId_test['sum_pageviews_per_day'] = gp_fullVisitorId_test.groupby(['day'])['pageviews'].transform('sum')\ngp_fullVisitorId_test = gp_fullVisitorId_test[['fullVisitorId', 'mean_hits_per_day', 'mean_pageviews_per_day', 'sum_hits_per_day', 'sum_pageviews_per_day']]\ntest = test.join(gp_fullVisitorId_test, on='fullVisitorId', how='inner', rsuffix='_')\ntest.drop(['fullVisitorId_'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d053ba72ff2b294e8e822ac3af53eda7201e9a30"},"cell_type":"markdown","source":"### Exploratory data analysis"},{"metadata":{"_uuid":"682cc1e37f809c372355c71adfbb3b1fa0cf97ca"},"cell_type":"markdown","source":"#### Let's take a look at our target value through the time."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"13b91461a4de7726ce42e91e9cc440c3fc2a095f"},"cell_type":"code","source":"time_agg = train.groupby('date')['transactionRevenue'].agg(['count', 'sum'])\nyear_agg = train.groupby('year')['transactionRevenue'].agg(['sum'])\nmonth_agg = train.groupby('month')['transactionRevenue'].agg(['sum'])\nday_agg = train.groupby('day')['transactionRevenue'].agg(['sum'])\nweekday_agg = train.groupby('weekday')['transactionRevenue'].agg(['count','sum'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6b49bf99cabb7f0d4a01f482ff7c116f225a429"},"cell_type":"markdown","source":"#### Here is sum of our tagert feature \"transactionRevenue\" through the time."},{"metadata":{"trusted":true,"_uuid":"61fece58d2e3470deb5f414cd61a419629f5e095","_kg_hide-input":true,"scrolled":true},"cell_type":"code","source":"plt.figure(figsize=(20,7))\nplt.ticklabel_format(axis='y', style='plain')\nplt.ylabel('Sum transactionRevenue', fontsize=12)\nplt.xlabel('Date', fontsize=12)\nplt.scatter(time_agg.index.values, time_agg['sum'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bd0d160eaacb0c7157d40e437785bc054eb025a"},"cell_type":"markdown","source":"Seems we had more transactions on late 2016 and early 2017, date features seems to be a good addition to our model."},{"metadata":{"_uuid":"8489d8eddf95d384f636c3a7f56218a6112802df"},"cell_type":"markdown","source":"#### And here count of our target feature \"transactionRevenue\"."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"034924a8b460cb50f92c382e666330d3625689f6"},"cell_type":"code","source":"plt.figure(figsize=(20,7))\nplt.ticklabel_format(axis='y', style='plain')\nplt.ylabel('Frequency', fontsize=12)\nplt.xlabel('Date', fontsize=12)\nplt.scatter(time_agg.index.values, time_agg['count'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a61cc057a1fc33401c07017aefe60b5bafe4504b"},"cell_type":"markdown","source":"Again we had higher frequency at a similar time period."},{"metadata":{"_uuid":"bfb09171f63bbed018afed67016ef6f46ce68d61"},"cell_type":"markdown","source":"#### Let's take a look at other time features."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"717f1bc22ba3f4bc16b0c6a8b6128c8d6278e6a6"},"cell_type":"code","source":"fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2, figsize=(20,7))\nax1.scatter(year_agg.index.values, year_agg['sum'])\nax1.locator_params(nbins=2)\nax1.ticklabel_format(axis='y', style='plain')\nax1.set_xlabel('Year', fontsize=12)\n\nax2.scatter(month_agg.index.values, month_agg['sum'])\nax2.locator_params(nbins=12)\nax2.ticklabel_format(axis='y', style='plain')\nax2.set_xlabel('Month', fontsize=12)\n\nax3.scatter(day_agg.index.values, day_agg['sum'])\nax3.locator_params(nbins=10)\nax3.ticklabel_format(axis='y', style='plain')\nax3.set_xlabel('Day', fontsize=12)\n\nax4.scatter(weekday_agg.index.values, weekday_agg['sum'])\nax4.locator_params(nbins=7)\nax4.ticklabel_format(axis='y', style='plain')\nax4.set_xlabel('Weekday', fontsize=12)\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"512191498dba9d5931436f83a03792c4b430abfb"},"cell_type":"markdown","source":"### About the engineered time features\n* Year: It seem transactions had a large increase from 2016 to 2017\n* Month: Lager transaction on december seems ok, but about months but im not sure why high values on april and august (maybe because of easter (april) or Tax-free weekend, back-to-school season(august)?)\n* Day: Here it seems that not really important is going on, seems this features can be discarded.\n* Weekday: Something strange is going on here, seems that weekends have less transactions?"},{"metadata":{"_uuid":"29ade30aca892241245ab0c4ae465fc90131d4c9"},"cell_type":"markdown","source":"### The let's do some cleaning"},{"metadata":{"trusted":true,"_uuid":"0a518b83d8655daf6011af0755112d7d76fb5f7f"},"cell_type":"code","source":"# Drop stange 'dict' column\ntrain = train.drop(['adwordsClickInfo'], axis=1)\ntest = test.drop(['adwordsClickInfo'], axis=1)\n# Drop column that exists only in train data\ntrain = train.drop(['campaignCode'], axis=1)\n# Input missing transactionRevenue values\ntrain[\"transactionRevenue\"].fillna(0, inplace=True)\n\ntest_ids = test[\"fullVisitorId\"].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83461e87e03bf7c1ed283ae1acd20b6fdd62f86e"},"cell_type":"markdown","source":"### Drop unwanted columns"},{"metadata":{"trusted":true,"_uuid":"aed9c5286317a4a99d6ad698da3f426d89482bc3"},"cell_type":"code","source":"# Unwanted columns\nunwanted_columns = ['fullVisitorId', 'sessionId', 'visitId', 'visitStartTime', \n                    'browser', 'browserSize', 'browserVersion', 'flashVersion', \n                    'mobileDeviceInfo', 'mobileDeviceMarketingName', 'mobileDeviceModel', \n                    'mobileInputSelector', 'operatingSystemVersion', 'screenColors', \n                    'metro','networkDomain', 'networkLocation', 'adContent', 'campaign', \n                    'isTrueDirect', 'keyword', 'referralPath', 'source', 'operatingSystem', 'day']\n\ntrain = train.drop(unwanted_columns, axis=1)\ntest = test.drop(unwanted_columns, axis=1)\n# Constant columns\nconstant_columns = [c for c in train.columns if train[c].nunique()<=1]\nprint('Columns with constant values: ', constant_columns)\ntrain = train.drop(constant_columns, axis=1)\ntest = test.drop(constant_columns, axis=1)\n# Columns with more than 50% null data\nhigh_null_columns = [c for c in train.columns if train[c].count()<=len(train) * 0.5]\nprint('Columns more than 50% null values: ', high_null_columns)\ntrain = train.drop(high_null_columns, axis=1)\ntest = test.drop(high_null_columns, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa67c4df2ffbabfdedf74e5c9a05cd0c2afc28fd"},"cell_type":"markdown","source":"### This is our new data with some cleaning and engineering."},{"metadata":{"trusted":true,"_uuid":"dc26547fca1279cc668ca6b0218ad8be84632f6c","_kg_hide-input":true},"cell_type":"code","source":"print('TRAIN SET')\nprint('Rows: %s' % train.shape[0])\nprint('Columns: %s' % train.shape[1])\nprint('Features: %s' % train.columns.values)\nprint()\nprint('TEST SET')\nprint('Rows: %s' % test.shape[0])\nprint('Columns: %s' % test.shape[1])\nprint('Features: %s' % test.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f09858c65539245776bd32982adca00a83cd0b8","_kg_hide-input":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b116a23b4e0b27e83a58b42b78aa20dae008ef3b"},"cell_type":"markdown","source":"### One-hot encode categorical data"},{"metadata":{"trusted":true,"_uuid":"ea46ffcd3a8a252581b82b34efea67bca1d7c323"},"cell_type":"code","source":"categorical_features = ['isMobile', 'month', 'weekday']\ntrain = pd.get_dummies(train, columns=categorical_features)\ntest = pd.get_dummies(test, columns=categorical_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3cad54cc1308c8d860dd9a7be62240ecf2e41a6"},"cell_type":"code","source":"# align both data sets (by outer join), to make they have the same amount of features,\n# this is required because of the mismatched categorical values in train and test sets\ntrain, test = train.align(test, join='outer', axis=1)\n\n# replace the nan values added by align for 0\ntrain.replace(to_replace=np.nan, value=0, inplace=True)\ntest.replace(to_replace=np.nan, value=0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7165c6e2ca825cc63ca5e731eacf74bf64dbe5e4"},"cell_type":"markdown","source":"### Split data in train and validation by date\n* This time based split will result in approximated 85% train and 15% validation."},{"metadata":{"trusted":true,"_uuid":"5e6053dd095976d91b58930556b81536dbb9e8ba","_kg_hide-output":true},"cell_type":"code","source":"X_train = train[train['date']<=datetime.date(2017, 5, 31)]\nX_val = train[train['date']>datetime.date(2017, 5, 31)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41a6ecec90bfdc51b04d8f7e576123c88bb76c03"},"cell_type":"code","source":"# Get labels\nY_train = X_train['transactionRevenue'].values\nY_val = X_val['transactionRevenue'].values\nX_train = X_train.drop(['transactionRevenue'], axis=1)\nX_val = X_val.drop(['transactionRevenue'], axis=1)\ntest = test.drop(['transactionRevenue'], axis=1)\n# Log transform the labels\nY_train = np.log1p(Y_train)\nY_val = np.log1p(Y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"319489402ef72e89cd3e755c56f4dad5fa66f621"},"cell_type":"code","source":"reduce_features = ['city', 'medium', 'channelGrouping', 'region', \n                   'subContinent', 'country', 'continent', 'deviceCategory', \n                   'year', 'date']\nX_train = X_train.drop(reduce_features, axis=1)\nX_val = X_val.drop(reduce_features, axis=1)\ntest = test.drop(reduce_features, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ea8506352342e8dfb29b72cb0300f12ea1882c8"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8ee14b132de2ddad8ed79bf2e1e4ce603e09c48"},"cell_type":"markdown","source":"### Normalize the data"},{"metadata":{"trusted":true,"_uuid":"fae2367dc4adc4314ca307403b133163e0c9b1ee","_kg_hide-output":true},"cell_type":"code","source":"normalized_features = ['visitNumber', 'hits', 'pageviews', \n                       'mean_hits_per_day', 'mean_pageviews_per_day', \n                       'sum_hits_per_day', 'sum_pageviews_per_day']\n\n# Normalize using Min-Max scaling\nscaler = preprocessing.MinMaxScaler()\nX_train[normalized_features] = scaler.fit_transform(X_train[normalized_features])\nX_val[normalized_features] = scaler.transform(X_val[normalized_features])\ntest[normalized_features] = scaler.transform(test[normalized_features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b1bb7a3525f706c1025995e45f75f1585ed7823","_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba7e8af998f613b559c168c8669e5a732c832d35"},"cell_type":"markdown","source":"### Model\n* Now let's try some deep learning to model our data."},{"metadata":{"trusted":true,"_uuid":"6835a447460f64c9e82379046dc9690e94679a01","_kg_hide-input":true},"cell_type":"code","source":"BATCH_SIZE = 64\nEPOCHS = 100\nLEARNING_RATE = 0.0003","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4458c97694da66e9547632accf0a474262e82de"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(256, kernel_initializer='glorot_normal', activation='relu', input_dim=X_train.shape[1]))\nmodel.add(Dense(128, kernel_initializer='glorot_normal', activation='relu'))\nmodel.add(Dense(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c12612ae50ad5de10ac61abbbc8366feffd62619"},"cell_type":"code","source":"adam = optimizers.adam(lr=LEARNING_RATE)\nmodel.compile(loss='mse', optimizer=adam)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"c74906e7da0ddf8c2f4de9d45154d718f64b1f0c"},"cell_type":"code","source":"print('Dataset size: %s' % X_train.shape[0])\nprint('Epochs: %s' % EPOCHS)\nprint('Learning rate: %s' % LEARNING_RATE)\nprint('Batch size: %s' % BATCH_SIZE)\nprint('Input dimension: %s' % X_train.shape[1])\nprint('Features used: %s' % X_train.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"fb5a065b3bf83364d4e4384e0055bf3927813033"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd8f201b71e6db7c61f22ffae5d182c5462fde7a","_kg_hide-output":true},"cell_type":"code","source":"history = model.fit(x=X_train.values, y=Y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, \n                    verbose=1, validation_data=(X_val.values, Y_val))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0db6c86387fb9d4800601a0590e3123533cf16e7"},"cell_type":"markdown","source":"### Model metrics and plot"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"1116d9301275a10be2f6ae81f29eebed3a12e382"},"cell_type":"code","source":"val_predictions = model.predict(X_val)\nmse = mean_squared_error(val_predictions, Y_val)\nrmse = np.sqrt(mean_squared_error(val_predictions, Y_val))\n\nprint('Model validation metrics')\nprint('MSE: %.2f' % mse)\nprint('RMSE: %.2f' % rmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f9a4b6061b48744e59627d62d3364af3c6aacdd"},"cell_type":"code","source":"plot_metrics(history.history['loss'], history.history['val_loss'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac573253c4ab0f3649a62bccbb534e785bc2b151","_kg_hide-output":true},"cell_type":"code","source":"predictions = model.predict(test)\n\nsubmission = pd.DataFrame({\"fullVisitorId\":test_ids})\npredictions[predictions<0] = 0\nsubmission[\"PredictedLogRevenue\"] = predictions\nsubmission = submission.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsubmission.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"]\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"2a90c17dd9515996604ddd21c90aae0d54d8ef23"},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}