{"nbformat_minor": 1, "nbformat": 4, "cells": [{"source": ["import time\n", "start_time_all = time.time()\n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "#import matplotlib.pyplot as plt"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "b66f1334-d238-45c8-b191-62712d218162", "collapsed": true, "_uuid": "6ababeef48dad092120f374a6286252a597056f0"}}, {"source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "train_set=pd.read_csv(\"../input/train.csv\")\n", "#oil_set=pd.read_csv(\"../input/oil.csv\")\n", "#mini_train=train_set\n", "#mini_train.to_csv(\"mini_train.csv\")\n", "#mini_train=pd.read_csv(\"../working/mini_train.csv\")\n", "# Any results you write to the current directory are saved as output.\n", "print('done!')"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "17edc9f9-ebd7-4649-bb18-6b797b5a7def", "_uuid": "4b52b17962076483b28094894925e7353e857691"}}, {"source": ["train_set.head()"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "ee14bae7-27f1-46c6-a0bb-0a2f1c189ac0", "scrolled": true, "_uuid": "4d266c17b91c81fe89459429269718184318559f"}}, {"source": ["size=train_set.shape[0]"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "5fd3f1ee-a590-4514-93b4-32464bc8cd1a", "_uuid": "203102154493b60899b43bd4f7e73bfde382a393"}}, {"source": ["from datetime import datetime\n"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "998e9d84-4406-401e-9441-981d16a05bc8", "collapsed": true, "_uuid": "5a0a095c3b9a1c053f9627d2ee2b24c5445486eb"}}, {"source": ["stores_set=pd.read_csv(\"../input/stores.csv\")\n", "#STORES ENCODE\n", "#city state type cluster\n", "from sklearn.preprocessing import LabelEncoder\n", "from collections import defaultdict\n", "dstores = defaultdict(LabelEncoder)\n", "df=stores_set.copy()\n", "sto_nbr=pd.DataFrame(df['store_nbr'])\n", "df=df.drop('store_nbr',axis=1)\n", "# Encoding the variable\n", "fite = df.apply(lambda x: dstores[x.name].fit_transform(x))\n", "stores_encoded=pd.merge(sto_nbr,fite, left_index=True, right_index=True)\n", "print(stores_encoded.head())\n", "del stores_set\n", "del fite\n", "del sto_nbr\n", "del df"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "316f9b85-02d9-4cdb-9f51-d359306ea4ed", "_uuid": "b72aa8525a90ee9664c3557b14deed1a3f5d0e20"}}, {"source": ["#ITEMS ENCODE\n", "#family class perishable\n", "items_set=pd.read_csv(\"../input/items.csv\")\n", "\n", "items_set.head()\n", "ditems = defaultdict(LabelEncoder)\n", "dfi=items_set.copy()\n", "ite_nbr=pd.DataFrame(dfi['item_nbr'])\n", "dfi=dfi.drop('item_nbr',axis=1)\n", "# Encoding the variable\n", "fite = dfi.apply(lambda x: ditems[x.name].fit_transform(x))\n", "items_encoded=pd.merge(ite_nbr,fite, left_index=True, right_index=True)\n", "print(items_encoded.head())\n", "del items_set\n", "del fite\n", "del ite_nbr\n", "del dfi"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "19b08bb2-263b-4fff-86d7-8b7ad77c0a21", "_uuid": "fc5317115f7687e7943d3562a743b56c3f1e1954"}}, {"source": ["from sklearn.metrics import mean_squared_error\n", "from math import sqrt\n", "from xgboost.sklearn import XGBRegressor\n", "max_depth = 3\n", "min_child_weight = 1#10\n", "subsample = 1#0.5\n", "colsample_bytree = 1#0.6\n", "objective = 'reg:linear'\n", "num_estimators = 1000\n", "learning_rate = 0.2\n", "silent=False\n", "#booster='gblinear'\n", "booster='gbtree'\n", "model = XGBRegressor(\n", "                    max_depth=max_depth, \n", "                    learning_rate=learning_rate, \n", "                    n_estimators=num_estimators, \n", "                    silent=silent, \n", "                    objective=objective, \n", "                    booster=booster, \n", "                    n_jobs=1, \n", "                    nthread=None, \n", "                    gamma=0, \n", "                    min_child_weight=min_child_weight, \n", "                    max_delta_step=0, \n", "                    subsample=subsample, \n", "                    colsample_bytree=colsample_bytree, \n", "                    colsample_bylevel=1, \n", "                    reg_alpha=0, \n", "                    reg_lambda=1, \n", "                    scale_pos_weight=1, \n", "                    base_score=0.5, \n", "                    random_state=0, seed=None,  missing=None\n", ")   "], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "f8538b1c-36c8-47c4-95e4-9e6876c9b6f0", "collapsed": true, "_uuid": "327ddc2bc8911320a2bcb9752023198752014423"}}, {"source": ["start_time = time.time()\n", "size=125497040\n", "inicio=0\n", "interval_size=1254970\n", "i_trainset=interval_size#0#125497040#    testset 3370464\n", "i=0\n", "while i<4:\n", "    i=i+1\n", "    print(i)\n", "    size=size-interval_size\n", "    if size>0:\n", "        mini_train=train_set.iloc[inicio:i_trainset,:]\n", "        inicio=i_trainset\n", "        i_trainset=i_trainset+interval_size\n", "        print(\"done1\")\n", "        mini_train=mini_train.drop('id',axis=1)\n", "        mini_train['Ano'] = mini_train['date'].map(lambda x: (datetime.strptime(x, '%Y-%M-%d')).year)\n", "        mini_train['Mes'] = mini_train['date'].map(lambda x: (datetime.strptime(x, '%Y-%M-%d')).month)\n", "        mini_train['Dia'] = mini_train['date'].map(lambda x: (datetime.strptime(x, '%Y-%M-%d')).day)\n", "        mini_train['onpromotion']=(mini_train.isnull()['onpromotion'])=0\n", "        mini_train.loc[mini_train['onpromotion'] ==False, 'onpromotion'] = 0\n", "        mini_train.loc[mini_train['onpromotion'] ==True, 'onpromotion'] = 1\n", "\n", "        t1=mini_train.merge(items_encoded,how='left', left_on=['item_nbr'], right_on = ['item_nbr'])\n", "        t2=t1.merge(stores_encoded,how='left', left_on=['store_nbr'], right_on = ['store_nbr'])\n", "        df=pd.DataFrame(t2)\n", "        del t1\n", "        del t2\n", "        del mini_train\n", "        print(\"Done4!\")\n", "        print(\"--- %s seconds ---\" % (time.time() - start_time))\n", "        print(\"--- %s seconds all---\" % (time.time() - start_time_all))\n", "        #create train set\n", "        #size=df.shape[0]\n", "        X_train = df.loc[:, (df.columns != 'unit_sales') & (df.columns != 'date') ]\n", "        y_train = df.loc[:,(df.columns == 'unit_sales')]\n", "        del df\n", "        print(\"go training\")\n", "        start_time = time.time()\n", "        model.fit(X_train, y_train)  \n", "        print(\"Done\")\n", "        print(\"--- %s seconds ---\" % (time.time() - start_time))\n", "        print(\"--- %s seconds all---\" % (time.time() - start_time_all))\n", "        print(\"train %d %d \" % (inicio , size))\n"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "1498221f-fdc5-4193-9198-fc50d71a8bc5", "_uuid": "a15d9e842e0930a98f45f6f09006dc56bc59bbef"}}, {"source": ["del X_train\n", "del y_train\n", "#del y_train_pred\n", "print(\"done\")"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "5c8d51b6-01da-4d9f-9bca-e7eff367a9c8", "_uuid": "9d87a0aea21ef14c48d456ac45b718900d10e8cb"}}, {"source": ["#preparacao do set de test\n", "start_time = time.time()\n", "test_set=pd.read_csv(\"../input/test.csv\")\n", "print(test_set.shape)\n", "test_set['Ano'] = test_set['date'].map(lambda x: (datetime.strptime(x, '%Y-%M-%d')).year)\n", "test_set['Mes'] = test_set['date'].map(lambda x: (datetime.strptime(x, '%Y-%M-%d')).month)\n", "test_set['Dia'] = test_set['date'].map(lambda x: (datetime.strptime(x, '%Y-%M-%d')).day)\n", "print(\"Done!\")\n", "test_set['onpromotion']=(test_set.isnull()['onpromotion'])=0\n", "test_set.loc[test_set['onpromotion'] ==False, 'onpromotion'] = 0\n", "test_set.loc[test_set['onpromotion'] ==True, 'onpromotion'] = 1\n", "print(\"Done!\")\n", "te1=test_set.merge(items_encoded,how='left', left_on=['item_nbr'], right_on = ['item_nbr'])\n", "te2=te1.merge(stores_encoded,how='left', left_on=['store_nbr'], right_on = ['store_nbr'])\n", "test_df=pd.DataFrame(te2)\n", "X_test = test_df.loc[:,(test_df.columns != 'id') & (test_df.columns != 'unit_sales') & (test_df.columns != 'date') ]\n", "del te1\n", "del te2\n", "del test_set\n", "del items_encoded\n", "del stores_encoded\n", "print(\"Done!\")\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "3bf33432-93f0-492b-81e6-92112830472c", "_uuid": "30c0a9200b195a595daace07fa2cd775d0ff820d"}}, {"source": ["start_time = time.time()\n", "y_test_pred=model.predict(X_test)  \n", "y_test_df=pd.DataFrame(y_test_pred,columns={'unit_sales'})\n", "y_test_df[y_test_df.unit_sales<0]=0\n", "output_set=pd.concat( (test_df,y_test_df),axis=1)[['id','date','store_nbr','item_nbr','onpromotion','unit_sales']]\n", "output_seta=output_set.groupby(['id']).agg({'unit_sales':'sum'})\n", "output_seta.to_csv(\"output.csv\")\n", "del output_set\n", "del output_seta\n", "print('done!')\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "943d839e-c428-4c03-a4b2-d8c959ba99c0", "_uuid": "75a5699e77c859f2a8100a28156f774a0e90a969"}}, {"source": ["#### THE END"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "5cd7caa0-26c4-4442-8967-b9d7eb34e884", "collapsed": true, "_uuid": "2969d71adf816b927f5e76343849f6e7798eaea7"}}, {"source": ["print(check_output([\"ls\", \"../working\"]).decode(\"utf8\"))\n", "print(\"--- %s seconds ---\" % (time.time() - start_time_all))"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "39dc03a4-4917-4837-b0ff-99d924cbfb67", "_uuid": "100542238fcaa3924a91019daf37df7de427f8ce"}}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"pygments_lexer": "ipython3", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "mimetype": "text/x-python", "version": "3.6.4", "nbconvert_exporter": "python", "name": "python"}}}