# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import xgboost as xgb
import math

# Product Mapping ID
producto_tabla = pd.read_csv('../input/producto_tabla.csv')
producto_tabla['First'] = producto_tabla['NombreProducto'].apply(lambda x: x.split(' ')[0])
producto_dict = dict()
for (i,x) in enumerate(pd.Series(producto_tabla['First'].unique())):
    producto_dict[x] = i
producto_tabla['MapFirst'] = producto_tabla['First'].apply(lambda x: producto_dict[x])
temp = producto_tabla[['Producto_ID', 'MapFirst']].values
producto_cluster = dict()
for i in range(temp.shape[0]):
    producto_cluster[temp[i,0]] = temp[i,1]

# Town State Mapping ID
town_state = pd.read_csv('../input/town_state.csv')
town_state['T_S'] = town_state['Town'] + ':' + town_state['State']
town_state_dict = dict()
s_t = town_state['T_S'].unique()
for i in range(s_t.shape[0]):
    ts = s_t[i].split(':')
    town_state_dict[(ts[0], ts[1])] = i
temp = town_state[['Agencia_ID', 'Town', 'State']].values
town_state_cluster = dict()
for i in range(temp.shape[0]):
    if town_state_cluster.get(temp[i,0], -1) == -1:
        town_state_cluster[temp[i,0]] = town_state_dict[(temp[i,1], temp[i,2])]
    else:
        pass
    
# Cliente ID Map
cliente_tabla = pd.read_csv('../input/cliente_tabla.csv')
cliente_tabla['First'] = cliente_tabla['NombreCliente'].apply(lambda x: x.split(' ')[0])
temp = dict()
tnp = cliente_tabla['First'].unique()
for i in range(tnp.shape[0]):
    temp[tnp[i]] = i
cliente_tabla['MapFirst'] = cliente_tabla['First'].apply(lambda x: temp[x])
temp = cliente_tabla[['Cliente_ID', 'MapFirst']].values
cliente_cluster = dict()
for i in range(temp.shape[0]):
    if cliente_cluster.get(temp[i,0], -1) == -1:
        cliente_cluster[temp[i,0]] = temp[i,1]
    else:
        pass
PC = producto_cluster           # producto dict
AC = town_state_cluster         # agencia dict
CC = cliente_cluster            # cliente dict

# Normalize train ##############################################################

# [ok]
dtypes_train = {#'Semana': np.int8, 
                'Agencia_ID': np.int16, 'Canal_ID': np.int8, 
                'Ruta_SAK' : np.int16, 'Cliente_ID':np.uint32, 
                'Producto_ID': np.uint16, 'Demanda_uni_equil': np.int16}
               
dtypes_test = {#'Semana': np.int8, 
               'id':np.int32, 
               'Agencia_ID': np.int16, 'Canal_ID': np.int8, 
               'Ruta_SAK' : np.int16, 'Cliente_ID':np.uint32, 
               'Producto_ID': np.uint16}
               
groupkeys = [#'Semana', 
             'Agencia_ID', 'Canal_ID','Ruta_SAK', 'Cliente_ID', 'Producto_ID']

reader = pd.read_csv('../input/train.csv', dtype = dtypes_train, 
                     usecols = dtypes_train.keys(), chunksize = 800000)
cnt = 0
df_train = pd.DataFrame()
df_count = pd.DataFrame()
for chunk in reader:
    chunk = pd.DataFrame(chunk)
    flag = chunk.shape[0]
    cnt += 1
    
    chunk['Agencia_ID'] = chunk['Agencia_ID'].apply(lambda x: AC[x])
    chunk['Producto_ID'] = chunk['Producto_ID'].apply(lambda x: PC[x])
    chunk['Cliente_ID'] = chunk['Cliente_ID'].apply(lambda x: CC[x])
    chunk['Count'] = 1
    
    df_train = df_train.append(chunk, ignore_index=True)
    if ((cnt != 0) and (cnt % 10 == 0)) or (flag < 800000):
        df_train = df_train.groupby(groupkeys)[['Demanda_uni_equil', 'Count']].sum()
        df_train = df_train.reset_index()
    else:
        pass
    
    print('cnt = ' + str(cnt) + ', ' + str(df_train.shape))
    # if cnt >= 10:
    #     break
    if flag < 800000:
        print(str(flag) + ', Loaded df_train.') 
        break
    else:
        pass

print(df_train.head())
print(df_train.shape)

# [ok]
df_test = pd.read_csv('../input/test.csv', dtype = dtypes_test, 
                      usecols = dtypes_test.keys())
df_test['Agencia_ID'] = df_test['Agencia_ID'].apply(lambda x: AC[x])
df_test['Producto_ID'] = df_test['Producto_ID'].apply(lambda x: PC[x])
df_test['Cliente_ID'] = df_test['Cliente_ID'].apply(lambda x: CC[x])
# print(df_test.head())
np_test = df_test[['Agencia_ID', 'Canal_ID','Ruta_SAK', 
                    'Cliente_ID', 'Producto_ID']].values
print('df_test: '+str(np_test.shape))

# [*]
Demanda_uni_equil = np.ones(df_test.shape[0])*(-1)
dict_range = range(0, df_train.shape[0]+4000000, 4000000)
for i in range(len(dict_range)-1):
    df_t = df_train[dict_range[i]:dict_range[i+1]][['Agencia_ID', 'Canal_ID','Ruta_SAK', 
                                                    'Cliente_ID', 'Producto_ID', 
                                                    'Demanda_uni_equil', 'Count']].values
    # print(df_train[dict_range[i]:dict_range[i]+1])
    # print(df_t[:1])
    # [ok]
    dict_t = dict()
    for i in range(df_t.shape[0]):
        x = df_t[i,:]
        dict_t[(x[0], x[1], x[2], x[3], x[4])] = [x[5], x[6]]
        # if i == 0:
        #     print(x)
    print('Length of dict: '+str(len(dict_t)))
    # [*]
    for i in range(np_test.shape[0]):
        if Demanda_uni_equil[i] == -1:
            x = np_test[i,:]
            vals = dict_t.get((x[0], x[1], x[2], x[3], x[4]), -1)

            if type(vals) == type([1, 2]):
                D, C = vals[0], vals[1]
                Demanda_uni_equil[i] = D/C if C != 0 else 0
            else:
                pass
            # print(Demanda_uni_equil[i])
        else:
            pass
# [*]
Demanda_uni_equil[Demanda_uni_equil==-1] = 0
Demanda_uni_equil = Demanda_uni_equil.round()
mean_submission = np.c_[df_test['id'].values.reshape((np_test.shape[0],1)), 
                        Demanda_uni_equil.reshape((np_test.shape[0],1))]
np.savetxt('mean_submission.csv', mean_submission, fmt='%.0f', delimiter=',',
           header='id,Demanda_uni_equil')