{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom datetime import timedelta\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport warnings\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.options.display.max_colwidth = 100\n\ndiasSemana = ['Lunes', 'Martes', 'Miercoles', 'Jueves', 'Viernes', 'Sabado', 'Domingo']\nmesesAnio = ['Enero','Febrero','Marzo','Abril','Mayo','Junio','Julio','Agosto','Septiembre','Octubre','Noviembre','Diciembre']\n\n\ndef get_categoria(serie_descripciones):\n    r = []\n    for descripcion in serie_descripciones:\n        string = descripcion.strip()\n        if(string[0] == '/'):\n            string = string.strip('/').split('/')[0]\n        else:\n            string = string.split(':')[0]\n        r.append(string.strip().upper())\n    return r\n\ndef poner_nombre_features(lista_features, dict_pages, dict_category, dict_category_bottom, dict_category_top, dict_site):\n    nombre_features = []\n    for feature in lista_features:\n        if(len(feature.split('_')) > 1):\n            feature_class = '_'.join(feature.split('_')[:-1])\n            feature_number = feature.split('_')[-1]\n            if(feature_number.isnumeric()):\n                feature_number = int(feature_number)\n            if(feature_class == \"page\"):\n                nombre_features.append(\"page_\"+dict_pages[feature_number])\n            elif(feature_class == \"content_category\"):\n                nombre_features.append(\"cat_\"+dict_category[feature_number])\n            elif(feature_class == \"content_category_bottom\"):\n                nombre_features.append(\"catbot_\"+dict_category_bottom[feature_number])\n            elif(feature_class == \"content_category_top\"):\n                nombre_features.append(\"cattop_\"+dict_category_top[feature_number])\n            elif(feature_class == \"site_id\"):\n                nombre_features.append(\"site_\"+dict_site[feature_number])\n            else:\n                nombre_features.append(feature)\n        else:\n            nombre_features.append(feature)\n    return nombre_features\n\ndef get_dict_jerarquias(serie_descripciones, serie_id):\n    jerarquias = {}\n    for descripcion, id_num in zip(serie_descripciones, serie_id):\n        string = descripcion.strip()\n        if(string[0] == '/'):\n            strings = string.strip('/').split('/')\n        else:\n            strings = string.strip(':').split(':')\n            strings = [s.strip() for s in strings]\n            \n        dict_actual = jerarquias\n        for s in strings:\n            if s not in dict_actual:\n                dict_actual[s] = {}\n            dict_actual = dict_actual[s]\n        dict_actual['ids'] = [id_num]\n    return jerarquias","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"../input/pageviews/pageviews.csv\", parse_dates=[\"FEC_EVENT\"])\ndata.columns = [s.lower() for s in data.columns]\n\ndata_complemento = pd.read_csv(\"../input/pageviews_complemento/pageviews_complemento.csv\", parse_dates=[\"FEC_EVENT\"], dtype=\"int\")\ndata_complemento.columns = [s.lower() for s in data_complemento.columns]\ndata_complemento = data_complemento[data.columns]\n\ndata = data.append(data_complemento, ignore_index=True)\n\n#cant_cada_pagina = data['page'].value_counts()\n#paginas_menores = list(cant_cada_pagina[cant_cada_pagina < 8].index)\n#data.loc[data['page'].isin(paginas_menores)]['page'] = 10000\n\npage = pd.read_csv(\"../input/page/PAGE.csv\")\npage.columns = [s.lower() for s in page.columns]\npage = page.set_index('page').reindex(data['page'].unique()).reset_index()\npage[\"categoria\"] = get_categoria(page[\"page_descripcion\"])\ncategorias_pages_dict = page.drop(['page_descripcion'], axis=1).set_index(\"page\").to_dict()['categoria']\ndata['categoria_pag'] = [categorias_pages_dict[k] for k in data['page']]\n\nconvs = pd.read_csv(\"../input/conversiones/conversiones.csv\", dtype=\"int\")\nconvs.columns = [s.lower() for s in convs.columns]\n#convs[\"trimestre\"] = ((convs[\"mes\"] - 1) // 3) + 1\n#conversiones_trimestrales = pd.crosstab(convs[\"user_id\"], convs[\"trimestre\"])\n#conversiones_trimestrales.columns = [\"convs_\" + str(c) + \"Q\" for c in conversiones_trimestrales.columns]\n\ncontent = pd.read_csv(\"../input/content_category/CONTENT_CATEGORY.csv\")\ncontent.columns = [s.lower() for s in content.columns]\n\ncontent_top = pd.read_csv(\"../input/content_category_top/CONTENT_CATEGORY_TOP.csv\")\ncontent_top.columns = [s.lower() for s in content_top.columns]\n\ncontent_bottom = pd.read_csv(\"../input/content_category_bottom/CONTENT_CATEGORY_BOTTOM.csv\")\ncontent_bottom.columns = [s.lower() for s in content_bottom.columns]\n\ndevice = pd.read_csv(\"../input/device_data/device_data.csv\", parse_dates=[\"FEC_EVENT\"])\ndevice.columns = [s.lower() for s in device.columns]\n\nsite_id = pd.read_csv(\"../input/site_id/SITE_ID.csv\")\nsite_id.columns = [s.lower() for s in site_id.columns]\n\ndict_pages = page.set_index(\"page\").to_dict()['page_descripcion']\ndict_pages[10000] = \"pagina_menor\"\ndict_category = content.set_index(\"content_category\").to_dict()['content_category_descripcion']\ndict_category_bottom = content_bottom.set_index(\"content_category_bottom\").to_dict()['content_category_bottom_descripcion']\ndict_category_top = content_top.set_index(\"content_category_top\").to_dict()['content_category_top_descripcion']\ndict_site = site_id.set_index(\"site_id\").to_dict()['site_id_descripcion']\n\njerarquiasPage = get_dict_jerarquias(page['page_descripcion'], page['page'])\n\ndata.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logins = [1809, 1808,    2, 1820, 1358, 1362, 1773, 1763, 1758, 1213, 1382,\n           1361, 1761, 1743, 1749, 1762,  947, 1456, 1214, 1734, 1174, 1457,\n           1835, 1774, 1041, 1221,  536,  168,  739,  596,  597,  897,    9,\n            518,   63,  366,  389,  368,  258,  308,  253,  421,   17,  312,\n            313,  478,  824,   14,  519,   74,  238,  497,  593,  323,  200,\n            758,   71,  388,   44,  231,  124,  358,  423,  492,  720, 1305,\n            945, 1026, 1832, 1032, 1447,  951,  711, 1027, 1085, 1314,  833]\ninicios = [3]\n\nerrores = [1188, 1815, 1001,   57, 1757,   99,  167,  612,    5, 1822, 1422,\n           1811, 1200, 1179, 1121, 1277, 1384, 1175, 1166,  337,  422,   18,\n             11,   12,  192,  193,  194,  701,  958,  355,  309,  592,   63,\n            366,  389,  368,  258,  308,  253,  421,   17,  594,  706,  104,\n            414,  101,  134,  216,  753,  106,  459,  105,  265,  266,  267,\n            608,  529, 1341,  538, 1267,   74,  238,  497,  593,  323,  200,\n            758,   71,  388,   44,  534,  606,   91,  641,  583,  302,  927,\n           1053, 1010,  532, 1007,  450, 1022, 1023, 1054, 1052, 1069,  996,\n            720, 1305,  945, 1026, 1832, 1032, 1447,  951,  711,  871, 1076,\n            722,  852,  728, 1354, 1040,  122,  973, 1074,  756, 1111, 1169,\n           1110,  983,  757]\n\nsobrantes = logins + inicios + errores\n\nprimeros_nueve_meses = data.loc[~data['page'].isin(sobrantes)]\nprimeros_nueve_meses = primeros_nueve_meses.loc[primeros_nueve_meses['fec_event'].dt.month < 10]\npages_prop = pd.crosstab(primeros_nueve_meses[\"user_id\"], primeros_nueve_meses['page'])\npages_prop = pages_prop.div(pages_prop.sum(axis=1), axis=0)\n\ny_anio = pd.read_csv(\"../input/conversiones/conversiones.csv\")\ny_anio.columns = [s.lower() for s in y_anio.columns]\npages_prop['target'] = 0\nidx = set(y_anio.loc[y_anio['mes'] >= 10][\"user_id\"].unique()).intersection(set(pages_prop.index))\npages_prop.loc[list(idx), ['target']] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fi = []\ni = 0\nscores_val = []\nscores_train = []\n\nfor train_idx, valid_idx in StratifiedKFold(n_splits=10, shuffle=True, random_state=42).split(pages_prop, pages_prop['target']):\n    i += 1\n    \n    Xyt = pages_prop.iloc[train_idx]\n    Xt = Xyt.drop('target', axis = 1)\n    yt = Xyt['target']\n\n    Xyv = pages_prop.iloc[valid_idx]\n    Xv = Xyv.drop('target', axis = 1)\n    yv = Xyv['target']\n    \n    learner = LGBMClassifier(n_estimators=800,\n                             max_depth=3,\n                             num_leaves=5,\n                             learning_rate=0.01,\n                             objective='binary',\n                             random_state=42\n                            )\n    \n    learner.fit(Xt, yt,  early_stopping_rounds=200, eval_metric=\"auc\",\n                eval_set=[(Xt, yt), (Xv, yv)], verbose=0)\n    \n    scores_val.append(learner.best_score_['valid_1']['auc'])\n    scores_train.append(learner.best_score_['training']['auc'])\n    \n    fi.append(pd.Series(learner.feature_importances_ / learner.feature_importances_.sum(), index=Xt.columns)) \n\nfi = pd.concat(fi, axis=1).mean(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Score promedio training: \" + str(np.around(np.mean(scores_train), decimals=4)))\nprint(\"Score promedio validacion: \" + str(np.around(np.mean(scores_val), decimals=4)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_paginas = 100\npaginas_importantes = fi[fi != 0].sort_values(ascending=False)\npaginas_importantes = list(paginas_importantes[:num_paginas].index)\nfi_nombres = fi[fi != 0].sort_values(ascending=False)[:num_paginas].copy()\nfi_nombres.index = [dict_pages[k] for k in list(fi_nombres.index)]\nfi_nombres[:25]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generar_features(data_periodo, device_periodo, convs_periodo):\n    ultimo_dia = data_periodo['fec_event'].dt.date.max()    \n    data_periodo = data_periodo.loc[~data_periodo['page'].isin(sobrantes)]\n    \n    def generar_cantidad_eventos_por_franja(data_periodo_temp, nombre_evento):\n        X = pd.DataFrame(index=sorted(data['user_id'].unique()))\n        \n        # CANTIDAD DE EVENTOS POR TRIMESTRE\n        \n        cantidad_ocurrencias_evento = data_periodo_temp.copy()\n        cantidad_ocurrencias_evento['fec_event_hora'] = cantidad_ocurrencias_evento['fec_event'].dt.floor('H')\n        cantidad_ocurrencias_evento['fec_event_trimestre'] = (cantidad_ocurrencias_evento['fec_event'].dt.month - 1) // 3\n        cantidad_ocurrencias_evento = cantidad_ocurrencias_evento.drop_duplicates(subset=['fec_event_hora', 'user_id'])\n        ocurrencias_por_trimestre = pd.crosstab(cantidad_ocurrencias_evento['user_id'], cantidad_ocurrencias_evento['fec_event_trimestre'])\n        ocurrencias_por_trimestre = ocurrencias_por_trimestre.loc[:, sorted(ocurrencias_por_trimestre.columns)]\n        ocurrencias_por_trimestre.columns = [\"ocurrencias_\" + nombre_evento + '_trimestre_' + str(i) for i in range(1, len(ocurrencias_por_trimestre.columns)+1)]\n        X = X.join(ocurrencias_por_trimestre, how = \"left\").fillna(0)\n        \n        # CANTIDAD DE EVENTOS ULTIMO MES\n        \n        cantidad_ocurrencias_evento_ult_mes = data_periodo_temp.loc[data_periodo_temp['fec_event'].dt.month == data_periodo_temp['fec_event'].dt.month.max()].copy()\n        cantidad_ocurrencias_evento_ult_mes['fec_event_hora'] = cantidad_ocurrencias_evento_ult_mes['fec_event'].dt.floor('H')\n        cantidad_ocurrencias_evento_ult_mes = cantidad_ocurrencias_evento_ult_mes.drop_duplicates(subset=['fec_event_hora', 'user_id'])['user_id'].value_counts().to_frame()\n        cantidad_ocurrencias_evento_ult_mes.columns = [nombre_evento + '_ultimo_mes']\n        X = X.join(cantidad_ocurrencias_evento_ult_mes, how = \"left\").fillna(0)\n        \n        # CANTIDAD DE EVENTOS ULTIMOS QUINCE DIAS\n        \n        inicio_ult_15_dias = ultimo_dia - timedelta(days=15)\n        cantidad_ocurrencias_evento_ult_15_dias = data_periodo_temp.loc[data_periodo_temp['fec_event'].dt.date > inicio_ult_15_dias].copy()\n        cantidad_ocurrencias_evento_ult_15_dias['fec_event_hora'] = cantidad_ocurrencias_evento_ult_15_dias['fec_event'].dt.floor('H')\n        cantidad_ocurrencias_evento_ult_15_dias = cantidad_ocurrencias_evento_ult_15_dias.drop_duplicates(subset=['fec_event_hora', 'user_id'])['user_id'].value_counts().to_frame()\n        cantidad_ocurrencias_evento_ult_15_dias.columns = [nombre_evento + '_ultimos_15_dias']\n        X = X.join(cantidad_ocurrencias_evento_ult_15_dias, how = \"left\").fillna(0)\n        \n        return X\n        \n    def generar_features_temporales(data_periodo_temp, nombre_evento):\n        X = pd.DataFrame(index=sorted(data['user_id'].unique()))\n\n        # CANTIDAD DE DIAS EN CADA TRIMESTRE QUE HUBO AL MENOS UNA OCURRENCIA DEL EVENTO PARA CADA USUARIO\n\n        dias_ocurrencia_evento = data_periodo_temp.copy()\n        dias_ocurrencia_evento['fec_event_dia'] = dias_ocurrencia_evento['fec_event'].dt.floor('D')\n        dias_ocurrencia_evento['fec_event_trimestre'] = (dias_ocurrencia_evento['fec_event'].dt.month - 1) // 3\n        dias_ocurrencia_evento = dias_ocurrencia_evento.drop_duplicates(subset=['fec_event_dia', 'user_id'])\n        dias_por_trimestre_ocurrencia_evento = pd.crosstab(dias_ocurrencia_evento['user_id'], dias_ocurrencia_evento['fec_event_trimestre'])\n        dias_por_trimestre_ocurrencia_evento = dias_por_trimestre_ocurrencia_evento.loc[:, sorted(dias_por_trimestre_ocurrencia_evento.columns)]\n        dias_por_trimestre_ocurrencia_evento.columns = [\"dias_actividad_\" + nombre_evento + '_trimestre_' + str(i) for i in range(1, len(dias_por_trimestre_ocurrencia_evento.columns)+1)]\n        X = X.join(dias_por_trimestre_ocurrencia_evento, how = \"left\").fillna(0)\n        \n        cantidades = generar_cantidad_eventos_por_franja(data_periodo_temp, nombre_evento)\n        cantidades_totales = generar_cantidad_eventos_por_franja(data_periodo, nombre_evento)\n        proporciones_por_periodo = cantidades.div(cantidades_totales)\n        proporciones_por_periodo.columns = [col.replace(\"ocurrencias\", \"proporcion\") for col in proporciones_por_periodo.columns]\n        proporciones_cada_trimestre = cantidades.filter(like=\"trimestre\")\n        proporciones_cada_trimestre = proporciones_cada_trimestre.div(proporciones_cada_trimestre.sum(axis=1), axis=0)\n        \n        X = X.join(proporciones_por_periodo, how = \"left\").fillna(0)\n        X = X.join(proporciones_cada_trimestre, how = \"left\").fillna(0)\n        \n        # SEMANAS DESDE ULTIMA ACTIVIDAD\n\n        data_ordenada_fecha = data_periodo_temp.sort_values(['fec_event'], ascending=True)\n\n        nombre_feature = 'semanas_desde_ultima_actividad_' + nombre_evento\n        semanas_desde = data_ordenada_fecha.drop_duplicates(subset=['user_id'], keep='last').copy()\n        semanas_desde[nombre_feature] = (data_periodo_temp['fec_event'].dt.dayofyear.max() - semanas_desde['fec_event'].dt.dayofyear) // 7\n        semanas_desde = semanas_desde.loc[:,['user_id', nombre_feature]].set_index('user_id')\n        X = X.join(semanas_desde, how = \"left\").fillna(1000)\n\n        # SEMANAS DESDE PRIMER ACTIVIDAD\n\n        nombre_feature = 'semanas_desde_primer_actividad_' + nombre_evento\n        semanas_desde = data_ordenada_fecha.drop_duplicates(subset=['user_id'], keep='first').copy()\n        semanas_desde[nombre_feature] = (data_periodo_temp['fec_event'].dt.dayofyear.max() - semanas_desde['fec_event'].dt.dayofyear) // 7\n        semanas_desde = semanas_desde.loc[:,['user_id', nombre_feature]].set_index('user_id')\n        X = X.join(semanas_desde, how = \"left\").fillna(1000)\n\n        # CANTIDAD TOTAL TODO EL PERIODO\n        \n        cantidad_ocurrencias_totales = data_periodo_temp.copy()\n        cantidad_ocurrencias_totales['fec_event_hora'] = cantidad_ocurrencias_totales['fec_event'].dt.floor('H')\n        cantidad_ocurrencias_totales = cantidad_ocurrencias_totales.drop_duplicates(subset=['fec_event_hora', 'user_id'])['user_id'].value_counts().to_frame()\n        cantidad_ocurrencias_totales.columns = [nombre_evento + '_totales']\n        X = X.join(cantidad_ocurrencias_totales, how = \"left\").fillna(0)\n        \n        return X\n    \n    \n    # CREACION DEL DATAFRAME\n    \n    X = pd.DataFrame(index=sorted(data['user_id'].unique()))\n    \n    #\n    # FEATURES DE PROPORCION DE ACCESO A CADA PAGE Y CATEGORIA \n    #\n    \n    prop_importantes = data_periodo.loc[data_periodo['page'].isin(paginas_importantes)]\n    prop_importantes = pd.crosstab(prop_importantes[\"user_id\"], prop_importantes['page'])\n    prop_importantes = prop_importantes.div(prop_importantes.sum(axis=1), axis=0)\n    X = X.join(prop_importantes, how='left')\n    \n    X.columns = [\"page_\" + str(i) for i in X.columns]\n    \n    prop_categorias = pd.crosstab(data_periodo[\"user_id\"], data_periodo['page'])\n    prop_categorias = prop_categorias.div(prop_categorias.sum(axis=1), axis=0)\n    page_periodo = page.set_index('page').reindex(data_periodo['page'].unique()).reset_index()\n    \n    pags_prestamos = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"prestamo\", case=False, regex=False)]['page'].values)\n    pags_hipotecario = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"hipotecario\", case=False, regex=False)]['page'].values)\n    pags_tarjetas = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"tarjeta\", case=False, regex=False)]['page'].values)\n    pags_moneda = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"moneda\", case=False, regex=False)]['page'].values)\n    pags_inversiones = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"inversiones\", case=False, regex=False)]['page'].values)\n    pags_plazo = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"fijo\", case=False, regex=False)]['page'].values)\n    pags_fima = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"fima\", case=False, regex=False)]['page'].values)\n    pags_transferencias = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"tranferencia\", case=False, regex=False)]['page'].values)\n    pags_agro = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"agro\", case=False, regex=False)]['page'].values)\n    pags_pyme = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"pyme\", case=False, regex=False)]['page'].values)\n    pags_haberes = list(page_periodo.loc[page_periodo['page_descripcion'].str.contains(\"haberes\", case=False, regex=False)]['page'].values)\n    \n    prop_categorias_suma = pd.DataFrame(index=prop_categorias.index)\n    \n    prop_categorias_suma['prestamos_prop'] =  prop_categorias.loc[:,pags_prestamos].sum(axis=1)\n    prop_categorias_suma['hipotecario_prop'] =  prop_categorias.loc[:,pags_hipotecario].sum(axis=1)\n    prop_categorias_suma['tarjetas_prop'] =  prop_categorias.loc[:,pags_tarjetas].sum(axis=1)\n    prop_categorias_suma['moneda_prop'] =  prop_categorias.loc[:,pags_moneda].sum(axis=1)\n    prop_categorias_suma['inversiones_prop'] =  prop_categorias.loc[:,pags_inversiones].sum(axis=1)\n    prop_categorias_suma['plazo_fijo_prop'] =  prop_categorias.loc[:,pags_plazo].sum(axis=1)\n    prop_categorias_suma['fima_prop'] =  prop_categorias.loc[:,pags_fima].sum(axis=1)\n    prop_categorias_suma['transferencias_prop'] =  prop_categorias.loc[:,pags_transferencias].sum(axis=1)\n    prop_categorias_suma['agro_prop'] =  prop_categorias.loc[:,pags_agro].sum(axis=1)\n    prop_categorias_suma['pyme_prop'] =  prop_categorias.loc[:,pags_pyme].sum(axis=1)\n    prop_categorias_suma['haberes_prop'] =  prop_categorias.loc[:,pags_haberes].sum(axis=1)\n    del prop_categorias\n    \n    X = X.join(prop_categorias_suma, how='left')\n    \n    #\n    # FEATURES TEMPORALES\n    #\n    \n    # TODOS LOS EVENTOS\n    \n    temp = generar_features_temporales(data_periodo, \"general\")\n    X = X.join(temp, how='left')\n    \n    # PRESTAMOS\n    \n    ## TODOS\n    \n    pags = page.loc[(page['page_descripcion'].str.contains(\"prestamo\", case=False, regex=False)) | (page['page_descripcion'].str.contains(\"pp\", case=False, regex=False))]['page'].values\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"prestamos\")\n    X = X.join(temp, how='left')\n    \n    ## HIPOTECARIOS\n    \n    pags = page.loc[page['page_descripcion'].str.contains('HIPOTECARIO', case=False, regex=False)]['page'].values\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"hipotecario\")\n    X = X.join(temp, how='left')\n    \n    ## PERSONALES EXITO\n    \n    temp = data_periodo.loc[data_periodo['page'] == 345]\n    temp = generar_features_temporales(temp, \"personales_exito\")\n    X = X.join(temp, how='left')\n    \n    ## HIPOTECARIO EXITO\n    \n    temp = data_periodo.loc[data_periodo['page'] == 1297]\n    temp = generar_features_temporales(temp, \"hipotecarios_exito\")\n    X = X.join(temp, how='left')\n    \n    # AUMENTO DE LIMITE TARJETAS\n    \n    temp = data_periodo.loc[data_periodo['page'].isin([159, 496, 830])]\n    temp = generar_features_temporales(temp, \"aumento_limite_tarj\")\n    X = X.join(temp, how='left')\n    \n    # TRANSFERENCIAS\n    \n    pags = page.loc[page['page_descripcion'].str.contains('TRANSFERENCIA', case=False, regex=False)]['page'].values\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"transferencias\")\n    X = X.join(temp, how='left')\n    \n    # TARJETAS\n    \n    ## PAGINAS DE RESUMENES Y CONSUMOS\n    pags = [1461,  328,  327, 1083,  506, 1224,  113,  235,  329,  330,  535, 1239,   23,   69,  409,  650, 65,  64, 213, 651]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"tarjetas\")\n    X = X.join(temp, how='left')\n    \n    # DOLARES\n    \n    ## COMPRA EXITO\n    \n    temp = data_periodo.loc[data_periodo['page'] == 173]\n    temp = generar_features_temporales(temp, \"compra_dolares_exito\")\n    X = X.join(temp, how='left')\n    \n    ## VENTA EXITO\n    \n    temp = data_periodo.loc[data_periodo['page'] == 318]\n    temp = generar_features_temporales(temp, \"venta_dolares_exito\")\n    X = X.join(temp, how='left')\n    \n    # BONOS Y ACCIONES\n    pags = [440,  442,  379,  377,  375,  376,  378, 1521, 1523, 1522,  737, 735,  733,  734,  736,  422,  753, 1010,  426,  513]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"inversiones\")\n    X = X.join(temp, how='left')\n    \n    # PLAZO FIJO\n    \n    ## TODOS\n    \n    pags = page.loc[page['page_descripcion'].str.contains('FIJO', case=False, regex=False)]['page'].values\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"plazo_fijo\")\n    X = X.join(temp, how='left')\n    \n    ## CONFIRMACION\n    \n    temp = data_periodo.loc[data_periodo['page'].isin([38, 172, 212, 491])]\n    temp = generar_features_temporales(temp, \"plazo_fijo_exito\")\n    X = X.join(temp, how='left')\n    \n    # FIMA\n    \n    ## TODOS\n    \n    pags = page.loc[page['page_descripcion'].str.contains('fima', case=False, regex=False)]['page'].values\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"fima\")\n    X = X.join(temp, how='left')\n    \n    ## RESCATE\n    \n    pags = [416,  417,  419,  418,  420, 1237, 1431,  309,  196,  538,  352, 996,  743]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"fima_rescate\")\n    X = X.join(temp, how='left')\n    \n    ## SUSCRIPCION\n    \n    pags = [284,  429,  343,  341,  431, 1081, 1072,  342,  430,  592,  307, 1267,  275,  714]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"fima_constitucion\")\n    X = X.join(temp, how='left')\n    \n    # HABERES\n    \n    pags = [603, 600, 846, 601, 507, 602, 599, 818]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"pago_haberes\")\n    X = X.join(temp, how='left')\n    \n    # AGRO\n    \n    pags = [1115,  900, 1332, 1132, 1681,  868, 1194, 1167,  243, 1410, 1796, 1767, 1016,  876]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"agro\")\n    X = X.join(temp, how='left')\n    \n    # PYME\n    \n    pags = [1051,  699,  398,  783,  840, 1304,  823,  779, 1791, 1282, 1364,1289, 1300, 1786, 1197, 1663,  203]\n    temp = data_periodo.loc[data_periodo['page'].isin(pags)]\n    temp = generar_features_temporales(temp, \"pyme\")\n    X = X.join(temp, how='left')\n    \n    # DETENER DEBITO AUTOMATICO\n    \n    ## TODOS\n    \n    temp = data_periodo.loc[data_periodo['categoria_pag'].isin([1338, 1340, 1339])]\n    temp = generar_features_temporales(temp, \"detener_debito\")\n    X = X.join(temp, how='left')\n    \n    ## CONFIRMACION\n    \n    temp = data_periodo.loc[data_periodo['categoria_pag'] == 1340]\n    temp = generar_features_temporales(temp, \"detener_debito_confirmacion\")\n    X = X.join(temp, how='left')\n    \n    # TARJETAS\n    \n    #\n    # OTRAS FEATURES\n    #\n    \n    # DEBITO AUTOMATICO\n    \n    ## DESADHESIONES\n    \n    ### TARJETAS\n    \n    #debito_automatico = data.loc[data['page'].isin([141, 486, 1340])].sort_values(by=['fec_event'], ascending=True).drop_duplicates(subset=['user_id'], keep='last')\n    \n    \n    ### PAGO\n    \n    \n    # CANTIDAD DE TARJETAS\n    \n    def generar_cantidad_tarjetas(df_X, arr_pages_tarjetas, nombre_feature):\n        df_X[nombre_feature] = 0\n        i = 1\n        for num_page in arr_pages_tarjetas:\n            usuarios_n_tarjetas = data.loc[data['page'] == num_page]['user_id'].unique()\n            df_X.loc[usuarios_n_tarjetas, [nombre_feature]] = i\n            i += 1\n    \n    ## CREDITO\n    \n    generar_cantidad_tarjetas(X, [23,  69, 409], 'cantidad_tarjetas_credito')\n    \n    ## DEBITO\n    \n    generar_cantidad_tarjetas(X, [113,  235,  329,  330,  535, 1239], 'cantidad_tarjetas_debito')\n    \n    ## ADICIONALES\n    \n    generar_cantidad_tarjetas(X, [328,  327], 'cantidad_tarjetas_adicionales')\n    \n    # CATEGORIA USUARIO\n    \n    paginas_web = [1,40,202,203,243,268,370,1492]\n    dict_pag_cat_usuario = {jerarquiasPage['WEB'][pag]['ids'][0]: pag for pag in jerarquiasPage['WEB']}\n    cat_usuarios = data.loc[data['page'].isin(paginas_web)].copy()\n    cat_usuarios['cat_usuario'] = [dict_pag_cat_usuario[k] for k in cat_usuarios['page']]\n    cat_usuarios = pd.crosstab(cat_usuarios[\"user_id\"], cat_usuarios['cat_usuario'])\n    cat_usuarios = cat_usuarios.div(cat_usuarios.sum(axis=1), axis=0)\n    X = X.join(cat_usuarios, how = \"left\").fillna(0)\n\n    # CONVERSIONES POR MES\n    \n    convs_por_mes = pd.crosstab(convs_periodo['user_id'], convs_periodo['mes'])\n    convs_por_mes = convs_por_mes.loc[:, sorted(convs_por_mes.columns)]\n    convs_por_mes.columns = ['convs_mes_' + str(i) for i in range(1, len(convs_por_mes.columns) + 1)]\n    X = X.join(convs_por_mes, how = \"left\").fillna(0)\n\n    # PORCENTAJE DE ENTRADAS POR MOVIL Y COMPUTADORA\n    \n    mobile = pd.crosstab(device_periodo['user_id'], device_periodo['is_mobile_device'])\n    mobile.columns = ['mobile_' + str(s) for s in mobile.columns]\n    mobile = mobile.div(mobile.sum(axis=1), axis=0)\n    X = X.join(mobile, how = \"left\").fillna(0)\n\n    # PORCENTAJE DE ENTRADA POR CADA CATEGORIA DE CONECTION SPEED\n    \n    conection_speed = pd.crosstab(device_periodo['user_id'], device_periodo['connection_speed'])\n    conection_speed.columns = ['con_speed_' + str(s) for s in conection_speed.columns]\n    conection_speed = conection_speed.div(conection_speed.sum(axis=1), axis=0)\n    X = X.join(conection_speed, how = \"left\").fillna(0)\n\n    # CANTIDAD DE SESIONES POR MES\n    \n    sesiones_por_mes = pd.crosstab(device_periodo['user_id'], device_periodo['fec_event'].dt.month)\n    sesiones_por_mes = sesiones_por_mes.loc[:, sorted(sesiones_por_mes.columns)]\n    sesiones_por_mes.columns = [\"sesiones_mes_\" + str(i) for i in range(1, len(sesiones_por_mes.columns)+1)]\n    X = X.join(sesiones_por_mes, how = \"left\").fillna(0)\n\n    # CANTIDAD DE SESIONES TOTALES DEL PERIODO\n    \n    sesiones_totales_periodo = device_periodo['user_id'].value_counts().to_frame()\n    sesiones_totales_periodo.columns = ['sesiones_totales_periodo']\n    X = X.join(sesiones_totales_periodo, how = \"left\").fillna(0)\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CANT_MESES_PERIODO = 9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generar test\n\nmes_inicio_test = 12 - CANT_MESES_PERIODO\ndata_test = data.loc[data[\"fec_event\"].dt.month > mes_inicio_test]\ndevice_test = device.loc[device[\"fec_event\"].dt.month > mes_inicio_test]\nconvs_test = convs.loc[convs[\"mes\"] > mes_inicio_test]\n\nX_test = generar_features(data_test, device_test, convs_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generar train\nX_train = []\ny_anio = pd.read_csv(\"../input/conversiones/conversiones.csv\")\ny_anio.columns = [s.lower() for s in y_anio.columns]\n\nmes_fin_train = 1 + CANT_MESES_PERIODO\n\ndata_train = data.loc[data[\"fec_event\"].dt.month < mes_fin_train]\ndevice_train = device.loc[device[\"fec_event\"].dt.month < mes_fin_train]\nconvs_train = convs.loc[convs[\"mes\"] < mes_fin_train]\n\nX_train = generar_features(data_train, device_train, convs_train)\ny_train = pd.Series(0, index=X_train.index)\nidx = set(y_anio.loc[y_anio['mes'] >= mes_fin_train][\"user_id\"].unique()).intersection(set(X_train.index))\ny_train.loc[list(idx)] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = list(set(X_train.columns).intersection(set(X_test.columns)))\nX_train = X_train[features]\nX_test = X_test[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/samplesubmission/sampleSubmission.csv\", index_col=0, header=0, names=[\"user_id\", \"score_ej\"])\nX_test = X_test.reindex(submission.index).fillna({'semanas_desde_ultima_actividad':1000, 'semanas_desde_primer_actividad':1000, 'semanas_desde_ultima_actividad_prestamos':1000, 'semanas_desde_primer_actividad_prestamos':1000}).fillna(0)\n\nfi = []\ntest_probs = []\ni = 0\nscores_val = []\nscores_train = []\n\nfor train_idx, valid_idx in StratifiedKFold(n_splits=10, shuffle=True, random_state=42).split(X_train, y_train):\n    i += 1\n    Xt = X_train.iloc[train_idx]\n    yt = y_train.loc[X_train.index].iloc[train_idx]\n\n    Xv = X_train.iloc[valid_idx]\n    yv = y_train.loc[X_train.index].iloc[valid_idx]\n\n    learner = LGBMClassifier(n_estimators=10000,\n                             max_depth=3,\n                             num_leaves=14,\n                             #boosting_type='dart',\n                             #bagging_freq=1, bagging_fraction=0.05\n                             #is_unbalance = True,\n                             learning_rate=0.01,\n                             #objective='binary',\n                             random_state=42\n                            )\n    \n    learner.fit(Xt, yt,  early_stopping_rounds=200, eval_metric=\"auc\",\n                eval_set=[(Xt, yt), (Xv, yv)], verbose=10000)\n    \n    scores_val.append(learner.best_score_['valid_1']['auc'])\n    scores_train.append(learner.best_score_['training']['auc'])\n    \n    test_probs.append(pd.Series(learner.predict_proba(X_test)[:, -1],\n                                index=X_test.index, name=\"fold_\" + str(i)))\n    fi.append(pd.Series(learner.feature_importances_ / learner.feature_importances_.sum(), index=Xt.columns))\n\ntest_probs = pd.concat(test_probs, axis=1).mean(axis=1)\ntest_probs.index.name=\"USER_ID\"\ntest_probs.name=\"SCORE\"\ntest_probs.to_csv(\"preds.csv\", header=True)\nfi = pd.concat(fi, axis=1).mean(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Score promedio training: \" + str(np.around(np.mean(scores_train), decimals=4)))\nprint(\"Score promedio validacion: \" + str(np.around(np.mean(scores_val), decimals=4)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_features = 50\n\nfeatures_importantes = fi[fi != 0].sort_values(ascending=False)\nfeatures_importantes = list(features_importantes[:num_features].index)\nfi_nombres = fi.copy()\nfi_nombres.index = poner_nombre_features(lista_features = fi_nombres.index,\n                                         dict_pages = dict_pages,\n                                         dict_category = dict_category,\n                                         dict_category_bottom = dict_category_bottom,\n                                         dict_category_top = dict_category_top,\n                                         dict_site = dict_site)\nfi_nombres[fi_nombres != 0].sort_values(ascending=False)[:num_features]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}