{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"","metadata":{"papermill":{"duration":0.014735,"end_time":"2023-08-04T06:32:18.401318","exception":false,"start_time":"2023-08-04T06:32:18.386583","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport heapq\ntqdm.pandas()\n\nfrom sklearn import preprocessing\nimport lightgbm as lgb\nfrom lightgbm import LGBMRanker\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\nfrom IPython.display import display\nfrom sklearn.metrics import f1_score\nfrom gensim.models import Word2Vec \n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.517184,"end_time":"2023-08-04T06:32:20.933511","exception":false,"start_time":"2023-08-04T06:32:18.416327","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.read_csv(\"/kaggle/input/datathon-alicorp-2023-b/train.csv\",sep=\"|\")\ndf_sub=pd.read_csv(\"/kaggle/input/datathon-alicorp-2023-b/submit_example.csv\")\ndf_rentabilidad=pd.read_csv(\"/kaggle/input/datathon-alicorp-2023-b/RentabilidadProduct.csv\",sep=\"|\")","metadata":{"papermill":{"duration":3.702019,"end_time":"2023-08-04T06:32:24.65008","exception":false,"start_time":"2023-08-04T06:32:20.948061","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_rentabilidad.head(3)","metadata":{"papermill":{"duration":0.041871,"end_time":"2023-08-04T06:32:24.707164","exception":false,"start_time":"2023-08-04T06:32:24.665293","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_rentabilidad[\"product_id\"].nunique()/len(df_rentabilidad)","metadata":{"papermill":{"duration":0.030869,"end_time":"2023-08-04T06:32:24.75313","exception":false,"start_time":"2023-08-04T06:32:24.722261","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"business_id\"].nunique()","metadata":{"papermill":{"duration":0.039556,"end_time":"2023-08-04T06:32:24.807783","exception":false,"start_time":"2023-08-04T06:32:24.768227","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"business_id\"].value_counts()","metadata":{"papermill":{"duration":0.051863,"end_time":"2023-08-04T06:32:24.875269","exception":false,"start_time":"2023-08-04T06:32:24.823406","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(\"customer_id\")[\"business_id\"].nunique().value_counts()","metadata":{"papermill":{"duration":0.540282,"end_time":"2023-08-04T06:32:25.431022","exception":false,"start_time":"2023-08-04T06:32:24.89074","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby([\"type_id\"])[\"business_id\"].unique()","metadata":{"papermill":{"duration":0.292455,"end_time":"2023-08-04T06:32:25.738498","exception":false,"start_time":"2023-08-04T06:32:25.446043","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df_train[\"monto_venta_transf\"]/df_train[\"cantidad_venta\"]).value_counts()","metadata":{"papermill":{"duration":0.145619,"end_time":"2023-08-04T06:32:25.899272","exception":false,"start_time":"2023-08-04T06:32:25.753653","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df_train[\"peso_venta_transf\"]/df_train[\"cantidad_venta\"]).value_counts()","metadata":{"papermill":{"duration":0.07979,"end_time":"2023-08-04T06:32:25.994336","exception":false,"start_time":"2023-08-04T06:32:25.914546","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"fecha_compra\"].min()","metadata":{"papermill":{"duration":0.193842,"end_time":"2023-08-04T06:32:26.203668","exception":false,"start_time":"2023-08-04T06:32:26.009826","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"channel_id\"].value_counts()","metadata":{"papermill":{"duration":0.139702,"end_time":"2023-08-04T06:32:26.359218","exception":false,"start_time":"2023-08-04T06:32:26.219516","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[df_train[\"channel_id\"]==\"V\",\"product_id\"].value_counts()","metadata":{"papermill":{"duration":0.221332,"end_time":"2023-08-04T06:32:26.596427","exception":false,"start_time":"2023-08-04T06:32:26.375095","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[df_train[\"channel_id\"]==\"B\",\"product_id\"].value_counts()","metadata":{"papermill":{"duration":0.20679,"end_time":"2023-08-04T06:32:26.819128","exception":false,"start_time":"2023-08-04T06:32:26.612338","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[df_train[\"channel_id\"]==\"A\",\"product_id\"].value_counts()","metadata":{"papermill":{"duration":0.196397,"end_time":"2023-08-04T06:32:27.031485","exception":false,"start_time":"2023-08-04T06:32:26.835088","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[df_train[\"channel_id\"]==\"V\",\"customer_id\"].value_counts()","metadata":{"papermill":{"duration":0.218421,"end_time":"2023-08-04T06:32:27.266142","exception":false,"start_time":"2023-08-04T06:32:27.047721","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[df_train[\"channel_id\"]==\"B\",\"customer_id\"].value_counts()","metadata":{"papermill":{"duration":0.214413,"end_time":"2023-08-04T06:32:27.496708","exception":false,"start_time":"2023-08-04T06:32:27.282295","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[df_train[\"channel_id\"]==\"A\",\"customer_id\"].value_counts()","metadata":{"papermill":{"duration":0.204051,"end_time":"2023-08-04T06:32:27.723303","exception":false,"start_time":"2023-08-04T06:32:27.519252","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"product_id\"].nunique()","metadata":{"papermill":{"duration":0.039789,"end_time":"2023-08-04T06:32:27.779445","exception":false,"start_time":"2023-08-04T06:32:27.739656","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for type_id in [\"BO\",\"GA\",\"PA\"]:\n    \n    print(type_id)\n    print(df_train.loc[df_train[\"type_id\"]==type_id,\"product_id\"].nunique())\n    print(\"\")\n    \n    ","metadata":{"papermill":{"duration":0.558357,"end_time":"2023-08-04T06:32:28.354289","exception":false,"start_time":"2023-08-04T06:32:27.795932","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"customer_id\"].nunique()","metadata":{"papermill":{"duration":0.042278,"end_time":"2023-08-04T06:32:28.413177","exception":false,"start_time":"2023-08-04T06:32:28.370899","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"product_id\"].value_counts()","metadata":{"papermill":{"duration":0.057223,"end_time":"2023-08-04T06:32:28.487112","exception":false,"start_time":"2023-08-04T06:32:28.429889","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"customer_id\"].nunique()","metadata":{"papermill":{"duration":0.040992,"end_time":"2023-08-04T06:32:28.544951","exception":false,"start_time":"2023-08-04T06:32:28.503959","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(\"customer_id\")[\"product_id\"].nunique()","metadata":{"papermill":{"duration":0.715345,"end_time":"2023-08-04T06:32:29.277617","exception":false,"start_time":"2023-08-04T06:32:28.562272","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(\"customer_id\")[\"product_id\"].nunique().describe()","metadata":{"papermill":{"duration":0.687241,"end_time":"2023-08-04T06:32:29.982834","exception":false,"start_time":"2023-08-04T06:32:29.295593","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.query(\"customer_id==13758\")","metadata":{"papermill":{"duration":0.058142,"end_time":"2023-08-04T06:32:30.059158","exception":false,"start_time":"2023-08-04T06:32:30.001016","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(pd.to_datetime(df_train[\"fecha_compra\"]).max()-pd.to_datetime(df_train[\"fecha_compra\"]).min())\nprint(df_train[\"fecha_compra\"].nunique())","metadata":{"papermill":{"duration":0.666732,"end_time":"2023-08-04T06:32:30.743611","exception":false,"start_time":"2023-08-04T06:32:30.076879","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef create_time_variables(df_train):\n\n    df_train[\"fecha_compra\"]=pd.to_datetime(df_train[\"fecha_compra\"])\n\n\n    df_temp=df_train[[\"fecha_compra\"]].drop_duplicates()\n\n    df_temp[\"max_fecha\"]=df_train[\"fecha_compra\"].max()\n\n    df_temp[\"order_day\"]=(df_temp[\"max_fecha\"]-df_temp[\"fecha_compra\"]).dt.days\n\n    df_temp[\"order_week\"]=df_temp[\"order_day\"]//7\n\n    df_train=df_train.merge(df_temp,on=\"fecha_compra\",how=\"left\")\n\n    del df_temp; gc.collect()\n    \n    return df_train","metadata":{"papermill":{"duration":1.03199,"end_time":"2023-08-04T06:32:31.793403","exception":false,"start_time":"2023-08-04T06:32:30.761413","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=create_time_variables(df_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.01747,"end_time":"2023-08-04T06:32:31.876671","exception":false,"start_time":"2023-08-04T06:32:31.859201","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.loc[df_train[\"customer_id\"].isin(df_sub[\"customer_id\"]),\"customer_id\"].nunique())\nprint(df_train.loc[~df_train[\"customer_id\"].isin(df_sub[\"customer_id\"]),\"customer_id\"].nunique())\nprint(df_sub[\"customer_id\"].nunique())\n","metadata":{"papermill":{"duration":0.092372,"end_time":"2023-08-04T06:32:31.986795","exception":false,"start_time":"2023-08-04T06:32:31.894423","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.query(\"order_week==0\").groupby(\"customer_id\")[\"product_id\"].nunique().describe())\n\nprint(df_train.query(\"order_week==1\").groupby(\"customer_id\")[\"product_id\"].nunique().describe())\n\nprint(df_train.query(\"order_week==2\").groupby(\"customer_id\")[\"product_id\"].nunique().describe())\n\n\n      \n      ","metadata":{"papermill":{"duration":0.251842,"end_time":"2023-08-04T06:32:32.256302","exception":false,"start_time":"2023-08-04T06:32:32.00446","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.017842,"end_time":"2023-08-04T06:32:32.292474","exception":false,"start_time":"2023-08-04T06:32:32.274632","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunkify(lst,n):\n    return [lst[i::n] for i in range(n)]\n\n\ndef create_dict_next_buy(df_train):\n    \n    customers_list=df_train[\"customer_id\"].unique().tolist()\n\n    customer_chunks=chunkify(customers_list,10)\n\n    df_next_week_buy=[]\n\n    for customer_chunk in tqdm(customer_chunks):\n\n        df_temp=df_train.loc[df_train[\"customer_id\"].isin(customer_chunk),[\"customer_id\",\"product_id\",\"order_week_customer\"]].reset_index(drop=True)\n\n\n        df_temp=df_temp.merge(df_temp,on=\"customer_id\",how=\"left\")\n\n        #df_temp=df_temp.loc[(df_temp[\"order_week_customer_y\"]-df_temp[\"order_week_customer_x\"])==1].reset_index(drop=True)\n        df_temp=df_temp.loc[(df_temp[\"order_week_customer_y\"]-df_temp[\"order_week_customer_x\"]) >= 0].reset_index(drop=True)\n\n        df_temp[\"unos\"]=1\n\n        df_temp=df_temp.groupby([\"product_id_x\",\"product_id_y\"])[\"unos\"].sum().reset_index()\n\n        df_temp=df_temp.rename(columns={\"unos\":\"venta_unitaria\"})\n\n        df_next_week_buy.append(df_temp)\n\n        del df_temp ; gc.collect()\n\n    df_next_week_buy=pd.concat(df_next_week_buy,ignore_index=True)\n\n    df_next_week_buy=df_next_week_buy.groupby([\"product_id_x\",\"product_id_y\"])[\"venta_unitaria\"].sum().reset_index()\n\n    df_next_week_buy[\"venta_unitaria_total\"]=df_next_week_buy.groupby(\"product_id_x\")[\"venta_unitaria\"].transform(\"sum\")\n\n    df_next_week_buy[\"score_next_buy\"]=df_next_week_buy[\"venta_unitaria\"]/df_next_week_buy.reset_index(drop=True)[\"venta_unitaria_total\"]\n\n    df_next_week_buy=df_next_week_buy.sort_values([\"product_id_x\",\"score_next_buy\"],ascending=[True,False]).reset_index(drop=True)\n    \n    df_next_week_buy=df_next_week_buy.groupby([\"product_id_x\"]).head(30).reset_index(drop=True)\n    \n\n    \n    df_next_week_buy=df_next_week_buy.rename(columns={\"product_id_x\":\"product_id\",\"product_id_y\":\"product_id_nextbuy\"})\n    \n    return df_next_week_buy\n\n\ndef prepare_dicts_rating(df_train,top=75):\n    \n\n    \n    #orden de semana de compra segun cliente\n    df_temp=df_train.drop_duplicates([\"customer_id\",\"order_week\"])\n    df_temp=df_temp.sort_values([\"customer_id\",\"order_week\"],ascending=[True,True]).reset_index(drop=True)\n    df_temp[\"unos\"]=1\n    df_temp[\"order_week_customer\"]=df_temp.groupby(\"customer_id\")[\"unos\"].cumsum()\n    df_temp[\"order_week_customer\"]=df_temp[\"order_week_customer\"]-1\n    df_train=df_train.merge(df_temp[[\"customer_id\",\"order_week\",\"order_week_customer\"]],on=[\"customer_id\",\"order_week\"],how=\"left\")\n    \n    #dictionary next buy\n\n    df_next_buy=create_dict_next_buy(df_train)\n    \n    df_train[\"venta_unitaria\"]=1\n\n    df_ven_cus_prod=df_train.groupby([\"customer_id\",\"product_id\"])[\"venta_unitaria\"].sum().reset_index()\n\n    df_ven_cus=df_train.groupby([\"customer_id\"])[\"venta_unitaria\"].sum().reset_index()\n\n    df_ven_cus=df_ven_cus.rename(columns={\"venta_unitaria\":\"venta_unitaria_total\"})\n\n    df_ven_cus_prod=df_ven_cus_prod.merge(df_ven_cus,on=\"customer_id\",how=\"left\")\n\n    df_ven_cus_prod[\"score\"]=df_ven_cus_prod[\"venta_unitaria\"]/df_ven_cus_prod[\"venta_unitaria_total\"]\n\n\n    df_ven_cus_prod=df_ven_cus_prod.sort_values([\"customer_id\",\"score\"],ascending=[True,False]).groupby([\"customer_id\"]).head(75).reset_index(drop=True)\n\n    dict_cus_prod=df_ven_cus_prod.set_index([\"customer_id\",\"product_id\"])[\"score\"].to_dict()\n    \n    dict_lastw_prod_cust=df_train.loc[df_train[\"order_week_customer\"]==0].drop_duplicates([\"customer_id\",\"product_id\"]).groupby(\"customer_id\")[\"product_id\"].agg(list).to_dict()\n    \n    df_ven_prod_type=df_train.query(\"order_week<2\").groupby([\"product_id\",\"type_id\"])[\"venta_unitaria\"].sum().reset_index()\n\n    df_ven_type=df_train.query(\"order_week<2\").groupby([\"type_id\"])[\"venta_unitaria\"].sum().reset_index()\n\n    df_ven_type=df_ven_type.rename(columns={\"venta_unitaria\":\"venta_unitaria_total\"})\n\n    df_ven_prod_type=df_ven_prod_type.merge(df_ven_type,on=\"type_id\",how=\"left\")\n\n    df_ven_prod_type[\"score\"]=df_ven_prod_type[\"venta_unitaria\"]/df_ven_prod_type[\"venta_unitaria_total\"]\n\n    \n    df_ven_prod_type=df_ven_prod_type.sort_values([\"type_id\",\"score\"],ascending=[True,False]).groupby([\"type_id\"]).head(top).reset_index(drop=True)\n\n    dict_prod_type=df_ven_prod_type.set_index([\"product_id\",\"type_id\"])[\"score\"].to_dict()\n    \n    print(df_ven_prod_type.query(\"type_id=='BO'\")[\"score\"].sum())\n    print(df_ven_prod_type.query(\"type_id=='GA'\")[\"score\"].sum())\n    print(df_ven_prod_type.query(\"type_id=='PA'\")[\"score\"].sum())\n    \n    \n    return dict_cus_prod,dict_prod_type,df_next_buy,dict_lastw_prod_cust\n\n\n\n\ndef assign_scores_prod_cus(df_train,dict_cus_prod,dict_prod_type,dict_next_buy,dict_lastw_prod_cust):\n    \n    dict_scores_customer_products={}\n\n\n    for type_product in df_train[\"type_id\"].unique().tolist():\n\n        customers_list=df_train.loc[df_train[\"type_id\"]==type_product,\"customer_id\"].unique().tolist()\n\n        score_products_populares_dict={key[0]:dict_prod_type[key] for key in dict_prod_type.keys() if key[1] == type_product}\n\n        for customer in tqdm(customers_list):\n\n            score_products_comprados_dict={key[1]:dict_cus_prod[key] for key in dict_cus_prod.keys() if key[0] == customer}\n\n            \n            products_comprados_lastw_list=dict_lastw_prod_cust[customer]\n            \n            score_next_buy_dict={key:dict_next_buy[key] for key in dict_next_buy.keys() if key[0] in products_comprados_lastw_list }\n            \n            score_product_next_buy_dict={}\n            \n            for pairs_products in score_next_buy_dict.keys():\n                \n                if pairs_products[1] not in score_product_next_buy_dict.keys():\n                    \n                    score_product_next_buy_dict[pairs_products[1]]=score_next_buy_dict[pairs_products]\n                    \n                else :\n                    \n                    if score_product_next_buy_dict[pairs_products[1]] < score_next_buy_dict[pairs_products] :\n                        \n                        score_product_next_buy_dict[pairs_products[1]] = score_next_buy_dict[pairs_products]\n                    \n                    \n            products_list=list(set(list(score_products_comprados_dict.keys())+list(score_products_populares_dict.keys())+list(score_product_next_buy_dict.keys())))\n\n            for product in products_list:\n\n\n                \n                if product in products_comprados_lastw_list:\n                    \n\n                    dict_scores_customer_products[(customer,product)]=1.25*score_products_comprados_dict.get(product,0)+score_products_populares_dict.get(product,0)+0.75*score_product_next_buy_dict.get(product,0)\n                    \n                else :\n                    \n\n                    dict_scores_customer_products[(customer,product)]=1.25*score_products_comprados_dict.get(product,0)+score_products_populares_dict.get(product,0)\n\n                    \n                    \n                \n    return dict_scores_customer_products\n\n\n\ndef assign_top_30_products(dict_scores_customer_products):\n    \n    # Step 1: Group the products by customer using defaultdict\n    customer_products = defaultdict(list)\n    for customer, product in dict_scores_customer_products.keys():\n        rating = dict_scores_customer_products[(customer, product)]\n        customer_products[customer].append((rating, product))\n\n    # Step 2: Select the top 20 products with the highest ratings for each customer using heapq\n    top_30_products_by_customer = {}\n    for customer, products in customer_products.items():\n        top_30_products = heapq.nlargest(30, products)\n        top_30_products_by_customer[customer] = [product for _, product in top_30_products]\n        \n        \n    return top_30_products_by_customer\n        \n    \n\n    \n        \n\n    \n    \n    \n    \n    ","metadata":{"papermill":{"duration":0.052867,"end_time":"2023-08-04T06:32:32.772322","exception":false,"start_time":"2023-08-04T06:32:32.719455","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def agrega_multiple(data,key,aggr,suffix='',threshold=0):\n    \n    temp=data.groupby(key).agg(aggr).reset_index()\n    \n    variable_name=\"_\".join(key)\n    \n    temp.columns=[variable_name+'-aggregation-'+x[0]+'-'+suffix+x[1] if x[0] not in key else x[0] for x in temp.columns]\n    \n    if threshold>0:\n        \n        temp_2=data.groupby(key).size().reset_index()\n\n        temp_2=temp_2.rename(columns={0:'size'})\n        \n        temp=temp.merge(temp_2,on=key,how='left')\n        \n        temp=temp.loc[temp['size']>threshold].reset_index(drop=True)\n\n        del temp['size'] ; gc.collect()\n        \n    return temp\n\ndef create_features_compra(df_train):\n    \n    df_train[\"semanas_recompra\"]=df_train[\"order_week\"]-df_train.groupby([\"customer_id\",\"product_id\"])[\"order_week\"].shift(1)\n    \n    df_train[\"order_semanas_recompra\"]=df_train[\"order_week_customer\"]-df_train.groupby([\"customer_id\",\"product_id\"])[\"order_week_customer\"].shift(1)\n    \n    \n    return df_train\n\n    \ndef create_cus_prod_features(df_train):\n    \n    df_temp=df_train.drop_duplicates([\"customer_id\",\"order_week\"])\n    df_temp=df_temp.sort_values([\"customer_id\",\"order_week\"],ascending=[True,True]).reset_index(drop=True)\n    df_temp[\"unos\"]=1\n    df_temp[\"order_week_customer\"]=df_temp.groupby(\"customer_id\")[\"unos\"].cumsum()\n    df_temp[\"order_week_customer\"]=df_temp[\"order_week_customer\"]-1\n    df_train=df_train.merge(df_temp[[\"customer_id\",\"order_week\",\"order_week_customer\"]],on=[\"customer_id\",\"order_week\"],how=\"left\")\n    df_train=pd.get_dummies(df_train,columns=[\"channel_id\"])\n    df_train=create_features_compra(df_train)\n    df_train[\"precio_monto\"]=df_train[\"monto_venta_transf\"]/df_train[\"cantidad_venta\"]\n    df_train[\"precio_peso\"]=df_train[\"peso_venta_transf\"]/df_train[\"cantidad_venta\"]\n    df_train[\"business_id_orig\"]=df_train[\"business_id\"].copy()\n    df_train=pd.get_dummies(df_train,columns=[\"business_id\"])\n    df_train=df_train.rename(columns={\"business_id_orig\":\"business_id\"})\n    df_train=df_train.merge(df_rentabilidad[[\"product_id\",\"tier_id\",\"utilidad_bruta_transf\"]],on=\"product_id\",how=\"left\")\n    \n    business_columns=[x for x in df_train.columns if \"business_id_\" in x]\n    \n    #\"business_id\":[\"nunique\"]\n    dict_stats_cus={\"product_id\":[\"nunique\"],\"business_id\":[\"nunique\"],\"cantidad_venta\":[\"size\",\"sum\",\"mean\"],\"monto_venta_transf\":[\"mean\",\"var\"],\n                \"peso_venta_transf\":[\"mean\",\"var\"],\"order_week\":[\"min\",\"max\",\"mean\"],\"order_week_customer\":[\"mean\",\"var\",\"max\"],\n               \"channel_id_A\":[\"mean\"],\"channel_id_B\":[\"mean\"],\"channel_id_V\":[\"mean\"],\n                   \"order_semanas_recompra\":[\"mean\",\"var\"], \"semanas_recompra\":[\"mean\",\"var\"],\n                   \"precio_monto\":[\"min\",\"max\",\"mean\",\"var\"],\"precio_peso\":[\"min\",\"max\",\"mean\",\"var\"],\n                   \"tier_id\":[\"mean\",\"var\"],\"utilidad_bruta_transf\":[\"mean\",\"var\",\"min\",\"max\",\"count\"]}\n    \n    for var in business_columns:\n        dict_stats_cus[var]=[\"mean\"]\n        \n    \n    dict_stats_cus_prod={\"business_id\":[\"nunique\"],\"cantidad_venta\":[\"size\",\"sum\",\"mean\"],\"monto_venta_transf\":[\"mean\",\"var\"],\n        \"peso_venta_transf\":[\"mean\",\"var\"],\"order_week\":[\"min\",\"max\",\"mean\"],\"order_week_customer\":[\"mean\",\"var\",\"max\"],\n        \"channel_id_A\":[\"mean\"],\"channel_id_B\":[\"mean\"],\"channel_id_V\":[\"mean\"],\n            \"order_semanas_recompra\":[\"mean\",\"var\"], \"semanas_recompra\":[\"mean\",\"var\"],\n            \"precio_monto\":[\"min\",\"max\",\"mean\",\"var\"],\"precio_peso\":[\"min\",\"max\",\"mean\",\"var\"]}\n    \n    df_agg_cus_prod=agrega_multiple(df_train,key=[\"customer_id\",\"product_id\"],aggr=dict_stats_cus_prod)\n    \n    df_agg_cus_prod[\"rank_product_in_customer\"]=df_agg_cus_prod.groupby(\"customer_id\")['customer_id_product_id-aggregation-cantidad_venta-size'].rank(method=\"max\",ascending=False).values\n    \n    df_agg_last3_cus_prod=agrega_multiple(df_train.query(\"order_week_customer < 3\"),key=[\"customer_id\",\"product_id\"],aggr=dict_stats_cus_prod,suffix=\"_last3weeks_\")\n    \n    df_agg_cus_prod=df_agg_cus_prod.merge(df_agg_last3_cus_prod,on=[\"customer_id\",\"product_id\"],how=\"left\")\n    \n    \n    df_agg_cus=agrega_multiple(df_train,key=[\"customer_id\"],aggr=dict_stats_cus)\n    \n\n    df_agg_cus_last3months=agrega_multiple(df_train.query(\"order_week_customer<4\"),key=[\"customer_id\"],aggr=dict_stats_cus,suffix=\"_last4weeks_\")\n    \n    df_agg_cus=df_agg_cus.merge(df_agg_cus_last3months,on=[\"customer_id\"],how=\"left\")\n    \n    \n    df_agg_cus_prod=df_agg_cus_prod.merge(df_agg_cus,on=[\"customer_id\"],how=\"left\")\n    \n    \n    df_counts=create_count_features(df_train)\n    \n    df_agg_cus_prod=df_agg_cus_prod.merge(df_counts,on=\"customer_id\",how=\"left\")\n    \n    return df_agg_cus_prod\n\n\ndef create_count_features(df_train):\n    \n    #products mas populares en las ultimas 12 semanas\n    pop_products=[1028, 772, 404, 916, 406, 149, 667, 972, 926, 414, 670, 671, 544, 673, 34, \n    37, 163, 679, 38, 425, 426, 683, 170, 172, 813, 174, 688, 689, 812, 177, 52, 178, 188, 190, \n    193, 194, 66, 452, 453, 965, 456, 328, 202, 203, 459, 77, 205, 206, 207, 977, 208, 210, 211, \n    213, 975, 976, 88, 216, 329, 987, 608, 98, 739, 493, 495, 116, 119, 120, 890, 891]\n    \n\n    \n\n    # Create the pivot table with counts of products for each customer\n    pivot_table = pd.pivot_table(\n        df_train.loc[df_train[\"product_id\"].isin(pop_products)],\n        values='order_week',\n        index='customer_id',\n        columns='product_id',\n        aggfunc=\"size\",\n        fill_value=0\n    )\n    \n    pivot_table.reset_index(inplace=True)  # Reset index to convert it to a column\n    pivot_table.rename_axis(None, axis=1, inplace=True)\n    pivot_table.columns=[f'{x}_product'if x!=\"customer_id\" else x for x in pivot_table.columns]\n        \n    print(pivot_table.head(5))\n    \n    return pivot_table\n\n\ndef create_prod_features(df_train):\n    \n    df_temp=df_train.drop_duplicates([\"customer_id\",\"order_week\"])\n    df_temp=df_temp.sort_values([\"customer_id\",\"order_week\"],ascending=[True,True]).reset_index(drop=True)\n    df_temp[\"unos\"]=1\n    df_temp[\"order_week_customer\"]=df_temp.groupby(\"customer_id\")[\"unos\"].cumsum()\n    df_temp[\"order_week_customer\"]=df_temp[\"order_week_customer\"]-1\n    df_train=df_train.merge(df_temp[[\"customer_id\",\"order_week\",\"order_week_customer\"]],on=[\"customer_id\",\"order_week\"],how=\"left\")\n    df_train=pd.get_dummies(df_train,columns=[\"channel_id\"])\n    df_train=pd.get_dummies(df_train,columns=[\"type_id\"])\n    df_train=create_features_compra(df_train)\n    df_train[\"precio_monto\"]=df_train[\"monto_venta_transf\"]/df_train[\"cantidad_venta\"]\n    df_train[\"precio_peso\"]=df_train[\"peso_venta_transf\"]/df_train[\"cantidad_venta\"]\n    \n    dict_stats_prod={\"customer_id\":[\"nunique\"],\"business_id\":[\"nunique\"],\"cantidad_venta\":[\"size\",\"sum\",\"mean\"],\"monto_venta_transf\":[\"mean\",\"var\"],\n                \"peso_venta_transf\":[\"mean\",\"var\"],\"order_week\":[\"min\",\"max\",\"mean\"],\"order_week_customer\":[\"mean\",\"max\",\"var\"],\n               \"channel_id_A\":[\"mean\"],\"channel_id_B\":[\"mean\"],\"channel_id_V\":[\"mean\"],\n                \"type_id_BO\":[\"mean\"],\"type_id_PA\":[\"mean\"],\"type_id_GA\":[\"mean\"],\n                \"order_semanas_recompra\":[\"mean\",\"var\",\"min\",\"max\"], \"semanas_recompra\":[\"mean\",\"var\",\"min\",\"max\"],\n                \"precio_monto\":[\"min\",\"max\",\"mean\",\"var\"],\"precio_peso\":[\"min\",\"max\",\"mean\",\"var\"]}\n    \n\n    df_agg_prod=agrega_multiple(df_train,key=[\"product_id\"],aggr=dict_stats_prod)\n    \n    df_agg_last3_prod=agrega_multiple(df_train.query(\"order_week<3\"),key=[\"product_id\"],aggr=dict_stats_prod,suffix=\"_last3weeks_\")\n\n    \n    dict_stats_prod_type={\"customer_id\":[\"nunique\"],\"business_id\":[\"nunique\"],\"cantidad_venta\":[\"size\",\"sum\",\"mean\"],\"monto_venta_transf\":[\"mean\",\"var\"],\n                \"peso_venta_transf\":[\"mean\",\"var\"],\"order_week\":[\"min\",\"max\",\"mean\"],\"order_week_customer\":[\"mean\",\"max\",\"var\"],\n               \"channel_id_A\":[\"mean\"],\"channel_id_B\":[\"mean\"],\"channel_id_V\":[\"mean\"],\n                \"order_semanas_recompra\":[\"min\",\"max\",\"mean\",\"var\"], \"semanas_recompra\":[\"mean\",\"var\",\"min\",\"max\"],\n                \"precio_monto\":[\"min\",\"max\",\"mean\",\"var\"],\"precio_peso\":[\"min\",\"max\",\"mean\",\"var\"]}\n    \n    df_agg_type_BO_prod=agrega_multiple(df_train.query(\"type_id_BO==1\"),key=[\"product_id\"],aggr=dict_stats_prod_type,suffix=\"_type_BO_\")\n    \n    df_agg_type_PA_prod=agrega_multiple(df_train.query(\"type_id_PA==1\"),key=[\"product_id\"],aggr=dict_stats_prod_type,suffix=\"_type_PA_\")\n    \n    df_agg_type_GA_prod=agrega_multiple(df_train.query(\"type_id_GA==1\"),key=[\"product_id\"],aggr=dict_stats_prod_type,suffix=\"_type_GA_\")\n    \n    df_agg_prod=df_agg_prod.merge(df_agg_last3_prod,on=[\"product_id\"],how=\"left\")\n        \n    df_agg_prod=df_agg_prod.merge(df_agg_type_BO_prod,on=[\"product_id\"],how=\"left\")\n    \n    df_agg_prod=df_agg_prod.merge(df_agg_type_PA_prod,on=[\"product_id\"],how=\"left\")\n    \n    df_agg_prod=df_agg_prod.merge(df_agg_type_GA_prod,on=[\"product_id\"],how=\"left\")\n    \n    \n        \n    return df_agg_prod\n\n\n\n","metadata":{"papermill":{"duration":0.058945,"end_time":"2023-08-04T06:34:52.617531","exception":false,"start_time":"2023-08-04T06:34:52.558586","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.018589,"end_time":"2023-08-04T06:34:52.655146","exception":false,"start_time":"2023-08-04T06:34:52.636557","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.019576,"end_time":"2023-08-04T06:35:11.534155","exception":false,"start_time":"2023-08-04T06:35:11.514579","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_candidate_buys(df_train):\n    \n    df_buy_candidates=df_train[[\"customer_id\",\"product_id\"]].drop_duplicates().reset_index(drop=True)\n    df_buy_candidates[\"type_candidate\"]=1\n    \n    return df_buy_candidates\n\n\ndef create_popularity_candidate(df_train,var_cat=\"type_id\",top=75):\n    \n    type_ids_list=df_train[var_cat].unique().tolist()\n    \n    df_popularity_total=[]\n    \n    for type_id in type_ids_list:\n    \n        df_train[\"unos\"]=1\n\n        df_popularity=df_train.loc[(df_train[var_cat]==type_id)&(df_train[\"order_week\"]<3)].groupby(\"product_id\")[\"unos\"].sum().reset_index().sort_values(\"unos\",ascending=False)\n\n        df_popularity=df_popularity.head(top).reset_index(drop=True)\n\n        df_popularity=df_popularity[[\"product_id\"]]\n\n        df_popularity['_tmpkey'] = 1\n\n\n        df_customer=df_train.loc[df_train[var_cat]==type_id,[\"customer_id\"]].drop_duplicates()\n\n        df_customer['_tmpkey'] = 1\n\n\n        df_popularity = pd.merge(df_popularity, df_customer, on='_tmpkey').drop('_tmpkey', axis=1)\n        \n        df_popularity_total.append(df_popularity)\n        \n    df_popularity_total=pd.concat(df_popularity_total,ignore_index=True).reset_index(drop=True)\n    \n    df_popularity_total[\"type_candidate\"]=2\n    \n    df_popularity_total=df_popularity_total[[\"customer_id\",\"product_id\",\"type_candidate\"]]\n    \n    \n    return df_popularity_total\n\ndef create_next_buy_candidates(df_train,df_next_buy):\n    \n    df_last_buy=df_train[[\"customer_id\",\"product_id\"]].drop_duplicates().reset_index(drop=True)\n    \n    df_last_buy=df_last_buy.merge(df_next_buy[[\"product_id\",\"product_id_nextbuy\"]],on=[\"product_id\"],how=\"left\")\n    \n    df_last_buy=df_last_buy.drop([\"product_id\"],axis=\"columns\")\n    \n    df_last_buy=df_last_buy.rename(columns={\"product_id_nextbuy\":\"product_id\"})\n    \n    df_last_buy=df_last_buy.drop_duplicates([\"customer_id\",\"product_id\"]).reset_index(drop=True)\n    \n    df_last_buy[\"type_candidate\"]=3\n    \n    return df_last_buy\n\n\ndef add_scores(df_candidates_features,dict_cus_prod,dict_prod_type):\n    \n    # Step 1: Convert the dictionary to a DataFrame\n    scores_df = pd.DataFrame.from_dict(dict_cus_prod, orient='index', columns=['score_cust_prod'])\n\n    # Step 2: Reset the index and extract customer_id and product_id from the tuple\n    scores_df.reset_index(inplace=True)\n    scores_df.rename(columns={'index': 'key'}, inplace=True)\n    scores_df[['customer_id', 'product_id']] = pd.DataFrame(scores_df['key'].tolist(), index=scores_df.index)\n\n    # Step 3: Merge with the existing DataFrame\n    df_candidates_features = pd.merge(df_candidates_features, scores_df[['customer_id', 'product_id', 'score_cust_prod']],\n                         on=['customer_id', 'product_id'], how='left')\n    \n    # Step 1: Convert the dictionary to a DataFrame\n    scores_df = pd.DataFrame.from_dict(dict_prod_type, orient='index', columns=['score_pop_prod_type'])\n\n    # Step 2: Reset the index and rename the index column to 'product_id'\n    scores_df.reset_index(inplace=True)\n    scores_df.rename(columns={'index': 'key'}, inplace=True)\n    scores_df[['product_id',\"type_id\"]] = pd.DataFrame(scores_df['key'].tolist(), index=scores_df.index)\n\n    # Step 3: Merge with the existing DataFrame based on 'product_id' column\n    df_candidates_features = pd.merge(df_candidates_features, scores_df[['product_id', 'type_id', 'score_pop_prod_type']],\n                         on=['product_id', 'type_id'], how='left')\n    \n    df_candidates_features=df_candidates_features.loc[(df_candidates_features[\"score_pop_prod_type\"].notnull())|(df_candidates_features[\"score_cust_prod\"].notnull())].reset_index(drop=True)\n\n    \n    return df_candidates_features\n\n\n\n\ndef apply_w2v_similarity(df_train,df_candidates_features):\n    \n    df_temp=df_train.drop_duplicates([\"customer_id\",\"order_week\"])\n    df_temp=df_temp.sort_values([\"customer_id\",\"order_week\"],ascending=[True,True]).reset_index(drop=True)\n    df_temp[\"unos\"]=1\n    df_temp[\"order_week_customer\"]=df_temp.groupby(\"customer_id\")[\"unos\"].cumsum()\n    df_temp[\"order_week_customer\"]=df_temp[\"order_week_customer\"]-1\n    df_train=df_train.merge(df_temp[[\"customer_id\",\"order_week\",\"order_week_customer\"]],on=[\"customer_id\",\"order_week\"],how=\"left\")\n    del df_temp \n    gc.collect()\n\n    df_product_sequences = df_train.loc[df_train[\"order_week_customer\"] < 3].drop_duplicates([\"customer_id\",\"product_id\"]).groupby('customer_id')['product_id'].agg(list)\n\n\n    model = Word2Vec(sentences=df_product_sequences.values, vector_size=100, window=5, min_count=10, workers=1)\n    \n        \n    df_product_sequences=df_product_sequences.reset_index()\n    df_product_sequences=df_product_sequences.rename(columns={\"product_id\":\"buy_products\"})\n    \n    df_similarity=df_candidates_features[[\"customer_id\",\"product_id\"]].drop_duplicates()\n    \n    df_similarity=df_similarity.merge(df_product_sequences,on=\"customer_id\",how=\"left\")\n    \n    del df_product_sequences ; gc.collect()\n    \n    print(df_similarity.head(5))\n    \n    print(\"applying similarity\")\n    \n    def calc_similarity_score(x):\n        try:\n            return model.wv.n_similarity(x['articles_buy_week_temp'],[x['article_id_temp']])\n        except:\n            return np.nan\n    \n    df_similarity[\"similarity_score\"]=df_similarity[[\"buy_products\",\"product_id\"]].apply(lambda row : model.wv.n_similarity(row[\"buy_products\"],[row[\"product_id\"]]),axis=1)\n\n    \n    df_candidates_features=df_candidates_features.merge(df_similarity[[\"customer_id\",\"product_id\",\"similarity_score\"]],on=[\"customer_id\",\"product_id\"],how=\"left\")\n    \n    return df_candidates_features\n\n\n\ndef reduce_mem_usage(df,no_usar=[]):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns :\n        col_type = df[col].dtype\n        \n        if col not in no_usar:\n        \n            if col_type != object:\n                c_min = df[col].min()\n                c_max = df[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float64)\n            #else:\n            #    df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n    \n    \n\n    \n    ","metadata":{"papermill":{"duration":0.048092,"end_time":"2023-08-04T06:35:11.601551","exception":false,"start_time":"2023-08-04T06:35:11.553459","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.019587,"end_time":"2023-08-04T06:38:56.937951","exception":false,"start_time":"2023-08-04T06:38:56.918364","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_target(df_candidates,df_test):\n    \n    df_temp=df_test[[\"customer_id\",\"product_id\"]].drop_duplicates()\n    \n    df_temp[\"target\"]=1\n    \n    df_candidates=df_candidates.merge(df_temp,on=[\"customer_id\",\"product_id\"],how=\"left\")\n    \n    df_candidates[\"target\"]=df_candidates[\"target\"].fillna(0)\n    \n    return df_candidates","metadata":{"papermill":{"duration":0.028248,"end_time":"2023-08-04T06:38:56.986092","exception":false,"start_time":"2023-08-04T06:38:56.957844","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features=[]\n\nfor order_week_test in [0,1]:\n    \n    print(f\"creating train and test : {order_week_test}\")\n    \n    df_test=df_train.loc[df_train[\"order_week\"]==0].reset_index(drop=True)\n\n    df_train=df_train.loc[df_train[\"order_week\"]>0].reset_index(drop=True)\n\n    df_train[\"order_week\"]=df_train[\"order_week\"]-1\n    \n    \n\n    print(\"creating dicts\")\n\n    dict_cus_prod,dict_prod_type,df_next_buy,dict_lastw_prod_cust=prepare_dicts_rating(df_train,top=75)\n\n\n    print(\"creating features\")\n\n    df_agg_cus_prod=create_cus_prod_features(df_train)\n\n    df_agg_prod=create_prod_features(df_train)\n\n\n    print(\"creating candidates\")\n\n    df_buy_candidates=create_candidate_buys(df_train)\n\n    df_popularity_candidates=create_popularity_candidate(df_train,top=75)\n\n\n\n    print(\"concatenating\")\n\n    df_candidates_features_temp=pd.concat([df_buy_candidates,df_popularity_candidates],ignore_index=True)\n\n\n    print(\"dropping duplicates\")\n    df_candidates_features_temp=df_candidates_features_temp.drop_duplicates([\"customer_id\",\"product_id\"]).reset_index(drop=True)\n\n    del df_buy_candidates\n\n    del df_popularity_candidates\n\n    gc.collect()\n\n\n    print(\"merging\")\n    df_candidates_features_temp=df_candidates_features_temp.merge(df_agg_cus_prod,on=[\"customer_id\",\"product_id\"],how=\"left\")\n\n    del df_agg_cus_prod ; gc.collect()\n\n    df_candidates_features_temp=df_candidates_features_temp.merge(df_agg_prod,on=[\"product_id\"],how=\"left\")\n\n    del df_agg_prod ; gc.collect()\n\n\n\n    df_candidates_features_temp=df_candidates_features_temp.merge(df_train[[\"customer_id\",\"type_id\",\"business_id\"]].drop_duplicates([\"customer_id\"]),on=\"customer_id\",how=\"left\")\n    df_candidates_features_temp=df_candidates_features_temp.merge(df_rentabilidad[[\"product_id\",\"tier_id\",\"utilidad_bruta_transf\"]],on=\"product_id\",how=\"left\")\n\n    df_candidates_features_temp=apply_w2v_similarity(df_train,df_candidates_features_temp)\n\n\n    print(\"adding scores\")\n    df_candidates_features_temp=add_scores(df_candidates_features_temp,dict_cus_prod,dict_prod_type)\n    \n    del dict_cus_prod\n    del dict_prod_type \n    del df_next_buy\n    del dict_lastw_prod_cust; gc.collect()\n    \n    \n    print(\"creating target\")\n    \n    \n          \n    df_candidates_features_temp[\"reference_week\"]=order_week_test\n    \n    gc.collect()\n    \n    df_candidates_features_temp=reduce_mem_usage(df_candidates_features_temp)\n    \n    df_candidates_features_temp=create_target(df_candidates_features_temp,df_test)\n\n    df_candidates_features.append(df_candidates_features_temp)\n    \n    del df_candidates_features_temp ; gc.collect()\n    \n    if order_week_test==0:\n        \n        df_test_validation=df_test[[\"customer_id\",\"product_id\"]].drop_duplicates().reset_index(drop=True)\n          \n          \n          \ndel df_train ; gc.collect()\n\n\n          ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features=pd.concat(df_candidates_features,ignore_index=True).reset_index(drop=True)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features[\"target\"].mean()","metadata":{"papermill":{"duration":0.03478,"end_time":"2023-08-04T06:39:00.511434","exception":false,"start_time":"2023-08-04T06:39:00.476654","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features.drop_duplicates([\"customer_id\",\"product_id\"])[\"target\"].mean()","metadata":{"papermill":{"duration":4.907726,"end_time":"2023-08-04T06:39:05.440215","exception":false,"start_time":"2023-08-04T06:39:00.532489","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def entrena_lgb_binary(data,test,features,categorical,target):\n\n    #kfold=StratifiedKFold(n_splits=5,shuffle=True)\n    kfold=GroupKFold(n_splits=5)\n\n\n    i=1\n\n    r=[]\n    \n    pred_test=np.zeros(len(test))\n\n    importancias=pd.DataFrame()\n\n    importancias['variable']=features\n    \n    list_models=[]\n    \n    \n    cat_ind=[features.index(x) for x in categorical if x in features]\n    \n    dict_cat={}\n    \n    categorical_numerical = data[categorical].dropna().select_dtypes(include=np.number).columns.tolist()\n    \n    categorical_transform=[x for x in categorical if x not in categorical_numerical]\n    \n    for l in categorical_transform:\n        le = preprocessing.LabelEncoder()\n        le.fit(list(data[l].dropna())+list(test[l].dropna()))\n\n        dict_cat[l]=le\n\n        data.loc[~data[l].isnull(),l]=le.transform(data.loc[~data[l].isnull(),l])\n        test.loc[~test[l].isnull(),l]=le.transform(test.loc[~test[l].isnull(),l])\n        \n        \n\n    for train_index,test_index in kfold.split(data,data[target],data['customer_id']):\n    #for train_index,test_index in kfold.split(data,data[target]):\n\n        lgb_data_train = lgb.Dataset(data.loc[train_index,features].values,data.loc[train_index,target].values)\n        lgb_data_eval = lgb.Dataset(data.loc[test_index,features].values,data.loc[test_index,target].values, reference=lgb_data_train)\n\n        params = {\n            'task': 'train',\n            'boosting_type': 'gbdt',\n            'objective': 'binary',\n            'metric': { 'auc'},\n            \"max_depth\":-1,\n            \"num_leaves\":32,\n            'learning_rate': 0.1,\n        \"min_child_samples\": 100,\n            'feature_fraction': 0.9,\n         \"bagging_freq\":1,\n            'bagging_fraction': 0.9,\n            \"lambda_l1\":10,\n            \"lambda_l2\":10,\n           #\"scale_pos_weight\":20,\n            #'min_data_per_group':10000,\n\n            'verbose': 1    \n        }\n\n\n\n\n        modelo = lgb.train(params,lgb_data_train,num_boost_round=4500,valid_sets=lgb_data_eval,early_stopping_rounds=50,verbose_eval=25,categorical_feature=cat_ind)\n\n        importancias['gain_'+str(i)]=modelo.feature_importance(importance_type=\"gain\")\n\n\n        data.loc[test_index,'estimator']=modelo.predict(data.loc[test_index,features].values, num_iteration=modelo.best_iteration)\n        \n        pred_test=pred_test+modelo.predict(test[features].values, num_iteration=modelo.best_iteration)\n        \n        list_models.append(modelo)\n\n        print (\"Fold_\"+str(i))\n        a= (roc_auc_score(data.loc[test_index,target],data.loc[test_index,'estimator']))\n        r.append(a)\n        print (a)\n        print (\"\")\n\n        i=i+1\n        \n    for l in categorical_transform:\n\n            data.loc[~data[l].isnull(),l]=dict_cat[l].inverse_transform(data.loc[~data[l].isnull(),l].astype(int))\n            \n            test.loc[~test[l].isnull(),l]=dict_cat[l].inverse_transform(test.loc[~test[l].isnull(),l].astype(int))\n            \n    var=[x for x in importancias.columns if 'gain_' in x]\n    importancias[\"gain_avg\"]=importancias[var].mean(axis=1)\n    importancias=importancias.sort_values(\"gain_avg\",ascending=False).reset_index(drop=True)\n    \n    pred_test=(pred_test/5)\n    \n    \n    oof=(roc_auc_score(data[target],data['estimator']))\n    \n    print (oof)\n    print (\"mean: \"+str(np.mean(np.array(r))))\n    print (\"std: \"+str(np.std(np.array(r))))\n    \n    dict_resultados={}\n    \n    dict_resultados['importancias']=importancias\n    \n    dict_resultados['predicciones']=pred_test\n    \n    dict_resultados['modelos']=list_models\n    \n    dict_resultados['categorical_transformer']=dict_cat\n    \n    \n    \n    return dict_resultados","metadata":{"papermill":{"duration":0.042244,"end_time":"2023-08-04T06:39:05.502662","exception":false,"start_time":"2023-08-04T06:39:05.460418","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features.columns","metadata":{"papermill":{"duration":0.030431,"end_time":"2023-08-04T06:39:05.553885","exception":false,"start_time":"2023-08-04T06:39:05.523454","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target=\"target\"\n\nno_usar=[\"target\",\"customer_id\",\"product_id\",\"type_candidate\",\"reference_week\"]\n\ncategorical=[\"type_candidate\",\"type_id\",\"business_id\"]\n\nfeatures=[x for x in df_candidates_features.columns if x not in no_usar]","metadata":{"papermill":{"duration":0.030235,"end_time":"2023-08-04T06:39:05.605214","exception":false,"start_time":"2023-08-04T06:39:05.574979","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.020062,"end_time":"2023-08-04T06:39:05.645714","exception":false,"start_time":"2023-08-04T06:39:05.625652","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_resultados=entrena_lgb_binary(df_candidates_features,df_candidates_features.head(5),features,categorical,target)\n","metadata":{"papermill":{"duration":1276.440774,"end_time":"2023-08-04T07:00:22.107243","exception":false,"start_time":"2023-08-04T06:39:05.666469","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_resultados[\"importancias\"]","metadata":{"papermill":{"duration":0.046483,"end_time":"2023-08-04T07:00:22.180671","exception":false,"start_time":"2023-08-04T07:00:22.134188","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(dict_resultados[\"importancias\"][\"gain_avg\"]>0).mean()","metadata":{"papermill":{"duration":0.037271,"end_time":"2023-08-04T07:00:22.244197","exception":false,"start_time":"2023-08-04T07:00:22.206926","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_resultados[\"importancias\"][\"variable\"].tolist()[:60]","metadata":{"papermill":{"duration":0.036404,"end_time":"2023-08-04T07:00:22.307154","exception":false,"start_time":"2023-08-04T07:00:22.27075","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_elegidos=df_candidates_features.query(\"reference_week==0\").sort_values([\"customer_id\",\"estimator\"],ascending=[True,False]).drop_duplicates([\"customer_id\",\"product_id\"],keep=\"first\").groupby(\"customer_id\").head(30)\n\ndel df_candidates_features ; gc.collect()\n\n","metadata":{"papermill":{"duration":12.587053,"end_time":"2023-08-04T07:00:34.983082","exception":false,"start_time":"2023-08-04T07:00:22.396029","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_30_products_by_customer=df_elegidos.groupby(\"customer_id\")[\"product_id\"].agg(list).to_dict()\n\ndict_comprados=df_test_validation.drop_duplicates([\"customer_id\",\"product_id\"]).groupby(\"customer_id\")[\"product_id\"].agg(list).to_dict()\n","metadata":{"papermill":{"duration":0.859109,"end_time":"2023-08-04T07:00:35.873686","exception":false,"start_time":"2023-08-04T07:00:35.014577","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_recalls=[]\n\nfor client in dict_comprados.keys():\n    \n    recall=len(set(dict_comprados[client]).intersection(top_30_products_by_customer[client]))/len(dict_comprados[client])\n    \n    list_recalls.append(recall)\n    \nprint(np.mean(list_recalls))\n\n\ndel df_test_validation ; gc.collect()","metadata":{"papermill":{"duration":0.065964,"end_time":"2023-08-04T07:00:35.972232","exception":false,"start_time":"2023-08-04T07:00:35.906268","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.read_csv(\"/kaggle/input/datathon-alicorp-2023-b/train.csv\",sep=\"|\")\ndf_sub=pd.read_csv(\"/kaggle/input/datathon-alicorp-2023-b/submit_example.csv\")\ndf_rentabilidad=pd.read_csv(\"/kaggle/input/datathon-alicorp-2023-b/RentabilidadProduct.csv\",sep=\"|\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=create_time_variables(df_train)","metadata":{"papermill":{"duration":0.031251,"end_time":"2023-08-04T07:00:37.671076","exception":false,"start_time":"2023-08-04T07:00:37.639825","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"creating dicts\")\n\ndict_cus_prod,dict_prod_type,df_next_buy,dict_lastw_prod_cust=prepare_dicts_rating(df_train,top=75)\n\n\nprint(\"creating features\")\n\ndf_agg_cus_prod=create_cus_prod_features(df_train)\n\ndf_agg_prod=create_prod_features(df_train)\n\n\nprint(\"creating candidates\")\n\ndf_buy_candidates=create_candidate_buys(df_train)\n\ndf_popularity_candidates=create_popularity_candidate(df_train,top=75)\n\n\n\nprint(\"concatenating\")\n\ndf_candidates_features=pd.concat([df_buy_candidates,df_popularity_candidates],ignore_index=True)\n\n\nprint(\"dropping duplicates\")\ndf_candidates_features=df_candidates_features.drop_duplicates([\"customer_id\",\"product_id\"]).reset_index(drop=True)\n\ndel df_buy_candidates\n\ndel df_popularity_candidates\n\ngc.collect()\n\n\nprint(\"merging\")\ndf_candidates_features=df_candidates_features.merge(df_agg_cus_prod,on=[\"customer_id\",\"product_id\"],how=\"left\")\n\ndel df_agg_cus_prod ; gc.collect()\n\ndf_candidates_features=df_candidates_features.merge(df_agg_prod,on=[\"product_id\"],how=\"left\")\n\ndel df_agg_prod ; gc.collect()\n\n\n\ndf_candidates_features=df_candidates_features.merge(df_train[[\"customer_id\",\"type_id\",\"business_id\"]].drop_duplicates([\"customer_id\"]),on=\"customer_id\",how=\"left\")\ndf_candidates_features=df_candidates_features.merge(df_rentabilidad[[\"product_id\",\"tier_id\",\"utilidad_bruta_transf\"]],on=\"product_id\",how=\"left\")\n\ndf_candidates_features=apply_w2v_similarity(df_train,df_candidates_features)\n\n\nprint(\"adding scores\")\ndf_candidates_features=add_scores(df_candidates_features,dict_cus_prod,dict_prod_type)\n","metadata":{"papermill":{"duration":175.904692,"end_time":"2023-08-04T07:03:33.607098","exception":false,"start_time":"2023-08-04T07:00:37.702406","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_resultados[\"categorical_transformer\"]","metadata":{"papermill":{"duration":0.143773,"end_time":"2023-08-04T07:07:22.833088","exception":false,"start_time":"2023-08-04T07:07:22.689315","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_transf=dict_resultados[\"categorical_transformer\"]\n\nfor l in [\"type_id\"]:\n    \n    transf=cat_transf[l]\n    \n    df_candidates_features.loc[~df_candidates_features[l].isnull(),l]=transf.transform(df_candidates_features.loc[~df_candidates_features[l].isnull(),l])\n    \n    ","metadata":{"papermill":{"duration":0.892042,"end_time":"2023-08-04T07:07:23.756207","exception":false,"start_time":"2023-08-04T07:07:22.864165","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_test=np.zeros(len(df_candidates_features))\n\n\nfor fold in [0,1,2,3,4]:\n\n    modelo=dict_resultados['modelos'][fold]\n\n    pred_test=pred_test+modelo.predict(df_candidates_features[features].values, num_iteration=modelo.best_iteration)/5\n    \n    \ndf_candidates_features[\"estimator\"]=pred_test","metadata":{"papermill":{"duration":60.291894,"end_time":"2023-08-04T07:08:24.07704","exception":false,"start_time":"2023-08-04T07:07:23.785146","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features[\"estimator\"]=pred_test","metadata":{"papermill":{"duration":0.039683,"end_time":"2023-08-04T07:08:24.146021","exception":false,"start_time":"2023-08-04T07:08:24.106338","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates_features[\"estimator\"].isnull().mean()","metadata":{"papermill":{"duration":0.042418,"end_time":"2023-08-04T07:08:24.217846","exception":false,"start_time":"2023-08-04T07:08:24.175428","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_elegidos=df_candidates_features[[\"customer_id\",\"product_id\",\"estimator\"]].sort_values([\"customer_id\",\"estimator\"],ascending=[True,False]).drop_duplicates([\"customer_id\",\"product_id\"],keep=\"first\").groupby(\"customer_id\").head(30)\n","metadata":{"papermill":{"duration":4.286283,"end_time":"2023-08-04T07:08:28.533878","exception":false,"start_time":"2023-08-04T07:08:24.247595","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = df_elegidos.groupby('customer_id')['product_id'].apply(lambda x: ' '.join(map(str, x))).reset_index()\n\ndf_submission.columns = ['customer_id', 'product_id']","metadata":{"papermill":{"duration":0.503465,"end_time":"2023-08-04T07:08:29.067335","exception":false,"start_time":"2023-08-04T07:08:28.56387","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv(\"submission.csv\",index=False)","metadata":{"papermill":{"duration":0.107397,"end_time":"2023-08-04T07:08:29.204104","exception":false,"start_time":"2023-08-04T07:08:29.096707","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}