{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-04T18:10:43.998074Z","iopub.execute_input":"2023-08-04T18:10:43.998750Z","iopub.status.idle":"2023-08-04T18:10:53.499441Z","shell.execute_reply.started":"2023-08-04T18:10:43.998707Z","shell.execute_reply":"2023-08-04T18:10:53.498431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# k_recommendations = 30\nbase_path = \"/kaggle/input/datathon-alicorp-2023-b/\"\n\nproduct_metadata = pd.read_csv(base_path + \"RentabilidadProduct.csv\", sep=\"|\")\ndata_raw = pd.read_csv(base_path + \"train.csv\", sep=\"|\")\ndata_raw[\"fecha_compra\"] = pd.to_datetime(data_raw[\"fecha_compra\"])\ndata_raw = data_raw.sort_values(by=[\"customer_id\", \"fecha_compra\"], ascending=[True, False])\ndata_raw[\"days\"] = (data_raw[\"fecha_compra\"].dt.year-2022) * 365 + data_raw[\"fecha_compra\"].dt.dayofyear\ndata_raw[\"week\"] = data_raw[\"fecha_compra\"].dt.year.astype(str) + data_raw[\"fecha_compra\"].dt.isocalendar().week.astype(str)\n\ndata_raw[\"dw\"] = data_raw.groupby([\"customer_id\", \"week\"])[\"days\"].transform(max)\ndata_raw[\"max_date\"] = data_raw.groupby([\"customer_id\", \"week\"])[\"fecha_compra\"].transform(min)\nproduct_metadata = product_metadata.merge(data_raw[[\"product_id\"]].drop_duplicates(), on=\"product_id\", how=\"outer\").fillna(0)\n\nweekly_data = data_raw.groupby([\"customer_id\", \"dw\", \"max_date\"])[\"product_id\"].unique().reset_index()\n\n\nweekly_data[\"next_purchase\"] = weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(-1)\n\nweekly_data[\"context\"] = [a+b+c+d+e+f+g+h+i+j for a,b,c,d,e,f,g,h,i,j in \nzip(\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(1).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(2).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(3).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(4).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(5).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(6).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(7).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(8).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(9).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n    weekly_data.groupby([\"customer_id\"])[\"product_id\"].shift(10).apply(lambda x: x.tolist() if x is not np.nan else []).tolist(),\n)]\n\nextra = pd.concat([\n    data_raw.groupby([\"customer_id\", \"dw\"])[\"type_id\"].unique().apply(lambda x: x.tolist() if x is not np.nan else []),\n    data_raw.groupby([\"customer_id\", \"dw\"])[\"business_id\"].unique().apply(lambda x: x.tolist() if x is not np.nan else []),\n    data_raw.groupby([\"customer_id\", \"dw\"])[\"channel_id\"].unique().apply(lambda x: x.tolist() if x is not np.nan else []),\n    #data_raw.groupby([\"customer_id\", \"dw\"])[\"monto_venta_transf\"].sum(),\n], axis=1).reset_index()\n\nweekly_data = weekly_data.merge(extra, on=[\"customer_id\", \"dw\"],how=\"left\")\n\nweekly_data[\"age\"] = weekly_data.groupby([\"customer_id\"])[\"dw\"].transform(lambda x: x - x.min())\nweekly_data[\"purchase_number\"] = weekly_data.groupby([\"customer_id\"])[\"dw\"].rank().astype(int)\nweekly_data[\"month\"] = weekly_data[\"max_date\"].dt.month\nweekly_data[\"day\"] = weekly_data[\"max_date\"].dt.day\n\nmask = weekly_data.groupby(\"customer_id\")[\"dw\"].rank(ascending=False)\ntrain_agg = weekly_data[mask>2].dropna()\nval_agg = weekly_data[mask==2].dropna()\nsubmit_agg = weekly_data[mask==1]\n\ntrain_agg[\"customer_id\"].nunique(), val_agg[\"customer_id\"].nunique(), submit_agg[\"customer_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:21:49.437653Z","iopub.execute_input":"2023-08-04T18:21:49.438015Z","iopub.status.idle":"2023-08-04T18:24:25.012256Z","shell.execute_reply.started":"2023-08-04T18:21:49.437978Z","shell.execute_reply":"2023-08-04T18:24:25.010054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weekly_data","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:21:17.813279Z","iopub.execute_input":"2023-08-04T18:21:17.813715Z","iopub.status.idle":"2023-08-04T18:21:17.863723Z","shell.execute_reply.started":"2023-08-04T18:21:17.813678Z","shell.execute_reply":"2023-08-04T18:21:17.862608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_product = data_raw[\"product_id\"].drop_duplicates().sort_values().astype(str).tolist()\nlookup_product = tf.keras.layers.TextVectorization(output_mode=\"multi_hot\", vocabulary=vocab_product)\n\nvocab_customer = train_agg[\"customer_id\"].drop_duplicates().sort_values().astype(str).tolist()\nlookup_customer = tf.keras.layers.TextVectorization(output_mode=\"count\", vocabulary=vocab_customer)\n\n\nvocab_type = train_agg[\"type_id\"].drop_duplicates().sort_values().astype(str).tolist()\nlookup_type = tf.keras.layers.TextVectorization(output_mode=\"count\", vocabulary=vocab_type)\n\nvocab_business = train_agg[\"business_id\"].drop_duplicates().sort_values().astype(str).tolist()\nlookup_business = tf.keras.layers.TextVectorization(output_mode=\"count\", vocabulary=vocab_business)\n\nvocab_channel = train_agg[\"channel_id\"].drop_duplicates().sort_values().astype(str).tolist()\nlookup_channel = tf.keras.layers.TextVectorization(output_mode=\"count\", vocabulary=vocab_channel)\n\nn_labels = lookup_product.vocabulary_size()\nn_customers = lookup_customer.vocabulary_size()\n\nn_type = lookup_type.vocabulary_size()\nn_business = lookup_business.vocabulary_size()\nn_channel = lookup_channel.vocabulary_size()\n\nn_customers, n_labels\n","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:26:41.135312Z","iopub.execute_input":"2023-08-04T18:26:41.135721Z","iopub.status.idle":"2023-08-04T18:26:41.995636Z","shell.execute_reply.started":"2023-08-04T18:26:41.135687Z","shell.execute_reply":"2023-08-04T18:26:41.994525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_agg","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:25:09.205915Z","iopub.execute_input":"2023-08-04T18:25:09.206295Z","iopub.status.idle":"2023-08-04T18:25:09.253385Z","shell.execute_reply.started":"2023-08-04T18:25:09.206263Z","shell.execute_reply":"2023-08-04T18:25:09.252394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset_raw = tf.data.Dataset.from_tensor_slices({\n    \"product_id\": train_agg[\"context\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"customer_id\": train_agg[\"customer_id\"].astype(str),\n    \"type_id\": train_agg[\"type_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"business_id\": train_agg[\"business_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"channel_id\": train_agg[\"channel_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"dw\": train_agg[\"dw\"]/400,\n    \"age\": train_agg[\"age\"]/train_agg[\"age\"].max(),\n    \"purchase_number\": train_agg[\"purchase_number\"]/train_agg[\"purchase_number\"].max(),\n    \"month\": train_agg[\"month\"]/12,\n    \"day\": train_agg[\"day\"]/31,\n    \"next_purchase\": train_agg[\"product_id\"].apply(lambda x: \" \".join([str(k) for k in x]))\n} )\n\nval_dataset_raw = tf.data.Dataset.from_tensor_slices({\n    \"product_id\": val_agg[\"context\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"customer_id\": val_agg[\"customer_id\"].astype(str),\n    \"type_id\": val_agg[\"type_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"business_id\": val_agg[\"business_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"channel_id\": val_agg[\"channel_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"dw\": val_agg[\"dw\"]/400,\n    \"age\": val_agg[\"age\"]/train_agg[\"age\"].max(),\n    \"purchase_number\": val_agg[\"purchase_number\"]/train_agg[\"purchase_number\"].max(),\n    \"month\": val_agg[\"month\"]/12,\n    \"day\": val_agg[\"day\"]/31,\n    \"next_purchase\": val_agg[\"product_id\"].apply(lambda x: \" \".join([str(k) for k in x]))\n} )\n\nsubmit_dataset_raw = tf.data.Dataset.from_tensor_slices({\n    \"product_id\": submit_agg[\"context\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"customer_id\": submit_agg[\"customer_id\"].astype(str),\n    \"type_id\": submit_agg[\"type_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"business_id\": submit_agg[\"business_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"channel_id\": submit_agg[\"channel_id\"].apply(lambda x: \" \".join([str(k) for k in x])),\n    \"dw\": submit_agg[\"dw\"]/400,\n    \"age\": submit_agg[\"age\"]/train_agg[\"age\"].max(),\n    \"purchase_number\": submit_agg[\"purchase_number\"]/train_agg[\"purchase_number\"].max(),\n    \"month\": submit_agg[\"month\"]/12,\n    \"day\": submit_agg[\"day\"]/31,\n} )\n\nbatch_size = 1024*8\nshuffle = 10_000\ntrain_dataset = train_dataset_raw.shuffle(buffer_size=shuffle).batch(batch_size).map(lambda x: ((lookup_product(x[\"product_id\"]), lookup_customer(x[\"customer_id\"]), lookup_type(x[\"type_id\"]), lookup_business(x[\"business_id\"]), lookup_channel(x[\"channel_id\"]), tf.reshape(x[\"dw\"], [-1,1]), tf.reshape(x[\"age\"], [-1,1]), tf.reshape(x[\"purchase_number\"], [-1,1]), tf.reshape(x[\"month\"], [-1,1]), tf.reshape(x[\"day\"], [-1,1])), lookup_product(x[\"next_purchase\"])), num_parallel_calls=tf.data.AUTOTUNE)\ntrain_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)\nval_dataset = val_dataset_raw.batch(batch_size).map(lambda x: ((lookup_product(x[\"product_id\"]), lookup_customer(x[\"customer_id\"]), lookup_type(x[\"type_id\"]), lookup_business(x[\"business_id\"]), lookup_channel(x[\"channel_id\"]), tf.reshape(x[\"dw\"], [-1,1]), tf.reshape(x[\"age\"], [-1,1]), tf.reshape(x[\"purchase_number\"], [-1,1]), tf.reshape(x[\"month\"], [-1,1]), tf.reshape(x[\"day\"], [-1,1])), lookup_product(x[\"next_purchase\"])), num_parallel_calls=tf.data.AUTOTUNE)\nval_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)\nsubmit_dataset = submit_dataset_raw.batch(batch_size).map(lambda x: ((lookup_product(x[\"product_id\"]), lookup_customer(x[\"customer_id\"]), lookup_type(x[\"type_id\"]), lookup_business(x[\"business_id\"]), lookup_channel(x[\"channel_id\"]), tf.reshape(x[\"dw\"], [-1,1]), tf.reshape(x[\"age\"], [-1,1]), tf.reshape(x[\"purchase_number\"], [-1,1]), tf.reshape(x[\"month\"], [-1,1]), tf.reshape(x[\"day\"], [-1,1])), ), num_parallel_calls=tf.data.AUTOTUNE)\nsubmit_dataset = submit_dataset.prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:28:29.642028Z","iopub.execute_input":"2023-08-04T18:28:29.642417Z","iopub.status.idle":"2023-08-04T18:28:39.562833Z","shell.execute_reply.started":"2023-08-04T18:28:29.642385Z","shell.execute_reply":"2023-08-04T18:28:39.561774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_past = tf.keras.layers.Input(shape=(n_labels))\ninput_customer = tf.keras.layers.Input(shape=(n_customers))\ninput_type = tf.keras.layers.Input(shape=(n_type))\ninput_business = tf.keras.layers.Input(shape=(n_business))\ninput_channel = tf.keras.layers.Input(shape=(n_channel))\ninput_dw = tf.keras.layers.Input(shape=(1), name=\"dw\")\ninput_age = tf.keras.layers.Input(shape=(1), name=\"age\")\ninput_purchase = tf.keras.layers.Input(shape=(1), name=\"purchase_number\")\ninput_month = tf.keras.layers.Input(shape=(1), name=\"month\")\ninput_day = tf.keras.layers.Input(shape=(1), name=\"day\")\n\nlayer = tf.keras.layers.Concatenate(name=\"concat\", axis=1)([input_past, input_customer, input_type, input_business, input_channel, input_dw, input_age, input_purchase,input_month, input_day])\nff = tf.keras.layers.Dense(300, activation=\"relu\")(layer)\n#ff = tf.keras.layers.Dense(300, activation=\"relu\")(ff)\nconcat = tf.keras.layers.Concatenate(name=\"concat\", axis=1)([layer, ff])\noutput = tf.keras.layers.Dense(n_labels, activation=\"sigmoid\")(layer)\nmodel = tf.keras.models.Model(inputs=[input_past, input_customer, input_type, input_business, input_channel, input_dw, input_age, input_purchase, input_month, input_day], outputs=output)\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam')\n","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:36:01.117717Z","iopub.execute_input":"2023-08-04T18:36:01.118102Z","iopub.status.idle":"2023-08-04T18:36:01.183918Z","shell.execute_reply.started":"2023-08-04T18:36:01.118069Z","shell.execute_reply":"2023-08-04T18:36:01.182900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    validation_data=val_dataset,\n    epochs=200,\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:36:01.469972Z","iopub.execute_input":"2023-08-04T18:36:01.471082Z","iopub.status.idle":"2023-08-04T18:36:29.748704Z","shell.execute_reply.started":"2023-08-04T18:36:01.471036Z","shell.execute_reply":"2023-08-04T18:36:29.747700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history).plot()","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:36:29.751613Z","iopub.execute_input":"2023-08-04T18:36:29.752003Z","iopub.status.idle":"2023-08-04T18:36:30.050713Z","shell.execute_reply.started":"2023-08-04T18:36:29.751965Z","shell.execute_reply":"2023-08-04T18:36:30.049604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#index = 101\n#x_obs = x_val[index]\n#y_obs = y_val[index]\n#pred = model.predict(x_obs.reshape(1,-1))\n\n#pd.DataFrame({\n#    \"item\": label_encoder.classes_,\n#    \"score\": pred[0],\n#    \"target\": y_obs,\n#}).sort_values(by=\"score\", ascending=False).head(20)","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:36:30.052327Z","iopub.execute_input":"2023-08-04T18:36:30.052718Z","iopub.status.idle":"2023-08-04T18:36:30.057787Z","shell.execute_reply.started":"2023-08-04T18:36:30.052681Z","shell.execute_reply":"2023-08-04T18:36:30.056657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"0.71 -> 0.69, 0.78 -> 0.74, ","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict(val_dataset)\n\ncol = lookup_product.get_vocabulary()\nindex = val_agg.customer_id\n\ntop_k = pd.DataFrame(y_pred, columns=col, index=index).drop(columns=\"[UNK]\").unstack().reset_index().rename(columns={\"level_0\": \"product_id\", 0: \"score\"})\ntop_k = top_k.sort_values([\"customer_id\", \"score\"], ascending=[True, False]).groupby(\"customer_id\").head(30)\n\nrecall = top_k.groupby(\"customer_id\")[\"product_id\"].apply(lambda x: x.tolist()).to_frame().join(val_agg.set_index(\"customer_id\")[\"product_id\"].to_frame(), rsuffix='_obs',)\nrecall = recall.dropna()\nrecall[\"product_id_obs\"] = recall[\"product_id_obs\"].apply(lambda x: [str(k) for k in x] )\nrecall.apply(lambda x: len(set(x.product_id if x.product_id else [] ).intersection(set(x.product_id_obs)))/len(x.product_id_obs), axis=1).mean()\n#","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:46:26.236146Z","iopub.execute_input":"2023-08-04T18:46:26.236499Z","iopub.status.idle":"2023-08-04T18:46:47.948328Z","shell.execute_reply.started":"2023-08-04T18:46:26.236470Z","shell.execute_reply":"2023-08-04T18:46:47.947200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(submit_dataset)\n\ncol = lookup_product.get_vocabulary()\nindex = submit_agg.customer_id\n\ntop_k = pd.DataFrame(y_pred, columns=col, index=index).drop(columns=\"[UNK]\").unstack().reset_index().rename(columns={\"level_0\": \"product_id\", 0: \"score\"})\ntop_k = top_k.sort_values([\"customer_id\", \"score\"], ascending=[True, False]).groupby(\"customer_id\").head(30)","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:47:05.193489Z","iopub.execute_input":"2023-08-04T18:47:05.193922Z","iopub.status.idle":"2023-08-04T18:47:25.474206Z","shell.execute_reply.started":"2023-08-04T18:47:05.193887Z","shell.execute_reply":"2023-08-04T18:47:25.473042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_result = top_k.groupby(\"customer_id\")[\"product_id\"].apply(lambda x: \" \".join(x.astype(str).tolist())).reset_index()\nsubmission_result.to_csv(\"submission.csv\", index=False)\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-08-04T18:47:25.476469Z","iopub.execute_input":"2023-08-04T18:47:25.476881Z","iopub.status.idle":"2023-08-04T18:47:27.989743Z","shell.execute_reply.started":"2023-08-04T18:47:25.476845Z","shell.execute_reply":"2023-08-04T18:47:27.988279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"customer_id,product_id\n* 1,456 414 404 406 453 459 452 418 426 116 915 405 120 203 1028 21 891 425 987 190 417 890 608 510 77 926 88 428 458 194\n* 2,414 404 456 425 406 119 459 453 426 452 337 496 315 405 890 427 587 27 99 120 418 88 428 417 987 635 531 77 21 295\n* 3,120 203 987 190 194 977 404 456 915 890 52 98 1028 77 904 116 926 88 452 414 930 337 406 405 891 916 453 635 418 426\n* 4,120 987 890 926 915 977 891 116 52 203 916 965 880 77 88 121 98 227 904 11 1028 190 254 194 452 131 51 252 128 414\n* 5,452 414 203 77 1028 456 904 572 803 194 120 404 88 987 890 809 999 453 190 406 915 1035 977 426 659 926 459 116 98 619\n* 6,120 987 890 926 915 203 88 52 77 904 880 116 337 891 190 414 977 194 98 659 1028 916 965 635 452 404 456 406 405 453\n* 7,452 456 404 414 453 459 405 406 418 417 426 915 425 450 427 88 890 120 428 194 904 458 987 203 190 21 77 926 116 52\n* 8,456 404 120 414 453 406 426 891 452 418 425 77 916 459 194 203 890 405 915 987 52 904 926 417 930 427 116 608 428 88\n* 9,120 77 116 452 203 414 88 916 456 891 987 194 890 426 404 926 52 453 406 190 997 98 977 915 11 459 1028 965 405 904","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}