{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Курсовой проект студента GeekBrains\n# Автор: Ельцов Артем Александрович\n# Ник на kaggle: virus_madison\n# Задание для курсового проекта\n# Метрика: MAP@5 - средневзвешенная точность рекомендаций\nСдача проекта:\n\nЦелевая метрика - MAP@5. Порог для уcпешной сдачи проекта - бэйзлайн в лидерборде (Most popular items baseline), если вам удастся его обойти вы успешно закрыли курс.\n\nДатасет разбит на две части private/public в равных пропорциях.\nУчтите что в тестовой выборке могут попстаться новые пользователи которые не видела ваша модель (для коллаборативной фильтрации)\nНЕ обязательно, но крайне желательно использовать 2-ух уровневые рекоммендательные системы в проекте\nВы сдаете код проекта в виде github репозитория и указываете свой ник в kaggle.\nДанные\nВсе данные знакомы вам из занятий:\n\n* hh_demographic.csv - фичи пользователей.\n* product.csv - фичи товарова.\n* retail_train.csv - взаимодействие пользователей с товарами.\n* test_users.csv - список пользователей для которого необходимо предоставить предсказания.\n* sample_submission.csv - пример данных для загрузки на kaggle для валидации.\n\nДля валидации необходимо загрузить предсказания для пользователей из файла test_users.csv.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-06T11:53:52.905125Z","iopub.execute_input":"2021-11-06T11:53:52.905435Z","iopub.status.idle":"2021-11-06T11:53:52.915925Z","shell.execute_reply.started":"2021-11-06T11:53:52.905405Z","shell.execute_reply":"2021-11-06T11:53:52.915316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\n\n# Для работы с матрицами\nfrom scipy.sparse import csr_matrix, coo_matrix\n\n# Матричная факторизация\nfrom implicit.als import AlternatingLeastSquares\nfrom implicit.nearest_neighbours import bm25_weight, tfidf_weight\nfrom implicit.bpr import BayesianPersonalizedRanking\nfrom implicit.nearest_neighbours import ItemItemRecommender  # нужен для одного трюка\nfrom lightfm import LightFM\nfrom lightfm.evaluation import precision_at_k, recall_at_k\n\n# Модель второго уровня\nfrom lightgbm import LGBMClassifier\n\n# Детерминированные алгоритмы\nfrom implicit.nearest_neighbours import ItemItemRecommender, CosineRecommender, TFIDFRecommender, BM25Recommender","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:52.917495Z","iopub.execute_input":"2021-11-06T11:53:52.917730Z","iopub.status.idle":"2021-11-06T11:53:52.928157Z","shell.execute_reply.started":"2021-11-06T11:53:52.917701Z","shell.execute_reply":"2021-11-06T11:53:52.927486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Для работы с матрицами\nfrom scipy.sparse import csr_matrix\n\n# Матричная факторизация\nfrom implicit.als import AlternatingLeastSquares\nfrom implicit.nearest_neighbours import ItemItemRecommender  # нужен для одного трюка\nfrom implicit.nearest_neighbours import bm25_weight, tfidf_weight\n\n\nclass MainRecommender:\n    \"\"\"Рекоммендации, которые можно получить из ALS\n\n    Input\n    -----\n    user_item_matrix: pd.DataFrame\n        Матрица взаимодействий user-item\n    \"\"\"\n\n    def __init__(self, data, weighting=True):\n\n        # Топ покупок каждого юзера\n        self.top_purchases = data.groupby(['user_id', 'item_id'])['quantity'].count().reset_index()\n        self.top_purchases.sort_values('quantity', ascending=False, inplace=True)\n        self.top_purchases = self.top_purchases[self.top_purchases['item_id'] != 999999]\n\n        # Топ покупок по всему датасету\n        self.overall_top_purchases = data.groupby('item_id')['quantity'].count().reset_index()\n        self.overall_top_purchases.sort_values('quantity', ascending=False, inplace=True)\n        self.overall_top_purchases = self.overall_top_purchases[self.overall_top_purchases['item_id'] != 999999]\n        self.overall_top_purchases = self.overall_top_purchases.item_id.tolist()\n\n        self.user_item_matrix = self._prepare_matrix(data)  # pd.DataFrame\n        self.id_to_itemid, self.id_to_userid, \\\n            self.itemid_to_id, self.userid_to_id = self._prepare_dicts(self.user_item_matrix)\n\n        if weighting:\n            self.user_item_matrix = bm25_weight(self.user_item_matrix.T).T\n\n        self.model = self.fit(self.user_item_matrix)\n        self.own_recommender = self.fit_own_recommender(self.user_item_matrix)\n\n    @staticmethod\n    def _prepare_matrix(data):\n        \"\"\"Готовит user-item матрицу\"\"\"\n        user_item_matrix = pd.pivot_table(data,\n                                          index='user_id', columns='item_id',\n                                          values='quantity',  # Можно пробовать другие варианты\n                                          aggfunc='count',\n                                          fill_value=0\n                                          )\n\n        user_item_matrix = user_item_matrix.astype(float)  # необходимый тип матрицы для implicit\n\n        return user_item_matrix\n\n    @staticmethod\n    def _prepare_dicts(user_item_matrix):\n        \"\"\"Подготавливает вспомогательные словари\"\"\"\n\n        userids = user_item_matrix.index.values\n        itemids = user_item_matrix.columns.values\n\n        matrix_userids = np.arange(len(userids))\n        matrix_itemids = np.arange(len(itemids))\n\n        id_to_itemid = dict(zip(matrix_itemids, itemids))\n        id_to_userid = dict(zip(matrix_userids, userids))\n\n        itemid_to_id = dict(zip(itemids, matrix_itemids))\n        userid_to_id = dict(zip(userids, matrix_userids))\n\n        return id_to_itemid, id_to_userid, itemid_to_id, userid_to_id\n\n    @staticmethod\n    def fit_own_recommender(user_item_matrix):\n        \"\"\"Обучает модель, которая рекомендует товары, среди товаров, купленных юзером\"\"\"\n\n        own_recommender = ItemItemRecommender(K=1, num_threads=4)\n        own_recommender.fit(csr_matrix(user_item_matrix).T.tocsr())\n\n        return own_recommender\n\n    @staticmethod\n    def fit(user_item_matrix, n_factors=50, regularization=0.001, iterations=15, num_threads=4):\n        \"\"\"Обучает ALS\"\"\"\n\n        model = AlternatingLeastSquares(factors=n_factors,\n                                        regularization=regularization,\n                                        iterations=iterations,\n                                        num_threads=num_threads)\n        model.fit(csr_matrix(user_item_matrix).T.tocsr())\n\n        return model\n\n    def _update_dict(self, user_id):\n        \"\"\"Если появился новыю user / item, то нужно обновить словари\"\"\"\n\n        if user_id not in self.userid_to_id.keys():\n\n            max_id = max(list(self.userid_to_id.values()))\n            max_id += 1\n\n            self.userid_to_id.update({user_id: max_id})\n            self.id_to_userid.update({max_id: user_id})\n\n    def _get_similar_item(self, item_id):\n        \"\"\"Находит товар, похожий на item_id\"\"\"\n        recs = self.model.similar_items(self.itemid_to_id[item_id], N=2)  # Товар похож на себя -> рекомендуем 2 товара\n        top_rec = recs[1][0]  # И берем второй (не товар из аргумента метода)\n        return self.id_to_itemid[top_rec]\n\n    def _extend_with_top_popular(self, recommendations, N=5):\n        \"\"\"Если кол-во рекоммендаций < N, то дополняем их топ-популярными\"\"\"\n\n        if len(recommendations) < N:\n            recommendations.extend(self.overall_top_purchases[:N])\n            recommendations = recommendations[:N]\n\n        return recommendations\n\n    def _get_recommendations(self, user, model, N=5):\n        \"\"\"Рекомендации через стардартные библиотеки implicit\"\"\"\n\n        self._update_dict(user_id=user)\n        res = [self.id_to_itemid[rec[0]] for rec in model.recommend(userid=self.userid_to_id[user],\n                                        user_items=csr_matrix(self.user_item_matrix).tocsr(),\n                                        N=N,\n                                        filter_already_liked_items=False,\n                                        filter_items=[self.itemid_to_id[999999]],\n                                        recalculate_user=True)]\n\n        res = self._extend_with_top_popular(res, N=N)\n\n        assert len(res) == N, 'Количество рекомендаций != {}'.format(N)\n        return res\n\n    def get_als_recommendations(self, user, N=5):\n        \"\"\"Рекомендации через стардартные библиотеки implicit\"\"\"\n\n        self._update_dict(user_id=user)\n        return self._get_recommendations(user, model=self.model, N=N)\n\n    def get_own_recommendations(self, user, N=5):\n        \"\"\"Рекомендуем товары среди тех, которые юзер уже купил\"\"\"\n\n        self._update_dict(user_id=user)\n        return self._get_recommendations(user, model=self.own_recommender, N=N)\n\n    def get_similar_items_recommendation(self, user, N=5):\n        \"\"\"Рекомендуем товары, похожие на топ-N купленных юзером товаров\"\"\"\n\n        top_users_purchases = self.top_purchases[self.top_purchases['user_id'] == user].head(N)\n\n        res = top_users_purchases['item_id'].apply(lambda x: self._get_similar_item(x)).tolist()\n        res = self._extend_with_top_popular(res, N=N) \n\n        assert len(res) == N, 'Количество рекомендаций != {}'.format(N)\n        return res\n\n    def get_similar_users_recommendation(self, user, N=5):\n        \"\"\"Рекомендуем топ-N товаров, среди купленных похожими юзерами\"\"\"\n\n        res = []\n\n        # Находим топ-N похожих пользователей\n        similar_users = self.model.similar_users(self.userid_to_id[user], N=N+1)\n        similar_users = [rec[0] for rec in similar_users]\n        similar_users = similar_users[1:]   # удалим юзера из запроса\n\n        for user in similar_users:\n            res.extend(self.get_own_recommendations(user, N=1))\n\n        res = self._extend_with_top_popular(res, N=N)\n\n        assert len(res) == N, 'Количество рекомендаций != {}'.format(N)\n        return res","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:52.944053Z","iopub.execute_input":"2021-11-06T11:53:52.944358Z","iopub.status.idle":"2021-11-06T11:53:52.983485Z","shell.execute_reply.started":"2021-11-06T11:53:52.944326Z","shell.execute_reply":"2021-11-06T11:53:52.982499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filters","metadata":{}},{"cell_type":"code","source":"def prefilter_items(data, take_n_popular=5000, item_features=None):\n    # Уберем самые популярные товары (их и так купят)\n    popularity = data.groupby('item_id')['user_id'].nunique().reset_index() / data['user_id'].nunique()\n    popularity.rename(columns={'user_id': 'share_unique_users'}, inplace=True)\n\n    top_popular = popularity[popularity['share_unique_users'] > 0.2].item_id.tolist()\n    data = data[~data['item_id'].isin(top_popular)]\n\n    # Уберем самые НЕ популярные товары (их и так НЕ купят)\n    top_notpopular = popularity[popularity['share_unique_users'] < 0.02].item_id.tolist()\n    data = data[~data['item_id'].isin(top_notpopular)]\n\n    # Уберем не интересные для рекоммендаций категории (department)\n    if item_features is not None:\n        department_size = pd.DataFrame(item_features.\\\n                                        groupby('department')['item_id'].nunique().\\\n                                        sort_values(ascending=False)).reset_index()\n\n        department_size.columns = ['department', 'n_items']\n        rare_departments = department_size[department_size['n_items'] < 150].department.tolist()\n        items_in_rare_departments = item_features[item_features['department'].isin(rare_departments)].item_id.unique().tolist()\n\n        data = data[~data['item_id'].isin(items_in_rare_departments)]\n\n\n    # Уберем слишком дешевые товары (на них не заработаем). 1 покупка из рассылок стоит 60 руб.\n    data['price'] = data['sales_value'] / (np.maximum(data['quantity'], 1))\n    data = data[data['price'] > 2]\n\n    # Уберем слишком дорогие товарыs\n    data = data[data['price'] < 50]\n\n    # Возьмем топ по популярности\n    popularity = data.groupby('item_id')['quantity'].sum().reset_index()\n    popularity.rename(columns={'quantity': 'n_sold'}, inplace=True)\n\n    top = popularity.sort_values('n_sold', ascending=False).head(take_n_popular).item_id.tolist()\n    \n    # Заведем фиктивный item_id (если юзер покупал товары из топ-5000, то он \"купил\" такой товар)\n    data.loc[~data['item_id'].isin(top), 'item_id'] = 999999\n    \n    # ...\n\n    return data\n    \n\ndef postfilter(recommendations, item_info, N=5):\n    \"\"\"Пост-фильтрация товаров\n    \n    Input\n    -----\n    recommendations: list\n        Ранжированный список item_id для рекомендаций\n    item_info: pd.DataFrame\n        Датафрейм с информацией о товарах\n    \"\"\"\n    \n    # Уникальность\n#     recommendations = list(set(recommendations)) - неверно! так теряется порядок\n    unique_recommendations = []\n    [unique_recommendations.append(item) for item in recommendations if item not in unique_recommendations]\n    \n    # Разные категории\n    categories_used = []\n    final_recommendations = []\n    \n    CATEGORY_NAME = 'sub_commodity_desc'\n    for item in unique_recommendations:\n        category = item_features.loc[item_features['item_id'] == item, CATEGORY_NAME].values[0]\n        \n        if category not in categories_used:\n            final_recommendations.append(item)\n            \n        unique_recommendations.remove(item)\n        categories_used.append(category)\n    \n    n_rec = len(final_recommendations)\n    if n_rec < N:\n        final_recommendations.extend(unique_recommendations[:N - n_rec])\n    else:\n        final_recommendations = final_recommendations[:N]\n    \n    assert len(final_recommendations) == N, 'Количество рекомендаций != {}'.format(N)\n    return final_recommendations","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:52.985437Z","iopub.execute_input":"2021-11-06T11:53:52.986055Z","iopub.status.idle":"2021-11-06T11:53:53.004587Z","shell.execute_reply.started":"2021-11-06T11:53:52.986019Z","shell.execute_reply":"2021-11-06T11:53:53.003929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"def precision_at_k(recommended_list, bought_list, k=5):\n    \n    bought_list = np.array(bought_list)\n    recommended_list = np.array(recommended_list)\n    \n    bought_list = bought_list  # Тут нет [:k] !!\n    recommended_list = recommended_list[:k]\n    \n    flags = np.isin(bought_list, recommended_list)\n    \n    precision = flags.sum() / len(recommended_list)\n    \n    \n    return precision","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:53.005623Z","iopub.execute_input":"2021-11-06T11:53:53.005999Z","iopub.status.idle":"2021-11-06T11:53:53.020780Z","shell.execute_reply.started":"2021-11-06T11:53:53.005965Z","shell.execute_reply":"2021-11-06T11:53:53.020169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ap_k(recommended_list, bought_list, k=5):\n    \n    bought_list = np.array(bought_list)\n    recommended_list = np.array(recommended_list)\n    \n    flags = np.isin(recommended_list, bought_list)\n    \n    if sum(flags) == 0:\n        return 0\n    \n    sum_ = 0\n    for i in range(1, k+1):\n        \n        if flags[i - 1] == True:\n            p_k = precision_at_k(recommended_list, bought_list, k=i)\n            \n            sum_ += p_k\n            \n    result = sum_ / min(len(recommended_list), k)\n    \n    return result","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:53.022442Z","iopub.execute_input":"2021-11-06T11:53:53.023066Z","iopub.status.idle":"2021-11-06T11:53:53.034535Z","shell.execute_reply.started":"2021-11-06T11:53:53.023024Z","shell.execute_reply":"2021-11-06T11:53:53.033739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_k(recommended_list_list, bought_list_list, k=5):\n    \n    ap = list()\n    for i in range(len(recommended_list_list)):\n        ap.append(ap_k(recommended_list_list[i], bought_list_list[i], k=k))\n    \n    map_metric = np.mean(ap)\n    \n    return map_metric","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:53.035467Z","iopub.execute_input":"2021-11-06T11:53:53.035974Z","iopub.status.idle":"2021-11-06T11:53:53.044290Z","shell.execute_reply.started":"2021-11-06T11:53:53.035940Z","shell.execute_reply":"2021-11-06T11:53:53.043692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ma_k_var2(predicted, actual, k=5):\n    return np.mean([ap_k(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:53.045291Z","iopub.execute_input":"2021-11-06T11:53:53.045791Z","iopub.status.idle":"2021-11-06T11:53:53.055569Z","shell.execute_reply.started":"2021-11-06T11:53:53.045760Z","shell.execute_reply":"2021-11-06T11:53:53.054931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Open data and their transform","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/gb-recsys-project/retail_train.csv')\nitem_features = pd.read_csv('/kaggle/input/gb-recsys-project/product.csv')\nuser_features = pd.read_csv('/kaggle/input/gb-recsys-project/hh_demographic.csv')\ntest_df = pd.read_csv('/kaggle/input/gb-recsys-project/test_users.csv')\n\n# column processing\nitem_features.columns = [col.lower() for col in item_features.columns]\nuser_features.columns = [col.lower() for col in user_features.columns]\n\nitem_features.rename(columns={'product_id': 'item_id'}, inplace=True)\nuser_features.rename(columns={'household_key': 'user_id'}, inplace=True)\n\n\n# Важна схема обучения и валидации!\n# -- давние покупки -- | -- 6 недель -- | -- 3 недель -- \n# подобрать размер 2-ого датасета (6 недель) --> learning curve (зависимость метрики recall@k от размера датасета)\nval_lvl_1_size_weeks = 6\nval_lvl_2_size_weeks = 3\n\ndata_train_lvl_1 = data[data['week_no'] < data['week_no'].max() - (val_lvl_1_size_weeks + val_lvl_2_size_weeks)]\ndata_val_lvl_1 = data[(data['week_no'] >= data['week_no'].max() - (val_lvl_1_size_weeks + val_lvl_2_size_weeks)) &\n                      (data['week_no'] < data['week_no'].max() - (val_lvl_2_size_weeks))]\n\ndata_train_lvl_2 = data_val_lvl_1.copy()  # Для наглядности. Далее мы добавим изменения, и они будут отличаться\ndata_val_lvl_2 = data[data['week_no'] >= data['week_no'].max() - val_lvl_2_size_weeks]\n\ndata_train_lvl_1.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:53.056615Z","iopub.execute_input":"2021-11-06T11:53:53.057369Z","iopub.status.idle":"2021-11-06T11:53:56.460083Z","shell.execute_reply.started":"2021-11-06T11:53:53.057334Z","shell.execute_reply":"2021-11-06T11:53:56.459166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"users, items, interactions = data.user_id.nunique(), data.item_id.nunique(), data.shape[0]\n\nprint('# users: ', users) #пользователи\nprint('# items: ', items) #товары\nprint('# interactions: ', interactions) #взаимодействия","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:56.461172Z","iopub.execute_input":"2021-11-06T11:53:56.461394Z","iopub.status.idle":"2021-11-06T11:53:56.513128Z","shell.execute_reply.started":"2021-11-06T11:53:56.461366Z","shell.execute_reply":"2021-11-06T11:53:56.512239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train-test split","metadata":{}},{"cell_type":"code","source":"# Используем train-test split по времени, а не случайно. Возьмем последние 3 недели в качестве теста\ntest_size_weeks = 3\n\ndata_train = data[data['week_no'] < data['week_no'].max() - test_size_weeks]\ndata_test = data[data['week_no'] >= data['week_no'].max() - test_size_weeks]","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:56.517065Z","iopub.execute_input":"2021-11-06T11:53:56.517309Z","iopub.status.idle":"2021-11-06T11:53:56.619555Z","shell.execute_reply.started":"2021-11-06T11:53:56.517278Z","shell.execute_reply":"2021-11-06T11:53:56.618618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train.shape[0], data_test.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:56.622306Z","iopub.execute_input":"2021-11-06T11:53:56.622548Z","iopub.status.idle":"2021-11-06T11:53:56.628833Z","shell.execute_reply.started":"2021-11-06T11:53:56.622519Z","shell.execute_reply":"2021-11-06T11:53:56.627831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Бейзлайны","metadata":{}},{"cell_type":"markdown","source":"Создадим датафрейм с покупками юзеров на тестовом датасете (последние 3 недели)","metadata":{}},{"cell_type":"code","source":"result = data_test.groupby('user_id')['item_id'].unique().reset_index()\nresult.columns=['user_id', 'actual']\nresult.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:56.630549Z","iopub.execute_input":"2021-11-06T11:53:56.630935Z","iopub.status.idle":"2021-11-06T11:53:56.763348Z","shell.execute_reply.started":"2021-11-06T11:53:56.630893Z","shell.execute_reply":"2021-11-06T11:53:56.762220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_users = result.shape[0]\nnew_test_users = len(set(data_test['user_id']) - set(data_train['user_id']))\n\nprint('В тестовом дата сете {} юзеров'.format(test_users))\nprint('В тестовом дата сете {} новых юзеров'.format(new_test_users))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:56.764602Z","iopub.execute_input":"2021-11-06T11:53:56.764832Z","iopub.status.idle":"2021-11-06T11:53:57.135896Z","shell.execute_reply.started":"2021-11-06T11:53:56.764794Z","shell.execute_reply":"2021-11-06T11:53:57.135010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.1 Popularity-based recommendation","metadata":{}},{"cell_type":"code","source":"popularity = data.groupby('item_id')['user_id'].nunique().reset_index() / data['user_id'].nunique()\npopularity.rename(columns={'user_id': 'share_unique_users'}, inplace=True)\n\ntop_popular = popularity[popularity['share_unique_users'] > 0.3].item_id.tolist()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:57.137170Z","iopub.execute_input":"2021-11-06T11:53:57.137672Z","iopub.status.idle":"2021-11-06T11:53:57.838649Z","shell.execute_reply.started":"2021-11-06T11:53:57.137630Z","shell.execute_reply":"2021-11-06T11:53:57.837763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def popularity_recommendation(data, n=5):\n    \"\"\"Топ-n популярных товаров\"\"\"\n    \n    popular = data.groupby('item_id')['sales_value'].sum().reset_index()\n    popular.sort_values('sales_value', ascending=False, inplace=True)\n    \n    recs = popular.head(n).item_id\n    \n    return recs.tolist()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:57.840420Z","iopub.execute_input":"2021-11-06T11:53:57.840654Z","iopub.status.idle":"2021-11-06T11:53:57.845945Z","shell.execute_reply.started":"2021-11-06T11:53:57.840624Z","shell.execute_reply":"2021-11-06T11:53:57.845180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Можно так делать, так как рекомендация не зависит от юзера\npopular_recs = popularity_recommendation(data_train, n=5)\n\nresult['popular_recommendation'] = result['user_id'].apply(lambda x: popular_recs)\nresult.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:57.846970Z","iopub.execute_input":"2021-11-06T11:53:57.847175Z","iopub.status.idle":"2021-11-06T11:53:57.974607Z","shell.execute_reply.started":"2021-11-06T11:53:57.847150Z","shell.execute_reply":"2021-11-06T11:53:57.973773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.2. Main Recommender","metadata":{}},{"cell_type":"code","source":"n_items_before = data_train_lvl_1['item_id'].nunique()\n\ndata_train_lvl_1 = prefilter_items(data_train_lvl_1, item_features=item_features, take_n_popular=5000)\n\nn_items_after = data_train_lvl_1['item_id'].nunique()\nprint('Decreased # items from {} to {}'.format(n_items_before, n_items_after))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:57.975929Z","iopub.execute_input":"2021-11-06T11:53:57.976158Z","iopub.status.idle":"2021-11-06T11:53:59.208095Z","shell.execute_reply.started":"2021-11-06T11:53:57.976130Z","shell.execute_reply":"2021-11-06T11:53:59.207184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommender = MainRecommender(data_train_lvl_1)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:53:59.209262Z","iopub.execute_input":"2021-11-06T11:53:59.209504Z","iopub.status.idle":"2021-11-06T11:54:08.885613Z","shell.execute_reply.started":"2021-11-06T11:53:59.209475Z","shell.execute_reply":"2021-11-06T11:54:08.884946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommender.get_als_recommendations(2375, N=5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:08.888992Z","iopub.execute_input":"2021-11-06T11:54:08.889708Z","iopub.status.idle":"2021-11-06T11:54:08.910159Z","shell.execute_reply.started":"2021-11-06T11:54:08.889671Z","shell.execute_reply":"2021-11-06T11:54:08.909216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommender.get_own_recommendations(2375, N=5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:08.911795Z","iopub.execute_input":"2021-11-06T11:54:08.916858Z","iopub.status.idle":"2021-11-06T11:54:08.943682Z","shell.execute_reply.started":"2021-11-06T11:54:08.916796Z","shell.execute_reply":"2021-11-06T11:54:08.942831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommender.get_similar_items_recommendation(2375, N=5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:08.945359Z","iopub.execute_input":"2021-11-06T11:54:08.950078Z","iopub.status.idle":"2021-11-06T11:54:08.972527Z","shell.execute_reply.started":"2021-11-06T11:54:08.950022Z","shell.execute_reply":"2021-11-06T11:54:08.971623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommender.get_similar_users_recommendation(2375, N=5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:08.974140Z","iopub.execute_input":"2021-11-06T11:54:08.981848Z","iopub.status.idle":"2021-11-06T11:54:09.035239Z","shell.execute_reply.started":"2021-11-06T11:54:08.981794Z","shell.execute_reply":"2021-11-06T11:54:09.034351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# В тестовом дата сетеесть \"холодные\" покупатели на которах модель не обучалась\n# Найдем данные по таким покупателям, для них по умолчанию будем рекомендовать самые популярные товары\nlist_out = result.loc[~result.user_id.isin(data_train_lvl_1.user_id), 'user_id'].tolist()\nlist_out","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:09.041349Z","iopub.execute_input":"2021-11-06T11:54:09.045361Z","iopub.status.idle":"2021-11-06T11:54:09.076499Z","shell.execute_reply.started":"2021-11-06T11:54:09.045304Z","shell.execute_reply":"2021-11-06T11:54:09.075639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rule(x, y, model):\n    if model == 'als':\n        if x in y:\n            return list([840361, 1029743, 995242, 981760, 1082185])\n        else:\n            return recommender.get_als_recommendations(x, N=5)\n    elif model == 'own':\n        if x in y:\n            return list([840361, 1029743, 995242, 981760, 1082185])\n        else:\n            return recommender.get_own_recommendations(x, N=5)\n    elif model == 'similar_items':\n        if x in y:\n            return list([840361, 1029743, 995242, 981760, 1082185])\n        else:\n            return recommender.get_similar_items_recommendation(x, N=5)\n    elif model == 'similar_users':\n        if x in y:\n            return list([840361, 1029743, 995242, 981760, 1082185])\n        else:\n            return recommender.get_similar_users_recommendation(x, N=5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:09.082579Z","iopub.execute_input":"2021-11-06T11:54:09.083118Z","iopub.status.idle":"2021-11-06T11:54:09.102470Z","shell.execute_reply.started":"2021-11-06T11:54:09.083063Z","shell.execute_reply":"2021-11-06T11:54:09.101371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['als'] = result['user_id'].apply(lambda x: rule(x, list_out, model='als'))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:09.107952Z","iopub.execute_input":"2021-11-06T11:54:09.108651Z","iopub.status.idle":"2021-11-06T11:54:40.648900Z","shell.execute_reply.started":"2021-11-06T11:54:09.108603Z","shell.execute_reply":"2021-11-06T11:54:40.647930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['own'] = result['user_id'].apply(lambda x: rule(x, list_out, model='own'))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:40.650672Z","iopub.execute_input":"2021-11-06T11:54:40.651214Z","iopub.status.idle":"2021-11-06T11:54:46.217077Z","shell.execute_reply.started":"2021-11-06T11:54:40.651164Z","shell.execute_reply":"2021-11-06T11:54:46.216225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['similar_items'] = result['user_id'].apply(lambda x: rule(x, list_out, model='similar_items'))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:46.218393Z","iopub.execute_input":"2021-11-06T11:54:46.218644Z","iopub.status.idle":"2021-11-06T11:54:52.062097Z","shell.execute_reply.started":"2021-11-06T11:54:46.218614Z","shell.execute_reply":"2021-11-06T11:54:52.061158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['als'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.070902Z","iopub.execute_input":"2021-11-06T11:54:52.071696Z","iopub.status.idle":"2021-11-06T11:54:52.255911Z","shell.execute_reply.started":"2021-11-06T11:54:52.071640Z","shell.execute_reply":"2021-11-06T11:54:52.255056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['own'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.257233Z","iopub.execute_input":"2021-11-06T11:54:52.257492Z","iopub.status.idle":"2021-11-06T11:54:52.410684Z","shell.execute_reply.started":"2021-11-06T11:54:52.257463Z","shell.execute_reply":"2021-11-06T11:54:52.409796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['similar_items'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.412419Z","iopub.execute_input":"2021-11-06T11:54:52.412720Z","iopub.status.idle":"2021-11-06T11:54:52.563061Z","shell.execute_reply.started":"2021-11-06T11:54:52.412680Z","shell.execute_reply":"2021-11-06T11:54:52.562116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.564704Z","iopub.execute_input":"2021-11-06T11:54:52.565034Z","iopub.status.idle":"2021-11-06T11:54:52.582535Z","shell.execute_reply.started":"2021-11-06T11:54:52.564990Z","shell.execute_reply":"2021-11-06T11:54:52.581583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Для примера генерации в тестовый дата сет\n# test_df['own'] = test_df['user_id'].apply(lambda x: rule(x, list_out, model='own'))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.584221Z","iopub.execute_input":"2021-11-06T11:54:52.584487Z","iopub.status.idle":"2021-11-06T11:54:52.592898Z","shell.execute_reply.started":"2021-11-06T11:54:52.584458Z","shell.execute_reply":"2021-11-06T11:54:52.591907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross validation","metadata":{}},{"cell_type":"code","source":"# user_item_matrix = recommender.user_item_matrix\n# list_out = data_test.loc[~data_test.user_id.isin(data_train_lvl_1.user_id), 'user_id'].tolist()\n# for factors in [50]:\n#     recommender.fit(user_item_matrix, n_factors=factors, regularization=0.001, iterations=15, num_threads=4)\n#     result['als'] = result['user_id'].apply(lambda x: rule(x, list_out, model='als'))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.596801Z","iopub.execute_input":"2021-11-06T11:54:52.597084Z","iopub.status.idle":"2021-11-06T11:54:52.604889Z","shell.execute_reply.started":"2021-11-06T11:54:52.597055Z","shell.execute_reply":"2021-11-06T11:54:52.603987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result.apply(lambda row: precision_at_k(row['als'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.606078Z","iopub.execute_input":"2021-11-06T11:54:52.606391Z","iopub.status.idle":"2021-11-06T11:54:52.615089Z","shell.execute_reply.started":"2021-11-06T11:54:52.606352Z","shell.execute_reply":"2021-11-06T11:54:52.614442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Обучаем модель 2-ого уровня на выбранных кандидатах","metadata":{}},{"cell_type":"code","source":"# -- давние покупки -- | -- 6 недель -- | -- 3 недель --\n\nusers_lvl_2 = pd.DataFrame(data_train_lvl_2['user_id'].unique())\nusers_lvl_2.columns = ['user_id']\n\n# Пока только warm start\ntrain_users = data_train_lvl_1['user_id'].unique()\nusers_lvl_2 = users_lvl_2[users_lvl_2['user_id'].isin(train_users)]\n\nusers_lvl_2['candidates'] = users_lvl_2['user_id'].apply(lambda x: recommender.get_own_recommendations(x, N=50))\nusers_lvl_2.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:52.616610Z","iopub.execute_input":"2021-11-06T11:54:52.617197Z","iopub.status.idle":"2021-11-06T11:54:58.639444Z","shell.execute_reply.started":"2021-11-06T11:54:52.617156Z","shell.execute_reply":"2021-11-06T11:54:58.638641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = users_lvl_2.apply(lambda x: pd.Series(x['candidates']), axis=1).stack().reset_index(level=1, drop=True)\ns.name = 'item_id'\n\nusers_lvl_2 = users_lvl_2.drop('candidates', axis=1).join(s)\nusers_lvl_2['flag'] = 1\n\nusers_lvl_2.head(4)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:58.640468Z","iopub.execute_input":"2021-11-06T11:54:58.640660Z","iopub.status.idle":"2021-11-06T11:54:59.125005Z","shell.execute_reply.started":"2021-11-06T11:54:58.640635Z","shell.execute_reply":"2021-11-06T11:54:59.124069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_lvl_2 = data_train_lvl_2[['user_id', 'item_id']].copy()\ntargets_lvl_2['target'] = 1  # тут только покупки \n\ntargets_lvl_2 = users_lvl_2.merge(targets_lvl_2, on=['user_id', 'item_id'], how='left')\n\ntargets_lvl_2['target'].fillna(0, inplace= True)\ntargets_lvl_2.drop('flag', axis=1, inplace=True)\ntargets_lvl_2.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.126479Z","iopub.execute_input":"2021-11-06T11:54:59.126879Z","iopub.status.idle":"2021-11-06T11:54:59.193757Z","shell.execute_reply.started":"2021-11-06T11:54:59.126816Z","shell.execute_reply":"2021-11-06T11:54:59.192930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = data_train.groupby('user_id')['item_id'].apply(list).reset_index()\nprint (df)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.195278Z","iopub.execute_input":"2021-11-06T11:54:59.195667Z","iopub.status.idle":"2021-11-06T11:54:59.943312Z","shell.execute_reply.started":"2021-11-06T11:54:59.195624Z","shell.execute_reply":"2021-11-06T11:54:59.942658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Фильтрация полученных рекомендаций\n# result = [postfilter(recommendations, item_info=item_features, N=5) for recommendations in df['item_id']]","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.944655Z","iopub.execute_input":"2021-11-06T11:54:59.945270Z","iopub.status.idle":"2021-11-06T11:54:59.948848Z","shell.execute_reply.started":"2021-11-06T11:54:59.945235Z","shell.execute_reply":"2021-11-06T11:54:59.948091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.0. Filter Items","metadata":{}},{"cell_type":"code","source":"# n_items_before = data_train['item_id'].nunique()\n\n# data_train = prefilter_items(data_train, item_features=item_features)\n\n# n_items_after = data_train['item_id'].nunique()\n# print('Decreased # items from {} to {}'.format(n_items_before, n_items_after))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.950189Z","iopub.execute_input":"2021-11-06T11:54:59.950492Z","iopub.status.idle":"2021-11-06T11:54:59.966460Z","shell.execute_reply.started":"2021-11-06T11:54:59.950464Z","shell.execute_reply":"2021-11-06T11:54:59.965587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.2. Возьмем 1 наиболее близкого соседа","metadata":{}},{"cell_type":"code","source":"# popularity = data_train.groupby('item_id')['quantity'].sum().reset_index()\n# popularity.rename(columns={'quantity': 'n_sold'}, inplace=True)\n\n# popularity.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.967710Z","iopub.execute_input":"2021-11-06T11:54:59.968414Z","iopub.status.idle":"2021-11-06T11:54:59.976900Z","shell.execute_reply.started":"2021-11-06T11:54:59.968377Z","shell.execute_reply":"2021-11-06T11:54:59.976187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# top_5000 = popularity.sort_values('n_sold', ascending=False).head(5000).item_id.tolist()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.978776Z","iopub.execute_input":"2021-11-06T11:54:59.979787Z","iopub.status.idle":"2021-11-06T11:54:59.987671Z","shell.execute_reply.started":"2021-11-06T11:54:59.979743Z","shell.execute_reply":"2021-11-06T11:54:59.987049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Заведем фиктивный item_id (если юзер покупал товары из топ-5000, то он \"купил\" такой товар)\n# data_train.loc[~data_train['item_id'].isin(top_5000), 'item_id'] = 999999\n\nuser_item_matrix = pd.pivot_table(data_train, \n                                  index='user_id', columns='item_id', \n                                  values='quantity',\n                                  aggfunc='count', \n                                  fill_value=0\n                                 )\n\nuser_item_matrix[user_item_matrix > 0] = 1 # так как в итоге хотим предсказать \nuser_item_matrix = user_item_matrix.astype(float) # необходимый тип матрицы для implicit\n\n# переведем в формат saprse matrix\nsparse_user_item = csr_matrix(user_item_matrix).tocsr()\n\nuser_item_matrix.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:54:59.989001Z","iopub.execute_input":"2021-11-06T11:54:59.989758Z","iopub.status.idle":"2021-11-06T11:55:54.103223Z","shell.execute_reply.started":"2021-11-06T11:54:59.989713Z","shell.execute_reply":"2021-11-06T11:55:54.102593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"userids = user_item_matrix.index.values\nitemids = user_item_matrix.columns.values\n\nmatrix_userids = np.arange(len(userids))\nmatrix_itemids = np.arange(len(itemids))\n\nid_to_itemid = dict(zip(matrix_itemids, itemids))\nid_to_userid = dict(zip(matrix_userids, userids))\n\nitemid_to_id = dict(zip(itemids, matrix_itemids))\nuserid_to_id = dict(zip(userids, matrix_userids))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:55:54.104139Z","iopub.execute_input":"2021-11-06T11:55:54.104836Z","iopub.status.idle":"2021-11-06T11:55:54.234320Z","shell.execute_reply.started":"2021-11-06T11:55:54.104764Z","shell.execute_reply":"2021-11-06T11:55:54.233540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel = ItemItemRecommender(K=1, num_threads=4) # K - кол-во ближайших соседей\n\nmodel.fit(csr_matrix(user_item_matrix).T.tocsr(), \n          show_progress=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:55:54.235581Z","iopub.execute_input":"2021-11-06T11:55:54.235828Z","iopub.status.idle":"2021-11-06T11:56:04.127713Z","shell.execute_reply.started":"2021-11-06T11:55:54.235800Z","shell.execute_reply":"2021-11-06T11:56:04.127038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nresult['own_purchases'] = result['user_id'].\\\n    apply(lambda x: [id_to_itemid[rec[0]] for rec in \n                    model.recommend(userid=userid_to_id[x], \n                                    user_items=sparse_user_item,   # на вход user-item matrix\n                                    N=5, \n                                    filter_already_liked_items=False, \n                                    filter_items=None,\n                                    recalculate_user=False)])","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:56:04.129156Z","iopub.execute_input":"2021-11-06T11:56:04.129743Z","iopub.status.idle":"2021-11-06T11:56:04.236102Z","shell.execute_reply.started":"2021-11-06T11:56:04.129697Z","shell.execute_reply":"2021-11-06T11:56:04.235035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:56:04.237398Z","iopub.execute_input":"2021-11-06T11:56:04.237626Z","iopub.status.idle":"2021-11-06T11:56:04.268097Z","shell.execute_reply.started":"2021-11-06T11:56:04.237598Z","shell.execute_reply":"2021-11-06T11:56:04.267196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.3. Alternating Least Squares (ALS)","metadata":{}},{"cell_type":"code","source":"def get_recommendations(user, model, N=5):\n    res = [id_to_itemid[rec[0]] for rec in \n                    model.recommend(userid=userid_to_id[user], \n                                    user_items=sparse_user_item,   # на вход user-item matrix\n                                    N=N, \n                                    filter_already_liked_items=False, \n                                    filter_items=None, \n                                    recalculate_user=True)]\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:56:04.269542Z","iopub.execute_input":"2021-11-06T11:56:04.269783Z","iopub.status.idle":"2021-11-06T11:56:04.275261Z","shell.execute_reply.started":"2021-11-06T11:56:04.269755Z","shell.execute_reply":"2021-11-06T11:56:04.274473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel = AlternatingLeastSquares(factors=50, \n                                regularization=0.001,\n                                iterations=15, \n                                calculate_training_loss=True, \n                                num_threads=4)\n\nmodel.fit(csr_matrix(user_item_matrix).T.tocsr(),  # На вход item-user matrix\n          show_progress=True)\n\nresult['als_var'] = result['user_id'].apply(lambda x: get_recommendations(x, model=model, N=5))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:56:04.276591Z","iopub.execute_input":"2021-11-06T11:56:04.277405Z","iopub.status.idle":"2021-11-06T11:58:14.791533Z","shell.execute_reply.started":"2021-11-06T11:56:04.277354Z","shell.execute_reply":"2021-11-06T11:58:14.790343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.3. TF-IDF взвешивание","metadata":{}},{"cell_type":"code","source":"user_item_matrix = tfidf_weight(user_item_matrix.T).T  # Применяется к item-user матрице ! ","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:58:14.793624Z","iopub.execute_input":"2021-11-06T11:58:14.794235Z","iopub.status.idle":"2021-11-06T11:58:16.862874Z","shell.execute_reply.started":"2021-11-06T11:58:14.794169Z","shell.execute_reply":"2021-11-06T11:58:16.862086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel = AlternatingLeastSquares(factors=50, \n                                regularization=0.001,\n                                iterations=15, \n                                calculate_training_loss=True, \n                                num_threads=4)\n\nmodel.fit(csr_matrix(user_item_matrix).T.tocsr(),  # На вход item-user matrix\n          show_progress=True)\n\nresult['als_tfidf'] = result['user_id'].apply(lambda x: get_recommendations(x, model=model, N=5))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T11:58:16.864062Z","iopub.execute_input":"2021-11-06T11:58:16.864401Z","iopub.status.idle":"2021-11-06T12:00:26.738913Z","shell.execute_reply.started":"2021-11-06T11:58:16.864372Z","shell.execute_reply":"2021-11-06T12:00:26.738040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.4. BM25 взвешивание","metadata":{}},{"cell_type":"code","source":"# Заведем фиктивный item_id (если юзер покупал товары из топ-5000, то он \"купил\" такой товар)\n# data_train.loc[~data_train['item_id'].isin(top_5000), 'item_id'] = 999999\n\nuser_item_matrix = pd.pivot_table(data_train, \n                                  index='user_id', columns='item_id', \n                                  values='quantity', # Можно пробоват ьдругие варианты\n                                  aggfunc='count', \n                                  fill_value=0\n                                 )\n\nuser_item_matrix = user_item_matrix.astype(float) # необходимый тип матрицы для implicit\n\n# переведем в формат saprse matrix\nsparse_user_item = csr_matrix(user_item_matrix).tocsr()\n\nuser_item_matrix.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:00:26.740394Z","iopub.execute_input":"2021-11-06T12:00:26.740983Z","iopub.status.idle":"2021-11-06T12:01:15.365813Z","shell.execute_reply.started":"2021-11-06T12:00:26.740938Z","shell.execute_reply":"2021-11-06T12:01:15.364927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_item_matrix = bm25_weight(user_item_matrix.T).T  # Применяется к item-user матрице ! ","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:01:15.366929Z","iopub.execute_input":"2021-11-06T12:01:15.367234Z","iopub.status.idle":"2021-11-06T12:01:17.390819Z","shell.execute_reply.started":"2021-11-06T12:01:15.367200Z","shell.execute_reply":"2021-11-06T12:01:17.389950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel = AlternatingLeastSquares(factors=50, \n                                regularization=0.001,\n                                iterations=15, \n                                calculate_training_loss=True, \n                                num_threads=4) # K - кол-во билжайших соседей\n\nmodel.fit(csr_matrix(user_item_matrix).T.tocsr(),  # На вход item-user matrix\n          show_progress=True)\n\nresult['als_bm25'] = result['user_id'].apply(lambda x: get_recommendations(x, model=model, N=5))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:01:17.392059Z","iopub.execute_input":"2021-11-06T12:01:17.392281Z","iopub.status.idle":"2021-11-06T12:03:26.894787Z","shell.execute_reply.started":"2021-11-06T12:01:17.392255Z","shell.execute_reply":"2021-11-06T12:03:26.893827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filter Items and model LFM","metadata":{}},{"cell_type":"code","source":"# n_items_before = data_train['item_id'].nunique()\n\n# data_train = prefilter_items(data_train, item_features=item_features)\n\n# n_items_after = data_train['item_id'].nunique()\n# print('Decreased # items from {} to {}'.format(n_items_before, n_items_after))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.896763Z","iopub.execute_input":"2021-11-06T12:03:26.897429Z","iopub.status.idle":"2021-11-06T12:03:26.903637Z","shell.execute_reply.started":"2021-11-06T12:03:26.897375Z","shell.execute_reply":"2021-11-06T12:03:26.902787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user_item_matrix = pd.pivot_table(data_train, \n#                                   index='user_id', columns='item_id', \n#                                   values='quantity', # Можно пробоват ьдругие варианты\n#                                   aggfunc='count', \n#                                   fill_value=0\n#                                  )\n\n# user_item_matrix = user_item_matrix.astype(float) # необходимый тип матрицы для implicit\n\n# # переведем в формат saprse matrix\n# sparse_user_item = csr_matrix(user_item_matrix).tocsr()\n\n# user_item_matrix.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.905204Z","iopub.execute_input":"2021-11-06T12:03:26.905703Z","iopub.status.idle":"2021-11-06T12:03:26.921221Z","shell.execute_reply.started":"2021-11-06T12:03:26.905657Z","shell.execute_reply":"2021-11-06T12:03:26.920175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_test = data_test[data_test['item_id'].isin(data_train['item_id'].unique())]\n# test_user_item_matrix = pd.pivot_table(data_test, \n#                                   index='user_id', columns='item_id', \n#                                   values='quantity', # Можно пробоват ьдругие варианты\n#                                   aggfunc='count', \n#                                   fill_value=0\n#                                  )\n\n# test_user_item_matrix = user_item_matrix.astype(float) # необходимый тип матрицы для implicit","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.922452Z","iopub.execute_input":"2021-11-06T12:03:26.922894Z","iopub.status.idle":"2021-11-06T12:03:26.934000Z","shell.execute_reply.started":"2021-11-06T12:03:26.922821Z","shell.execute_reply":"2021-11-06T12:03:26.932956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# userids = user_item_matrix.index.values\n# itemids = user_item_matrix.columns.values\n\n# matrix_userids = np.arange(len(userids))\n# matrix_itemids = np.arange(len(itemids))\n\n# id_to_itemid = dict(zip(matrix_itemids, itemids))\n# id_to_userid = dict(zip(matrix_userids, userids))\n\n# itemid_to_id = dict(zip(itemids, matrix_itemids))\n# userid_to_id = dict(zip(userids, matrix_userids))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.935228Z","iopub.execute_input":"2021-11-06T12:03:26.935922Z","iopub.status.idle":"2021-11-06T12:03:26.945469Z","shell.execute_reply.started":"2021-11-06T12:03:26.935846Z","shell.execute_reply":"2021-11-06T12:03:26.944496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user_feat = pd.DataFrame(user_item_matrix.index)\n# user_feat = user_feat.merge(user_features, on='user_id', how='left')\n# user_feat.set_index('user_id', inplace=True)\n\n# item_feat = pd.DataFrame(user_item_matrix.columns)\n# item_feat = item_feat.merge(item_features, on='item_id', how='left')\n# item_feat.set_index('item_id', inplace=True)\n\n# user_feat.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.947625Z","iopub.execute_input":"2021-11-06T12:03:26.948353Z","iopub.status.idle":"2021-11-06T12:03:26.956976Z","shell.execute_reply.started":"2021-11-06T12:03:26.948298Z","shell.execute_reply":"2021-11-06T12:03:26.956058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user_feat_lightfm = pd.get_dummies(user_feat, columns=user_feat.columns.tolist())\n# item_feat_lightfm = pd.get_dummies(item_feat, columns=item_feat.columns.tolist())","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.959056Z","iopub.execute_input":"2021-11-06T12:03:26.959744Z","iopub.status.idle":"2021-11-06T12:03:26.975273Z","shell.execute_reply.started":"2021-11-06T12:03:26.959686Z","shell.execute_reply":"2021-11-06T12:03:26.974261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user_feat_lightfm.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.976583Z","iopub.execute_input":"2021-11-06T12:03:26.976945Z","iopub.status.idle":"2021-11-06T12:03:26.987181Z","shell.execute_reply.started":"2021-11-06T12:03:26.976900Z","shell.execute_reply":"2021-11-06T12:03:26.986234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = LightFM(no_components=30,\n#                 loss='bpr', # 'warp'\n#                 learning_rate=0.05, \n#                 item_alpha=0.1, user_alpha=0.1, \n#                 random_state=42)\n\n# model.fit((sparse_user_item > 0) * 1,  # user-item matrix из 0 и 1\n#           sample_weight=coo_matrix(user_item_matrix),\n#           user_features=csr_matrix(user_feat_lightfm.values).tocsr(),\n#           item_features=csr_matrix(item_feat_lightfm.values).tocsr(),\n#           epochs=15, \n#           num_threads=4) ","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:26.988904Z","iopub.execute_input":"2021-11-06T12:03:26.989251Z","iopub.status.idle":"2021-11-06T12:03:27.000679Z","shell.execute_reply.started":"2021-11-06T12:03:26.989205Z","shell.execute_reply":"2021-11-06T12:03:26.999714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user_emb = model.get_user_representations(features=csr_matrix(user_feat_lightfm.values).tocsr())","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.001987Z","iopub.execute_input":"2021-11-06T12:03:27.002340Z","iopub.status.idle":"2021-11-06T12:03:27.011618Z","shell.execute_reply.started":"2021-11-06T12:03:27.002294Z","shell.execute_reply":"2021-11-06T12:03:27.010956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_precision = precision_at_k(model, sparse_user_item, \n#                                  user_features=csr_matrix(user_feat_lightfm.values).tocsr(),\n#                                  item_features=csr_matrix(item_feat_lightfm.values).tocsr(),\n#                                  k=5).mean()\n\n# train_precision","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.012532Z","iopub.execute_input":"2021-11-06T12:03:27.012733Z","iopub.status.idle":"2021-11-06T12:03:27.024318Z","shell.execute_reply.started":"2021-11-06T12:03:27.012707Z","shell.execute_reply":"2021-11-06T12:03:27.023286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Измерим качество по precision@5","metadata":{}},{"cell_type":"code","source":"result.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.025716Z","iopub.execute_input":"2021-11-06T12:03:27.026067Z","iopub.status.idle":"2021-11-06T12:03:27.057320Z","shell.execute_reply.started":"2021-11-06T12:03:27.026024Z","shell.execute_reply":"2021-11-06T12:03:27.056481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['popular_recommendation'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.058547Z","iopub.execute_input":"2021-11-06T12:03:27.059468Z","iopub.status.idle":"2021-11-06T12:03:27.200211Z","shell.execute_reply.started":"2021-11-06T12:03:27.059433Z","shell.execute_reply":"2021-11-06T12:03:27.199438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['own_purchases'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.201643Z","iopub.execute_input":"2021-11-06T12:03:27.202384Z","iopub.status.idle":"2021-11-06T12:03:27.353313Z","shell.execute_reply.started":"2021-11-06T12:03:27.202338Z","shell.execute_reply":"2021-11-06T12:03:27.352470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['als_var'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.354557Z","iopub.execute_input":"2021-11-06T12:03:27.355505Z","iopub.status.idle":"2021-11-06T12:03:27.509189Z","shell.execute_reply.started":"2021-11-06T12:03:27.355462Z","shell.execute_reply":"2021-11-06T12:03:27.508137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['als_tfidf'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.510916Z","iopub.execute_input":"2021-11-06T12:03:27.511228Z","iopub.status.idle":"2021-11-06T12:03:27.680534Z","shell.execute_reply.started":"2021-11-06T12:03:27.511185Z","shell.execute_reply":"2021-11-06T12:03:27.679544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: precision_at_k(row['als_bm25'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.681791Z","iopub.execute_input":"2021-11-06T12:03:27.682356Z","iopub.status.idle":"2021-11-06T12:03:27.824077Z","shell.execute_reply.started":"2021-11-06T12:03:27.682322Z","shell.execute_reply":"2021-11-06T12:03:27.823237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Измерим качество по MAP@k (Mean Average Precision@k)  \n- Показывает средневзвешенную точность рекомендаций","metadata":{}},{"cell_type":"code","source":"result.apply(lambda row: ap_k(row['popular_recommendation'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:27.825265Z","iopub.execute_input":"2021-11-06T12:03:27.825907Z","iopub.status.idle":"2021-11-06T12:03:28.128640Z","shell.execute_reply.started":"2021-11-06T12:03:27.825875Z","shell.execute_reply":"2021-11-06T12:03:28.127830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: ap_k(row['own_purchases'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:28.129949Z","iopub.execute_input":"2021-11-06T12:03:28.130181Z","iopub.status.idle":"2021-11-06T12:03:28.477731Z","shell.execute_reply.started":"2021-11-06T12:03:28.130152Z","shell.execute_reply":"2021-11-06T12:03:28.476909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: ap_k(row['als'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:28.478934Z","iopub.execute_input":"2021-11-06T12:03:28.479304Z","iopub.status.idle":"2021-11-06T12:03:28.756158Z","shell.execute_reply.started":"2021-11-06T12:03:28.479270Z","shell.execute_reply":"2021-11-06T12:03:28.755303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: ap_k(row['als_tfidf'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:28.757806Z","iopub.execute_input":"2021-11-06T12:03:28.758141Z","iopub.status.idle":"2021-11-06T12:03:29.075097Z","shell.execute_reply.started":"2021-11-06T12:03:28.758099Z","shell.execute_reply":"2021-11-06T12:03:29.074220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.apply(lambda row: ap_k(row['als_bm25'], row['actual']), axis=1).mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:29.076400Z","iopub.execute_input":"2021-11-06T12:03:29.077085Z","iopub.status.idle":"2021-11-06T12:03:29.388805Z","shell.execute_reply.started":"2021-11-06T12:03:29.077039Z","shell.execute_reply":"2021-11-06T12:03:29.387937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. Прогнозирование на тестовом датасете","metadata":{}},{"cell_type":"markdown","source":"Для удобства перевода данных в подходящий для загрузки формат:","metadata":{}},{"cell_type":"code","source":"def transform_data_for_eval(dataset, rec_col, user_col='user_id'):\n    '''\n    Func for transforming recommendations into kaggle evaluation format\n\n    Parameters:\n    dataset (pd.DataFrame): Dataset with 2 required columns:\n        rec_col - column with recommendations should be iterable\n        user_col - columns with user id\n\n    rec_col (str): name of column in dataset with recommendations\n\n    user_col (str): name of column in dataset with user id\n\n    Returns:\n    pd.DataFrame: DataFrame in suitable format\n\n   '''\n    eval_dataset = dataset[[user_col, rec_col]].copy()\n    eval_dataset[rec_col] = eval_dataset[rec_col].apply(lambda x: ' '.join([str(i) for i in x]))\n    eval_dataset.rename(columns={\n        user_col: 'UserId',\n        rec_col: 'Predicted'\n    }, inplace=True)\n    return eval_dataset","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:29.390300Z","iopub.execute_input":"2021-11-06T12:03:29.391270Z","iopub.status.idle":"2021-11-06T12:03:29.397891Z","shell.execute_reply.started":"2021-11-06T12:03:29.391231Z","shell.execute_reply":"2021-11-06T12:03:29.396937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = pd.read_csv('/kaggle/input/gb-recsys-project/sample_submission.csv')\nsubmit.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:29.399214Z","iopub.execute_input":"2021-11-06T12:03:29.399596Z","iopub.status.idle":"2021-11-06T12:03:29.422819Z","shell.execute_reply.started":"2021-11-06T12:03:29.399553Z","shell.execute_reply":"2021-11-06T12:03:29.421954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Эффективней других оказалась идея найти 1 ближашего соседа и фильтровать по топ 5000, ее и применим.","metadata":{}},{"cell_type":"code","source":"def get_recommendations(user, model, N=5):\n    res = [id_to_itemid[rec[0]] for rec in \n                    model.recommend(userid=userid_to_id[user], \n                                    user_items=sparse_user_item,   # на вход user-item matrix\n                                    N=N, \n                                    filter_already_liked_items=False, \n                                    filter_items=None, \n                                    recalculate_user=True)]\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:29.423995Z","iopub.execute_input":"2021-11-06T12:03:29.425606Z","iopub.status.idle":"2021-11-06T12:03:29.431193Z","shell.execute_reply.started":"2021-11-06T12:03:29.425566Z","shell.execute_reply":"2021-11-06T12:03:29.430500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_item_matrix = pd.pivot_table(data_train, \n                                  index='user_id', columns='item_id', \n                                  values='quantity',\n                                  aggfunc='count', \n                                  fill_value=0\n                                 )\n\nuser_item_matrix[user_item_matrix > 0] = 1 # так как в итоге хотим предсказать \nuser_item_matrix = user_item_matrix.astype(float) # необходимый тип матрицы для implicit\n\n# переведем в формат saprse matrix\nsparse_user_item = csr_matrix(user_item_matrix).tocsr()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:03:29.442196Z","iopub.execute_input":"2021-11-06T12:03:29.442541Z","iopub.status.idle":"2021-11-06T12:04:19.219971Z","shell.execute_reply.started":"2021-11-06T12:03:29.442497Z","shell.execute_reply":"2021-11-06T12:04:19.219084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"userids = user_item_matrix.index.values\nitemids = user_item_matrix.columns.values\n\nmatrix_userids = np.arange(len(userids))\nmatrix_itemids = np.arange(len(itemids))\n\nid_to_itemid = dict(zip(matrix_itemids, itemids))\nid_to_userid = dict(zip(matrix_userids, userids))\n\nitemid_to_id = dict(zip(itemids, matrix_itemids))\nuserid_to_id = dict(zip(userids, matrix_userids))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:19.220982Z","iopub.execute_input":"2021-11-06T12:04:19.221193Z","iopub.status.idle":"2021-11-06T12:04:19.328559Z","shell.execute_reply.started":"2021-11-06T12:04:19.221167Z","shell.execute_reply":"2021-11-06T12:04:19.327392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel = ItemItemRecommender(K=1, num_threads=4) # K - кол-во ближайших соседей\n\nmodel.fit(csr_matrix(user_item_matrix).T.tocsr(), \n          show_progress=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:19.329925Z","iopub.execute_input":"2021-11-06T12:04:19.330190Z","iopub.status.idle":"2021-11-06T12:04:29.220528Z","shell.execute_reply.started":"2021-11-06T12:04:19.330157Z","shell.execute_reply":"2021-11-06T12:04:29.219819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.221761Z","iopub.execute_input":"2021-11-06T12:04:29.225616Z","iopub.status.idle":"2021-11-06T12:04:29.230771Z","shell.execute_reply.started":"2021-11-06T12:04:29.225576Z","shell.execute_reply":"2021-11-06T12:04:29.230176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_out = test_df.loc[~test_df.user_id.isin(data_train.user_id), 'user_id'].tolist()\nlist_out","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.232000Z","iopub.execute_input":"2021-11-06T12:04:29.232406Z","iopub.status.idle":"2021-11-06T12:04:29.263405Z","shell.execute_reply.started":"2021-11-06T12:04:29.232372Z","shell.execute_reply":"2021-11-06T12:04:29.262404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rule(x, y):\n    if x in y:\n         return list([840361, 1029743, 995242, 981760, 1082185])\n    else:\n         return get_recommendations(x, model=model, N=5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.265010Z","iopub.execute_input":"2021-11-06T12:04:29.265258Z","iopub.status.idle":"2021-11-06T12:04:29.270475Z","shell.execute_reply.started":"2021-11-06T12:04:29.265229Z","shell.execute_reply":"2021-11-06T12:04:29.269588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['own_purchases'] = test_df['user_id'].apply(lambda x: rule(x, list_out))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.272034Z","iopub.execute_input":"2021-11-06T12:04:29.272799Z","iopub.status.idle":"2021-11-06T12:04:29.371741Z","shell.execute_reply.started":"2021-11-06T12:04:29.272750Z","shell.execute_reply":"2021-11-06T12:04:29.370876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.372994Z","iopub.execute_input":"2021-11-06T12:04:29.373319Z","iopub.status.idle":"2021-11-06T12:04:29.385822Z","shell.execute_reply.started":"2021-11-06T12:04:29.373269Z","shell.execute_reply":"2021-11-06T12:04:29.384942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampl_df = transform_data_for_eval(test_df, rec_col='own_purchases', user_col='user_id')","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.387156Z","iopub.execute_input":"2021-11-06T12:04:29.388068Z","iopub.status.idle":"2021-11-06T12:04:29.408899Z","shell.execute_reply.started":"2021-11-06T12:04:29.388024Z","shell.execute_reply":"2021-11-06T12:04:29.408245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampl_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.410102Z","iopub.execute_input":"2021-11-06T12:04:29.410910Z","iopub.status.idle":"2021-11-06T12:04:29.426585Z","shell.execute_reply.started":"2021-11-06T12:04:29.410843Z","shell.execute_reply":"2021-11-06T12:04:29.425776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampl_df.to_csv('sample_test_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T12:04:29.428027Z","iopub.execute_input":"2021-11-06T12:04:29.428656Z","iopub.status.idle":"2021-11-06T12:04:29.442894Z","shell.execute_reply.started":"2021-11-06T12:04:29.428622Z","shell.execute_reply":"2021-11-06T12:04:29.441982Z"},"trusted":true},"execution_count":null,"outputs":[]}]}