{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3288,"databundleVersionId":31051,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport dask.dataframe as dd\n\n# Загрузка данных с использованием Dask с указанием типов данных\ntrain = dd.read_csv('/kaggle/input/event-recommendation-engine-challenge/train.csv')\ntest = dd.read_csv('/kaggle/input/event-recommendation-engine-challenge/test.csv')\nusers = dd.read_csv('/kaggle/input/event-recommendation-engine-challenge/users.csv', dtype={'birthyear': 'object', 'timezone': 'float64'})\nuser_friends = dd.read_csv('/kaggle/input/event-recommendation-engine-challenge/user_friends.csv.gz')\nevents = dd.read_csv('/kaggle/input/event-recommendation-engine-challenge/events.csv.gz', dtype={'city': 'object', 'country': 'object', 'state': 'object', 'zip': 'object'})\nevent_attendees = dd.read_csv('/kaggle/input/event-recommendation-engine-challenge/event_attendees.csv.gz')\n\n# Предварительный просмотр данных\nprint(\"Train.csv:\")\nprint(train.head(), \"\\n\")\n\nprint(\"Test.csv:\")\nprint(test.head(), \"\\n\")\n\nprint(\"Users.csv:\")\nprint(users.head(), \"\\n\")\n\nprint(\"User_friends.csv:\")\nprint(user_friends.head(), \"\\n\")\n\nprint(\"Events.csv:\")\nprint(events.head(), \"\\n\")\n\nprint(\"Event_attendees.csv:\")\nprint(event_attendees.head(), \"\\n\")\n\n# Предобработка данных\ntrain['timestamp'] = dd.to_datetime(train['timestamp'], errors='coerce')\ntest['timestamp'] = dd.to_datetime(test['timestamp'], errors='coerce')\nusers['joinedAt'] = dd.to_datetime(users['joinedAt'], errors='coerce')\nevents['start_time'] = dd.to_datetime(events['start_time'], errors='coerce')\n\n# Преобразование данных в Pandas DataFrame для EDA и дальнейшей работы\ntrain_pd = train.compute()\ntest_pd = test.compute()\nusers_pd = users.compute()\nuser_friends_pd = user_friends.compute()\nevents_pd = events.compute()\nevent_attendees_pd = event_attendees.compute()\n\n# EDA Analysis\nprint(\"Train Data Info\")\nprint(train_pd.info())\n\nprint(\"\\nTest Data Info\")\nprint(test_pd.info())\n\nprint(\"\\nUsers Data Info\")\nprint(users_pd.info())\n\nprint(\"\\nUser Friends Data Info\")\nprint(user_friends_pd.info())\n\nprint(\"\\nEvents Data Info\")\nprint(events_pd.info())\n\nprint(\"\\nEvent Attendees Data Info\")\nprint(event_attendees_pd.info())\n\n# Basic statistics\nprint(\"\\nTrain Data Statistics\")\nprint(train_pd.describe())\n\nprint(\"\\nTest Data Statistics\")\nprint(test_pd.describe())\n\nprint(\"\\nUsers Data Statistics\")\nprint(users_pd.describe())\n\nprint(\"\\nEvents Data Statistics\")\nprint(events_pd.describe())\n\n# Visualizing missing values\ndef plot_missing_values(df, title):\n    missing = df.isnull().mean()\n    missing = missing[missing > 0]\n    missing.sort_values(inplace=True)\n    missing.plot.bar()\n    plt.title(title)\n    plt.show()\n\nplot_missing_values(train_pd, \"Missing Values in Train Data\")\nplot_missing_values(test_pd, \"Missing Values in Test Data\")\nplot_missing_values(users_pd, \"Missing Values in Users Data\")\nplot_missing_values(events_pd, \"Missing Values in Events Data\")\n\n# Distribution of interested and not_interested in train data\ntrain_pd['interested'].value_counts().plot(kind='bar', title='Distribution of Interested and Not Interested')\nplt.show()\n\n# Создание маппинга пользователей и событий\nuser_ids = users_pd['user_id'].unique().tolist()\nevent_ids = train_pd['event'].unique().tolist()\n\nuser_to_index = {x: i for i, x in enumerate(user_ids)}\nevent_to_index = {x: i for i, x in enumerate(event_ids)}\n\ntrain_pd['user'] = train_pd['user'].map(user_to_index)\ntrain_pd['event'] = train_pd['event'].map(event_to_index)\ntest_pd['user'] = test_pd['user'].map(user_to_index)\ntest_pd['event'] = test_pd['event'].map(event_to_index)\n\n# Проверка на соответствие пользователей и событий в тестовом наборе\ntest_users = test_pd['user'].unique()\ntest_events = test_pd['event'].unique()\n\nvalid_users = np.isin(test_users, list(user_to_index.values()))\nvalid_events = np.isin(test_events, list(event_to_index.values()))\n\nprint(f\"Число пользователей в тестовом наборе: {len(test_users)}\")\nprint(f\"Число событий в тестовом наборе: {len(test_events)}\")\nprint(f\"Число пользователей в тестовом наборе, которые есть в обучающем: {np.sum(valid_users)}\")\nprint(f\"Число событий в тестовом наборе, которые есть в обучающем: {np.sum(valid_events)}\")\n\n# Преобразование тестовых данных\ntest_pd = test_pd[test_pd['user'].isin(user_to_index.values()) & test_pd['event'].isin(event_to_index.values())]\n\n# Проверка, что после фильтрации тестовый набор не пустой\nif test_pd.empty:\n    print(\"После фильтрации тестовый набор пуст. Убедитесь, что данные корректны.\")\nelse:\n    num_users = len(user_ids)\n    num_events = len(event_ids)\n\n    # Подготовка данных для модели\n    X = train_pd[['user', 'event']].values\n    y = train_pd['interested'].values\n\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n    # Создание модели факторизации матрицы\n    class RecommenderNet(tf.keras.Model):\n        def __init__(self, num_users, num_events, embedding_size=50, **kwargs):\n            super(RecommenderNet, self).__init__(**kwargs)\n            self.user_embedding = tf.keras.layers.Embedding(\n                num_users, embedding_size,\n                embeddings_initializer='he_normal',\n                embeddings_regularizer=tf.keras.regularizers.l2(1e-6)\n            )\n            self.event_embedding = tf.keras.layers.Embedding(\n                num_events, embedding_size,\n                embeddings_initializer='he_normal',\n                embeddings_regularizer=tf.keras.regularizers.l2(1e-6)\n            )\n            self.user_bias = tf.keras.layers.Embedding(num_users, 1)\n            self.event_bias = tf.keras.layers.Embedding(num_events, 1)\n\n        def call(self, inputs):\n            user_vector = self.user_embedding(inputs[:, 0])\n            event_vector = self.event_embedding(inputs[:, 1])\n            \n            user_bias = self.user_bias(inputs[:, 0])\n            event_bias = self.event_bias(inputs[:, 1])\n            \n            dot_user_event = tf.tensordot(user_vector, event_vector, 2)\n            \n            x = dot_user_event + user_bias + event_bias\n            \n            return tf.nn.sigmoid(x)\n\n    # Параметры модели\n    embedding_size = 50\n\n    model = RecommenderNet(num_users, num_events, embedding_size)\n    model.compile(\n        loss=tf.keras.losses.BinaryCrossentropy(),\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n        metrics=[tf.keras.metrics.AUC()]\n    )\n\n    # Обучение модели\n    history = model.fit(\n        x=X_train,\n        y=y_train,\n        batch_size=64,\n        epochs=10,\n        validation_data=(X_val, y_val)\n    )\n\n    # Функция для рекомендаций\n    def recommend(user_id, model, num_recommendations=10):\n        if user_id not in user_to_index:\n            print(f\"User ID {user_id} не найден.\")\n            return []\n            \n        user_index = user_to_index[user_id]\n        event_indices = np.arange(num_events)\n        \n        user_array = np.array([user_index] * num_events)\n        event_array = event_indices\n        \n        predictions = model.predict(np.vstack([user_array, event_array]).T).flatten()\n        \n        top_indices = predictions.argsort()[-num_recommendations:][::-1]\n        recommended_events = [event_ids[i] for i in top_indices]\n        \n        return recommended_events\n\n    # Создание файла submission\n    submission = []\n\n    for user_id in test_pd['user'].unique():\n        user_id = int(user_id)  # Преобразование в целое число\n        original_user_id = user_ids[user_id]\n        recommendations = recommend(original_user_id, model, num_recommendations=200)\n        submission.append({\n            'User': original_user_id,\n            'Events': ' '.join(map(str, recommendations))\n        })\n\n    submission_df = pd.DataFrame(submission)\n    submission_df.to_csv('submission_TensorFlow.csv', index=False)\n    print(\"Файл submission создан.\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-29T19:26:08.107861Z","iopub.execute_input":"2024-06-29T19:26:08.108299Z","iopub.status.idle":"2024-06-29T19:39:18.699809Z","shell.execute_reply.started":"2024-06-29T19:26:08.108263Z","shell.execute_reply":"2024-06-29T19:39:18.698487Z"},"trusted":true},"execution_count":null,"outputs":[]}]}