{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Input, Flatten,Dropout\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop, Adagrad, Adadelta\nfrom tensorflow.keras import regularizers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:06.446169Z","iopub.execute_input":"2024-12-02T21:08:06.447058Z","iopub.status.idle":"2024-12-02T21:08:06.453576Z","shell.execute_reply.started":"2024-12-02T21:08:06.447000Z","shell.execute_reply":"2024-12-02T21:08:06.452568Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Початкові налаштування","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Завантажимо датасет в DataFrame","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(X_train, y_train), (X_test, y_test) = boston_housing.load_data()\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:07.207160Z","iopub.execute_input":"2024-12-02T21:08:07.208096Z","iopub.status.idle":"2024-12-02T21:08:07.217065Z","shell.execute_reply.started":"2024-12-02T21:08:07.208049Z","shell.execute_reply":"2024-12-02T21:08:07.216095Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Нормалізуємо дані","metadata":{}},{"cell_type":"code","source":"print(X_train.shape)\nprint(X_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:07.465938Z","iopub.execute_input":"2024-12-02T21:08:07.466352Z","iopub.status.idle":"2024-12-02T21:08:07.471399Z","shell.execute_reply.started":"2024-12-02T21:08:07.466310Z","shell.execute_reply":"2024-12-02T21:08:07.470434Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Сформуємо модель","metadata":{}},{"cell_type":"code","source":"def create_model(regularizer, dropout_rate):\n    model = Sequential([\n        Input(shape=(X_train.shape[1],)),\n        Flatten(),\n        Dense(64, activation='relu', kernel_regularizer=regularizer),\n        Dropout(dropout_rate),\n        Dense(64, activation='relu', kernel_regularizer=regularizer),\n        Dropout(dropout_rate),\n        Dense(1)\n    ])\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:07.706935Z","iopub.execute_input":"2024-12-02T21:08:07.707796Z","iopub.status.idle":"2024-12-02T21:08:07.712438Z","shell.execute_reply.started":"2024-12-02T21:08:07.707759Z","shell.execute_reply":"2024-12-02T21:08:07.711529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Створимо функцію для тренування усіх моделей на одному оптимізаторі","metadata":{}},{"cell_type":"code","source":"def train_model(optimizer, regularizer = None,dropout_rate = 0):\n    model = create_model(regularizer, dropout_rate)\n    model.compile(optimizer=optimizer, loss='mse', metrics=['mae'])\n\n    history = model.fit(\n        X_train, y_train, \n        validation_data=(X_test, y_test),\n        batch_size=16, \n        epochs=50, \n        verbose=0,\n        ) \n    return history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:07.969575Z","iopub.execute_input":"2024-12-02T21:08:07.969928Z","iopub.status.idle":"2024-12-02T21:08:07.974999Z","shell.execute_reply.started":"2024-12-02T21:08:07.969896Z","shell.execute_reply":"2024-12-02T21:08:07.974196Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Задамо змінні","metadata":{}},{"cell_type":"code","source":"regularizerslist = {\n    \"None\": None,\n    \"Dropout\": None,\n    \"L1\": regularizers.l1(0.01),\n    \"L2\": regularizers.l2(0.01)\n}\nhistory_results = None\n\noptimizers = [\n    (\"SGD\", SGD),\n    (\"Adam\", Adam),\n    (\"RMSprop\", RMSprop)\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:08.522755Z","iopub.execute_input":"2024-12-02T21:08:08.523652Z","iopub.status.idle":"2024-12-02T21:08:08.528578Z","shell.execute_reply.started":"2024-12-02T21:08:08.523615Z","shell.execute_reply":"2024-12-02T21:08:08.527578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_results = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:08.858149Z","iopub.execute_input":"2024-12-02T21:08:08.858557Z","iopub.status.idle":"2024-12-02T21:08:08.862743Z","shell.execute_reply.started":"2024-12-02T21:08:08.858526Z","shell.execute_reply":"2024-12-02T21:08:08.861764Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Почнемо використовувати оптимізатори","metadata":{}},{"cell_type":"code","source":"for optimizer_name, optimizer_class in optimizers:\n    for regularizer_name, regularizer in regularizerslist.items():\n        print(f\"Training with {optimizer_name}, regularization {regularizer_name}...\")\n        optimizer_instance = optimizer_class(learning_rate=0.01)\n        if regularizer_name == \"Dropout\":\n            history = train_model(optimizer_instance, dropout_rate=0.2)\n        else:\n            history = train_model(optimizer_instance, regularizer=regularizer)\n        if 'loss' in history.history and 'val_loss' in history.history:\n            history_results.append((optimizer_name, regularizer_name, history))\n        else:\n            print(f\"Warning: No data in history for {optimizer_name} with {regularizer_name}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:08:09.975081Z","iopub.execute_input":"2024-12-02T21:08:09.975938Z","iopub.status.idle":"2024-12-02T21:09:22.139442Z","shell.execute_reply.started":"2024-12-02T21:08:09.975900Z","shell.execute_reply":"2024-12-02T21:09:22.138471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_results","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:09:22.141509Z","iopub.execute_input":"2024-12-02T21:09:22.141820Z","iopub.status.idle":"2024-12-02T21:09:22.148418Z","shell.execute_reply.started":"2024-12-02T21:09:22.141790Z","shell.execute_reply":"2024-12-02T21:09:22.147510Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef plot_results(history_results):\n    optimizer_results = {}\n\n    for optimizer_name, regularizer_name, history in history_results:\n        if optimizer_name not in optimizer_results:\n            optimizer_results[optimizer_name] = []\n        optimizer_results[optimizer_name].append((regularizer_name, history))\n\n    num_rows = len(optimizer_results)\n    num_cols = 2 \n\n    fig, axes = plt.subplots(num_rows, num_cols, figsize=(12, 6 * num_rows))\n\n    if num_rows == 1:\n        axes = [axes]\n    if num_cols == 1:\n        axes = [[ax] for ax in axes]\n\n    for i, (optimizer_name, results) in enumerate(optimizer_results.items()):\n        for regularizer_name, history in results:\n            if regularizer_name != \"None\":\n                axes[i][0].plot(history.history['loss'], label=f'Train: {regularizer_name}')\n        axes[i][0].set_title(f'{optimizer_name} - Training Loss')\n        axes[i][0].set_xlabel('Epochs')\n        axes[i][0].set_ylabel('Loss')\n        axes[i][0].legend()\n\n        for regularizer_name, history in results:\n            if regularizer_name != \"None\":\n                axes[i][1].plot(history.history['val_loss'], label=f'Validation: {regularizer_name}')\n        axes[i][1].set_title(f'{optimizer_name} - Validation Loss')\n        axes[i][1].set_xlabel('Epochs')\n        axes[i][1].set_ylabel('Loss')\n        axes[i][1].legend()\n\n    fig_none, ax_none = plt.subplots(figsize=(12, 6))\n\n    colormap = cm.get_cmap('tab10')\n    color_map = {optimizer_name: colormap(i) for i, optimizer_name in enumerate(optimizer_results.keys())}\n\n    for optimizer_name, results in optimizer_results.items():\n        for regularizer_name, history in results:\n            if regularizer_name == \"None\":\n                ax_none.plot(history.history['loss'], label=f'Train: None ({optimizer_name})', color=color_map[optimizer_name])\n\n    ax_none.set_title('None Regularizer - Training Loss for All Optimizers')\n    ax_none.set_xlabel('Epochs')\n    ax_none.set_ylabel('Loss')\n    ax_none.legend()\n\n    fig_none_val, ax_none_val = plt.subplots(figsize=(12, 6))\n    for optimizer_name, results in optimizer_results.items():\n        for regularizer_name, history in results:\n            if regularizer_name == \"None\":\n                ax_none_val.plot(history.history['val_loss'], label=f'Validation: None ({optimizer_name})', color=color_map[optimizer_name])\n\n    ax_none_val.set_title('None Regularizer - Validation Loss for All Optimizers')\n    ax_none_val.set_xlabel('Epochs')\n    ax_none_val.set_ylabel('Loss')\n    ax_none_val.legend()\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:09:22.161679Z","iopub.execute_input":"2024-12-02T21:09:22.161929Z","iopub.status.idle":"2024-12-02T21:09:22.175572Z","shell.execute_reply.started":"2024-12-02T21:09:22.161905Z","shell.execute_reply":"2024-12-02T21:09:22.174809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_results(history_results)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T21:09:22.176545Z","iopub.execute_input":"2024-12-02T21:09:22.176910Z","iopub.status.idle":"2024-12-02T21:09:23.766334Z","shell.execute_reply.started":"2024-12-02T21:09:22.176883Z","shell.execute_reply":"2024-12-02T21:09:23.765434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}