{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Предобработка и исследование данных","metadata":{}},{"cell_type":"markdown","source":"## Табличные данные","metadata":{}},{"cell_type":"code","source":"#импорты\nimport os\nimport zipfile\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nrandom.seed(42)\nnp.random.seed(42)\ntorch.manual_seed(42)\npd.set_option('display.max_columns', 30)\n\nprint('PyTorch:', torch.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:49:01.850402Z","iopub.execute_input":"2026-09-20T19:49:01.851285Z","iopub.status.idle":"2026-09-20T19:49:09.760288Z","shell.execute_reply.started":"2026-09-20T19:49:01.85125Z","shell.execute_reply":"2026-09-20T19:49:09.759334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#сырой CSV\ncsv_path = '/kaggle/input/competitions/titanic/train.csv'\n\nwith open(csv_path) as f:\n    for _ in range(3):\n        print(f.readline())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:49:38.201584Z","iopub.execute_input":"2026-09-20T19:49:38.20249Z","iopub.status.idle":"2026-09-20T19:49:38.215086Z","shell.execute_reply.started":"2026-09-20T19:49:38.202457Z","shell.execute_reply":"2026-09-20T19:49:38.21369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#читаем таблицу через pandas\ndf = pd.read_csv(csv_path)\nprint('Размер таблицы:', df.shape)   # (строки, столбцы)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:49:45.284796Z","iopub.execute_input":"2026-09-20T19:49:45.285498Z","iopub.status.idle":"2026-09-20T19:49:45.34467Z","shell.execute_reply.started":"2026-09-20T19:49:45.285463Z","shell.execute_reply":"2026-09-20T19:49:45.343954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()\nprint()\nprint(df.describe())\nprint()\ndf.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:50:57.229708Z","iopub.execute_input":"2026-09-20T19:50:57.230122Z","iopub.status.idle":"2026-09-20T19:50:57.261187Z","shell.execute_reply.started":"2026-09-20T19:50:57.230092Z","shell.execute_reply":"2026-09-20T19:50:57.26033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Работа с пропусками в данных на примере Титаника\n#Возраст: заполняем медианой \ndf['Age'] = df['Age'].fillna(df['Age'].median())\n#Порт посадки: заполняем модой (самое частое значение)\ndf['Embarked'] = df['Embarked'].fillna(df['Embarked'].mode()[0])\n#Каюта пропущена у 687 из 891, удаляем столбец\ndf = df.drop(columns=['Cabin'])\n\ndf.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:51:11.815512Z","iopub.execute_input":"2026-09-20T19:51:11.815907Z","iopub.status.idle":"2026-09-20T19:51:11.831615Z","shell.execute_reply.started":"2026-09-20T19:51:11.815869Z","shell.execute_reply":"2026-09-20T19:51:11.830815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Извлечение признаков из текста (титул пассажиров Титаника)\n#'Braund, Mr. Owen Harris'  ->  'Mr'\ndf['Title'] = (\n    df['Name']\n    .str.split(',').str[1]\n    .str.split('.').str[0]\n    .str.strip()\n)\nprint(df['Title'].value_counts())\n\n#Редкие титулы (Dr, Rev, Major) объединяем в 'Other'\ncommon = ['Mr', 'Mrs', 'Miss', 'Master']\ndf['Title'] = df['Title'].apply(lambda t: t if t in common else 'Other')\ndf['Title'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:51:16.042147Z","iopub.execute_input":"2026-09-20T19:51:16.042798Z","iopub.status.idle":"2026-09-20T19:51:16.059774Z","shell.execute_reply.started":"2026-09-20T19:51:16.042766Z","shell.execute_reply":"2026-09-20T19:51:16.058898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Создание новых признаков из имеющихся\ndf['FamilySize'] = df['SibSp'] + df['Parch'] + 1\ndf['IsAlone'] = (df['FamilySize'] == 1).astype(int)\n\ndf[['SibSp', 'Parch', 'FamilySize', 'IsAlone']].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:51:32.803132Z","iopub.execute_input":"2026-09-20T19:51:32.803445Z","iopub.status.idle":"2026-09-20T19:51:32.815598Z","shell.execute_reply.started":"2026-09-20T19:51:32.80342Z","shell.execute_reply":"2026-09-20T19:51:32.814638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df.groupby('Sex')['Survived'].mean(), '\\n')\nprint(df.groupby('Pclass')['Survived'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:51:41.277846Z","iopub.execute_input":"2026-09-20T19:51:41.278169Z","iopub.status.idle":"2026-09-20T19:51:41.290895Z","shell.execute_reply.started":"2026-09-20T19:51:41.278143Z","shell.execute_reply":"2026-09-20T19:51:41.290041Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Визуализация","metadata":{}},{"cell_type":"code","source":"#Простая столбчатая диаграмма\nsurv_sex = df.groupby('Sex')['Survived'].mean()\n\nplt.figure(figsize=(6, 4))\nplt.bar(surv_sex.index, surv_sex.values, color=['#3498db', '#e67e22'])\nplt.title('Доля выживших: женщины против мужчин')\nplt.xlabel('Пол')\nplt.ylabel('Доля выживших')\nplt.ylim(0, 1)\nfor i, v in enumerate(surv_sex.values):\n    plt.text(i, v + 0.02, f'{v:.0%}', ha='center')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:51:52.200083Z","iopub.execute_input":"2026-09-20T19:51:52.200411Z","iopub.status.idle":"2026-09-20T19:51:52.399387Z","shell.execute_reply.started":"2026-09-20T19:51:52.200382Z","shell.execute_reply":"2026-09-20T19:51:52.398551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Гистограмма на примере возрастов\nplt.figure(figsize=(10, 4))\nplt.hist(df.loc[df['Survived'] == 1, 'Age'], bins=30, alpha=0.6,\n         label='Выжили', color='#27ae60')\nplt.hist(df.loc[df['Survived'] == 0, 'Age'], bins=30, alpha=0.6,\n         label='Погибли', color='#c0392b')\nplt.xlabel('Возраст')\nplt.ylabel('Количество пассажиров')\nplt.title('Возраст и выживаемость')\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:51:56.633247Z","iopub.execute_input":"2026-09-20T19:51:56.633597Z","iopub.status.idle":"2026-09-20T19:51:56.885538Z","shell.execute_reply.started":"2026-09-20T19:51:56.633566Z","shell.execute_reply":"2026-09-20T19:51:56.884754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Перевод текста в числовой формат\n#Пол: 2 значения -> 0/1\ndf['Sex_code'] = df['Sex'].map({'male': 0, 'female': 1})\n#Порт и титул: несколько значений -> one-hot (по столбцу на каждое значение)\ndf = pd.get_dummies(df, columns=['Embarked', 'Title'], dtype=int)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:52:21.091632Z","iopub.execute_input":"2026-09-20T19:52:21.092385Z","iopub.status.idle":"2026-09-20T19:52:21.119047Z","shell.execute_reply.started":"2026-09-20T19:52:21.092352Z","shell.execute_reply":"2026-09-20T19:52:21.118137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Сохранение CSV\ndf.to_csv('titanic_clean.csv', index=False)   # index=False - без лишнего столбца\n\ncheck = pd.read_csv('titanic_clean.csv')\nprint(check.shape)\ncheck.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:52:25.357309Z","iopub.execute_input":"2026-09-20T19:52:25.357647Z","iopub.status.idle":"2026-09-20T19:52:25.393458Z","shell.execute_reply.started":"2026-09-20T19:52:25.357619Z","shell.execute_reply":"2026-09-20T19:52:25.392659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Работа с изображениями на примере Dogs vs. Cats","metadata":{}},{"cell_type":"code","source":"#Распаковка .zip\nzip_path = '/kaggle/input/competitions/dogs-vs-cats/train.zip'\nprint(f'Размер архива: {os.path.getsize(zip_path) / 1024**2:.1f} МБ')\n\nwith zipfile.ZipFile(zip_path) as zf:   #просмотреть без распаковки\n    names = [n for n in zf.namelist() if n.endswith('.jpg')]\n\nEXTRACT_DIR = '/kaggle/working/cats_dogs'\nN_FILES = 4000   #распаковать всё: zf.extractall(...)\n\nrandom.seed(42)\nsample = random.sample(names, N_FILES)   \n\nwith zipfile.ZipFile(zip_path) as zf:\n    for name in sample:\n        zf.extract(name, EXTRACT_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:53:57.266798Z","iopub.execute_input":"2026-09-20T19:53:57.267128Z","iopub.status.idle":"2026-09-20T19:54:11.001711Z","shell.execute_reply.started":"2026-09-20T19:53:57.2671Z","shell.execute_reply":"2026-09-20T19:54:11.000971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Извлечение меток\nimg_dir = os.path.join(EXTRACT_DIR, 'train')\nif not os.path.isdir(img_dir):      #если в архиве нет вложенной папки\n    img_dir = EXTRACT_DIR\n\nfiles = sorted(f for f in os.listdir(img_dir) if f.endswith('.jpg'))\nprint('Распаковано картинок:', len(files))\n\nn_cats = sum(1 for f in files if f.startswith('cat'))\nn_dogs = sum(1 for f in files if f.startswith('dog'))\nprint(f'Кошек: {n_cats}, Собак: {n_dogs}')\n\ndef get_label(filename):\n    # кошка - 0, собака - 1\n    return 1 if filename.split('.')[0] == 'dog' else 0\n\nfor f in files[:3]:\n    print(f, '- метка', get_label(f))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:54:17.696501Z","iopub.execute_input":"2026-09-20T19:54:17.696845Z","iopub.status.idle":"2026-09-20T19:54:17.71041Z","shell.execute_reply.started":"2026-09-20T19:54:17.696817Z","shell.execute_reply":"2026-09-20T19:54:17.709609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img = Image.open(os.path.join(img_dir, files[0]))\nprint('Размер (ширина × высота):', img.size)\nprint('Режим (каналы):', img.mode)   #RGB \nimg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:54:44.565955Z","iopub.execute_input":"2026-09-20T19:54:44.566874Z","iopub.status.idle":"2026-09-20T19:54:44.63376Z","shell.execute_reply.started":"2026-09-20T19:54:44.566836Z","shell.execute_reply":"2026-09-20T19:54:44.632812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Показ через matplotlib\nplt.figure(figsize=(4, 4))\nplt.imshow(img)\nplt.title(f'{files[0]} (метка: {get_label(files[0])})')\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:55:06.36813Z","iopub.execute_input":"2026-09-20T19:55:06.368931Z","iopub.status.idle":"2026-09-20T19:55:06.498948Z","shell.execute_reply.started":"2026-09-20T19:55:06.368893Z","shell.execute_reply":"2026-09-20T19:55:06.498088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Обработка и приведение к одному виду\ntransform = transforms.Compose([\n    transforms.Resize((64, 64)),   #одинаковый размер\n    transforms.ToTensor(),         #приведение к диапазону [0, 1]\n])\n\nimg_tensor = transform(img)\nprint('Форма тензора:', img_tensor.shape)            #[3, 64, 64]\nprint('Минимум:', img_tensor.min().item(),\n      '| Максимум:', img_tensor.max().item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:55:18.143371Z","iopub.execute_input":"2026-09-20T19:55:18.144186Z","iopub.status.idle":"2026-09-20T19:55:18.230191Z","shell.execute_reply.started":"2026-09-20T19:55:18.144136Z","shell.execute_reply":"2026-09-20T19:55:18.229205Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset и DataLoader","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, folder, transform=None):\n        self.paths = [os.path.join(folder, f)\n                      for f in os.listdir(folder) if f.endswith('.jpg')]\n        self.transform = transform\n\n    def __len__(self):                 #сколько всего примеров\n        return len(self.paths)\n\n    def __getitem__(self, idx):        #достаём один пример по номеру\n        path = self.paths[idx]\n        label = get_label(os.path.basename(path))\n        image = Image.open(path).convert('RGB')   #может быть ч/б или RGBA\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = CatsDogsDataset(img_dir, transform=transform)\n\nprint('Примеров в датасете:', len(dataset))\nx, y = dataset[100]\nprint('Одна картинка:', x.shape, '| метка:', y)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"loader = DataLoader(dataset, batch_size=16, shuffle=True)\n\nimages, labels = next(iter(loader))\nprint('Батч картинок:', images.shape)      #[16, 3, 64, 64]\nprint('Метки:', labels.tolist())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor i in range(16):\n    plt.subplot(4, 4, i + 1)\n    plt.imshow(images[i].permute(1, 2, 0).numpy())  #из (C,H,W) в (H,W,C) для matplotlib\n    plt.title('dog' if labels[i] == 1 else 'cat', fontsize=10)\n    plt.axis('off')\nplt.suptitle('Один батч из DataLoader (shuffle=True)', fontsize=14)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Аугментация\naugment = transforms.Compose([\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomRotation(20),\n    transforms.ColorJitter(brightness=0.3, contrast=0.3),\n    transforms.Resize((64, 64)),\n    transforms.ToTensor(),\n])\n\nsome_img = Image.open(os.path.join(img_dir, files[7]))\n\nplt.figure(figsize=(12, 4))\nfor i in range(5):\n    plt.subplot(1, 5, i + 1)\n    plt.imshow(augment(some_img).permute(1, 2, 0).numpy())\n    plt.axis('off')\nplt.suptitle('Одна картинка - пять случайных вариантов')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-20T19:48:35.174278Z","iopub.execute_input":"2026-09-20T19:48:35.174607Z","iopub.status.idle":"2026-09-20T19:48:35.179588Z","shell.execute_reply.started":"2026-09-20T19:48:35.174578Z","shell.execute_reply":"2026-09-20T19:48:35.178862Z"}},"outputs":[],"execution_count":null}]}