{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":117193,"databundleVersionId":14001224,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nANIP-Reconnaissance Faciale - Estimation Age & OCR\nComplete Solution for All Three Tasks - KAGGLE COMPATIBLE\n\"\"\"\n\n# ============================================================================\n# INSTALLATIONS - Run these first in Kaggle\n# ============================================================================\n\nimport sys\nimport subprocess\n\ndef install_packages():\n    \"\"\"Install required packages\"\"\"\n    packages = [\n        'timm',\n        'opencv-python',\n        'albumentations',\n    ]\n    \n    for package in packages:\n        print(f\"Installing {package}...\")\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", package])\n    print(\"✓ All packages installed successfully!\")\n\n# Install packages\ninstall_packages()\n\n# ============================================================================\n# IMPORTS\n# ============================================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nimport json\nimport cv2\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Deep Learning imports\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nimport torchvision.transforms as T\n\n# Additional libraries\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, f1_score, mean_absolute_error\nfrom tqdm.auto import tqdm\nimport timm\n\n# Set device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"CUDA Version: {torch.version.cuda}\")\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\n\n# Kaggle paths - adjust if needed\nBASE_PATH = '/kaggle/input/anip-reconnaissance-faciale-estimation-ages-ocr'\nOUTPUT_PATH = '/kaggle/working'\n\n# Create output directory\nos.makedirs(OUTPUT_PATH, exist_ok=True)\n\nBATCH_SIZE = 32\nEPOCHS = 15\nLEARNING_RATE = 0.001\nIMG_SIZE = 224\n\n# Set random seeds for reproducibility\ntorch.manual_seed(42)\nnp.random.seed(42)\n\n# ============================================================================\n# HELPER FUNCTIONS\n# ============================================================================\n\ndef check_dataset_structure(base_path):\n    \"\"\"Check if dataset exists and print structure\"\"\"\n    print(\"\\n\" + \"=\"*80)\n    print(\"DATASET STRUCTURE CHECK\")\n    print(\"=\"*80)\n    \n    if not os.path.exists(base_path):\n        print(f\"❌ ERROR: Base path does not exist: {base_path}\")\n        print(\"\\nPlease ensure you've added the dataset to your Kaggle notebook.\")\n        print(\"Go to: Add Data -> Search for the competition dataset\")\n        return False\n    \n    print(f\"✓ Base path exists: {base_path}\")\n    \n    # Check each task\n    tasks = {\n        'Task 1 (Face Recognition)': 'dataset_tache_1',\n        'Task 2 (Age Estimation)': 'dataset_tache_2',\n        'Task 3 (OCR/Fraud Detection)': 'dataset_tache_3'\n    }\n    \n    for task_name, task_folder in tasks.items():\n        task_path = os.path.join(base_path, task_folder)\n        if os.path.exists(task_path):\n            print(f\"✓ {task_name}: Found at {task_folder}\")\n        else:\n            print(f\"❌ {task_name}: NOT FOUND at {task_folder}\")\n    \n    print(\"=\"*80 + \"\\n\")\n    return True\n\n# Check dataset\nif not check_dataset_structure(BASE_PATH):\n    print(\"\\n⚠️ DATASET NOT FOUND!\")\n    print(\"Please add the competition dataset to your Kaggle notebook before running.\")\n    print(\"Stopping execution...\")\n    sys.exit(1)\n\n# ============================================================================\n# TASK 1: FACE RECOGNITION / MATCHING\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TASK 1: FACE RECOGNITION AND MATCHING\")\nprint(\"=\"*80)\n\n# Load training data\ntrain_path_t1 = os.path.join(BASE_PATH, 'dataset_tache_1/dataset_tache_1/train')\ntest_path_t1 = os.path.join(BASE_PATH, 'dataset_tache_1/dataset_tache_1/test')\n\n# Alternative paths if structure is different\nif not os.path.exists(train_path_t1):\n    train_path_t1 = os.path.join(BASE_PATH, 'dataset_tache_1/train')\n    test_path_t1 = os.path.join(BASE_PATH, 'dataset_tache_1/test')\n\nprint(f\"Train path: {train_path_t1}\")\nprint(f\"Test path: {test_path_t1}\")\n\n# Get all training images\ntrain_images_t1 = sorted([f for f in os.listdir(train_path_t1) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))])\ntest_images_t1 = sorted([f for f in os.listdir(test_path_t1) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))])\n\nprint(f\"\\nTraining images: {len(train_images_t1)}\")\nprint(f\"Test images: {len(test_images_t1)}\")\n\n# Parse training data - extract person IDs\ntrain_data_t1 = []\nfor img_name in train_images_t1:\n    parts = img_name.split('_')\n    person_id = parts[0]\n    photo_num = parts[1].split('.')[0]\n    train_data_t1.append({\n        'filename': img_name,\n        'person_id': person_id,\n        'photo_num': photo_num\n    })\n\ndf_train_t1 = pd.DataFrame(train_data_t1)\n\nprint(\"\\n--- EDA for Task 1 ---\")\nprint(f\"Number of unique persons: {df_train_t1['person_id'].nunique()}\")\nprint(f\"Photos per person: {df_train_t1.groupby('person_id').size().describe()}\")\n\n# Visualize sample images\nfig, axes = plt.subplots(2, 4, figsize=(15, 8))\nfig.suptitle('Task 1: Sample Face Pairs', fontsize=16)\n\nsample_persons = df_train_t1['person_id'].unique()[:4]\nfor idx, person_id in enumerate(sample_persons):\n    person_imgs = df_train_t1[df_train_t1['person_id'] == person_id]['filename'].values\n    \n    if len(person_imgs) >= 2:\n        img1 = cv2.imread(os.path.join(train_path_t1, person_imgs[0]))\n        img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)\n        img2 = cv2.imread(os.path.join(train_path_t1, person_imgs[1]))\n        img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)\n        \n        axes[0, idx].imshow(img1)\n        axes[0, idx].set_title(f'Person {person_id} - Photo 0')\n        axes[0, idx].axis('off')\n        \n        axes[1, idx].imshow(img2)\n        axes[1, idx].set_title(f'Person {person_id} - Photo 1')\n        axes[1, idx].axis('off')\n\nplt.tight_layout()\nplt.savefig(os.path.join(OUTPUT_PATH, 'task1_samples.png'), dpi=100, bbox_inches='tight')\nplt.close()\nprint(\"✓ Saved sample images visualization\")\n\n\n# Dataset for Task 1\nclass FaceRecognitionDataset(Dataset):\n    def __init__(self, image_dir, image_list, person_ids, transform=None):\n        self.image_dir = image_dir\n        self.image_list = image_list\n        self.person_ids = person_ids\n        self.transform = transform\n        \n        # Create person_id to label mapping\n        unique_ids = sorted(list(set(person_ids)))\n        self.id_to_label = {pid: idx for idx, pid in enumerate(unique_ids)}\n        \n    def __len__(self):\n        return len(self.image_list)\n    \n    def __getitem__(self, idx):\n        img_name = self.image_list[idx]\n        img_path = os.path.join(self.image_dir, img_name)\n        \n        image = Image.open(img_path).convert('RGB')\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        label = self.id_to_label[self.person_ids[idx]]\n        \n        return image, label\n\n\n# Test dataset for Task 1\nclass FaceTestDataset(Dataset):\n    def __init__(self, image_dir, image_list, transform=None):\n        self.image_dir = image_dir\n        self.image_list = image_list\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.image_list)\n    \n    def __getitem__(self, idx):\n        img_name = self.image_list[idx]\n        img_path = os.path.join(self.image_dir, img_name)\n        \n        image = Image.open(img_path).convert('RGB')\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, img_name\n\n\n# Face Recognition Model (Feature Extraction)\nclass FaceRecognitionModel(nn.Module):\n    def __init__(self, num_classes, embedding_size=512):\n        super(FaceRecognitionModel, self).__init__()\n        \n        # Use pre-trained ResNet50\n        self.backbone = models.resnet50(pretrained=True)\n        \n        # Remove the final FC layer\n        self.backbone.fc = nn.Identity()\n        \n        # Add embedding layer\n        self.embedding = nn.Sequential(\n            nn.Linear(2048, embedding_size),\n            nn.BatchNorm1d(embedding_size),\n            nn.ReLU(),\n            nn.Dropout(0.5)\n        )\n        \n        # Classification head\n        self.classifier = nn.Linear(embedding_size, num_classes)\n        \n    def forward(self, x):\n        features = self.backbone(x)\n        embeddings = self.embedding(features)\n        logits = self.classifier(embeddings)\n        return embeddings, logits\n    \n    def get_embedding(self, x):\n        features = self.backbone(x)\n        embeddings = self.embedding(features)\n        return embeddings\n\n\n# Transforms\ntransform_train = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(10),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\ntransform_test = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# Create dataset\ntrain_dataset_t1 = FaceRecognitionDataset(\n    train_path_t1, \n    df_train_t1['filename'].values,\n    df_train_t1['person_id'].values,\n    transform=transform_train\n)\n\ntrain_loader_t1 = DataLoader(train_dataset_t1, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n\n# Initialize model\nnum_classes_t1 = df_train_t1['person_id'].nunique()\nmodel_t1 = FaceRecognitionModel(num_classes=num_classes_t1).to(device)\n\ncriterion_t1 = nn.CrossEntropyLoss()\noptimizer_t1 = torch.optim.Adam(model_t1.parameters(), lr=LEARNING_RATE)\nscheduler_t1 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_t1, mode='min', patience=2, factor=0.5)\n\n# Training loop for Task 1\nprint(\"\\n--- Training Task 1 Model ---\")\ntrain_losses_t1 = []\ntrain_accs_t1 = []\n\nfor epoch in range(EPOCHS):\n    model_t1.train()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    \n    pbar = tqdm(train_loader_t1, desc=f'Epoch {epoch+1}/{EPOCHS}')\n    for images, labels in pbar:\n        images, labels = images.to(device), labels.to(device)\n        \n        optimizer_t1.zero_grad()\n        \n        embeddings, logits = model_t1(images)\n        loss = criterion_t1(logits, labels)\n        \n        loss.backward()\n        optimizer_t1.step()\n        \n        running_loss += loss.item()\n        \n        _, predicted = torch.max(logits, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        \n        pbar.set_postfix({'loss': f'{loss.item():.4f}'})\n    \n    epoch_loss = running_loss / len(train_loader_t1)\n    epoch_acc = 100 * correct / total\n    \n    train_losses_t1.append(epoch_loss)\n    train_accs_t1.append(epoch_acc)\n    \n    print(f'Epoch {epoch+1}: Loss = {epoch_loss:.4f}, Accuracy = {epoch_acc:.2f}%')\n    \n    scheduler_t1.step(epoch_loss)\n\n# Plot training curves\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nax1.plot(train_losses_t1)\nax1.set_title('Task 1: Training Loss')\nax1.set_xlabel('Epoch')\nax1.set_ylabel('Loss')\nax1.grid(True)\n\nax2.plot(train_accs_t1)\nax2.set_title('Task 1: Training Accuracy')\nax2.set_xlabel('Epoch')\nax2.set_ylabel('Accuracy (%)')\nax2.grid(True)\n\nplt.tight_layout()\nplt.savefig(os.path.join(OUTPUT_PATH, 'task1_training_curves.png'), dpi=100, bbox_inches='tight')\nplt.close()\nprint(\"✓ Saved training curves\")\n\n# Generate embeddings for all training images\nprint(\"\\n--- Generating embeddings for matching ---\")\nmodel_t1.eval()\ntrain_embeddings = {}\n\ntrain_loader_embedding = DataLoader(train_dataset_t1, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\nwith torch.no_grad():\n    batch_idx = 0\n    for images, labels in tqdm(train_loader_embedding, desc='Extracting embeddings'):\n        images = images.to(device)\n        embeddings = model_t1.get_embedding(images)\n        \n        for i, emb in enumerate(embeddings):\n            global_idx = batch_idx * BATCH_SIZE + i\n            if global_idx < len(df_train_t1):\n                train_embeddings[df_train_t1.iloc[global_idx]['filename']] = emb.cpu().numpy()\n        batch_idx += 1\n\n# Generate predictions for test set\nprint(\"\\n--- Generating predictions for test set ---\")\ntest_dataset_t1 = FaceTestDataset(test_path_t1, test_images_t1, transform=transform_test)\ntest_loader_t1 = DataLoader(test_dataset_t1, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\ntest_embeddings = {}\n\nwith torch.no_grad():\n    for images, filenames in tqdm(test_loader_t1, desc='Test embeddings'):\n        images = images.to(device)\n        embeddings = model_t1.get_embedding(images)\n        \n        for filename, emb in zip(filenames, embeddings):\n            test_embeddings[filename] = emb.cpu().numpy()\n\n# Match test images to training images\ndef cosine_similarity(a, b):\n    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n\n# Create matching pairs\nmatches = []\nfor test_img in test_images_t1:\n    test_emb = test_embeddings[test_img]\n    \n    best_match = None\n    best_score = -1\n    \n    for train_img, train_emb in train_embeddings.items():\n        score = cosine_similarity(test_emb, train_emb)\n        if score > best_score:\n            best_score = score\n            best_match = train_img\n    \n    # Extract person ID from best match\n    person_id = best_match.split('_')[0]\n    matches.append({\n        'test_image': test_img,\n        'matched_train_image': best_match,\n        'person_id': person_id,\n        'similarity_score': best_score\n    })\n\ndf_matches_t1 = pd.DataFrame(matches)\n\n# Save predictions\ndf_matches_t1[['test_image', 'person_id']].to_csv(os.path.join(OUTPUT_PATH, 'task1_predictions.csv'), index=False)\nprint(\"✓ Saved Task 1 predictions\")\n\n# Save model\ntorch.save(model_t1.state_dict(), os.path.join(OUTPUT_PATH, 'task1_model.pth'))\nprint(\"✓ Saved Task 1 model weights\")\n\n\n# ============================================================================\n# TASK 2: AGE ESTIMATION\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TASK 2: AGE ESTIMATION\")\nprint(\"=\"*80)\n\ntrain_path_t2 = os.path.join(BASE_PATH, 'dataset_tache_2/dataset_tache_2/train')\ntest_path_t2 = os.path.join(BASE_PATH, 'dataset_tache_2/dataset_tache_2/test')\n\n# Alternative paths\nif not os.path.exists(train_path_t2):\n    train_path_t2 = os.path.join(BASE_PATH, 'dataset_tache_2/train')\n    test_path_t2 = os.path.join(BASE_PATH, 'dataset_tache_2/test')\n\n# Load training data\ntrain_images_t2 = sorted([f for f in os.listdir(train_path_t2) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))])\ntest_images_t2 = sorted([f for f in os.listdir(test_path_t2) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))])\n\nprint(f\"\\nTraining images: {len(train_images_t2)}\")\nprint(f\"Test images: {len(test_images_t2)}\")\n\n# Parse training data\ntrain_data_t2 = []\nfor img_name in train_images_t2:\n    try:\n        # Format: XXXXXX_YZWW\n        parts = img_name.split('.')[0].split('_')\n        if len(parts) >= 2:\n            person_id = parts[0]\n            info = parts[1]\n            \n            if len(info) >= 3:\n                photo_num = info[0]\n                gender = info[1]\n                age = int(info[2:])\n                \n                train_data_t2.append({\n                    'filename': img_name,\n                    'person_id': person_id,\n                    'photo_num': photo_num,\n                    'gender': gender,\n                    'age': age\n                })\n    except Exception as e:\n        continue\n\ndf_train_t2 = pd.DataFrame(train_data_t2)\n\nprint(\"\\n--- EDA for Task 2 ---\")\nprint(f\"Number of images: {len(df_train_t2)}\")\nprint(f\"Age range: {df_train_t2['age'].min()} - {df_train_t2['age'].max()}\")\nprint(f\"Mean age: {df_train_t2['age'].mean():.2f}\")\nprint(f\"\\nGender distribution:\")\nprint(df_train_t2['gender'].value_counts())\n\n# Visualizations\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# Age distribution\naxes[0].hist(df_train_t2['age'], bins=50, edgecolor='black', alpha=0.7)\naxes[0].set_title('Age Distribution')\naxes[0].set_xlabel('Age')\naxes[0].set_ylabel('Frequency')\naxes[0].grid(True, alpha=0.3)\n\n# Age by gender\ndf_train_t2.boxplot(column='age', by='gender', ax=axes[1])\naxes[1].set_title('Age Distribution by Gender')\naxes[1].set_xlabel('Gender')\naxes[1].set_ylabel('Age')\n\nplt.tight_layout()\nplt.savefig(os.path.join(OUTPUT_PATH, 'task2_eda.png'), dpi=100, bbox_inches='tight')\nplt.close()\n\n# Sample images\nsample_indices = np.random.choice(len(df_train_t2), min(8, len(df_train_t2)), replace=False)\nfig2, axes2 = plt.subplots(2, 4, figsize=(16, 8))\nfig2.suptitle('Task 2: Sample Images with Ages', fontsize=16)\n\nfor idx, ax in enumerate(axes2.flat):\n    if idx < len(sample_indices):\n        sample_idx = sample_indices[idx]\n        img_info = df_train_t2.iloc[sample_idx]\n        \n        img = cv2.imread(os.path.join(train_path_t2, img_info['filename']))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        ax.imshow(img)\n        ax.set_title(f\"Age: {img_info['age']}, Gender: {img_info['gender']}\")\n        ax.axis('off')\n\nplt.tight_layout()\nplt.savefig(os.path.join(OUTPUT_PATH, 'task2_samples.png'), dpi=100, bbox_inches='tight')\nplt.close()\nprint(\"✓ Saved sample images\")\n\n# Dataset for Task 2\nclass AgeEstimationDataset(Dataset):\n    def __init__(self, image_dir, dataframe, transform=None, is_test=False):\n        self.image_dir = image_dir\n        self.dataframe = dataframe if dataframe is not None else None\n        self.transform = transform\n        self.is_test = is_test\n        \n        if is_test:\n            self.image_list = sorted([f for f in os.listdir(image_dir) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))])\n        \n    def __len__(self):\n        if self.is_test:\n            return len(self.image_list)\n        return len(self.dataframe)\n    \n    def __getitem__(self, idx):\n        if self.is_test:\n            img_name = self.image_list[idx]\n            img_path = os.path.join(self.image_dir, img_name)\n            image = Image.open(img_path).convert('RGB')\n            \n            if self.transform:\n                image = self.transform(image)\n            \n            return image, img_name\n        else:\n            img_name = self.dataframe.iloc[idx]['filename']\n            age = self.dataframe.iloc[idx]['age']\n            \n            img_path = os.path.join(self.image_dir, img_name)\n            image = Image.open(img_path).convert('RGB')\n            \n            if self.transform:\n                image = self.transform(image)\n            \n            return image, torch.tensor(age, dtype=torch.float32)\n\n\n# Age Estimation Model\nclass AgeEstimationModel(nn.Module):\n    def __init__(self):\n        super(AgeEstimationModel, self).__init__()\n        \n        # Use EfficientNet\n        self.backbone = timm.create_model('efficientnet_b0', pretrained=True)\n        \n        # Get the number of features\n        n_features = self.backbone.classifier.in_features\n        \n        # Replace classifier\n        self.backbone.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            nn.Linear(n_features, 512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, 1)\n        )\n        \n    def forward(self, x):\n        return self.backbone(x).squeeze()\n\n\n# Split data\ntrain_df, val_df = train_test_split(df_train_t2, test_size=0.15, random_state=42)\n\ntrain_dataset_t2 = AgeEstimationDataset(train_path_t2, train_df, transform=transform_train)\nval_dataset_t2 = AgeEstimationDataset(train_path_t2, val_df, transform=transform_test)\n\ntrain_loader_t2 = DataLoader(train_dataset_t2, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader_t2 = DataLoader(val_dataset_t2, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n# Initialize model\nmodel_t2 = AgeEstimationModel().to(device)\n\ncriterion_t2 = nn.L1Loss()  # MAE\noptimizer_t2 = torch.optim.Adam(model_t2.parameters(), lr=LEARNING_RATE)\nscheduler_t2 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_t2, mode='min', patience=2, factor=0.5)\n\n# Training loop\nprint(\"\\n--- Training Task 2 Model ---\")\ntrain_losses_t2 = []\nval_losses_t2 = []\n\nbest_val_loss = float('inf')\n\nfor epoch in range(EPOCHS):\n    # Training\n    model_t2.train()\n    running_loss = 0.0\n    \n    pbar = tqdm(train_loader_t2, desc=f'Epoch {epoch+1}/{EPOCHS} [Train]')\n    for images, ages in pbar:\n        images, ages = images.to(device), ages.to(device)\n        \n        optimizer_t2.zero_grad()\n        \n        outputs = model_t2(images)\n        loss = criterion_t2(outputs, ages)\n        \n        loss.backward()\n        optimizer_t2.step()\n        \n        running_loss += loss.item()\n        pbar.set_postfix({'loss': f'{loss.item():.4f}'})\n    \n    train_loss = running_loss / len(train_loader_t2)\n    train_losses_t2.append(train_loss)\n    \n    # Validation\n    model_t2.eval()\n    val_loss = 0.0\n    \n    with torch.no_grad():\n        for images, ages in val_loader_t2:\n            images, ages = images.to(device), ages.to(device)\n            outputs = model_t2(images)\n            loss = criterion_t2(outputs, ages)\n            val_loss += loss.item()\n    \n    val_loss = val_loss / len(val_loader_t2)\n    val_losses_t2.append(val_loss)\n    \n    print(f'Epoch {epoch+1}: Train Loss = {train_loss:.4f}, Val Loss = {val_loss:.4f}')\n    \n    # Save best model\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        torch.save(model_t2.state_dict(), os.path.join(OUTPUT_PATH, 'task2_model_best.pth'))\n    \n    scheduler_t2.step(val_loss)\n\n# Plot training curves\nfig, ax = plt.subplots(figsize=(10, 6))\nax.plot(train_losses_t2, label='Train Loss')\nax.plot(val_losses_t2, label='Val Loss')\nax.set_title('Task 2: Training and Validation Loss (MAE)')\nax.set_xlabel('Epoch')\nax.set_ylabel('Loss')\nax.legend()\nax.grid(True)\nplt.savefig(os.path.join(OUTPUT_PATH, 'task2_training_curves.png'), dpi=100, bbox_inches='tight')\nplt.close()\nprint(\"✓ Saved training curves\")\n\n# Load best model\nmodel_t2.load_state_dict(torch.load(os.path.join(OUTPUT_PATH, 'task2_model_best.pth')))\n\n# Generate predictions\nprint(\"\\n--- Generating predictions for test set ---\")\ntest_dataset_t2 = AgeEstimationDataset(test_path_t2, None, transform=transform_test, is_test=True)\ntest_loader_t2 = DataLoader(test_dataset_t2, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\npredictions_t2 = []\nmodel_t2.eval()\n\nwith torch.no_grad():\n    for images, filenames in tqdm(test_loader_t2, desc='Predicting'):\n        images = images.to(device)\n        outputs = model_t2(images)\n        \n        for filename, age in zip(filenames, outputs):\n            predictions_t2.append({\n                'filename': filename,\n                'predicted_age': round(age.item())\n            })\n\ndf_predictions_t2 = pd.DataFrame(predictions_t2)\ndf_predictions_t2.to_csv(os.path.join(OUTPUT_PATH, 'task2_predictions.csv'), index=False)\nprint(\"✓ Saved Task 2 predictions\")\n\n# Save model\ntorch.save(model_t2.state_dict(), os.path.join(OUTPUT_PATH, 'task2_model.pth'))\nprint(\"✓ Saved Task 2 model weights\")\n\n\n# ============================================================================\n# TASK 3: OCR AND FRAUD DETECTION\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TASK 3: OCR AND FRAUD DETECTION\")\nprint(\"=\"*80)\n\ntrain_path_t3 = os.path.join(BASE_PATH, 'dataset_tache_3/dataset_tache_3/train')\ntest_path_t3 = os.path.join(BASE_PATH, 'dataset_tache_3/dataset_tache_3/test')\n\n# Alternative paths\nif not os.path.exists(train_path_t3):\n    train_path_t3 = os.path.join(BASE_PATH, 'dataset_tache_3/train')\n    test_path_t3 = os.path.join(BASE_PATH, 'dataset_tache_3/test')\n\n# Explore training data structure\ncountries = ['spain', 'estonia', 'russia', 'arizona']\ndoc_types = ['normal', 'forgery_1', 'forgery_2', 'forgery_3', 'forgery_4']\n\nprint(\"\\n--- EDA for Task 3 ---\")\n\n# Count images per category\ndata_summary = []\nfor country in countries:\n    country_path = os.path.join(train_path_t3, country)\n    if os.path.exists(country_path):\n        for doc_type in doc_types:\n            type_path = os.path.join(country_path, doc_type)\n            if os.path.exists(type_path):\n                images = [f for f in os.listdir(type_path) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))]\n                data_summary.append({\n                    'country': country,\n                    'type': doc_type,\n                    'count': len(images)\n                })\n\ndf_summary_t3 = pd.DataFrame(data_summary)\nprint(\"\\nDataset summary:\")\nprint(df_summary_t3.pivot(index='country', columns='type', values='count'))\n\n# Visualize samples\nfig, axes = plt.subplots(len(countries), 5, figsize=(20, 16))\nfig.suptitle('Task 3: Document Samples by Country and Type', fontsize=16)\n\nfor i, country in enumerate(countries):\n    for j, doc_type in enumerate(doc_types):\n        type_path = os.path.join(train_path_t3, country, doc_type)\n        if os.path.exists(type_path):\n            images = [f for f in os.listdir(type_path) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))]\n            if images:\n                img = cv2.imread(os.path.join(type_path, images[0]))\n                if img is not None:\n                    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n                    axes[i, j].imshow(img)\n        \n        axes[i, j].set_title(f'{country}\\n{doc_type}')\n        axes[i, j].axis('off')\n\nplt.tight_layout()\nplt.savefig(os.path.join(OUTPUT_PATH, 'task3_samples.png'), dpi=100, bbox_inches='tight')\nplt.close()\nprint(\"✓ Saved sample images\")\n\n# Prepare dataset for classification\ntrain_data_t3 = []\n\nfor country in countries:\n    country_path = os.path.join(train_path_t3, country)\n    if os.path.exists(country_path):\n        for doc_type in doc_types:\n            type_path = os.path.join(country_path, doc_type)\n            if os.path.exists(type_path):\n                images = [f for f in os.listdir(type_path) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))]\n                for img in images:\n                    train_data_t3.append({\n                        'filename': img,\n                        'country': country,\n                        'type': doc_type,\n                        'path': os.path.join(type_path, img)\n                    })\n\ndf_train_t3 = pd.DataFrame(train_data_t3)\n\n# Create label mapping\nlabel_mapping = {label: idx for idx, label in enumerate(doc_types)}\ndf_train_t3['label'] = df_train_t3['type'].map(label_mapping)\n\nprint(f\"\\nTotal training images: {len(df_train_t3)}\")\nprint(f\"\\nClass distribution:\")\nprint(df_train_t3['type'].value_counts())\n\n\n# Dataset for Task 3\nclass DocumentDataset(Dataset):\n    def __init__(self, dataframe, transform=None, is_test=False, test_path=None):\n        self.dataframe = dataframe\n        self.transform = transform\n        self.is_test = is_test\n        self.test_path = test_path\n        \n        if is_test:\n            self.image_list = sorted([f for f in os.listdir(test_path) if f.endswith(('.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG'))])\n        \n    def __len__(self):\n        if self.is_test:\n            return len(self.image_list)\n        return len(self.dataframe)\n    \n    def __getitem__(self, idx):\n        if self.is_test:\n            img_name = self.image_list[idx]\n            img_path = os.path.join(self.test_path, img_name)\n            image = Image.open(img_path).convert('RGB')\n            \n            if self.transform:\n                image = self.transform(image)\n            \n            return image, img_name\n        else:\n            row = self.dataframe.iloc[idx]\n            img_path = row['path']\n            label = row['label']\n            \n            image = Image.open(img_path).convert('RGB')\n            \n            if self.transform:\n                image = self.transform(image)\n            \n            return image, label\n\n\n# Document Classification Model\nclass DocumentClassifier(nn.Module):\n    def __init__(self, num_classes=5):\n        super(DocumentClassifier, self).__init__()\n        \n        # Use ResNet50\n        self.backbone = models.resnet50(pretrained=True)\n        n_features = self.backbone.fc.in_features\n        \n        self.backbone.fc = nn.Sequential(\n            nn.Dropout(0.3),\n            nn.Linear(n_features, 512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, num_classes)\n        )\n        \n    def forward(self, x):\n        return self.backbone(x)\n\n\n# Split data\ntrain_df_t3, val_df_t3 = train_test_split(df_train_t3, test_size=0.15, random_state=42, stratify=df_train_t3['label'])\n\ntrain_dataset_t3 = DocumentDataset(train_df_t3, transform=transform_train)\nval_dataset_t3 = DocumentDataset(val_df_t3, transform=transform_test)\n\ntrain_loader_t3 = DataLoader(train_dataset_t3, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader_t3 = DataLoader(val_dataset_t3, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n# Initialize model\nmodel_t3 = DocumentClassifier(num_classes=5).to(device)\n\ncriterion_t3 = nn.CrossEntropyLoss()\noptimizer_t3 = torch.optim.Adam(model_t3.parameters(), lr=LEARNING_RATE)\nscheduler_t3 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_t3, mode='max', patience=2, factor=0.5)\n\n# Training loop\nprint(\"\\n--- Training Task 3 Model ---\")\ntrain_losses_t3 = []\ntrain_accs_t3 = []\nval_accs_t3 = []\n\nbest_val_acc = 0.0\n\nfor epoch in range(EPOCHS):\n    # Training\n    model_t3.train()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    \n    pbar = tqdm(train_loader_t3, desc=f'Epoch {epoch+1}/{EPOCHS} [Train]')\n    for images, labels in pbar:\n        images, labels = images.to(device), labels.to(device)\n        \n        optimizer_t3.zero_grad()\n        \n        outputs = model_t3(images)\n        loss = criterion_t3(outputs, labels)\n        \n        loss.backward()\n        optimizer_t3.step()\n        \n        running_loss += loss.item()\n        \n        _, predicted = torch.max(outputs, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        \n        pbar.set_postfix({'loss': f'{loss.item():.4f}'})\n    \n    train_loss = running_loss / len(train_loader_t3)\n    train_acc = 100 * correct / total\n    \n    train_losses_t3.append(train_loss)\n    train_accs_t3.append(train_acc)\n    \n    # Validation\n    model_t3.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for images, labels in val_loader_t3:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model_t3(images)\n            \n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    \n    val_acc = 100 * correct / total\n    val_accs_t3.append(val_acc)\n    \n    print(f'Epoch {epoch+1}: Train Loss = {train_loss:.4f}, Train Acc = {train_acc:.2f}%, Val Acc = {val_acc:.2f}%')\n    \n    # Save best model\n    if val_acc > best_val_acc:\n        best_val_acc = val_acc\n        torch.save(model_t3.state_dict(), os.path.join(OUTPUT_PATH, 'task3_model_best.pth'))\n    \n    scheduler_t3.step(val_acc)\n\n# Plot training curves\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))\n\nax1.plot(train_losses_t3, label='Train Loss')\nax1.set_title('Task 3: Training Loss')\nax1.set_xlabel('Epoch')\nax1.set_ylabel('Loss')\nax1.grid(True)\nax1.legend()\n\nax2.plot(train_accs_t3, label='Train Accuracy')\nax2.plot(val_accs_t3, label='Val Accuracy')\nax2.set_title('Task 3: Training and Validation Accuracy')\nax2.set_xlabel('Epoch')\nax2.set_ylabel('Accuracy (%)')\nax2.grid(True)\nax2.legend()\n\nplt.tight_layout()\nplt.savefig(os.path.join(OUTPUT_PATH, 'task3_training_curves.png'), dpi=100, bbox_inches='tight')\nplt.close()\nprint(\"✓ Saved training curves\")\n\n# Load best model\nmodel_t3.load_state_dict(torch.load(os.path.join(OUTPUT_PATH, 'task3_model_best.pth')))\n\n# Generate predictions\nprint(\"\\n--- Generating predictions for test set ---\")\ntest_dataset_t3 = DocumentDataset(None, transform=transform_test, is_test=True, test_path=test_path_t3)\ntest_loader_t3 = DataLoader(test_dataset_t3, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\npredictions_t3 = []\nmodel_t3.eval()\n\n# Reverse label mapping\nreverse_label_mapping = {v: k for k, v in label_mapping.items()}\n\nwith torch.no_grad():\n    for images, filenames in tqdm(test_loader_t3, desc='Predicting'):\n        images = images.to(device)\n        outputs = model_t3(images)\n        _, predicted = torch.max(outputs, 1)\n        \n        for filename, pred in zip(filenames, predicted):\n            predictions_t3.append({\n                'filename': filename,\n                'predicted_class': reverse_label_mapping[pred.item()]\n            })\n\ndf_predictions_t3 = pd.DataFrame(predictions_t3)\ndf_predictions_t3.to_csv(os.path.join(OUTPUT_PATH, 'task3_predictions.csv'), index=False)\nprint(\"✓ Saved Task 3 predictions\")\n\n# Save model\ntorch.save(model_t3.state_dict(), os.path.join(OUTPUT_PATH, 'task3_model.pth'))\nprint(\"✓ Saved Task 3 model weights\")\n\n\n# ============================================================================\n# SUMMARY\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TRAINING COMPLETE - SUMMARY\")\nprint(\"=\"*80)\n\nprint(\"\\nTask 1 - Face Recognition:\")\nprint(f\"  - Final training accuracy: {train_accs_t1[-1]:.2f}%\")\nprint(f\"  - Matched {len(df_matches_t1)} test images\")\nprint(f\"  - Output: task1_predictions.csv\")\nprint(f\"  - Model: task1_model.pth\")\n\nprint(\"\\nTask 2 - Age Estimation:\")\nprint(f\"  - Best validation MAE: {best_val_loss:.2f} years\")\nprint(f\"  - Predictions for {len(df_predictions_t2)} test images\")\nprint(f\"  - Output: task2_predictions.csv\")\nprint(f\"  - Model: task2_model_best.pth\")\n\nprint(\"\\nTask 3 - Document Classification:\")\nprint(f\"  - Best validation accuracy: {best_val_acc:.2f}%\")\nprint(f\"  - Predictions for {len(df_predictions_t3)} test images\")\nprint(f\"  - Output: task3_predictions.csv\")\nprint(f\"  - Model: task3_model_best.pth\")\n\nprint(\"\\n\" + \"=\"*80)\nprint(f\"All files saved in {OUTPUT_PATH}\")\nprint(\"=\"*80)\n\nprint(\"\\nGenerated files:\")\nprint(\"  Visualizations:\")\nprint(\"    - task1_samples.png\")\nprint(\"    - task1_training_curves.png\")\nprint(\"    - task2_samples.png\")\nprint(\"    - task2_eda.png\")\nprint(\"    - task2_training_curves.png\")\nprint(\"    - task3_samples.png\")\nprint(\"    - task3_training_curves.png\")\nprint(\"\\n  Predictions:\")\nprint(\"    - task1_predictions.csv\")\nprint(\"    - task2_predictions.csv\")\nprint(\"    - task3_predictions.csv\")\nprint(\"\\n  Model Weights:\")\nprint(\"    - task1_model.pth\")\nprint(\"    - task2_model_best.pth\")\nprint(\"    - task3_model_best.pth\")\n\nprint(\"\\n✓ ALL TASKS COMPLETED SUCCESSFULLY!\")\nprint(\"\\nTo download your results in Kaggle:\")\nprint(\"1. Click on the 'Output' tab on the right\")\nprint(\"2. Download the prediction CSV files\")\nprint(\"3. Submit to the competition!\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}