{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.8.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Random Forest Dogs vs Cats\n\n## Plan\n\nUse Random Forest with engineered features for color, edge and texture.\n\n**Features:**\n- **Color**: RGB histograms (768 dims) - capture color distribution across R, G, B channels\n- **Edges**: Canny edge detection (102 dims) - spatial edge density in 10×10 grid + global stats\n- **Texture**: Local Binary Patterns (256 dims) - capture fur texture patterns\n\n**Strategy:**\n1. Start with small subset (100 cats + 100 dogs) to iterate quickly\n2. Build feature extraction pipeline and cache features to disk\n3. Train Random Forest with cross-validation\n4. If performance is promising, scale to full 25k training images","metadata":{}},{"cell_type":"markdown","source":"## Phase 1: Quick Iteration (200 images)","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom skimage import feature, color\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import cross_validate, GridSearchCV\nimport joblib\nimport pandas as pd","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_image_subset(data_dir, n_cats=100, n_dogs=100):\n    data_path = Path(data_dir)\n    images = []\n    labels = []\n    filenames = []\n    \n    cat_files = sorted([f for f in os.listdir(data_path) if f.startswith('cat.')])[:n_cats]\n    print(f\"Loading {len(cat_files)} cat images...\")\n    \n    for filename in cat_files:\n        img = Image.open(data_path / filename)\n        img_array = np.array(img)\n        \n        if len(img_array.shape) == 2:\n            continue\n        \n        images.append(img_array)\n        labels.append(0)\n        filenames.append(filename)\n    \n    dog_files = sorted([f for f in os.listdir(data_path) if f.startswith('dog.')])[:n_dogs]\n    print(f\"Loading {len(dog_files)} dog images...\")\n    \n    for filename in dog_files:\n        img = Image.open(data_path / filename)\n        img_array = np.array(img)\n        \n        if len(img_array.shape) == 2:\n            continue\n        \n        images.append(img_array)\n        labels.append(1)\n        filenames.append(filename)\n    \n    print(f\"Loaded {len(images)} images total\")\n    return images, labels, filenames","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_rgb_histogram(img_array, bins=256):\n    r_hist, _ = np.histogram(img_array[:, :, 0], bins=bins, range=(0, 256))\n    g_hist, _ = np.histogram(img_array[:, :, 1], bins=bins, range=(0, 256))\n    b_hist, _ = np.histogram(img_array[:, :, 2], bins=bins, range=(0, 256))\n    \n    total_pixels = img_array.shape[0] * img_array.shape[1]\n    features = np.concatenate([r_hist / total_pixels, g_hist / total_pixels, b_hist / total_pixels])\n    \n    return features.astype(np.float32)\n\ndef extract_edge_features(img_array, sigma=1.0, grid_size=10):\n    if len(img_array.shape) == 3:\n        gray = color.rgb2gray(img_array)\n    else:\n        gray = img_array\n    \n    edges = feature.canny(gray, sigma=sigma)\n    h, w = edges.shape\n    cell_h = h // grid_size\n    cell_w = w // grid_size\n    \n    grid_densities = []\n    for i in range(grid_size):\n        for j in range(grid_size):\n            cell = edges[i*cell_h:(i+1)*cell_h, j*cell_w:(j+1)*cell_w]\n            density = cell.sum() / cell.size if cell.size > 0 else 0.0\n            grid_densities.append(density)\n    \n    overall_density = edges.sum() / edges.size\n    edge_concentration = np.std(grid_densities)\n    features = np.array(grid_densities + [overall_density, edge_concentration])\n    \n    return features.astype(np.float32)\n\ndef extract_lbp_features(img_array, n_points=8, radius=1, bins=256):\n    if len(img_array.shape) == 3:\n        gray = color.rgb2gray(img_array)\n    else:\n        gray = img_array\n    \n    lbp = feature.local_binary_pattern(gray, n_points, radius, method='default')\n    hist, _ = np.histogram(lbp.ravel(), bins=bins, range=(0, 256), density=True)\n    \n    return hist.astype(np.float32)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dir = \"../data/train\"\nimages, labels, filenames = load_image_subset(train_dir, n_cats=100, n_dogs=100)\n\nprint(f\"\\nDataset: {len(images)} images\")\nprint(f\"Sample shape: {images[0].shape}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Extracting features...\")\nrgb_features = []\nedge_features = []\nlbp_features = []\n\nfor i, img in enumerate(images):\n    rgb_features.append(extract_rgb_histogram(img))\n    edge_features.append(extract_edge_features(img))\n    lbp_features.append(extract_lbp_features(img))\n    \n    if (i + 1) % 50 == 0:\n        print(f\"  {i + 1}/{len(images)}\")\n\nX_rgb = np.array(rgb_features)\nX_edge = np.array(edge_features)\nX_lbp = np.array(lbp_features)\nX_combined = np.hstack([X_rgb, X_edge, X_lbp])\ny = np.array(labels)\n\nprint(f\"\\nCombined features shape: {X_combined.shape}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_dir = Path(\"../data/features\")\nfeatures_dir.mkdir(exist_ok=True)\n\nnp.save(features_dir / \"rgb_histogram_features.npy\", X_rgb)\nnp.save(features_dir / \"edge_features.npy\", X_edge)\nnp.save(features_dir / \"lbp_features.npy\", X_lbp)\nnp.save(features_dir / \"labels.npy\", y)\n\nprint(f\"Saved to {features_dir}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Feature shape: {X_combined.shape}\")\nprint(f\"Expected: (200, 1126)\")\nprint(f\"\\nRGB sum per image: {X_rgb.sum(axis=1).mean():.4f}\")\nprint(f\"LBP sum per image: {X_lbp.sum(axis=1).mean():.4f}\")\nprint(f\"\\nLabel split: {(y==0).sum()} cats, {(y==1).sum()} dogs\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rf = RandomForestClassifier(random_state=42, n_jobs=-1)\n\nprint(\"Running 5-fold CV...\")\ncv_results = cross_validate(\n    rf, X_combined, y, cv=5,\n    scoring=['accuracy', 'neg_log_loss'],\n    return_train_score=False\n)\n\nacc = cv_results['test_accuracy']\nloss = -cv_results['test_neg_log_loss']\n\nprint(f\"\\nAccuracy: {acc.mean():.4f} ± {acc.std():.4f}\")\nprint(f\"Log Loss: {loss.mean():.4f} ± {loss.std():.4f}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_grid = {\n    'n_estimators': [100, 200, 300],\n    'max_depth': [None, 10, 20, 30],\n    'min_samples_split': [2, 5, 10]\n}\n\nprint(\"Grid search...\")\ngrid = GridSearchCV(\n    RandomForestClassifier(random_state=42, n_jobs=-1),\n    param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1\n)\n\ngrid.fit(X_combined, y)\n\nprint(f\"\\nBest accuracy: {grid.best_score_:.4f}\")\nprint(f\"Best params: {grid.best_params_}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_rf = grid.best_estimator_\n\ncv_best = cross_validate(\n    best_rf, X_combined, y, cv=5,\n    scoring=['accuracy', 'neg_log_loss'],\n    return_train_score=False\n)\n\nacc_best = cv_best['test_accuracy']\nloss_best = -cv_best['test_neg_log_loss']\n\nprint(f\"Best model accuracy: {acc_best.mean():.4f} ± {acc_best.std():.4f}\")\nprint(f\"Best model log loss: {loss_best.mean():.4f} ± {loss_best.std():.4f}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Phase 2: Full Dataset (25K images)","metadata":{}},{"cell_type":"code","source":"images_full, labels_full, filenames_full = load_image_subset(train_dir, n_cats=12500, n_dogs=12500)\n\nprint(f\"\\nDataset: {len(images_full)} images\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Extracting features from full dataset...\")\nrgb_features_full = []\nedge_features_full = []\nlbp_features_full = []\n\nfor i, img in enumerate(images_full):\n    rgb_features_full.append(extract_rgb_histogram(img))\n    edge_features_full.append(extract_edge_features(img))\n    lbp_features_full.append(extract_lbp_features(img))\n    \n    if (i + 1) % 1000 == 0:\n        print(f\"  {i + 1}/{len(images_full)}\")\n\nX_rgb_full = np.array(rgb_features_full)\nX_edge_full = np.array(edge_features_full)\nX_lbp_full = np.array(lbp_features_full)\nX_combined_full = np.hstack([X_rgb_full, X_edge_full, X_lbp_full])\ny_full = np.array(labels_full)\n\nprint(f\"\\nCombined features shape: {X_combined_full.shape}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.save(features_dir / \"full_rgb_histogram_features.npy\", X_rgb_full)\nnp.save(features_dir / \"full_edge_features.npy\", X_edge_full)\nnp.save(features_dir / \"full_lbp_features.npy\", X_lbp_full)\nnp.save(features_dir / \"full_labels.npy\", y_full)\n\nprint(f\"Saved full features to {features_dir}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Feature shape: {X_combined_full.shape}\")\nprint(f\"Expected: (25000, 1126)\")\nprint(f\"\\nLabel split: {(y_full==0).sum()} cats, {(y_full==1).sum()} dogs\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Training final model on full dataset...\")\nprint(f\"Using best params from phase 1: {grid.best_params_}\")\n\nfinal_model = RandomForestClassifier(\n    n_estimators=grid.best_params_['n_estimators'],\n    max_depth=grid.best_params_['max_depth'],\n    min_samples_split=grid.best_params_['min_samples_split'],\n    random_state=42,\n    n_jobs=-1\n)\n\nfinal_model.fit(X_combined_full, y_full)\n\nmodels_dir = Path(\"../models\")\nmodels_dir.mkdir(exist_ok=True)\nmodel_path = models_dir / \"random_forest_final.pkl\"\n\njoblib.dump(final_model, model_path)\nprint(f\"\\nModel saved to {model_path}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_test_images(data_dir):\n    data_path = Path(data_dir)\n    test_files = sorted([f for f in os.listdir(data_path) if f.endswith('.jpg')], \n                       key=lambda x: int(x.split('.')[0]))\n    \n    print(f\"Processing {len(test_files)} test images...\")\n    \n    test_ids = []\n    rgb_feats = []\n    edge_feats = []\n    lbp_feats = []\n    \n    for i, filename in enumerate(test_files):\n        img_id = int(filename.split('.')[0])\n        test_ids.append(img_id)\n        \n        img = Image.open(data_path / filename)\n        img_array = np.array(img)\n        \n        if len(img_array.shape) == 2:\n            img_array = np.stack([img_array] * 3, axis=-1)\n        \n        rgb_feats.append(extract_rgb_histogram(img_array))\n        edge_feats.append(extract_edge_features(img_array))\n        lbp_feats.append(extract_lbp_features(img_array))\n        \n        if (i + 1) % 1000 == 0:\n            print(f\"  {i + 1}/{len(test_files)}\")\n    \n    X_test = np.hstack([np.array(rgb_feats), np.array(edge_feats), np.array(lbp_feats)])\n    print(f\"\\nTest features shape: {X_test.shape}\")\n    \n    return test_ids, X_test\n\ntest_dir = \"../data/test\"\ntest_ids, X_test = load_test_images(test_dir)\n\nnp.save(features_dir / \"test_features.npy\", X_test)\nnp.save(features_dir / \"test_ids.npy\", np.array(test_ids))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Generating predictions...\")\n\npredictions = final_model.predict_proba(X_test)\ndog_probs = predictions[:, 1]\n\nsubmission = pd.DataFrame({\n    'id': test_ids,\n    'label': dog_probs\n})\n\nsubmission = submission.sort_values('id').reset_index(drop=True)\nsubmission_path = Path(\"../submission.csv\")\nsubmission.to_csv(submission_path, index=False)\n\nprint(f\"Submission saved to {submission_path}\")\nprint(f\"\\nPreview:\")\nprint(submission.head(10))","metadata":{},"outputs":[],"execution_count":null}]}