{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport requests\nimport time\nimport sys\n\n# --- CONFIGURATION ---\nstates = {\n    'nsw': 'New South Wales',\n    'vic': 'Victoria',\n    'qld': 'Queensland',\n    'sa':  'South Australia',\n    'wa':  'Western Australia',\n    'tas': 'Tasmania',\n    'nt':  'Northern Territory'\n}\n\nmonths = {\n    '01': 'Jan', '02': 'Feb', '03': 'Mar', '04': 'Apr', \n    '05': 'May', '06': 'Jun', '07': 'Jul', '08': 'Aug', \n    '09': 'Sep', '10': 'Oct', '11': 'Nov', '12': 'Dec'\n}\n\n# BOM requires a browser User-Agent\nheaders = {\n    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'\n}\n\ntarget_year = \"2015\"\n# PRIORITY 1: The 'web01' server is the current active host for these text files.\n# PRIORITY 2: The 'archive' server.\nbase_url_options = [\n    \"http://www.bom.gov.au/web01/ncc/www/cli_chg/timeseries/rain\",\n    \"http://www.bom.gov.au/archive/ncc/www/cli_chg/timeseries/rain\"\n]\n\nresults = []\n\nprint(f\"Starting scrape for {target_year}...\")\nprint(\"-\" * 40)\n\n# 1. Determine the working server first (using Annual Australia data as a probe)\nactive_base_url = None\nfor url in base_url_options:\n    probe_url = f\"{url}/0112/aus/latest.txt\"\n    try:\n        print(f\"Probing server: {url} ...\")\n        r = requests.get(probe_url, headers=headers, timeout=5)\n        if r.status_code == 200:\n            active_base_url = url\n            print(f\" -> Connection Established with {url}\")\n            break\n    except:\n        pass\n\nif not active_base_url:\n    print(\"[!] CRITICAL: Could not connect to BOM text servers. They may be blocking your IP or the structure has changed globally.\")\n    sys.exit(1)\n\n# 2. Main Scrape Loop\nfor state_code, state_name in states.items():\n    state_data = {'State': state_name}\n    print(f\"Processing {state_name} ({state_code})...\")\n    \n    for month_code, month_name in months.items():\n        # Try 'latest.txt' first, then specific year file if needed\n        # URL Structure: BASE / MONTH / STATE / latest.txt\n        file_url = f\"{active_base_url}/{month_code}/{state_code}/latest.txt\"\n        \n        try:\n            response = requests.get(file_url, headers=headers, timeout=10)\n            \n            if response.status_code == 200:\n                # Parse the text file\n                lines = response.text.splitlines()\n                found = False\n                for line in lines:\n                    # Format usually: 2015  20150101  45.2\n                    if line.strip().startswith(target_year):\n                        parts = line.split()\n                        # Robust parsing: take the last element as the value\n                        if len(parts) >= 2:\n                            try:\n                                val = float(parts[-1])\n                                state_data[month_name] = val\n                                found = True\n                            except ValueError:\n                                pass\n                        break\n                \n                if not found:\n                    state_data[month_name] = None\n            else:\n                print(f\"  [x] Failed to fetch {month_name} (HTTP {response.status_code})\")\n                state_data[month_name] = None\n\n        except Exception as e:\n            print(f\"  [!] Error: {e}\")\n            state_data[month_name] = None\n            \n        # Be polite\n        time.sleep(0.1)\n\n    results.append(state_data)\n\n# 3. Save Output\nif results:\n    df = pd.DataFrame(results)\n    \n    # Calculate Annual Total (Sum only if we have data)\n    numeric_cols = list(months.values())\n    df['Annual_Total'] = df[numeric_cols].sum(axis=1, min_count=1) \n    \n    output_file = 'australia_rainfall_2015_by_state.csv'\n    df.to_csv(output_file, index=False)\n    \n    print(\"\\n\" + \"=\"*40)\n    print(f\"Success! File saved to: {output_file}\")\n    print(\"=\"*40)\n    print(df.to_string())\nelse:\n    print(\"No data collected.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-22T04:15:07.643274Z","iopub.execute_input":"2025-11-22T04:15:07.643653Z","iopub.status.idle":"2025-11-22T04:15:55.146849Z","shell.execute_reply.started":"2025-11-22T04:15:07.643627Z","shell.execute_reply":"2025-11-22T04:15:55.145615Z"}},"outputs":[],"execution_count":null}]}