{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ! pip install beautifulsoup4","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This gets monthly means by reporting station.  I grabbed one from each state.  If you plan to use this you would want to\ntry and get stations near the locations - have to do that by eye ball.","metadata":{}},{"cell_type":"markdown","source":"Did not attempt to get mulitple years - it was painful enough getting one year to run.  Rerun for each year you want saving to a new file name.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport requests\nimport io\nimport time\nimport re\n\n# Capital City Stations (Proxy for State Climatology)\nstations = {\n    'New South Wales': '066062',    # Sydney (Observatory Hill)\n    'Victoria': '086071',           # Melbourne (Regional Office)\n    'Queensland': '040913',         # Brisbane Aero\n    'South Australia': '023090',    # Adelaide (Kent Town)\n    'Western Australia': '009021',  # Perth Metro\n    'Tasmania': '094029',           # Hobart (Ellerslie Rd)\n    'Northern Territory': '014015', # Darwin Airport\n    'ACT': '070351'                 # Canberra Airport\n}\n\nbase_url = \"http://www.bom.gov.au/climate/averages/tables/cw_{}.shtml\"\n\nheaders = {\n    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'\n}\n\ndef extract_first_12_floats(row_items):\n    \"\"\"\n    Helper: Extract the first 12 valid numbers from a list of row items.\n    Handles formatting like '25.4' or '18.2*'.\n    \"\"\"\n    nums = []\n    for item in row_items:\n        try:\n            # Clean string: remove '*' (incomplete data marker) or spaces\n            s = str(item).replace(\"*\", \"\").strip()\n            val = float(s)\n            nums.append(val)\n        except ValueError:\n            continue\n    \n    # We need exactly 12 months (Jan-Dec). \n    # BOM rows usually have 13+ numbers (Jan-Dec + Annual + Years).\n    # We strictly take the first 12.\n    if len(nums) >= 12:\n        return nums[:12]\n    return None\n\nresults = []\n\nprint(\"Fetching Long-term Monthly Averages (Climatology)...\")\nprint(\"-\" * 50)\n\nfor state_name, station_id in stations.items():\n    url = base_url.format(station_id)\n    print(f\"Processing {state_name}...\", end=\" \")\n    \n    try:\n        response = requests.get(url, headers=headers, timeout=15)\n        \n        if response.status_code == 200:\n            # Parse all tables on the page\n            try:\n                dfs = pd.read_html(io.StringIO(response.text))\n            except ValueError:\n                print(\"[!] No tables found.\")\n                continue\n\n            max_temps = None\n            min_temps = None\n            \n            # BRUTE FORCE SEARCH: Scan every row of every table\n            found_in_tables = False\n            for df in dfs:\n                # Convert entire dataframe to string for searching\n                for idx, row in df.iterrows():\n                    # Join row to a single string to check for keywords\n                    row_text = \" \".join(row.astype(str)).lower()\n                    \n                    # Check for Maximum Temperature row\n                    if \"mean\" in row_text and \"maximum\" in row_text and \"temperature\" in row_text:\n                        extracted = extract_first_12_floats(row.tolist())\n                        if extracted:\n                            max_temps = extracted\n\n                    # Check for Minimum Temperature row\n                    if \"mean\" in row_text and \"minimum\" in row_text and \"temperature\" in row_text:\n                        extracted = extract_first_12_floats(row.tolist())\n                        if extracted:\n                            min_temps = extracted\n                \n                # If we found both in this table (or across tables), we are good\n                if max_temps and min_temps:\n                    found_in_tables = True\n                    break \n            \n            if max_temps and min_temps:\n                # Calculate Mean = (Max + Min) / 2\n                monthly_mean = [(mx + mn) / 2 for mx, mn in zip(max_temps, min_temps)]\n                \n                # Build Result Row\n                data_row = {'State': state_name}\n                month_names = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']\n                \n                for i, m in enumerate(month_names):\n                    data_row[m] = round(monthly_mean[i], 1)\n                \n                # Calculate Annual Average from the monthly means\n                data_row['Annual_Avg'] = round(sum(monthly_mean) / 12, 1)\n                \n                results.append(data_row)\n                print(\"Success.\")\n            else:\n                print(\"[!] Data rows not found (Structure mismatch).\")\n                \n        else:\n            print(f\"[!] HTTP {response.status_code}\")\n            \n    except Exception as e:\n        print(f\"[!] Error: {e}\")\n        \n    time.sleep(0.5)\n\n# Save to CSV\nif results:\n    df = pd.DataFrame(results)\n    output_file = 'australia_capital_cities_mean_temp.csv'\n    df.to_csv(output_file, index=False)\n    \n    print(\"\\n\" + \"=\"*60)\n    print(f\"Success! File saved to: {output_file}\")\n    print(\"Data: Climatological Mean Temperature (°C) [(Max+Min)/2]\")\n    print(\"=\"*60)\n    print(df.to_string())\nelse:\n    print(\"\\nNo data collected. The BOM website structure may have changed drastically.\")","metadata":{},"outputs":[],"execution_count":null}]}