{"cells":[{"metadata":{"id":"ZqQ0WNGv608i"},"cell_type":"markdown","source":"# **Explore Earth Engine Data**","execution_count":null},{"metadata":{"id":"6DZldYs2608k"},"cell_type":"markdown","source":"**Step 1: Explore the power plant data**","execution_count":null},{"metadata":{"_kg_hide-input":true,"id":"weDvKvFh608l","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport rasterio as rio\nimport folium\n        \ndef plot_points_on_map(dataframe,begin_index,end_index,latitude_column,latitude_value,longitude_column,longitude_value,zoom):\n    df = dataframe[begin_index:end_index]\n    location = [latitude_value,longitude_value]\n    plot = folium.Map(location=location,zoom_start=zoom)\n    for i in range(0,len(df)):\n        popup = folium.Popup(str(df.primary_fuel[i:i+1]))\n        folium.Marker([df[latitude_column].iloc[i],df[longitude_column].iloc[i]],popup=popup).add_to(plot)\n    return(plot)\n\ndef overlay_image_on_puerto_rico(file_name,band_layer):\n    band = rio.open(file_name).read(band_layer)\n    m = folium.Map([lat, lon], zoom_start=8)\n    folium.raster_layers.ImageOverlay(\n        image=band,\n        bounds = [[18.6,-67.3,],[17.9,-65.2]],\n        colormap=lambda x: (1, 0, 0, x),\n    ).add_to(m)\n    return m\n\ndef plot_scaled(file_name):\n    vmin, vmax = np.nanpercentile(file_name, (5,95))  # 5-95% stretch\n    img_plt = plt.imshow(file_name, cmap='gray', vmin=vmin, vmax=vmax)\n    plt.show()\n\ndef split_column_into_new_columns(dataframe,column_to_split,new_column_one,begin_column_one,end_column_one):\n    for i in range(0, len(dataframe)):\n        dataframe.loc[i, new_column_one] = dataframe.loc[i, column_to_split][begin_column_one:end_column_one]\n    return dataframe","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"id":"K1DlwAqB608o","outputId":"91b580a9-17db-4f63-ec3a-784f3334aaf5","trusted":true},"cell_type":"code","source":"power_plants = pd.read_csv('/kaggle/input/ds4g-environmental-insights-explorer/eie_data/gppd/gppd_120_pr.csv')\npower_plants = split_column_into_new_columns(power_plants,'.geo','latitude',50,66)\npower_plants = split_column_into_new_columns(power_plants,'.geo','longitude',31,48)\npower_plants['latitude'] = power_plants['latitude'].astype(float)\na = np.array(power_plants['latitude'].values.tolist()) # 18 instead of 8\npower_plants['latitude'] = np.where(a < 10, a+10, a).tolist() \nlat=18.200178; lon=-66.664513\nplot_points_on_map(power_plants,0,425,'latitude',lat,'longitude',lon,9)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"id":"0eDVlNlC608r","outputId":"72f1995e-1183-4ad9-8659-02ed8806a31b","trusted":true},"cell_type":"code","source":"power_plants_df = power_plants.sort_values('capacity_mw',ascending=False).reset_index()\npower_plants_df[['name','latitude','longitude','primary_fuel','capacity_mw','estimated_generation_gwh']]","execution_count":null,"outputs":[]},{"metadata":{"id":"G1ca13u6608t"},"cell_type":"markdown","source":"**Step 2: Explore the nO2 emissions data**","execution_count":null},{"metadata":{"_kg_hide-input":true,"id":"BLzBQe_N608u","outputId":"088d622c-ae5e-4c73-e5e6-9fb783f2e7e5","trusted":true},"cell_type":"code","source":"image = '/kaggle/input/ds4g-environmental-insights-explorer/eie_data/s5p_no2/s5p_no2_20180708T172237_20180714T190743.tif'\nimage_band = rio.open(image).read(1)\nplot_scaled(image_band)\noverlay_image_on_puerto_rico(image,band_layer=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"1cWq-KgX608w"},"cell_type":"markdown","source":"**Step 3: Explore the weather data**","execution_count":null},{"metadata":{"_kg_hide-input":true,"id":"L7M5yI_9608x","outputId":"df4fb3d2-9cac-4b10-c77a-399576e2cf94","trusted":true},"cell_type":"code","source":"image = '/kaggle/input/ds4g-environmental-insights-explorer/eie_data/gldas/gldas_20180702_1500.tif'\nimage_band = rio.open(image).read(3)\nplot_scaled(image_band)\n\nimage = '/kaggle/input/ds4g-environmental-insights-explorer/eie_data/gfs/gfs_2018072118.tif'\nimage_band = rio.open(image).read(3)\nplot_scaled(image_band)\n\noverlay_image_on_puerto_rico(image,band_layer=3)","execution_count":null,"outputs":[]},{"metadata":{"id":"-13q1iOg608z"},"cell_type":"markdown","source":"**Step 4: Connect to the Google Earth Engine API**","execution_count":null},{"metadata":{"_kg_hide-input":true,"id":"TFXRp_Hy6082","trusted":true},"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nfrom google.oauth2.credentials import Credentials\nimport ee\nimport folium\n\ndef add_ee_layer(self, ee_image_object, vis_params, name):\n  # https://github.com/google/earthengine-api/blob/master/python/examples/ipynb/ee-api-colab-setup.ipynb\n  map_id_dict = ee.Image(ee_image_object).getMapId(vis_params)\n  folium.raster_layers.TileLayer(\n    tiles = map_id_dict['tile_fetcher'].url_format,\n    attr = 'Map Data &copy; <a href=\"https://earthengine.google.com/\">Google Earth Engine</a>',\n    name = name,\n    overlay = True,\n    control = True\n  ).add_to(self)\n\ndef plot_ee_data_on_map(dataset,column,begin_date,end_date,minimum_value,maximum_value,latitude,longitude,zoom):\n    # https://github.com/google/earthengine-api/blob/master/python/examples/ipynb/ee-api-colab-setup.ipynb\n    folium.Map.add_ee_layer = add_ee_layer\n    vis_params = {\n      'min': minimum_value,\n      'max': maximum_value,\n      'palette': ['006633', 'E5FFCC', '662A00', 'D8D8D8', 'F5F5F5']}\n    my_map = folium.Map(location=[latitude,longitude], zoom_start=zoom, height=500)\n    s5p = ee.ImageCollection(dataset).filterDate(\n        begin_date, end_date)\n    my_map.add_ee_layer(s5p.first().select(column), vis_params, 'Color')\n    my_map.add_child(folium.LayerControl())\n    display(my_map)","execution_count":null,"outputs":[]},{"metadata":{"id":"GmnTyaUu6085"},"cell_type":"markdown","source":"After registering at https://earthengine.google.com/signup/, navigate to the add-on menu of the notebook editor and create a new user secret called \"earth_engine\" that contains the refresh token from ee.Authenticate(). This step only needs to be run once. You can generate the refresh_token by running the following line in a new (internet-enabled) code cell: ee.Authenticate().  Next, follow the instructions, paste the value into the input box, and run the following command in a new code cell: !cat ~/.config/earthengine/credentials.  This should return to you a refresh token that can then be saved as your Kaggle user secret.\n\nIn summary:\n* Step 0: Register your account using both of the following two links: [Link #1](https://earthengine.google.com/signup/), [Link #2](https://docs.google.com/forms/d/e/1FAIpQLScFk_pkrrDDF4O8imsEBMaryLDU-Ghf44eHbgujIAl_SXJTJQ/viewform) \n* Step 1: Open an **internet-enabled** notebook and then retrieve your token by opening the ee.Authenticate() link **in a new tab**.  You will also need to copy/paste that value into the relevant **input box**, and you will need to enable the internet (in the settings menu of the notebook editor) in order for the value to get created in the first place.\n* Step 2: After completing Step 1, retrieve your **refresh-token** by running `!cat ~/.config/earthengine/credentials`\n* Step 3: Save your refresh-token as a **Kaggle user secret**\n* Step 4: Run the code snippet that contains ee.Initialize()\n\nSteps #0, #1, #2, and #3 only need to be performed once (for the initial setup). Step #4 is run every time that you run your code. Hopefully that helps!","execution_count":null},{"metadata":{"id":"aqCw05AR6086","trusted":true},"cell_type":"code","source":"user_secret = \"earth_engine\" # Your user secret, defined in the add-on menu of the notebook editor\nrefresh_token = UserSecretsClient().get_secret(user_secret)\ncredentials = Credentials(\n        None,\n        refresh_token=refresh_token,\n        token_uri=ee.oauth.TOKEN_URI,\n        client_id=ee.oauth.CLIENT_ID,\n        client_secret=ee.oauth.CLIENT_SECRET,\n        scopes=ee.oauth.SCOPES)\nee.Initialize(credentials=credentials)","execution_count":null,"outputs":[]},{"metadata":{"id":"4Mrhh2896088"},"cell_type":"markdown","source":"Congratulations! You are connected!","execution_count":null},{"metadata":{"id":"5XUi-MLC6089"},"cell_type":"markdown","source":"### ** Step 5: Explore the emissions data using the Google Earth Engine API**\n* Plots will exist for 2 days or until they are regenerated.","execution_count":null},{"metadata":{"_kg_hide-input":true,"id":"EILEV9yp608-","outputId":"7c157a29-ff4d-4824-85db-4e9648faa3ce","trusted":true},"cell_type":"code","source":"dataset = \"COPERNICUS/S5P/NRTI/L3_NO2\"\ncolumn = 'absorbing_aerosol_index'\nbegin_date = '2018-07-08'\nend_date = '2018-07-14'\nminimum_value = 0.0000000001\nmaximum_value = 1\nlatitude = 60.17\nlongitude = 25.94\nzoom = 5\nplot_ee_data_on_map(dataset,column,begin_date,end_date,minimum_value,maximum_value,latitude,longitude,zoom)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"YeLxgsiI609A"},"cell_type":"markdown","source":"## ** Step 6: Explore the weather data data using the Google Earth Engine API**\n* Plots will exist for 2 days or until they are regenerated.","execution_count":null},{"metadata":{"_kg_hide-input":true,"id":"ddLBiG9D609B","outputId":"5c537320-b109-4120-aac5-c8d136f2b74a","trusted":true},"cell_type":"code","source":"dataset = \"NOAA/GFS0P25\"\ncolumn = 'temperature_2m_above_ground'\nbegin_date = '2018-07-08'\nend_date = '2018-07-14'\nminimum_value = 0\nmaximum_value = 50\nlatitude = 18.20\nlongitude = -66.66\nzoom = 1\nplot_ee_data_on_map(dataset,column,begin_date,end_date,minimum_value,maximum_value,latitude,longitude,zoom)\n\ndataset = \"NASA/GLDAS/V021/NOAH/G025/T3H\"\ncolumn = 'Tair_f_inst'\nbegin_date = '2018-07-08'\nend_date = '2018-07-14'\nminimum_value = 270\nmaximum_value = 310\nlatitude = 18.20\nlongitude = -66.66\nzoom = 6\nplot_ee_data_on_map(dataset,column,begin_date,end_date,minimum_value,maximum_value,latitude,longitude,zoom)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**More getting started material is available here:**\n* [Explore Image Metadata](https://www.kaggle.com/paultimothymooney/explore-image-metadata-s5p-gfs-gldas)\n - bounding boxes, band names, etc\n* [Overview of the EIE Analytics Challenge](https://www.kaggle.com/paultimothymooney/overview-of-the-eie-analytics-challenge) \n - submission instructions, evaluation criteria, etc\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}