{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\ndata_folder = os.getcwd()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from datetime import datetime, timedelta\nimport attr\nimport pandas as pd\nimport pytz\nimport requests\nfrom itertools import groupby\n\ndef fetch_historic_weather_data(site_data, api_key:str, file_path:str='./input')-> pd.DataFrame:\n    \"\"\"\n    Fetch Historic Weather data for the given site\n\n    id: Job Id\n    site_id: Site identifier\n    start_datetime: Start date from which the historic weather data needs to be fetched\n    end_datetime: End date for the historic weather data fetch\n    \"\"\"\n\n    if site_data.get('end_datetime') == None:\n        site_data['end_datetime'] = datetime.now()\n\n    weather_connector = DarkSkyWeatherConnector(site_id=site_data.get('site_id'),\n                                                latitude=site_data.get('latitude'),\n                                                longitude=site_data.get('longitude'),\n                                                start_date=site_data.get('start_datetime'),\n                                                end_date=site_data.get('end_datetime'),\n                                                exclude_existing_data=False,\n                                                include_solar=site_data.get('pv'),\n                                                api_key=api_key,\n                                                )\n\n    weather_data = weather_connector.extract_weather_in_range()\n    weather_data.to_csv(f'{file_path}/site{site_data.get(\"site_id\")}_darksky_weather.csv')\n    return weather_data\n\n\n@attr.s\nclass DarkSkyWeatherConnector(object):\n    \"\"\"\n    Class used for pulling weather data from DarkSky's API.  `self.extract_weather_in_range` will pull hourly weather\n    data from `self.start_date` to `self.end_date` for `self.latitude` and `self.longitude`, parse returned json for\n    self.desired_columns, and construct and return a pd.DataFrame.\n\n    Their documentation is concise and comprehensive:\n        https://darksky.net/dev/docs\n\n    The basic http request for either historical data or forecasted data is:\n        https://api.darksky.net/forecast/[key]/[latitude],[longitude],[time]\n    eg\n        https://api.darksky.net/forecast/0123456789abcdef9876543210fedcba/42.3601,-71.0589,255657600\n\n    By default, we use the following params with the above request:\n        exclude: 'currently,minutely,daily,alerts,flags'\n        solar: 1\n\n    Setting exclude to 'currently,minutely,daily,alerts,flags' strips out data that we do not use; therefore, reducing\n    the amount of data that we transfer.\n\n    We presently do not use minutely data because it is only for the current hour, and when we pull weather data it\n    is for historical simulations.\n\n    By specifying solar = 1, we get the following additional data:\n        'azimuth', 'altitude', 'dni', 'ghi', 'dhi', 'etr'\n\n    We currently only use dni, but should continue integrating other data in the future.\n    \"\"\"\n\n    site_id = attr.ib()\n    latitude = attr.ib()\n    longitude = attr.ib()\n    start_date = attr.ib()\n    end_date = attr.ib()\n    exclude_existing_data = attr.ib(default=False)\n\n    api_key = attr.ib(default=\"4b6c6722e6c612fd5f789cee71aa7135\")\n    base_url = attr.ib(default=\"https://api.darksky.net/forecast\")\n    # Weather data that will be parsed from api request and stored in returned pd.DataFrame\n    desired_columns = attr.ib(default=['time', 'dni', 'ghi', 'dhi', 'apparentTemperature', 'temperature', 'precipIntensity',\n                                       'humidity', 'dewPoint', 'pressure', 'cloudCover', 'windSpeed', 'windGust', 'windBearing','visibility'])\n    # will remove data points from returned json (reduces I/O burden)\n    to_exclude = attr.ib(default='minutely,daily,alerts,flags')\n    # If true, api request will return 'azimuth', 'altitude', 'dni', 'ghi', 'dhi', 'etr'\n    include_solar = attr.ib(default=True)\n\n    current_weather_data_dict = attr.ib(init=False, default=None)\n    weather_df = attr.ib(init=True, default=pd.DataFrame())\n\n\n    def fetch_current_weather_data(self):\n        \"\"\"\n        Pull current weather data from the weather data provider DarkSky\n        :return:\n        \"\"\"\n\n        url = f\"{self.base_url}/{self.api_key}/{self.latitude},{self.longitude}\"\n\n        # Exclude everything except \"currently\"\n        to_exclude = 'minutely,hourly,daily,alerts,flags'\n        params = {\"exclude\": to_exclude, \"solar\": 1 if self.include_solar else 0}\n\n        self.current_weather_data_dict = requests.get(url, params=params).json()['currently']\n\n\n    def make_request_for_date(self, date):\n        \"\"\"\n        Makes api request for specified date and returns json\n        :param date: <datetime.datetime or datetime.date> If datetime, request will ignore time and pull for 00:00\n        :return:\n        \"\"\"\n        str_date = date.strftime('%Y-%m-%dT00:00:00')\n        url = f\"{self.base_url}/{self.api_key}/{self.latitude},{self.longitude},{str_date}\"\n        params = {\"exclude\": self.to_exclude, \"solar\": 1 if self.include_solar else 0}\n        return requests.get(url, params=params).json()\n\n\n    def construct_daily_df(self, date):\n        \"\"\"\n        Calls `self.make_request_for_date` and converts returned json into a pd.DataFrame\n        :param date: <datetime.datetime or datetime.date> If datetime, request will ignore time and pull for 00:00\n        :return:\n        \"\"\"\n        raw_json = self.make_request_for_date(date=date)\n\n        #Get currently data\n        self.current_weather_data_dict = raw_json['currently']\n        flat_daily_df = []\n        for hd in raw_json['hourly']['data']:\n            hd['time'] = datetime.fromtimestamp(hd['time'], pytz.timezone(raw_json['timezone']))\n            flat_daily_df.append({k:v for k,v in hd.items() if k!='solar' and k in self.desired_columns})\n            flat_daily_df[-1].update(hd.get('solar',{}))\n\n\n        weather_df = pd.DataFrame(flat_daily_df).set_index('time')\n        weather_df[['dni', 'dhi', 'ghi']] = weather_df[['dni', 'dhi', 'ghi']].fillna(value=0)\n\n        return weather_df[[c for c in self.desired_columns if c!='time']]\n\n\n    def extract_weather_in_range(self):\n        \"\"\"\n        extract_weather_in_range will pull hourly weather data for every day from start_date to\n        end_date (inclusive)\n\n        Steps:\n            1. Creates a date_range based on `self.start_date` and `self.end_date`\n            2. Makes api requests and constructs a pd.DataFrame for date_range\n        :return:\n        \"\"\"\n\n        if self.exclude_existing_data:\n            # Exclude weather data fetch for those days, for which we have 24 hours of data in the DB.\n            exclusion_list = self.get_exclusion_list()\n            date_range = [x.date() for x in pd.date_range(start=self.start_date.date(), end=self.end_date.date(), freq='D').to_pydatetime()]\n            date_range = [x for x in date_range if x not in exclusion_list]\n            if len(date_range) == 0:\n                return None\n        else:\n            # Fetch all data irrespective of whether they exist in the DF or not. Existing data gets overwritten\n            date_range = pd.date_range(start=self.start_date, end=self.end_date, freq='D')\n\n        return pd.concat([self.construct_daily_df(date) for date in date_range])\n\n\n    def get_exclusion_list(self, nhours:int=24) -> list:\n        \"\"\"\n        returns a list of dates for which we have 24 hours of data in the DF\n        \"\"\"\n\n        weather_data_qs = self.weather_df[self.site_id]\n        dates_list = [weather_data.timestamp.date() for weather_data in weather_data_qs]\n\n        # Dates for which we have 24 hours of data\n        exclusion_list = [key for key, group in groupby(dates_list) if len(list(group)) >= nhours]\n\n        return exclusion_list\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\ndef clean_weather_data(weather_filenm:str, method:str='linear', gap_limit:int=None, limit_direction:str='forward', save_filenm=None):\n    \"\"\"\n    Assumes weather_filenm is of the format ASHRAE provided\n    \n    :param weather_filenm: \n    :param method : {‘linear’, ‘time’, ‘index’, ‘values’, ‘nearest’, ‘zero’, ‘slinear’, ‘quadratic’, ‘cubic’, ‘barycentric’, ‘krogh’, ‘polynomial’, ‘spline’, ‘piecewise_polynomial’, ‘from_derivatives’, ‘pchip’, ‘akima’} \n    :param gap_limit: Maximum number of consecutive hours to fill. Must be greater than 0.\n    :param limit_direction: forward/backward/both\n    :return: \n    \"\"\"\n    df_weather_dtypes = {'site_id': np.int8, 'air_temperature': np.float32, 'cloud_coverage': np.float32, 'dew_temperature': np.float32,\n                     'precip_depth_1_hr': np.float32, 'sea_level_pressure': np.float32, 'wind_direction': np.float32, 'wind_speed': np.float32}\n\n    weather_df = pd.read_csv(weather_filenm, dtype=df_weather_dtypes, parse_dates=['timestamp'])\n    grouped_weather_df = weather_df.groupby('site_id').apply(lambda group: group.interpolate(method=method, limit=gap_limit, limit_direction=limit_direction))\n    \n    if 'cloud_coverage' in grouped_weather_df.columns:\n        grouped_weather_df['cloud_coverage'] = grouped_weather_df['cloud_coverage'].round(decimals=0).clip(0,8)\n        \n    grouped_weather_df.reset_index(inplace=True)\n    if save_filenm!=None:\n        grouped_weather_df.to_csv(save_filenm)\n\n    return grouped_weather_df\n\n\nweather_train_filenm = f'{data_folder}/input/weather_train.csv'\n\ninterp_weather_train_filenm = f'{data_folder}/fully_interpolated_weather_train.csv'\ngrouped_weather_train = clean_weather_data(weather_train_filenm, method='linear', gap_limit=None, save_filenm=interp_weather_train_filenm)\n\npartially_interp_weather_train_filenm = f'{data_folder}/partially_interpolated_weather_train.csv'\ngrouped_weather_train_with_gap_limit = clean_weather_data(weather_train_filenm, method='linear', gap_limit=3, save_filenm=partially_interp_weather_train_filenm)\n\nprint(grouped_weather_train.head(10))\nprint(grouped_weather_train_with_gap_limit.head(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# from darksky_weather_connector import *\n\nlocations = {\n                0: {'name': 'Orlando', 'lat':28.538336, 'lon':-81.379234},\n                1: {'name': 'London', 'lat':51.507351, 'lon':-0.127758},\n                2: {'name': 'Phoenix', 'lat':33.448376, 'lon':-112.074036},\n                3: {'name': 'DC', 'lat':38.907192, 'lon':-77.036873},\n                4: {'name': 'San Francisco', 'lat':37.774929, 'lon':-122.419418},\n                5: {'name': 'Loughborough', 'lat':52.770771, 'lon':-1.204350},\n                6: {'name': 'Philadelphia', 'lat':39.952583, 'lon':-75.165222},\n                7: {'name': 'Montreal', 'lat':45.501690, 'lon':-73.567253},\n                8: {'name': 'Orlando', 'lat':28.538336, 'lon':-81.379234},\n                9: {'name': 'Austin', 'lat':30.267153, 'lon':-97.743057},\n                10: {'name': 'Las Vegas', 'lat':36.169941, 'lon':-115.139832},\n                11: {'name': 'Toronto', 'lat':43.653225, 'lon':-79.383186},\n                12: {'name': 'Dublin', 'lat':53.349804, 'lon':-6.260310},\n                13: {'name': 'Minneapolis', 'lat':44.977753, 'lon':-93.265015},\n                14: {'name': 'Charlottesville', 'lat':38.029305, 'lon':-78.476677},\n                15: {'name': 'Toronto', 'lat':43.653225, 'lon':-79.383186}\n            }\n\nAPI_KEY = 'GET YOUR OWN: https://darksky.net/dev/register'\nTRAINING_YEAR = 2016\nTARGET_SITES = [2]\n\nfor siteid,location in locations.items():\n    if siteid in TARGET_SITES:\n        SITE_DATA = {'site_id': siteid,\n                     'start_datetime':datetime.strptime(f'{TRAINING_YEAR}-01-01', '%Y-%m-%d'),\n                     'end_datetime':datetime.strptime(f'{TRAINING_YEAR+1}-01-01', '%Y-%m-%d'),\n                     'latitude':location['lat'],\n                     'longitude':location['lon'],\n                     'pv':True,\n                     }\n        wd = fetch_historic_weather_data(SITE_DATA, api_key=API_KEY, file_path=data_folder)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"siteid = 2\n# timezone = 'America/New_York'\n# timezone = 'Europe/London'\ntimezone = 'America/Phoenix'\nds_data = pd.read_csv(f'{data_folder}/input/site{siteid}_darksky_weather.csv', parse_dates=['time'])\n# match darksky cols to training cols\nds_data = ds_data.rename(columns={'time':'timestamp','temperature':'air_temperature', 'cloudCover':'cloud_coverage', 'windSpeed':'wind_speed'})\ntraining_data = pd.read_csv(f'{data_folder}/input/fully_interpolated_weather_train.csv', parse_dates=['timestamp'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ds_data['timestamp'] =  pd.to_datetime(ds_data['timestamp'], utc=True)\nd_data = ds_data.set_index('timestamp')\n# F to C\nd_data['air_temperature'] = (d_data['air_temperature'] - 32) * 5.0/9.0 \n\n# quantify the difference between the two ts\nrmse = {}\nDST_TIMES = [\"2016-03-13 02:00:00\", \"2016-03-27 01:00:00\"]\nfor n in range(15):\n    t_data = training_data.loc[training_data['site_id']==n].set_index('timestamp')\n    t_data = t_data.drop([pd.Timestamp(a) for a in DST_TIMES if a in t_data.index])\n    t_data = t_data.tz_localize(timezone, ambiguous='NaT')\n    df= pd.merge(d_data[['air_temperature']],t_data[['air_temperature']], how='inner',on=['timestamp'])\n    # df= pd.concat([d_data[['air_temperature']],t_data[['air_temperature']]],axis=1)\n    rmse[n] = ((df.air_temperature_x - df.air_temperature_y) ** 2).mean() ** .5\n\n# print the results\nprint(rmse)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}