From acfa74ce0d0e6eefe51a2d72e1d3ed161c7f015f Mon Sep 17 00:00:00 2001 From: Joshua Wise Date: Thu, 3 Jun 2021 03:41:50 -0400 Subject: [PATCH 1/6] update to new IBM/TWC API --- weather/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/weather/__init__.py b/weather/__init__.py index 1291fe5..bd4387a 100644 --- a/weather/__init__.py +++ b/weather/__init__.py @@ -17,7 +17,8 @@ if os.environ.get('HONEYCOMB_KEY'): HoneyMiddleware(app, db_events = True) auth_internal = os.environ['REBBLE_AUTH_URL_INT'] -ibm_root = os.environ['IBM_API_ROOT'] +ibm_root = os.environ.get('IBM_API_ROOT', 'https://api.weather.com') +ibm_key = os.environ['IBM_API_KEY'] http_protocol = os.environ.get('HTTP_PROTOCOL', 'https') # For some reason, the standard float converter rejects negative numbers @@ -62,11 +63,11 @@ def geocode(latitude, longitude): beeline.add_context_field("weather.language", language) beeline.add_context_field("weather.units", units) - forecast_req = requests.get(f"{ibm_root}/geocode/{latitude}/{longitude}/forecast/daily/7day.json?language={language}&units={units}") + forecast_req = requests.get(f"{ibm_root}/v3/wx/forecast/daily/7day?geocode={latitude},{longitude}&format=json&units={units}&language={language}&apiKey={ibm_key}") forecast_req.raise_for_status() forecast = forecast_req.json() - current_req = requests.get(f"{ibm_root}/geocode/{latitude}/{longitude}/observations.json?language={language}&units={units}") + current_req = requests.get(f"{ibm_root}/v1/geocode/{latitude}/{longitude}/observations.json?language={language}&units={units}&apiKey={ibm_key}") current_req.raise_for_status() current = current_req.json() observation = current['observation'] From 80f225c7a8c0a889436947c93a9b66e90431f385 Mon Sep 17 00:00:00 2001 From: Joshua Wise Date: Thu, 3 Jun 2021 03:58:24 -0400 Subject: [PATCH 2/6] Bump MarkupSafe, darn it. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50fbb9e..ffbcf17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ Flask==1.0.2 idna==2.7 itsdangerous==0.24 Jinja2==2.10 -MarkupSafe==1.0 +MarkupSafe==1.1.1 requests==2.19.1 urllib3==1.23 Werkzeug==0.14.1 From 193fcfa290b1e6e79752afd631d84968819c9388 Mon Sep 17 00:00:00 2001 From: Joshua Wise Date: Mon, 7 Jun 2021 20:33:25 -0400 Subject: [PATCH 3/6] Convert new IBM API format to old IBM API format, roughly. --- weather/__init__.py | 128 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/weather/__init__.py b/weather/__init__.py index bd4387a..13b0361 100644 --- a/weather/__init__.py +++ b/weather/__init__.py @@ -33,6 +33,130 @@ app.url_map.converters['float'] = SignedFloatConverter def format_date(date): return date.strftime("%Y-%m-%dT%H:%M:%S") +def day_night_for_lang(day_night, language): + # It's not clear that the Weather Channel API *ever* did this, at least, + # as long as Rebble has been involved. But yet, the Android app seems + # to care about this, according to a decompilation. + if day_night == 'D' and language[0:2] == 'de': + return 'T' + return day_night + +def mangle_daypart(language, units, day, daypart): + return { + 'fcst_valid': day['validTimeUtc'], # THIS IS NOT QUITE CORRECT + 'fcst_valid_local': day['validTimeLocal'], + 'day_ind': day_night_for_lang(daypart['dayOrNight'], language), + 'thunder_enum': daypart['thunderIndex'], # probably? + 'daypart_name': daypart['daypartName'], # "Tonight" + 'long_daypart_name': daypart['daypartName'], # THIS IS NOT QUITE CORRECT: should be "Thursday night" + 'alt_daypart_name': daypart['daypartName'], # "Tonight" + # 'num' is an enumerator, not used by app + 'thunder_enum_phrase': daypart['thunderCategory'], + 'temp': daypart['temperature'], + 'hi': daypart['temperatureHeatIndex'], + 'wc': daypart['temperatureWindChill'], + 'pop': daypart['precipChance'], # XXX: is this correct? + 'icon_extd': daypart['iconCodeExtend'], + 'icon_code': daypart['iconCode'], + # 'wxman' not used by app + 'phrase_12char': daypart['wxPhraseShort'], + 'phrase_22char': daypart['wxPhraseLong'], + 'phrase_32char': daypart['wxPhraseLong'], + # 'subphrase_pt1' not used by app + # 'subphrase_pt2' not used by app + # 'subphrase_pt3' not used by app + 'precip_type': daypart['precipType'], + 'rh': daypart['relativeHumidity'], + 'wspd': daypart['windSpeed'], + 'wdir': daypart['windDirection'], + 'wdir_cardinal': daypart['windDirectionCardinal'], + 'clds': daypart['cloudCover'], + # 'pop_phrase' not used by app + 'temp_phrase': f"{'High' if daypart['dayOrNight'] == 'D' else 'Low'} {daypart['temperature']}{'F' if units == 'e' else 'C'}.", # XXX: i18n + # 'accumulation_phrase' not used by app + 'wind_phrase': daypart['windPhrase'], + 'shortcast': daypart['wxPhraseLong'], + 'narrative': daypart['narrative'], + 'qpf': daypart['qpf'], + 'snow_qpf': daypart['qpfSnow'], + 'snow_range': daypart['snowRange'], + # 'snow_phrase' not used by app + # 'snow_code' not used by app + # 'vocal_key' not used by app + 'qualifier_code': daypart['qualifierCode'], + 'qualifier': daypart['qualifierPhrase'], + 'uv_index_raw': daypart['uvIndex'], # THIS IS NOT QUITE CORRECT, 9.7 vs 10 + 'uv_index': daypart['uvIndex'], + # 'uv_warning' not used by app + 'uv_desc': daypart['uvDescription'], + # 'golf_index' not used by app + # 'golf_category' not used by app + 'golf_category': 'boring sports' + } + +def new_ibm_to_old_ibm(language, units, forecast): + # Invert the bizarre IBM dictionary-of-arrays. + forecast_inv = [] + for k in forecast: + if k == 'daypart': + continue + + for day, v in enumerate(forecast[k]): + if day >= len(forecast_inv): + forecast_inv.append({}) + forecast_inv[day][k] = v + + # The day parts are even more brain damaged. Invert them, too. + if len(forecast['daypart']) != 1: + raise ValueError(f"forecast['daypart'] had wrong number of values {len(forecast['daypart'])}") + + for k in forecast['daypart'][0]: + for halfday, v in enumerate(forecast['daypart'][0][k]): + # v == None? + day = halfday // 2 + dn = "day" if ((halfday % 2) == 0) else "night" + shouldbe = "D" if ((halfday % 2) == 0) else "N" + if k == "dayOrNight" and v != shouldbe and v != None: + raise ValueError(f"halfday {halfday} should be {dn}, but dayOrNight is {v}") + if day >= len(forecast_inv): + # There is no day to append this daypart to. + continue + if forecast_inv[day].get(dn) is None and v == None: + continue + forecast_inv[day][dn] = forecast_inv[day].get(dn, {}) + forecast_inv[day][dn][k] = v + + return [{ + 'class': 'fod_long_range_daily', + 'expire_time_gmt': day['expirationTimeUtc'], + 'fcst_valid': day['validTimeUtc'], + 'fcst_valid_local': day['validTimeLocal'], + # 'num' is an enumerator, not used by app + 'max_temp': day['temperatureMax'], + 'min_temp': day['temperatureMin'], + # 'torcon' not used by app + # 'stormcon' not used by app + # 'blurb' not used by app + # 'blurb_author' not used by app + 'lunar_phase_day': day['moonPhaseDay'], + 'dow': day['dayOfWeek'], + 'lunar_phase': day['moonPhase'], + 'lunar_phase_code': day['moonPhaseCode'], + 'sunrise': day['sunriseTimeLocal'], + 'sunset': day['sunsetTimeLocal'], + 'moonrise': day['moonriseTimeLocal'], + 'moonset': day['moonsetTimeLocal'], + # qualifier_code is a property of a daypart now, not used by app + # qualifier is a property of a daypart now, not used by app + 'qpf': day['qpf'], + 'snow_qpf': day['qpfSnow'], + # snow_range is a property of a daypart now, not used by app + # snow_phrase not used by app + # snow_code not used by app + **({'day': mangle_daypart(language, units, day, day['day'])} if day.get('day') else {}), + **({'night': mangle_daypart(language, units, day, day['night'])} if day.get('night') else {}), + } for day in forecast_inv] + class HTTPPaymentRequired(HTTPException): def __init__(self, description=None, response=None): @@ -67,6 +191,8 @@ def geocode(latitude, longitude): forecast_req.raise_for_status() forecast = forecast_req.json() + old_style_fcstdaily7 = { 'forecasts': new_ibm_to_old_ibm(language, units, forecast) } + current_req = requests.get(f"{ibm_root}/v1/geocode/{latitude}/{longitude}/observations.json?language={language}&units={units}&apiKey={ibm_key}") current_req.raise_for_status() current = current_req.json() @@ -125,7 +251,7 @@ def geocode(latitude, longitude): return jsonify( fcstdaily7={ 'errors': False, - 'data': forecast, + 'data': old_style_fcstdaily7, }, conditions={ 'errors': False, From c1f996c675e5247444885f1b8317286f2b3f3f08 Mon Sep 17 00:00:00 2001 From: Joshua Wise Date: Mon, 7 Jun 2021 21:00:38 -0400 Subject: [PATCH 4/6] add context field for API version for honeycomb lookups --- weather/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/weather/__init__.py b/weather/__init__.py index 13b0361..84a3fde 100644 --- a/weather/__init__.py +++ b/weather/__init__.py @@ -186,6 +186,7 @@ def geocode(latitude, longitude): beeline.add_context_field("weather.language", language) beeline.add_context_field("weather.units", units) + beeline.add_context_field("weather.api_version", "v2") forecast_req = requests.get(f"{ibm_root}/v3/wx/forecast/daily/7day?geocode={latitude},{longitude}&format=json&units={units}&language={language}&apiKey={ibm_key}") forecast_req.raise_for_status() From 7e183d48b3f032ce99323241023d0af9e0613f79 Mon Sep 17 00:00:00 2001 From: Joshua Wise Date: Mon, 26 Jul 2021 00:08:52 -0400 Subject: [PATCH 5/6] our new API key does not allow 7-day. it does allow 15-day. go figure. --- weather/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weather/__init__.py b/weather/__init__.py index 84a3fde..d78e02d 100644 --- a/weather/__init__.py +++ b/weather/__init__.py @@ -188,7 +188,7 @@ def geocode(latitude, longitude): beeline.add_context_field("weather.units", units) beeline.add_context_field("weather.api_version", "v2") - forecast_req = requests.get(f"{ibm_root}/v3/wx/forecast/daily/7day?geocode={latitude},{longitude}&format=json&units={units}&language={language}&apiKey={ibm_key}") + forecast_req = requests.get(f"{ibm_root}/v3/wx/forecast/daily/15day?geocode={latitude},{longitude}&format=json&units={units}&language={language}&apiKey={ibm_key}") forecast_req.raise_for_status() forecast = forecast_req.json() From 9bce9e812837e8c8801c950ee74112525e97e3fd Mon Sep 17 00:00:00 2001 From: Joshua Wise Date: Mon, 26 Jul 2021 01:27:53 -0400 Subject: [PATCH 6/6] update to use the new IBM observations API --- weather/__init__.py | 90 +++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/weather/__init__.py b/weather/__init__.py index d78e02d..c799086 100644 --- a/weather/__init__.py +++ b/weather/__init__.py @@ -194,56 +194,74 @@ def geocode(latitude, longitude): old_style_fcstdaily7 = { 'forecasts': new_ibm_to_old_ibm(language, units, forecast) } - current_req = requests.get(f"{ibm_root}/v1/geocode/{latitude}/{longitude}/observations.json?language={language}&units={units}&apiKey={ibm_key}") + current_req = requests.get(f"{ibm_root}/v3/wx/observations/current?geocode={latitude},{longitude}&language={language}&units={units}&format=json&apiKey={ibm_key}") current_req.raise_for_status() current = current_req.json() - observation = current['observation'] old_style_conditions = { - 'metadata': current['metadata'], + 'metadata': { + 'language': language, + 'transaction_id': 'lol!', + 'version': '1', + 'latitude': latitude, + 'longitude': longitude, + 'units': units, + 'expire_time_gmt': current['expirationTimeUtc'], + 'status_code': 200, + }, 'observation': { - 'class': observation['class'], - 'expire_time_gmt': observation['expire_time_gmt'], - 'obs_time': observation['valid_time_gmt'], + 'class': 'observation', + 'expire_time_gmt': current['expirationTimeUtc'], + 'obs_time': current['validTimeUtc'], # 'obs_time_local': we don't know. - 'wdir': observation['wdir'], - 'icon_code': observation['wx_icon'], - 'icon_extd': observation['icon_extd'], + 'wdir': current['windDirection'], + 'icon_code': current['iconCode'], + 'icon_extd': current['iconCodeExtend'], # sunrise: we don't know these, but we could yank them out of the forecast for today. # sunset - 'day_ind': observation['day_ind'], - 'uv_index': observation['uv_index'], + 'day_ind': current['dayOrNight'], + 'uv_index': current['uvIndex'], # uv_warning: I don't even know what this is. Apparently numeric. # wxman: ??? - 'obs_qualifier_code': observation['qualifier'], - 'ptend_code': observation['pressure_tend'], - 'dow': datetime.datetime.utcfromtimestamp(observation['valid_time_gmt']).strftime('%A'), - 'wdir_cardinal': observation['wdir_cardinal'], # sometimes this is "CALM", don't know if that's okay - 'uv_desc': observation['uv_desc'], + 'obs_qualifier_code': current['obsQualifierCode'], + 'ptend_code': current['pressureTendencyCode'], + 'dow': current['dayOfWeek'], + 'wdir_cardinal': current['windDirectionCardinal'], # sometimes this is "CALM", don't know if that's okay + 'uv_desc': current['uvDescription'], # I'm just guessing at how the three phrases map. - 'phrase_12char': observation['blunt_phrase'] or observation['wx_phrase'], - 'phrase_22char': observation['terse_phrase'] or observation['wx_phrase'], - 'phrase_32char': observation['wx_phrase'], - 'ptend_desc': observation['pressure_desc'], + 'phrase_12char': current['wxPhraseShort'], + 'phrase_22char': current['wxPhraseMedium'], + 'phrase_32char': current['wxPhraseLong'], + 'ptend_desc': current['pressureTendencyTrend'], # sky_cover: we don't seem to get a description of this? - 'clds': observation['clds'], - 'obs_qualifier_severity': observation['qualifier_svrty'], + 'clds': current['cloudCoverPhrase'], # old was 'CLR', new is 'Partly Cloudy' + 'obs_qualifier_severity': current['obsQualifierSeverity'], # vocal_key: we don't get one of these {'e': 'imperial', 'm': 'metric', 'h': 'uk_hybrid'}[units]: { - 'wspd': observation['wspd'], - 'gust': observation['gust'], - 'vis': observation['vis'], - # mslp: don't know what this is but it doesn't map to anything - 'altimeter': observation['pressure'], - 'temp': observation['temp'], - 'dewpt': observation['dewPt'], - 'rh': observation['rh'], - 'wc': observation['wc'], - 'hi': observation['heat_index'], - 'feels_like': observation['feels_like'], - # temp_change_24hour, temp_max_24hour, temp_min_24hour, pchange: don't get any of these - # {snow,precip}_{{1,6,24}hour,mtd,season,{2,3,7}day}: don't get these either - # ceiling, obs_qualifier_{100,50,32}char: or these. + 'wspd': current['windSpeed'], + 'gust': current['windGust'], + 'vis': current['visibility'], + 'mslp': current['pressureMeanSeaLevel'], + 'altimeter': current['pressureAltimeter'], + 'temp': current['temperature'], + 'dewpt': current['temperatureDewPoint'], + 'rh': current['relativeHumidity'], + 'wc': current['temperatureWindChill'], + 'hi': current['temperatureHeatIndex'], + 'feels_like': current['temperatureFeelsLike'], + 'temp_change_24hour': current['temperatureChange24Hour'], + 'temp_max_24hour': current['temperatureMax24Hour'], + 'temp_min_24hour': current['temperatureMin24Hour'], + 'pchange': current['pressureChange'], + 'snow_1hour': current['snow1Hour'], + 'snow_6hour': current['snow6Hour'], + 'snow_24hour': current['snow24Hour'], + 'precip_1hour': current['precip1Hour'], + 'precip_6hour': current['precip6Hour'], + 'precip_24hour': current['precip24Hour'], + # {snow,precip}_{mtd,season,{2,3,7}day}: don't get these either + 'ceiling': current['cloudCeiling'], + # obs_qualifier_{100,50,32}char: or these. # these are all now in their own request that you can pay extra to retrieve. }, }