From f3595f790a637055fe0ff6ebb3558037badcaa8b Mon Sep 17 00:00:00 2001 From: Johann Kellerman Date: Thu, 3 Nov 2016 04:34:12 +0200 Subject: [PATCH] Async version of Yr.no (#4158) * initial * feedback * More feedback. Still need to fix match_url * url_match * split_lines --- homeassistant/components/sensor/yr.py | 181 +++++++++++++++----------- pylintrc | 1 + tests/components/sensor/test_yr.py | 108 +++++++-------- tests/test_util/aiohttp.py | 36 ++++- 4 files changed, 186 insertions(+), 140 deletions(-) diff --git a/homeassistant/components/sensor/yr.py b/homeassistant/components/sensor/yr.py index 6fe6b429990d..05412131679d 100644 --- a/homeassistant/components/sensor/yr.py +++ b/homeassistant/components/sensor/yr.py @@ -4,9 +4,13 @@ Support for Yr.no weather service. For more details about this platform, please refer to the documentation at https://home-assistant.io/components/sensor.yr/ """ +import asyncio +from datetime import timedelta import logging +from xml.parsers.expat import ExpatError -import requests +import async_timeout +from aiohttp.web import HTTPException import voluptuous as vol import homeassistant.helpers.config_validation as cv @@ -15,8 +19,10 @@ from homeassistant.const import ( CONF_LATITUDE, CONF_LONGITUDE, CONF_ELEVATION, CONF_MONITORED_CONDITIONS, ATTR_ATTRIBUTION) from homeassistant.helpers.entity import Entity +from homeassistant.helpers.event import async_track_point_in_utc_time from homeassistant.util import dt as dt_util + REQUIREMENTS = ['xmltodict==0.10.2'] _LOGGER = logging.getLogger(__name__) @@ -43,15 +49,15 @@ SENSOR_TYPES = { } PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ - vol.Optional(CONF_MONITORED_CONDITIONS, default=[]): - [vol.In(SENSOR_TYPES.keys())], + vol.Optional(CONF_MONITORED_CONDITIONS, default=['symbol']): vol.All( + cv.ensure_list, vol.Length(min=1), [vol.In(SENSOR_TYPES.keys())]), vol.Optional(CONF_LATITUDE): cv.latitude, vol.Optional(CONF_LONGITUDE): cv.longitude, vol.Optional(CONF_ELEVATION): vol.Coerce(int), }) -def setup_platform(hass, config, add_devices, discovery_info=None): +def async_setup_platform(hass, config, async_add_devices, discovery_info=None): """Setup the Yr.no sensor.""" latitude = config.get(CONF_LATITUDE, hass.config.latitude) longitude = config.get(CONF_LONGITUDE, hass.config.longitude) @@ -63,32 +69,25 @@ def setup_platform(hass, config, add_devices, discovery_info=None): coordinates = dict(lat=latitude, lon=longitude, msl=elevation) - weather = YrData(coordinates) - dev = [] for sensor_type in config[CONF_MONITORED_CONDITIONS]: - dev.append(YrSensor(sensor_type, weather)) + dev.append(YrSensor(sensor_type)) + yield from async_add_devices(dev) - # add symbol as default sensor - if len(dev) == 0: - dev.append(YrSensor("symbol", weather)) - add_devices(dev) + weather = YrData(hass, coordinates, dev) + yield from weather.async_update() class YrSensor(Entity): """Representation of an Yr.no sensor.""" - def __init__(self, sensor_type, weather): + def __init__(self, sensor_type): """Initialize the sensor.""" self.client_name = 'yr' self._name = SENSOR_TYPES[sensor_type][0] self.type = sensor_type self._state = None - self._weather = weather self._unit_of_measurement = SENSOR_TYPES[self.type][1] - self._update = None - - self.update() @property def name(self): @@ -100,6 +99,11 @@ class YrSensor(Entity): """Return the state of the device.""" return self._state + @property + def should_poll(self): # pylint: disable=no-self-use + """No polling needed.""" + return False + @property def entity_picture(self): """Weather symbol if type is symbol.""" @@ -120,78 +124,97 @@ class YrSensor(Entity): """Return the unit of measurement of this entity, if any.""" return self._unit_of_measurement - def update(self): - """Get the latest data from yr.no and updates the states.""" - now = dt_util.utcnow() - # Check if data should be updated - if self._update is not None and now <= self._update: - return - - self._weather.update() - - # Find sensor - for time_entry in self._weather.data['product']['time']: - valid_from = dt_util.parse_datetime(time_entry['@from']) - valid_to = dt_util.parse_datetime(time_entry['@to']) - - loc_data = time_entry['location'] - - if self.type not in loc_data or now >= valid_to: - continue - - self._update = valid_to - - if self.type == 'precipitation' and valid_from < now: - self._state = loc_data[self.type]['@value'] - break - elif self.type == 'symbol' and valid_from < now: - self._state = loc_data[self.type]['@number'] - break - elif self.type in ('temperature', 'pressure', 'humidity', - 'dewpointTemperature'): - self._state = loc_data[self.type]['@value'] - break - elif self.type in ('windSpeed', 'windGust'): - self._state = loc_data[self.type]['@mps'] - break - elif self.type == 'windDirection': - self._state = float(loc_data[self.type]['@deg']) - break - elif self.type in ('fog', 'cloudiness', 'lowClouds', - 'mediumClouds', 'highClouds'): - self._state = loc_data[self.type]['@percent'] - break - class YrData(object): """Get the latest data and updates the states.""" - def __init__(self, coordinates): + def __init__(self, hass, coordinates, devices): """Initialize the data object.""" self._url = 'http://api.yr.no/weatherapi/locationforecast/1.9/?' \ 'lat={lat};lon={lon};msl={msl}'.format(**coordinates) - self._nextrun = None + self.devices = devices self.data = {} - self.update() + self.hass = hass - def update(self): + @asyncio.coroutine + def async_update(self): """Get the latest data from yr.no.""" - # Check if new will be available - if self._nextrun is not None and dt_util.utcnow() <= self._nextrun: - return - try: - with requests.Session() as sess: - response = sess.get(self._url) - except requests.RequestException: - return - if response.status_code != 200: - return - data = response.text + def try_again(err: str): + """Schedule again later.""" + _LOGGER.warning('Retrying in 15 minutes: %s', err) + nxt = dt_util.utcnow() + timedelta(minutes=15) + async_track_point_in_utc_time(self.hass, self.async_update, nxt) - import xmltodict - self.data = xmltodict.parse(data)['weatherdata'] - model = self.data['meta']['model'] - if '@nextrun' not in model: - model = model[0] - self._nextrun = dt_util.parse_datetime(model['@nextrun']) + try: + with async_timeout.timeout(10, loop=self.hass.loop): + resp = yield from self.hass.websession.get(self._url) + if resp.status != 200: + try_again('{} returned {}'.format(self._url, resp.status)) + return + text = yield from resp.text() + self.hass.loop.create_task(resp.release()) + except asyncio.TimeoutError as err: + try_again(err) + return + except HTTPException as err: + resp.close() + try_again(err) + return + + try: + import xmltodict + self.data = xmltodict.parse(text)['weatherdata'] + model = self.data['meta']['model'] + if '@nextrun' not in model: + model = model[0] + next_run = dt_util.parse_datetime(model['@nextrun']) + except (ExpatError, IndexError) as err: + try_again(err) + return + + # Schedule next execution + async_track_point_in_utc_time(self.hass, self.async_update, next_run) + + now = dt_util.utcnow() + + tasks = [] + # Update all devices + for dev in self.devices: + # Find sensor + for time_entry in self.data['product']['time']: + valid_from = dt_util.parse_datetime(time_entry['@from']) + valid_to = dt_util.parse_datetime(time_entry['@to']) + + loc_data = time_entry['location'] + + if dev.type not in loc_data or now >= valid_to: + continue + + if dev.type == 'precipitation' and valid_from < now: + new_state = loc_data[dev.type]['@value'] + break + elif dev.type == 'symbol' and valid_from < now: + new_state = loc_data[dev.type]['@number'] + break + elif dev.type in ('temperature', 'pressure', 'humidity', + 'dewpointTemperature'): + new_state = loc_data[dev.type]['@value'] + break + elif dev.type in ('windSpeed', 'windGust'): + new_state = loc_data[dev.type]['@mps'] + break + elif dev.type == 'windDirection': + new_state = float(loc_data[dev.type]['@deg']) + break + elif dev.type in ('fog', 'cloudiness', 'lowClouds', + 'mediumClouds', 'highClouds'): + new_state = loc_data[dev.type]['@percent'] + break + + # pylint: disable=protected-access + if new_state != dev._state: + dev._state = new_state + tasks.append(dev.async_update_ha_state()) + + yield from asyncio.gather(*tasks, loop=self.hass.loop) diff --git a/pylintrc b/pylintrc index 627524fc2409..710f392e95f7 100644 --- a/pylintrc +++ b/pylintrc @@ -27,6 +27,7 @@ disable= too-many-instance-attributes, too-many-locals, too-many-public-methods, + too-many-return-statements, too-many-statements, too-few-public-methods, diff --git a/tests/components/sensor/test_yr.py b/tests/components/sensor/test_yr.py index 7df47a996883..8d54037a3799 100644 --- a/tests/components/sensor/test_yr.py +++ b/tests/components/sensor/test_yr.py @@ -1,77 +1,69 @@ """The tests for the Yr sensor platform.""" +import asyncio from datetime import datetime from unittest.mock import patch -from homeassistant.bootstrap import setup_component +from homeassistant.bootstrap import async_setup_component import homeassistant.util.dt as dt_util -from tests.common import get_test_home_assistant, load_fixture +from tests.common import assert_setup_component, load_fixture -class TestSensorYr: - """Test the Yr sensor.""" +NOW = datetime(2016, 6, 9, 1, tzinfo=dt_util.UTC) - def setup_method(self): - """Setup things to be run when tests are started.""" - self.hass = get_test_home_assistant() - self.hass.config.latitude = 32.87336 - self.hass.config.longitude = 117.22743 - def teardown_method(self): - """Stop everything that was started.""" - self.hass.stop() +@asyncio.coroutine +def test_default_setup(hass, aioclient_mock): + """Test the default setup.""" + aioclient_mock.get('http://api.yr.no/weatherapi/locationforecast/1.9/', + text=load_fixture('yr.no.json')) + config = {'platform': 'yr', + 'elevation': 0} + hass.allow_pool = True + with patch('homeassistant.components.sensor.yr.dt_util.utcnow', + return_value=NOW), assert_setup_component(1): + yield from async_setup_component(hass, 'sensor', {'sensor': config}) - def test_default_setup(self, requests_mock): - """Test the default setup.""" - requests_mock.get('http://api.yr.no/weatherapi/locationforecast/1.9/', - text=load_fixture('yr.no.json')) - now = datetime(2016, 6, 9, 1, tzinfo=dt_util.UTC) + state = hass.states.get('sensor.yr_symbol') - with patch('homeassistant.components.sensor.yr.dt_util.utcnow', - return_value=now): - assert setup_component(self.hass, 'sensor', { - 'sensor': {'platform': 'yr', - 'elevation': 0}}) + assert state.state == '3' + assert state.attributes.get('unit_of_measurement') is None - state = self.hass.states.get('sensor.yr_symbol') - assert '3' == state.state - assert state.state.isnumeric() - assert state.attributes.get('unit_of_measurement') is None +@asyncio.coroutine +def test_custom_setup(hass, aioclient_mock): + """Test a custom setup.""" + aioclient_mock.get('http://api.yr.no/weatherapi/locationforecast/1.9/', + text=load_fixture('yr.no.json')) - def test_custom_setup(self, requests_mock): - """Test a custom setup.""" - requests_mock.get('http://api.yr.no/weatherapi/locationforecast/1.9/', - text=load_fixture('yr.no.json')) - now = datetime(2016, 6, 9, 1, tzinfo=dt_util.UTC) + config = {'platform': 'yr', + 'elevation': 0, + 'monitored_conditions': [ + 'pressure', + 'windDirection', + 'humidity', + 'fog', + 'windSpeed']} + hass.allow_pool = True + with patch('homeassistant.components.sensor.yr.dt_util.utcnow', + return_value=NOW), assert_setup_component(1): + yield from async_setup_component(hass, 'sensor', {'sensor': config}) - with patch('homeassistant.components.sensor.yr.dt_util.utcnow', - return_value=now): - assert setup_component(self.hass, 'sensor', { - 'sensor': {'platform': 'yr', - 'elevation': 0, - 'monitored_conditions': [ - 'pressure', - 'windDirection', - 'humidity', - 'fog', - 'windSpeed']}}) + state = hass.states.get('sensor.yr_pressure') + assert state.attributes.get('unit_of_measurement') == 'hPa' + assert state.state == '1009.3' - state = self.hass.states.get('sensor.yr_pressure') - assert 'hPa' == state.attributes.get('unit_of_measurement') - assert '1009.3' == state.state + state = hass.states.get('sensor.yr_wind_direction') + assert state.attributes.get('unit_of_measurement') == '°' + assert state.state == '103.6' - state = self.hass.states.get('sensor.yr_wind_direction') - assert '°' == state.attributes.get('unit_of_measurement') - assert '103.6' == state.state + state = hass.states.get('sensor.yr_humidity') + assert state.attributes.get('unit_of_measurement') == '%' + assert state.state == '55.5' - state = self.hass.states.get('sensor.yr_humidity') - assert '%' == state.attributes.get('unit_of_measurement') - assert '55.5' == state.state + state = hass.states.get('sensor.yr_fog') + assert state.attributes.get('unit_of_measurement') == '%' + assert state.state == '0.0' - state = self.hass.states.get('sensor.yr_fog') - assert '%' == state.attributes.get('unit_of_measurement') - assert '0.0' == state.state - - state = self.hass.states.get('sensor.yr_wind_speed') - assert 'm/s', state.attributes.get('unit_of_measurement') - assert '3.5' == state.state + state = hass.states.get('sensor.yr_wind_speed') + assert state.attributes.get('unit_of_measurement') == 'm/s' + assert state.state == '3.5' diff --git a/tests/test_util/aiohttp.py b/tests/test_util/aiohttp.py index 9de94b50df85..7cf0fe9378d2 100644 --- a/tests/test_util/aiohttp.py +++ b/tests/test_util/aiohttp.py @@ -4,6 +4,7 @@ from contextlib import contextmanager import functools import json as _json from unittest import mock +from urllib.parse import urlparse, parse_qs class AiohttpClientMocker: @@ -57,7 +58,8 @@ class AiohttpClientMocker: return len(self.mock_calls) @asyncio.coroutine - def match_request(self, method, url, *, auth=None): + def match_request(self, method, url, *, auth=None): \ + # pylint: disable=unused-variable """Match a request against pre-registered requests.""" for response in self._mocks: if response.match_request(method, url): @@ -74,13 +76,41 @@ class AiohttpClientMockResponse: def __init__(self, method, url, status, response): """Initialize a fake response.""" self.method = method - self.url = url + self._url = url + self._url_parts = (None if hasattr(url, 'search') + else urlparse(url.lower())) self.status = status self.response = response def match_request(self, method, url): """Test if response answers request.""" - return method == self.method and url == self.url + if method.lower() != self.method.lower(): + return False + + # regular expression matching + if self._url_parts is None: + return self._url.search(url) is not None + + req = urlparse(url.lower()) + + if self._url_parts.scheme and req.scheme != self._url_parts.scheme: + return False + if self._url_parts.netloc and req.netloc != self._url_parts.netloc: + return False + if (req.path or '/') != (self._url_parts.path or '/'): + return False + + # Ensure all query components in matcher are present in the request + request_qs = parse_qs(req.query) + matcher_qs = parse_qs(self._url_parts.query) + for key, vals in matcher_qs.items(): + for val in vals: + try: + request_qs.get(key, []).remove(val) + except ValueError: + return False + + return True @asyncio.coroutine def read(self):