Extract ruamel.yaml to util with secrets, lovelace ws decorators (#17958)

* Extract ruamel.yaml to util, ws decorators, secrets

* lint

* Extend SafeConstructor

Somehow my last commit is gone after rebase...

* lint

* Woof...

* Woof woof...

* Cleanup type hints

* Update homeassistant/scripts/check_config.py

* lint

* typing

homeassistant/components/lovelace/__init__.py

@@ -1,19 +1,17 @@
 """Lovelace UI."""
 import logging
 import uuid
-import os
-from os import O_CREAT, O_TRUNC, O_WRONLY
-from collections import OrderedDict
+from functools import wraps
 from typing import Dict, List, Union
 
 import voluptuous as vol
 
+import homeassistant.util.ruamel_yaml as yaml
 from homeassistant.components import websocket_api
 from homeassistant.exceptions import HomeAssistantError
 
 _LOGGER = logging.getLogger(__name__)
 DOMAIN = 'lovelace'
-REQUIREMENTS = ['ruamel.yaml==0.15.72']
 
 LOVELACE_CONFIG_FILE = 'ui-lovelace.yaml'
 JSON_TYPE = Union[List, Dict, str]  # pylint: disable=invalid-name
@@ -77,10 +75,6 @@ SCHEMA_DELETE_CARD = websocket_api.BASE_COMMAND_MESSAGE_SCHEMA.extend({
 })
 
 
-class WriteError(HomeAssistantError):
-    """Error writing the data."""
-
-
 class CardNotFoundError(HomeAssistantError):
     """Card not found in data."""
 
@@ -89,87 +83,25 @@ class ViewNotFoundError(HomeAssistantError):
     """View not found in data."""
 
 
-class UnsupportedYamlError(HomeAssistantError):
-    """Unsupported YAML."""
-
-
 class DuplicateIdError(HomeAssistantError):
     """Duplicate ID's."""
 
 
-def save_yaml(fname: str, data: JSON_TYPE):
-    """Save a YAML file."""
-    from ruamel.yaml import YAML
-    from ruamel.yaml.error import YAMLError
-    yaml = YAML(typ='rt')
-    yaml.indent(sequence=4, offset=2)
-    tmp_fname = fname + "__TEMP__"
-    try:
-        with open(os.open(tmp_fname, O_WRONLY | O_CREAT | O_TRUNC, 0o644),
-                  'w', encoding='utf-8') as temp_file:
-            yaml.dump(data, temp_file)
-        os.replace(tmp_fname, fname)
-    except YAMLError as exc:
-        _LOGGER.error(str(exc))
-        raise HomeAssistantError(exc)
-    except OSError as exc:
-        _LOGGER.exception('Saving YAML file %s failed: %s', fname, exc)
-        raise WriteError(exc)
-    finally:
-        if os.path.exists(tmp_fname):
-            try:
-                os.remove(tmp_fname)
-            except OSError as exc:
-                # If we are cleaning up then something else went wrong, so
-                # we should suppress likely follow-on errors in the cleanup
-                _LOGGER.error("YAML replacement cleanup failed: %s", exc)
-
-
-def _yaml_unsupported(loader, node):
-    raise UnsupportedYamlError(
-        'Unsupported YAML, you can not use {} in ui-lovelace.yaml'
-        .format(node.tag))
-
-
-def load_yaml(fname: str) -> JSON_TYPE:
-    """Load a YAML file."""
-    from ruamel.yaml import YAML
-    from ruamel.yaml.constructor import RoundTripConstructor
-    from ruamel.yaml.error import YAMLError
-
-    RoundTripConstructor.add_constructor(None, _yaml_unsupported)
-
-    yaml = YAML(typ='rt')
-
-    try:
-        with open(fname, encoding='utf-8') as conf_file:
-            # If configuration file is empty YAML returns None
-            # We convert that to an empty dict
-            return yaml.load(conf_file) or OrderedDict()
-    except YAMLError as exc:
-        _LOGGER.error("YAML error in %s: %s", fname, exc)
-        raise HomeAssistantError(exc)
-    except UnicodeDecodeError as exc:
-        _LOGGER.error("Unable to read file %s: %s", fname, exc)
-        raise HomeAssistantError(exc)
-
-
 def load_config(fname: str) -> JSON_TYPE:
     """Load a YAML file."""
-    return load_yaml(fname)
+    return yaml.load_yaml(fname, False)
 
 
-def migrate_config(fname: str) -> JSON_TYPE:
-    """Load a YAML file and adds id to views and cards if not present."""
-    config = load_yaml(fname)
-    # Check if all views and cards have a unique id or else add one
+def migrate_config(fname: str) -> None:
+    """Add id to views and cards if not present and check duplicates."""
+    config = yaml.load_yaml(fname, True)
     updated = False
     seen_card_ids = set()
     seen_view_ids = set()
     index = 0
     for view in config.get('views', []):
-        view_id = view.get('id')
-        if view_id is None:
+        view_id = str(view.get('id', ''))
+        if not view_id:
             updated = True
             view.insert(0, 'id', index,
                         comment="Automatically created id")
@@ -179,8 +111,8 @@ def migrate_config(fname: str) -> JSON_TYPE:
                     'ID `{}` has multiple occurances in views'.format(view_id))
             seen_view_ids.add(view_id)
         for card in view.get('cards', []):
-            card_id = card.get('id')
-            if card_id is None:
+            card_id = str(card.get('id', ''))
+            if not card_id:
                 updated = True
                 card.insert(0, 'id', uuid.uuid4().hex,
                             comment="Automatically created id")
@@ -192,48 +124,22 @@ def migrate_config(fname: str) -> JSON_TYPE:
                 seen_card_ids.add(card_id)
         index += 1
     if updated:
-        save_yaml(fname, config)
-    return config
-
-
-def object_to_yaml(data: JSON_TYPE) -> str:
-    """Create yaml string from object."""
-    from ruamel.yaml import YAML
-    from ruamel.yaml.error import YAMLError
-    from ruamel.yaml.compat import StringIO
-    yaml = YAML(typ='rt')
-    yaml.indent(sequence=4, offset=2)
-    stream = StringIO()
-    try:
-        yaml.dump(data, stream)
-        return stream.getvalue()
-    except YAMLError as exc:
-        _LOGGER.error("YAML error: %s", exc)
-        raise HomeAssistantError(exc)
-
-
-def yaml_to_object(data: str) -> JSON_TYPE:
-    """Create object from yaml string."""
-    from ruamel.yaml import YAML
-    from ruamel.yaml.error import YAMLError
-    yaml = YAML(typ='rt')
-    try:
-        return yaml.load(data)
-    except YAMLError as exc:
-        _LOGGER.error("YAML error: %s", exc)
-        raise HomeAssistantError(exc)
+        yaml.save_yaml(fname, config)
 
 
 def get_card(fname: str, card_id: str, data_format: str = FORMAT_YAML)\
         -> JSON_TYPE:
     """Load a specific card config for id."""
-    config = load_yaml(fname)
+    round_trip = data_format == FORMAT_YAML
+
+    config = yaml.load_yaml(fname, round_trip)
+
     for view in config.get('views', []):
         for card in view.get('cards', []):
-            if str(card.get('id')) != card_id:
+            if str(card.get('id', '')) != card_id:
                 continue
             if data_format == FORMAT_YAML:
-                return object_to_yaml(card)
+                return yaml.object_to_yaml(card)
             return card
 
     raise CardNotFoundError(
@@ -241,17 +147,17 @@ def get_card(fname: str, card_id: str, data_format: str = FORMAT_YAML)\
 
 
 def update_card(fname: str, card_id: str, card_config: str,
-                data_format: str = FORMAT_YAML):
+                data_format: str = FORMAT_YAML) -> None:
     """Save a specific card config for id."""
-    config = load_yaml(fname)
+    config = yaml.load_yaml(fname, True)
     for view in config.get('views', []):
         for card in view.get('cards', []):
-            if str(card.get('id')) != card_id:
+            if str(card.get('id', '')) != card_id:
                 continue
             if data_format == FORMAT_YAML:
-                card_config = yaml_to_object(card_config)
+                card_config = yaml.yaml_to_object(card_config)
             card.update(card_config)
-            save_yaml(fname, config)
+            yaml.save_yaml(fname, config)
             return
 
     raise CardNotFoundError(
@@ -259,39 +165,39 @@ def update_card(fname: str, card_id: str, card_config: str,
 
 
 def add_card(fname: str, view_id: str, card_config: str,
-             position: int = None, data_format: str = FORMAT_YAML):
+             position: int = None, data_format: str = FORMAT_YAML) -> None:
     """Add a card to a view."""
-    config = load_yaml(fname)
+    config = yaml.load_yaml(fname, True)
     for view in config.get('views', []):
-        if str(view.get('id')) != view_id:
+        if str(view.get('id', '')) != view_id:
             continue
         cards = view.get('cards', [])
         if data_format == FORMAT_YAML:
-            card_config = yaml_to_object(card_config)
+            card_config = yaml.yaml_to_object(card_config)
         if position is None:
             cards.append(card_config)
         else:
             cards.insert(position, card_config)
-        save_yaml(fname, config)
+        yaml.save_yaml(fname, config)
         return
 
     raise ViewNotFoundError(
         "View with ID: {} was not found in {}.".format(view_id, fname))
 
 
-def move_card(fname: str, card_id: str, position: int = None):
+def move_card(fname: str, card_id: str, position: int = None) -> None:
     """Move a card to a different position."""
     if position is None:
         raise HomeAssistantError('Position is required if view is not\
                                  specified.')
-    config = load_yaml(fname)
+    config = yaml.load_yaml(fname, True)
     for view in config.get('views', []):
         for card in view.get('cards', []):
-            if str(card.get('id')) != card_id:
+            if str(card.get('id', '')) != card_id:
                 continue
             cards = view.get('cards')
             cards.insert(position, cards.pop(cards.index(card)))
-            save_yaml(fname, config)
+            yaml.save_yaml(fname, config)
             return
 
     raise CardNotFoundError(
@@ -299,14 +205,14 @@ def move_card(fname: str, card_id: str, position: int = None):
 
 
 def move_card_view(fname: str, card_id: str, view_id: str,
-                   position: int = None):
+                   position: int = None) -> None:
     """Move a card to a different view."""
-    config = load_yaml(fname)
+    config = yaml.load_yaml(fname, True)
     for view in config.get('views', []):
-        if str(view.get('id')) == view_id:
+        if str(view.get('id', '')) == view_id:
             destination = view.get('cards')
         for card in view.get('cards'):
-            if str(card.get('id')) != card_id:
+            if str(card.get('id', '')) != card_id:
                 continue
             origin = view.get('cards')
             card_to_move = card
@@ -325,19 +231,19 @@ def move_card_view(fname: str, card_id: str, view_id: str,
     else:
         destination.insert(position, card_to_move)
 
-    save_yaml(fname, config)
+    yaml.save_yaml(fname, config)
 
 
-def delete_card(fname: str, card_id: str, position: int = None):
+def delete_card(fname: str, card_id: str, position: int = None) -> None:
     """Delete a card from view."""
-    config = load_yaml(fname)
+    config = yaml.load_yaml(fname, True)
     for view in config.get('views', []):
         for card in view.get('cards', []):
-            if str(card.get('id')) != card_id:
+            if str(card.get('id', '')) != card_id:
                 continue
             cards = view.get('cards')
             cards.pop(cards.index(card))
-            save_yaml(fname, config)
+            yaml.save_yaml(fname, config)
             return
 
     raise CardNotFoundError(
@@ -382,193 +288,100 @@ async def async_setup(hass, config):
     return True
 
 
+def handle_yaml_errors(func):
+    """Handle error with websocket calls."""
+    @wraps(func)
+    async def send_with_error_handling(hass, connection, msg):
+        error = None
+        try:
+            result = await func(hass, connection, msg)
+            message = websocket_api.result_message(
+                msg['id'], result
+            )
+        except FileNotFoundError:
+            error = ('file_not_found',
+                     'Could not find ui-lovelace.yaml in your config dir.')
+        except yaml.UnsupportedYamlError as err:
+            error = 'unsupported_error', str(err)
+        except yaml.WriteError as err:
+            error = 'write_error', str(err)
+        except CardNotFoundError as err:
+            error = 'card_not_found', str(err)
+        except ViewNotFoundError as err:
+            error = 'view_not_found', str(err)
+        except HomeAssistantError as err:
+            error = 'error', str(err)
+
+        if error is not None:
+            message = websocket_api.error_message(msg['id'], *error)
+
+        connection.send_message(message)
+
+    return send_with_error_handling
+
+
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_config(hass, connection, msg):
     """Send lovelace UI config over websocket config."""
-    error = None
-    try:
-        config = await hass.async_add_executor_job(
-            load_config, hass.config.path(LOVELACE_CONFIG_FILE))
-        message = websocket_api.result_message(
-            msg['id'], config
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except HomeAssistantError as err:
-        error = 'load_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        load_config, hass.config.path(LOVELACE_CONFIG_FILE))
 
 
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_migrate_config(hass, connection, msg):
     """Migrate lovelace UI config."""
-    error = None
-    try:
-        config = await hass.async_add_executor_job(
-            migrate_config, hass.config.path(LOVELACE_CONFIG_FILE))
-        message = websocket_api.result_message(
-            msg['id'], config
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except HomeAssistantError as err:
-        error = 'load_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        migrate_config, hass.config.path(LOVELACE_CONFIG_FILE))
 
 
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_get_card(hass, connection, msg):
     """Send lovelace card config over websocket config."""
-    error = None
-    try:
-        card = await hass.async_add_executor_job(
-            get_card, hass.config.path(LOVELACE_CONFIG_FILE), msg['card_id'],
-            msg.get('format', FORMAT_YAML))
-        message = websocket_api.result_message(
-            msg['id'], card
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except CardNotFoundError as err:
-        error = 'card_not_found', str(err)
-    except HomeAssistantError as err:
-        error = 'load_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        get_card, hass.config.path(LOVELACE_CONFIG_FILE), msg['card_id'],
+        msg.get('format', FORMAT_YAML))
 
 
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_update_card(hass, connection, msg):
     """Receive lovelace card config over websocket and save."""
-    error = None
-    try:
-        await hass.async_add_executor_job(
-            update_card, hass.config.path(LOVELACE_CONFIG_FILE),
-            msg['card_id'], msg['card_config'], msg.get('format', FORMAT_YAML))
-        message = websocket_api.result_message(
-            msg['id']
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except CardNotFoundError as err:
-        error = 'card_not_found', str(err)
-    except HomeAssistantError as err:
-        error = 'save_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        update_card, hass.config.path(LOVELACE_CONFIG_FILE),
+        msg['card_id'], msg['card_config'], msg.get('format', FORMAT_YAML))
 
 
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_add_card(hass, connection, msg):
     """Add new card to view over websocket and save."""
-    error = None
-    try:
-        await hass.async_add_executor_job(
-            add_card, hass.config.path(LOVELACE_CONFIG_FILE),
-            msg['view_id'], msg['card_config'], msg.get('position'),
-            msg.get('format', FORMAT_YAML))
-        message = websocket_api.result_message(
-            msg['id']
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except ViewNotFoundError as err:
-        error = 'view_not_found', str(err)
-    except HomeAssistantError as err:
-        error = 'save_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        add_card, hass.config.path(LOVELACE_CONFIG_FILE),
+        msg['view_id'], msg['card_config'], msg.get('position'),
+        msg.get('format', FORMAT_YAML))
 
 
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_move_card(hass, connection, msg):
     """Move card to different position over websocket and save."""
-    error = None
-    try:
-        if 'new_view_id' in msg:
-            await hass.async_add_executor_job(
-                move_card_view, hass.config.path(LOVELACE_CONFIG_FILE),
-                msg['card_id'], msg['new_view_id'], msg.get('new_position'))
-        else:
-            await hass.async_add_executor_job(
-                move_card, hass.config.path(LOVELACE_CONFIG_FILE),
-                msg['card_id'], msg.get('new_position'))
+    if 'new_view_id' in msg:
+        return await hass.async_add_executor_job(
+            move_card_view, hass.config.path(LOVELACE_CONFIG_FILE),
+            msg['card_id'], msg['new_view_id'], msg.get('new_position'))
 
-        message = websocket_api.result_message(
-            msg['id']
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except ViewNotFoundError as err:
-        error = 'view_not_found', str(err)
-    except CardNotFoundError as err:
-        error = 'card_not_found', str(err)
-    except HomeAssistantError as err:
-        error = 'save_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        move_card, hass.config.path(LOVELACE_CONFIG_FILE),
+        msg['card_id'], msg.get('new_position'))
 
 
 @websocket_api.async_response
+@handle_yaml_errors
 async def websocket_lovelace_delete_card(hass, connection, msg):
     """Delete card from lovelace over websocket and save."""
-    error = None
-    try:
-        await hass.async_add_executor_job(
-            delete_card, hass.config.path(LOVELACE_CONFIG_FILE),
-            msg['card_id'])
-        message = websocket_api.result_message(
-            msg['id']
-        )
-    except FileNotFoundError:
-        error = ('file_not_found',
-                 'Could not find ui-lovelace.yaml in your config dir.')
-    except UnsupportedYamlError as err:
-        error = 'unsupported_error', str(err)
-    except CardNotFoundError as err:
-        error = 'card_not_found', str(err)
-    except HomeAssistantError as err:
-        error = 'save_error', str(err)
-
-    if error is not None:
-        message = websocket_api.error_message(msg['id'], *error)
-
-    connection.send_message(message)
+    return await hass.async_add_executor_job(
+        delete_card, hass.config.path(LOVELACE_CONFIG_FILE),
+        msg['card_id'])

homeassistant/package_constraints.txt

@@ -11,6 +11,7 @@ pip>=8.0.3
 pytz>=2018.04
 pyyaml>=3.13,<4
 requests==2.20.0
+ruamel.yaml==0.15.72
 voluptuous==0.11.5
 voluptuous-serialize==2.0.0
 

homeassistant/scripts/check_config.py

@@ -30,7 +30,7 @@ _LOGGER = logging.getLogger(__name__)
 MOCKS = {
     'load': ("homeassistant.util.yaml.load_yaml", yaml.load_yaml),
     'load*': ("homeassistant.config.load_yaml", yaml.load_yaml),
-    'secrets': ("homeassistant.util.yaml._secret_yaml", yaml._secret_yaml),
+    'secrets': ("homeassistant.util.yaml.secret_yaml", yaml.secret_yaml),
 }
 SILENCE = (
     'homeassistant.scripts.check_config.yaml.clear_secret_cache',
@@ -198,7 +198,7 @@ def check(config_dir, secrets=False):
 
     if secrets:
         # Ensure !secrets point to the patched function
-        yaml.yaml.SafeLoader.add_constructor('!secret', yaml._secret_yaml)
+        yaml.yaml.SafeLoader.add_constructor('!secret', yaml.secret_yaml)
 
     try:
         hass = core.HomeAssistant()
@@ -223,7 +223,7 @@ def check(config_dir, secrets=False):
             pat.stop()
         if secrets:
             # Ensure !secrets point to the original function
-            yaml.yaml.SafeLoader.add_constructor('!secret', yaml._secret_yaml)
+            yaml.yaml.SafeLoader.add_constructor('!secret', yaml.secret_yaml)
         bootstrap.clear_secret_cache()
 
     return res

homeassistant/util/ruamel_yaml.py

@@ -0,0 +1,134 @@
+"""ruamel.yaml utility functions."""
+import logging
+import os
+from os import O_CREAT, O_TRUNC, O_WRONLY
+from collections import OrderedDict
+from typing import Union, List, Dict
+
+import ruamel.yaml
+from ruamel.yaml import YAML
+from ruamel.yaml.constructor import SafeConstructor
+from ruamel.yaml.error import YAMLError
+from ruamel.yaml.compat import StringIO
+
+from homeassistant.util.yaml import secret_yaml
+from homeassistant.exceptions import HomeAssistantError
+
+_LOGGER = logging.getLogger(__name__)
+
+JSON_TYPE = Union[List, Dict, str]  # pylint: disable=invalid-name
+
+
+class ExtSafeConstructor(SafeConstructor):
+    """Extended SafeConstructor."""
+
+
+class UnsupportedYamlError(HomeAssistantError):
+    """Unsupported YAML."""
+
+
+class WriteError(HomeAssistantError):
+    """Error writing the data."""
+
+
+def _include_yaml(constructor: SafeConstructor, node: ruamel.yaml.nodes.Node) \
+        -> JSON_TYPE:
+    """Load another YAML file and embeds it using the !include tag.
+
+    Example:
+        device_tracker: !include device_tracker.yaml
+    """
+    fname = os.path.join(os.path.dirname(constructor.name), node.value)
+    return load_yaml(fname, False)
+
+
+def _yaml_unsupported(constructor: SafeConstructor, node:
+                      ruamel.yaml.nodes.Node) -> None:
+    raise UnsupportedYamlError(
+        'Unsupported YAML, you can not use {} in {}'
+        .format(node.tag, os.path.basename(constructor.name)))
+
+
+def object_to_yaml(data: JSON_TYPE) -> str:
+    """Create yaml string from object."""
+    yaml = YAML(typ='rt')
+    yaml.indent(sequence=4, offset=2)
+    stream = StringIO()
+    try:
+        yaml.dump(data, stream)
+        result = stream.getvalue()  # type: str
+        return result
+    except YAMLError as exc:
+        _LOGGER.error("YAML error: %s", exc)
+        raise HomeAssistantError(exc)
+
+
+def yaml_to_object(data: str) -> JSON_TYPE:
+    """Create object from yaml string."""
+    yaml = YAML(typ='rt')
+    try:
+        result = yaml.load(data)  # type: Union[List, Dict, str]
+        return result
+    except YAMLError as exc:
+        _LOGGER.error("YAML error: %s", exc)
+        raise HomeAssistantError(exc)
+
+
+def load_yaml(fname: str, round_trip: bool = False) -> JSON_TYPE:
+    """Load a YAML file."""
+    if round_trip:
+        yaml = YAML(typ='rt')
+        yaml.preserve_quotes = True
+    else:
+        ExtSafeConstructor.name = fname
+        yaml = YAML(typ='safe')
+        yaml.Constructor = ExtSafeConstructor
+
+    try:
+        with open(fname, encoding='utf-8') as conf_file:
+            # If configuration file is empty YAML returns None
+            # We convert that to an empty dict
+            return yaml.load(conf_file) or OrderedDict()
+    except YAMLError as exc:
+        _LOGGER.error("YAML error in %s: %s", fname, exc)
+        raise HomeAssistantError(exc)
+    except UnicodeDecodeError as exc:
+        _LOGGER.error("Unable to read file %s: %s", fname, exc)
+        raise HomeAssistantError(exc)
+
+
+def save_yaml(fname: str, data: JSON_TYPE) -> None:
+    """Save a YAML file."""
+    yaml = YAML(typ='rt')
+    yaml.indent(sequence=4, offset=2)
+    tmp_fname = fname + "__TEMP__"
+    try:
+        file_stat = os.stat(fname)
+        with open(os.open(tmp_fname, O_WRONLY | O_CREAT | O_TRUNC,
+                          file_stat.st_mode), 'w', encoding='utf-8') \
+                as temp_file:
+            yaml.dump(data, temp_file)
+        os.replace(tmp_fname, fname)
+        try:
+            os.chown(fname, file_stat.st_uid, file_stat.st_gid)
+        except OSError:
+            pass
+    except YAMLError as exc:
+        _LOGGER.error(str(exc))
+        raise HomeAssistantError(exc)
+    except OSError as exc:
+        _LOGGER.exception('Saving YAML file %s failed: %s', fname, exc)
+        raise WriteError(exc)
+    finally:
+        if os.path.exists(tmp_fname):
+            try:
+                os.remove(tmp_fname)
+            except OSError as exc:
+                # If we are cleaning up then something else went wrong, so
+                # we should suppress likely follow-on errors in the cleanup
+                _LOGGER.error("YAML replacement cleanup failed: %s", exc)
+
+
+ExtSafeConstructor.add_constructor(u'!secret', secret_yaml)
+ExtSafeConstructor.add_constructor(u'!include', _include_yaml)
+ExtSafeConstructor.add_constructor(None, _yaml_unsupported)

homeassistant/util/yaml.py

@@ -272,8 +272,8 @@ def _load_secret_yaml(secret_path: str) -> JSON_TYPE:
     return secrets
 
 
-def _secret_yaml(loader: SafeLineLoader,
-                 node: yaml.nodes.Node) -> JSON_TYPE:
+def secret_yaml(loader: SafeLineLoader,
+                node: yaml.nodes.Node) -> JSON_TYPE:
     """Load secrets and embed it into the configuration YAML."""
     secret_path = os.path.dirname(loader.name)
     while True:
@@ -322,7 +322,7 @@ yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
 yaml.SafeLoader.add_constructor(
     yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq)
 yaml.SafeLoader.add_constructor('!env_var', _env_var_yaml)
-yaml.SafeLoader.add_constructor('!secret', _secret_yaml)
+yaml.SafeLoader.add_constructor('!secret', secret_yaml)
 yaml.SafeLoader.add_constructor('!include_dir_list', _include_dir_list_yaml)
 yaml.SafeLoader.add_constructor('!include_dir_merge_list',
                                 _include_dir_merge_list_yaml)

requirements_all.txt

@@ -12,6 +12,7 @@ pip>=8.0.3
 pytz>=2018.04
 pyyaml>=3.13,<4
 requests==2.20.0
+ruamel.yaml==0.15.72
 voluptuous==0.11.5
 voluptuous-serialize==2.0.0
 
@@ -1313,9 +1314,6 @@ roombapy==1.3.1
 # homeassistant.components.switch.rpi_rf
 # rpi-rf==0.9.6
 
-# homeassistant.components.lovelace
-ruamel.yaml==0.15.72
-
 # homeassistant.components.media_player.russound_rnet
 russound==0.1.9
 

requirements_test_all.txt

@@ -214,9 +214,6 @@ rflink==0.0.37
 # homeassistant.components.ring
 ring_doorbell==0.2.2
 
-# homeassistant.components.lovelace
-ruamel.yaml==0.15.72
-
 # homeassistant.components.media_player.yamaha
 rxv==0.5.1
 

setup.py

@@ -46,6 +46,7 @@ REQUIRES = [
     'pytz>=2018.04',
     'pyyaml>=3.13,<4',
     'requests==2.20.0',
+    'ruamel.yaml==0.15.72',
     'voluptuous==0.11.5',
     'voluptuous-serialize==2.0.0',
 ]

tests/components/lovelace/test_init.py

@@ -1,17 +1,12 @@
 """Test the Lovelace initialization."""
-import os
-import unittest
 from unittest.mock import patch
-from tempfile import mkdtemp
-import pytest
 from ruamel.yaml import YAML
 
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.setup import async_setup_component
 from homeassistant.components.websocket_api.const import TYPE_RESULT
-from homeassistant.components.lovelace import (load_yaml, migrate_config,
-                                               save_yaml,
-                                               UnsupportedYamlError)
+from homeassistant.components.lovelace import migrate_config
+from homeassistant.util.ruamel_yaml import UnsupportedYamlError
 
 TEST_YAML_A = """\
 title: My Awesome Home
@@ -118,63 +113,33 @@ views:
 """
 
 
-class TestYAML(unittest.TestCase):
-    """Test lovelace.yaml save and load."""
+def test_add_id():
+    """Test if id is added."""
+    yaml = YAML(typ='rt')
 
-    def setUp(self):
-        """Set up for tests."""
-        self.tmp_dir = mkdtemp()
-        self.yaml = YAML(typ='rt')
+    fname = "dummy.yaml"
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
+               return_value=yaml.load(TEST_YAML_A)), \
+            patch('homeassistant.util.ruamel_yaml.save_yaml') \
+            as save_yaml_mock:
+        migrate_config(fname)
 
-    def tearDown(self):
-        """Clean up after tests."""
-        for fname in os.listdir(self.tmp_dir):
-            os.remove(os.path.join(self.tmp_dir, fname))
-        os.rmdir(self.tmp_dir)
+    result = save_yaml_mock.call_args_list[0][0][1]
+    assert 'id' in result['views'][0]['cards'][0]
+    assert 'id' in result['views'][1]
 
-    def _path_for(self, leaf_name):
-        return os.path.join(self.tmp_dir, leaf_name+".yaml")
 
-    def test_save_and_load(self):
-        """Test saving and loading back."""
-        fname = self._path_for("test1")
-        save_yaml(fname, self.yaml.load(TEST_YAML_A))
-        data = load_yaml(fname)
-        assert data == self.yaml.load(TEST_YAML_A)
+def test_id_not_changed():
+    """Test if id is not changed if already exists."""
+    yaml = YAML(typ='rt')
 
-    def test_overwrite_and_reload(self):
-        """Test that we can overwrite an existing file and read back."""
-        fname = self._path_for("test3")
-        save_yaml(fname, self.yaml.load(TEST_YAML_A))
-        save_yaml(fname, self.yaml.load(TEST_YAML_B))
-        data = load_yaml(fname)
-        assert data == self.yaml.load(TEST_YAML_B)
-
-    def test_load_bad_data(self):
-        """Test error from trying to load unserialisable data."""
-        fname = self._path_for("test5")
-        with open(fname, "w") as fh:
-            fh.write(TEST_BAD_YAML)
-        with pytest.raises(HomeAssistantError):
-            load_yaml(fname)
-
-    def test_add_id(self):
-        """Test if id is added."""
-        fname = self._path_for("test6")
-        with patch('homeassistant.components.lovelace.load_yaml',
-                   return_value=self.yaml.load(TEST_YAML_A)), \
-                patch('homeassistant.components.lovelace.save_yaml'):
-            data = migrate_config(fname)
-        assert 'id' in data['views'][0]['cards'][0]
-        assert 'id' in data['views'][1]
-
-    def test_id_not_changed(self):
-        """Test if id is not changed if already exists."""
-        fname = self._path_for("test7")
-        with patch('homeassistant.components.lovelace.load_yaml',
-                   return_value=self.yaml.load(TEST_YAML_B)):
-            data = migrate_config(fname)
-        assert data == self.yaml.load(TEST_YAML_B)
+    fname = "dummy.yaml"
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
+               return_value=yaml.load(TEST_YAML_B)), \
+            patch('homeassistant.util.ruamel_yaml.save_yaml') \
+            as save_yaml_mock:
+        migrate_config(fname)
+    assert save_yaml_mock.call_count == 0
 
 
 async def test_deprecated_lovelace_ui(hass, hass_ws_client):
@@ -231,7 +196,7 @@ async def test_deprecated_lovelace_ui_load_err(hass, hass_ws_client):
     assert msg['id'] == 5
     assert msg['type'] == TYPE_RESULT
     assert msg['success'] is False
-    assert msg['error']['code'] == 'load_error'
+    assert msg['error']['code'] == 'error'
 
 
 async def test_lovelace_ui(hass, hass_ws_client):
@@ -288,7 +253,7 @@ async def test_lovelace_ui_load_err(hass, hass_ws_client):
     assert msg['id'] == 5
     assert msg['type'] == TYPE_RESULT
     assert msg['success'] is False
-    assert msg['error']['code'] == 'load_error'
+    assert msg['error']['code'] == 'error'
 
 
 async def test_lovelace_ui_load_json_err(hass, hass_ws_client):
@@ -316,7 +281,7 @@ async def test_lovelace_get_card(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)):
         await client.send_json({
             'id': 5,
@@ -337,7 +302,7 @@ async def test_lovelace_get_card_not_found(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)):
         await client.send_json({
             'id': 5,
@@ -357,7 +322,7 @@ async def test_lovelace_get_card_bad_yaml(hass, hass_ws_client):
     await async_setup_component(hass, 'lovelace')
     client = await hass_ws_client(hass)
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                side_effect=HomeAssistantError):
         await client.send_json({
             'id': 5,
@@ -369,7 +334,7 @@ async def test_lovelace_get_card_bad_yaml(hass, hass_ws_client):
     assert msg['id'] == 5
     assert msg['type'] == TYPE_RESULT
     assert msg['success'] is False
-    assert msg['error']['code'] == 'load_error'
+    assert msg['error']['code'] == 'error'
 
 
 async def test_lovelace_update_card(hass, hass_ws_client):
@@ -378,9 +343,9 @@ async def test_lovelace_update_card(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,
@@ -404,7 +369,7 @@ async def test_lovelace_update_card_not_found(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)):
         await client.send_json({
             'id': 5,
@@ -426,9 +391,9 @@ async def test_lovelace_update_card_bad_yaml(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.yaml_to_object',
+        patch('homeassistant.util.ruamel_yaml.yaml_to_object',
               side_effect=HomeAssistantError):
         await client.send_json({
             'id': 5,
@@ -441,7 +406,7 @@ async def test_lovelace_update_card_bad_yaml(hass, hass_ws_client):
     assert msg['id'] == 5
     assert msg['type'] == TYPE_RESULT
     assert msg['success'] is False
-    assert msg['error']['code'] == 'save_error'
+    assert msg['error']['code'] == 'error'
 
 
 async def test_lovelace_add_card(hass, hass_ws_client):
@@ -450,9 +415,9 @@ async def test_lovelace_add_card(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,
@@ -476,9 +441,9 @@ async def test_lovelace_add_card_position(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,
@@ -503,9 +468,9 @@ async def test_lovelace_move_card_position(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,
@@ -529,9 +494,9 @@ async def test_lovelace_move_card_view(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,
@@ -555,9 +520,9 @@ async def test_lovelace_move_card_view_position(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,
@@ -582,9 +547,9 @@ async def test_lovelace_delete_card(hass, hass_ws_client):
     client = await hass_ws_client(hass)
     yaml = YAML(typ='rt')
 
-    with patch('homeassistant.components.lovelace.load_yaml',
+    with patch('homeassistant.util.ruamel_yaml.load_yaml',
                return_value=yaml.load(TEST_YAML_A)), \
-        patch('homeassistant.components.lovelace.save_yaml') \
+        patch('homeassistant.util.ruamel_yaml.save_yaml') \
             as save_yaml_mock:
         await client.send_json({
             'id': 5,

tests/util/test_ruamel_yaml.py

@@ -0,0 +1,158 @@
+"""Test Home Assistant ruamel.yaml loader."""
+import os
+import unittest
+from tempfile import mkdtemp
+import pytest
+
+from ruamel.yaml import YAML
+
+from homeassistant.exceptions import HomeAssistantError
+import homeassistant.util.ruamel_yaml as util_yaml
+
+
+TEST_YAML_A = """\
+title: My Awesome Home
+# Include external resources
+resources:
+  - url: /local/my-custom-card.js
+    type: js
+  - url: /local/my-webfont.css
+    type: css
+
+# Exclude entities from "Unused entities" view
+excluded_entities:
+  - weblink.router
+views:
+    # View tab title.
+  - title: Example
+    # Optional unique id for direct access /lovelace/${id}
+    id: example
+    # Optional background (overwrites the global background).
+    background: radial-gradient(crimson, skyblue)
+    # Each view can have a different theme applied.
+    theme: dark-mode
+    # The cards to show on this view.
+    cards:
+        # The filter card will filter entities for their state
+      - type: entity-filter
+        entities:
+          - device_tracker.paulus
+          - device_tracker.anne_there
+        state_filter:
+          - 'home'
+        card:
+          type: glance
+          title: People that are home
+
+        # The picture entity card will represent an entity with a picture
+      - type: picture-entity
+        image: https://www.home-assistant.io/images/default-social.png
+        entity: light.bed_light
+
+    # Specify a tab icon if you want the view tab to be an icon.
+  - icon: mdi:home-assistant
+    # Title of the view. Will be used as the tooltip for tab icon
+    title: Second view
+    cards:
+      - id: test
+        type: entities
+        title: Test card
+        # Entities card will take a list of entities and show their state.
+      - type: entities
+        # Title of the entities card
+        title: Example
+        # The entities here will be shown in the same order as specified.
+        # Each entry is an entity ID or a map with extra options.
+        entities:
+          - light.kitchen
+          - switch.ac
+          - entity: light.living_room
+            # Override the name to use
+            name: LR Lights
+
+        # The markdown card will render markdown text.
+      - type: markdown
+        title: Lovelace
+        content: >
+          Welcome to your **Lovelace UI**.
+"""
+
+TEST_YAML_B = """\
+title: Home
+views:
+  - title: Dashboard
+    id: dashboard
+    icon: mdi:home
+    cards:
+      - id: testid
+        type: vertical-stack
+        cards:
+          - type: picture-entity
+            entity: group.sample
+            name: Sample
+            image:  /local/images/sample.jpg
+            tap_action: toggle
+"""
+
+# Test data that can not be loaded as YAML
+TEST_BAD_YAML = """\
+title: Home
+views:
+  - title: Dashboard
+    icon: mdi:home
+    cards:
+      - id: testid
+          type: vertical-stack
+"""
+
+# Test unsupported YAML
+TEST_UNSUP_YAML = """\
+title: Home
+views:
+  - title: Dashboard
+    icon: mdi:home
+    cards: !include cards.yaml
+"""
+
+
+class TestYAML(unittest.TestCase):
+    """Test lovelace.yaml save and load."""
+
+    def setUp(self):
+        """Set up for tests."""
+        self.tmp_dir = mkdtemp()
+        self.yaml = YAML(typ='rt')
+
+    def tearDown(self):
+        """Clean up after tests."""
+        for fname in os.listdir(self.tmp_dir):
+            os.remove(os.path.join(self.tmp_dir, fname))
+        os.rmdir(self.tmp_dir)
+
+    def _path_for(self, leaf_name):
+        return os.path.join(self.tmp_dir, leaf_name+".yaml")
+
+    def test_save_and_load(self):
+        """Test saving and loading back."""
+        fname = self._path_for("test1")
+        open(fname, "w+")
+        util_yaml.save_yaml(fname, self.yaml.load(TEST_YAML_A))
+        data = util_yaml.load_yaml(fname, True)
+        assert data == self.yaml.load(TEST_YAML_A)
+
+    def test_overwrite_and_reload(self):
+        """Test that we can overwrite an existing file and read back."""
+        fname = self._path_for("test2")
+        open(fname, "w+")
+        util_yaml.save_yaml(fname, self.yaml.load(TEST_YAML_A))
+        util_yaml.save_yaml(fname, self.yaml.load(TEST_YAML_B))
+        data = util_yaml.load_yaml(fname, True)
+        assert data == self.yaml.load(TEST_YAML_B)
+
+    def test_load_bad_data(self):
+        """Test error from trying to load unserialisable data."""
+        fname = self._path_for("test3")
+        with open(fname, "w") as fh:
+            fh.write(TEST_BAD_YAML)
+        with pytest.raises(HomeAssistantError):
+            util_yaml.load_yaml(fname, True)