Split up yaml loaders into multiple files (#23774)
* Start moving parts of yaml utils to own module Move parts of yaml loader out of the single large file and start to create the structure of the yaml loaders in Ansible [0]. [0]: https://github.com/ansible/ansible/tree/devel/lib/ansible/parsing/yaml * Finish yaml migration, update tests and mocks * Move code around to finish the migration * Update the mocks so that `open` is patched in `homeassistant.util.yaml.loader` instead of `homeassistant.util.yaml`. * Updated mypy ignores * Updated external API of `homeasistant.util.yaml`, see below: Checked what part of the api of `homeassistant.util.yaml` was actually called from outside the tests and added an `__ALL__` that contains only these elements. Updated the tests so that references to internal parts of the API (e.g. the yaml module imported into `homeassistant.util.yaml.loader`) are referenced directly from `homeassistant.util.yaml.loader`. In `tests/test_yaml.py` the import `yaml` refers to `homeassistant.util.yaml` and `yaml_loader` refers to `~.loader`. Future work that remains for the next iteration is to create a custom SafeConstructor and refers to that instead of monkey patching `yaml` with custom loaders. * Update mocks in yaml dumper, check_config
This commit is contained in:
1 parent
118d3bc11c
commit
4004867eda
11 files changed
+179
-139
No files matched your search
@@ -0,0 +1,15 @@
|
||||
"""YAML utility functions."""
|
||||
from .const import (
|
||||
SECRET_YAML, _SECRET_NAMESPACE
|
||||
)
|
||||
from .dumper import dump, save_yaml
|
||||
from .loader import (
|
||||
clear_secret_cache, load_yaml, secret_yaml
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'SECRET_YAML', '_SECRET_NAMESPACE',
|
||||
'dump', 'save_yaml',
|
||||
'clear_secret_cache', 'load_yaml', 'secret_yaml',
|
||||
]
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Constants."""
|
||||
SECRET_YAML = 'secrets.yaml'
|
||||
|
||||
_SECRET_NAMESPACE = 'homeassistant'
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Custom dumper and representers."""
|
||||
from collections import OrderedDict
|
||||
import yaml
|
||||
|
||||
from .objects import NodeListClass
|
||||
|
||||
|
||||
def dump(_dict: dict) -> str:
|
||||
"""Dump YAML to a string and remove null."""
|
||||
return yaml.safe_dump(
|
||||
_dict, default_flow_style=False, allow_unicode=True) \
|
||||
.replace(': null\n', ':\n')
|
||||
|
||||
|
||||
def save_yaml(path: str, data: dict) -> None:
|
||||
"""Save YAML to a file."""
|
||||
# Dump before writing to not truncate the file if dumping fails
|
||||
str_data = dump(data)
|
||||
with open(path, 'w', encoding='utf-8') as outfile:
|
||||
outfile.write(str_data)
|
||||
|
||||
|
||||
# From: https://gist.github.com/miracle2k/3184458
|
||||
# pylint: disable=redefined-outer-name
|
||||
def represent_odict(dump, tag, mapping, # type: ignore
|
||||
flow_style=None) -> yaml.MappingNode:
|
||||
"""Like BaseRepresenter.represent_mapping but does not issue the sort()."""
|
||||
value = [] # type: list
|
||||
node = yaml.MappingNode(tag, value, flow_style=flow_style)
|
||||
if dump.alias_key is not None:
|
||||
dump.represented_objects[dump.alias_key] = node
|
||||
best_style = True
|
||||
if hasattr(mapping, 'items'):
|
||||
mapping = mapping.items()
|
||||
for item_key, item_value in mapping:
|
||||
node_key = dump.represent_data(item_key)
|
||||
node_value = dump.represent_data(item_value)
|
||||
if not (isinstance(node_key, yaml.ScalarNode) and not node_key.style):
|
||||
best_style = False
|
||||
if not (isinstance(node_value, yaml.ScalarNode) and
|
||||
not node_value.style):
|
||||
best_style = False
|
||||
value.append((node_key, node_value))
|
||||
if flow_style is None:
|
||||
if dump.default_flow_style is not None:
|
||||
node.flow_style = dump.default_flow_style
|
||||
else:
|
||||
node.flow_style = best_style
|
||||
return node
|
||||
|
||||
|
||||
yaml.SafeDumper.add_representer(
|
||||
OrderedDict,
|
||||
lambda dumper, value:
|
||||
represent_odict(dumper, 'tag:yaml.org,2002:map', value))
|
||||
|
||||
yaml.SafeDumper.add_representer(
|
||||
NodeListClass,
|
||||
lambda dumper, value:
|
||||
dumper.represent_sequence('tag:yaml.org,2002:seq', value))
|
||||
@@ -1,4 +1,4 @@
|
||||
"""YAML utility functions."""
|
||||
"""Custom loader."""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
@@ -7,6 +7,7 @@ from collections import OrderedDict
|
||||
from typing import Union, List, Dict, Iterator, overload, TypeVar
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
import keyring
|
||||
except ImportError:
|
||||
@@ -19,25 +20,23 @@ except ImportError:
|
||||
|
||||
from homeassistant.exceptions import HomeAssistantError
|
||||
|
||||
from .const import _SECRET_NAMESPACE, SECRET_YAML
|
||||
from .objects import NodeListClass, NodeStrClass
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
_SECRET_NAMESPACE = 'homeassistant'
|
||||
SECRET_YAML = 'secrets.yaml'
|
||||
__SECRET_CACHE = {} # type: Dict[str, JSON_TYPE]
|
||||
|
||||
JSON_TYPE = Union[List, Dict, str] # pylint: disable=invalid-name
|
||||
DICT_T = TypeVar('DICT_T', bound=Dict) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
class NodeListClass(list):
|
||||
"""Wrapper class to be able to add attributes on a list."""
|
||||
def clear_secret_cache() -> None:
|
||||
"""Clear the secret cache.
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NodeStrClass(str):
|
||||
"""Wrapper class to be able to add attributes on a string."""
|
||||
|
||||
pass
|
||||
Async friendly.
|
||||
"""
|
||||
__SECRET_CACHE.clear()
|
||||
|
||||
|
||||
# pylint: disable=too-many-ancestors
|
||||
@@ -54,6 +53,21 @@ class SafeLineLoader(yaml.SafeLoader):
|
||||
return node
|
||||
|
||||
|
||||
def load_yaml(fname: str) -> JSON_TYPE:
|
||||
"""Load a YAML file."""
|
||||
try:
|
||||
with open(fname, encoding='utf-8') as conf_file:
|
||||
# If configuration file is empty YAML returns None
|
||||
# We convert that to an empty dict
|
||||
return yaml.load(conf_file, Loader=SafeLineLoader) or OrderedDict()
|
||||
except yaml.YAMLError as exc:
|
||||
_LOGGER.error(str(exc))
|
||||
raise HomeAssistantError(exc)
|
||||
except UnicodeDecodeError as exc:
|
||||
_LOGGER.error("Unable to read file %s: %s", fname, exc)
|
||||
raise HomeAssistantError(exc)
|
||||
|
||||
|
||||
# pylint: disable=pointless-statement
|
||||
@overload
|
||||
def _add_reference(obj: Union[list, NodeListClass],
|
||||
@@ -86,44 +100,6 @@ def _add_reference(obj, loader: SafeLineLoader, # type: ignore # noqa: F811
|
||||
return obj
|
||||
|
||||
|
||||
def load_yaml(fname: str) -> JSON_TYPE:
|
||||
"""Load a YAML file."""
|
||||
try:
|
||||
with open(fname, encoding='utf-8') as conf_file:
|
||||
# If configuration file is empty YAML returns None
|
||||
# We convert that to an empty dict
|
||||
return yaml.load(conf_file, Loader=SafeLineLoader) or OrderedDict()
|
||||
except yaml.YAMLError as exc:
|
||||
_LOGGER.error(str(exc))
|
||||
raise HomeAssistantError(exc)
|
||||
except UnicodeDecodeError as exc:
|
||||
_LOGGER.error("Unable to read file %s: %s", fname, exc)
|
||||
raise HomeAssistantError(exc)
|
||||
|
||||
|
||||
def dump(_dict: dict) -> str:
|
||||
"""Dump YAML to a string and remove null."""
|
||||
return yaml.safe_dump(
|
||||
_dict, default_flow_style=False, allow_unicode=True) \
|
||||
.replace(': null\n', ':\n')
|
||||
|
||||
|
||||
def save_yaml(path: str, data: dict) -> None:
|
||||
"""Save YAML to a file."""
|
||||
# Dump before writing to not truncate the file if dumping fails
|
||||
str_data = dump(data)
|
||||
with open(path, 'w', encoding='utf-8') as outfile:
|
||||
outfile.write(str_data)
|
||||
|
||||
|
||||
def clear_secret_cache() -> None:
|
||||
"""Clear the secret cache.
|
||||
|
||||
Async friendly.
|
||||
"""
|
||||
__SECRET_CACHE.clear()
|
||||
|
||||
|
||||
def _include_yaml(loader: SafeLineLoader,
|
||||
node: yaml.nodes.Node) -> JSON_TYPE:
|
||||
"""Load another YAML file and embeds it using the !include tag.
|
||||
@@ -331,43 +307,3 @@ yaml.SafeLoader.add_constructor('!include_dir_merge_list',
|
||||
yaml.SafeLoader.add_constructor('!include_dir_named', _include_dir_named_yaml)
|
||||
yaml.SafeLoader.add_constructor('!include_dir_merge_named',
|
||||
_include_dir_merge_named_yaml)
|
||||
|
||||
|
||||
# From: https://gist.github.com/miracle2k/3184458
|
||||
# pylint: disable=redefined-outer-name
|
||||
def represent_odict(dump, tag, mapping, # type: ignore
|
||||
flow_style=None) -> yaml.MappingNode:
|
||||
"""Like BaseRepresenter.represent_mapping but does not issue the sort()."""
|
||||
value = [] # type: list
|
||||
node = yaml.MappingNode(tag, value, flow_style=flow_style)
|
||||
if dump.alias_key is not None:
|
||||
dump.represented_objects[dump.alias_key] = node
|
||||
best_style = True
|
||||
if hasattr(mapping, 'items'):
|
||||
mapping = mapping.items()
|
||||
for item_key, item_value in mapping:
|
||||
node_key = dump.represent_data(item_key)
|
||||
node_value = dump.represent_data(item_value)
|
||||
if not (isinstance(node_key, yaml.ScalarNode) and not node_key.style):
|
||||
best_style = False
|
||||
if not (isinstance(node_value, yaml.ScalarNode) and
|
||||
not node_value.style):
|
||||
best_style = False
|
||||
value.append((node_key, node_value))
|
||||
if flow_style is None:
|
||||
if dump.default_flow_style is not None:
|
||||
node.flow_style = dump.default_flow_style
|
||||
else:
|
||||
node.flow_style = best_style
|
||||
return node
|
||||
|
||||
|
||||
yaml.SafeDumper.add_representer(
|
||||
OrderedDict,
|
||||
lambda dumper, value:
|
||||
represent_odict(dumper, 'tag:yaml.org,2002:map', value))
|
||||
|
||||
yaml.SafeDumper.add_representer(
|
||||
NodeListClass,
|
||||
lambda dumper, value:
|
||||
dumper.represent_sequence('tag:yaml.org,2002:seq', value))
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Custom yaml object types."""
|
||||
|
||||
|
||||
class NodeListClass(list):
|
||||
"""Wrapper class to be able to add attributes on a list."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NodeStrClass(str):
|
||||
"""Wrapper class to be able to add attributes on a string."""
|
||||
|
||||
pass
|
||||
Reference in new issue
Block a user