Async bootstrap / component init (#3991)
* Async bootstrap * Adress comments * Fix tests * More fixes * Tests fixes
This commit is contained in:
1 parent
d9999f36e8
commit
d5368f6f78
54 files changed
+1431
-1131
No files matched your search
+233
-105
@@ -1,11 +1,10 @@
|
||||
"""Provides methods to bootstrap a home assistant instance."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from threading import RLock
|
||||
|
||||
from types import ModuleType
|
||||
from typing import Any, Optional, Dict
|
||||
@@ -19,6 +18,8 @@ import homeassistant.config as conf_util
|
||||
import homeassistant.core as core
|
||||
import homeassistant.loader as loader
|
||||
import homeassistant.util.package as pkg_util
|
||||
from homeassistant.util.async import (
|
||||
run_coroutine_threadsafe, run_callback_threadsafe)
|
||||
from homeassistant.util.yaml import clear_secret_cache
|
||||
from homeassistant.const import EVENT_COMPONENT_LOADED, PLATFORM_FORMAT
|
||||
from homeassistant.exceptions import HomeAssistantError
|
||||
@@ -26,7 +27,6 @@ from homeassistant.helpers import (
|
||||
event_decorators, service, config_per_platform, extract_domain_configs)
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
_SETUP_LOCK = RLock()
|
||||
_CURRENT_SETUP = []
|
||||
|
||||
ATTR_COMPONENT = 'component'
|
||||
@@ -39,11 +39,23 @@ _PERSISTENT_VALIDATION = set()
|
||||
def setup_component(hass: core.HomeAssistant, domain: str,
|
||||
config: Optional[Dict]=None) -> bool:
|
||||
"""Setup a component and all its dependencies."""
|
||||
return run_coroutine_threadsafe(
|
||||
async_setup_component(hass, domain, config), loop=hass.loop).result()
|
||||
|
||||
|
||||
@asyncio.coroutine
|
||||
def async_setup_component(hass: core.HomeAssistant, domain: str,
|
||||
config: Optional[Dict]=None) -> bool:
|
||||
"""Setup a component and all its dependencies.
|
||||
|
||||
This method need to run in a executor.
|
||||
"""
|
||||
if domain in hass.config.components:
|
||||
_LOGGER.debug('Component %s already set up.', domain)
|
||||
return True
|
||||
|
||||
_ensure_loader_prepared(hass)
|
||||
if not loader.PREPARED:
|
||||
yield from hass.loop.run_in_executor(None, loader.prepare, hass)
|
||||
|
||||
if config is None:
|
||||
config = defaultdict(dict)
|
||||
@@ -55,7 +67,8 @@ def setup_component(hass: core.HomeAssistant, domain: str,
|
||||
return False
|
||||
|
||||
for component in components:
|
||||
if not _setup_component(hass, component, config):
|
||||
res = yield from _async_setup_component(hass, component, config)
|
||||
if not res:
|
||||
_LOGGER.error('Component %s failed to setup', component)
|
||||
return False
|
||||
|
||||
@@ -64,7 +77,11 @@ def setup_component(hass: core.HomeAssistant, domain: str,
|
||||
|
||||
def _handle_requirements(hass: core.HomeAssistant, component,
|
||||
name: str) -> bool:
|
||||
"""Install the requirements for a component."""
|
||||
"""Install the requirements for a component.
|
||||
|
||||
Asyncio don't support file operation jet.
|
||||
This method need to run in a executor.
|
||||
"""
|
||||
if hass.config.skip_pip or not hasattr(component, 'REQUIREMENTS'):
|
||||
return True
|
||||
|
||||
@@ -77,65 +94,82 @@ def _handle_requirements(hass: core.HomeAssistant, component,
|
||||
return True
|
||||
|
||||
|
||||
def _setup_component(hass: core.HomeAssistant, domain: str, config) -> bool:
|
||||
"""Setup a component for Home Assistant."""
|
||||
@asyncio.coroutine
|
||||
def _async_setup_component(hass: core.HomeAssistant,
|
||||
domain: str, config) -> bool:
|
||||
"""Setup a component for Home Assistant.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
# pylint: disable=too-many-return-statements,too-many-branches
|
||||
# pylint: disable=too-many-statements
|
||||
if domain in hass.config.components:
|
||||
return True
|
||||
|
||||
with _SETUP_LOCK:
|
||||
# It might have been loaded while waiting for lock
|
||||
if domain in hass.config.components:
|
||||
return True
|
||||
if domain in _CURRENT_SETUP:
|
||||
_LOGGER.error('Attempt made to setup %s during setup of %s',
|
||||
domain, domain)
|
||||
return False
|
||||
|
||||
if domain in _CURRENT_SETUP:
|
||||
_LOGGER.error('Attempt made to setup %s during setup of %s',
|
||||
domain, domain)
|
||||
config = yield from async_prepare_setup_component(hass, config, domain)
|
||||
|
||||
if config is None:
|
||||
return False
|
||||
|
||||
component = loader.get_component(domain)
|
||||
_CURRENT_SETUP.append(domain)
|
||||
|
||||
try:
|
||||
if hasattr(component, 'async_setup'):
|
||||
result = yield from component.async_setup(hass, config)
|
||||
else:
|
||||
result = yield from hass.loop.run_in_executor(
|
||||
None, component.setup, hass, config)
|
||||
|
||||
if result is False:
|
||||
_LOGGER.error('component %s failed to initialize', domain)
|
||||
return False
|
||||
|
||||
config = prepare_setup_component(hass, config, domain)
|
||||
|
||||
if config is None:
|
||||
elif result is not True:
|
||||
_LOGGER.error('component %s did not return boolean if setup '
|
||||
'was successful. Disabling component.', domain)
|
||||
loader.set_component(domain, None)
|
||||
return False
|
||||
except Exception: # pylint: disable=broad-except
|
||||
_LOGGER.exception('Error during setup of component %s', domain)
|
||||
return False
|
||||
finally:
|
||||
_CURRENT_SETUP.remove(domain)
|
||||
|
||||
component = loader.get_component(domain)
|
||||
_CURRENT_SETUP.append(domain)
|
||||
hass.config.components.append(component.DOMAIN)
|
||||
|
||||
try:
|
||||
result = component.setup(hass, config)
|
||||
if result is False:
|
||||
_LOGGER.error('component %s failed to initialize', domain)
|
||||
return False
|
||||
elif result is not True:
|
||||
_LOGGER.error('component %s did not return boolean if setup '
|
||||
'was successful. Disabling component.', domain)
|
||||
loader.set_component(domain, None)
|
||||
return False
|
||||
except Exception: # pylint: disable=broad-except
|
||||
_LOGGER.exception('Error during setup of component %s', domain)
|
||||
return False
|
||||
finally:
|
||||
_CURRENT_SETUP.remove(domain)
|
||||
# Assumption: if a component does not depend on groups
|
||||
# it communicates with devices
|
||||
if 'group' not in getattr(component, 'DEPENDENCIES', []) and \
|
||||
hass.pool.worker_count <= 10:
|
||||
hass.pool.add_worker()
|
||||
|
||||
hass.config.components.append(component.DOMAIN)
|
||||
hass.bus.async_fire(
|
||||
EVENT_COMPONENT_LOADED, {ATTR_COMPONENT: component.DOMAIN}
|
||||
)
|
||||
|
||||
# Assumption: if a component does not depend on groups
|
||||
# it communicates with devices
|
||||
if 'group' not in getattr(component, 'DEPENDENCIES', []) and \
|
||||
hass.pool.worker_count <= 10:
|
||||
hass.pool.add_worker()
|
||||
|
||||
hass.bus.fire(
|
||||
EVENT_COMPONENT_LOADED, {ATTR_COMPONENT: component.DOMAIN}
|
||||
)
|
||||
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
domain: str):
|
||||
"""Prepare setup of a component and return processed config."""
|
||||
return run_coroutine_threadsafe(
|
||||
async_prepare_setup_component(hass, config, domain), loop=hass.loop
|
||||
).result()
|
||||
|
||||
|
||||
@asyncio.coroutine
|
||||
def async_prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
domain: str):
|
||||
"""Prepare setup of a component and return processed config.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
# pylint: disable=too-many-return-statements
|
||||
component = loader.get_component(domain)
|
||||
missing_deps = [dep for dep in getattr(component, 'DEPENDENCIES', [])
|
||||
@@ -151,7 +185,7 @@ def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
try:
|
||||
config = component.CONFIG_SCHEMA(config)
|
||||
except vol.Invalid as ex:
|
||||
log_exception(ex, domain, config, hass)
|
||||
async_log_exception(ex, domain, config, hass)
|
||||
return None
|
||||
|
||||
elif hasattr(component, 'PLATFORM_SCHEMA'):
|
||||
@@ -161,7 +195,7 @@ def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
try:
|
||||
p_validated = component.PLATFORM_SCHEMA(p_config)
|
||||
except vol.Invalid as ex:
|
||||
log_exception(ex, domain, config, hass)
|
||||
async_log_exception(ex, domain, config, hass)
|
||||
continue
|
||||
|
||||
# Not all platform components follow same pattern for platforms
|
||||
@@ -171,8 +205,8 @@ def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
platforms.append(p_validated)
|
||||
continue
|
||||
|
||||
platform = prepare_setup_platform(hass, config, domain,
|
||||
p_name)
|
||||
platform = yield from async_prepare_setup_platform(
|
||||
hass, config, domain, p_name)
|
||||
|
||||
if platform is None:
|
||||
continue
|
||||
@@ -180,10 +214,11 @@ def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
# Validate platform specific schema
|
||||
if hasattr(platform, 'PLATFORM_SCHEMA'):
|
||||
try:
|
||||
# pylint: disable=no-member
|
||||
p_validated = platform.PLATFORM_SCHEMA(p_validated)
|
||||
except vol.Invalid as ex:
|
||||
log_exception(ex, '{}.{}'.format(domain, p_name),
|
||||
p_validated, hass)
|
||||
async_log_exception(ex, '{}.{}'.format(domain, p_name),
|
||||
p_validated, hass)
|
||||
continue
|
||||
|
||||
platforms.append(p_validated)
|
||||
@@ -195,7 +230,9 @@ def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
if key not in filter_keys}
|
||||
config[domain] = platforms
|
||||
|
||||
if not _handle_requirements(hass, component, domain):
|
||||
res = yield from hass.loop.run_in_executor(
|
||||
None, _handle_requirements, hass, component, domain)
|
||||
if not res:
|
||||
return None
|
||||
|
||||
return config
|
||||
@@ -204,7 +241,22 @@ def prepare_setup_component(hass: core.HomeAssistant, config: dict,
|
||||
def prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
|
||||
platform_name: str) -> Optional[ModuleType]:
|
||||
"""Load a platform and makes sure dependencies are setup."""
|
||||
_ensure_loader_prepared(hass)
|
||||
return run_coroutine_threadsafe(
|
||||
async_prepare_setup_platform(hass, config, domain, platform_name),
|
||||
loop=hass.loop
|
||||
).result()
|
||||
|
||||
|
||||
@asyncio.coroutine
|
||||
def async_prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
|
||||
platform_name: str) \
|
||||
-> Optional[ModuleType]:
|
||||
"""Load a platform and makes sure dependencies are setup.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
if not loader.PREPARED:
|
||||
yield from hass.loop.run_in_executor(None, loader.prepare, hass)
|
||||
|
||||
platform_path = PLATFORM_FORMAT.format(domain, platform_name)
|
||||
|
||||
@@ -218,7 +270,7 @@ def prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
|
||||
message = ('Unable to find the following platforms: ' +
|
||||
', '.join(list(_PERSISTENT_PLATFORMS)) +
|
||||
'(please check your configuration)')
|
||||
persistent_notification.create(
|
||||
persistent_notification.async_create(
|
||||
hass, message, 'Invalid platforms', 'platform_errors')
|
||||
return None
|
||||
|
||||
@@ -228,14 +280,17 @@ def prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
|
||||
|
||||
# Load dependencies
|
||||
for component in getattr(platform, 'DEPENDENCIES', []):
|
||||
if not setup_component(hass, component, config):
|
||||
res = yield from async_setup_component(hass, component, config)
|
||||
if not res:
|
||||
_LOGGER.error(
|
||||
'Unable to prepare setup for platform %s because '
|
||||
'dependency %s could not be initialized', platform_path,
|
||||
component)
|
||||
return None
|
||||
|
||||
if not _handle_requirements(hass, platform, platform_path):
|
||||
res = yield from hass.loop.run_in_executor(
|
||||
None, _handle_requirements, hass, platform, platform_path)
|
||||
if not res:
|
||||
return None
|
||||
|
||||
return platform
|
||||
@@ -261,15 +316,50 @@ def from_config_dict(config: Dict[str, Any],
|
||||
hass.config.config_dir = config_dir
|
||||
mount_local_lib_path(config_dir)
|
||||
|
||||
@asyncio.coroutine
|
||||
def _async_init_from_config_dict(future):
|
||||
try:
|
||||
re_hass = yield from async_from_config_dict(
|
||||
config, hass, config_dir, enable_log, verbose, skip_pip,
|
||||
log_rotate_days)
|
||||
future.set_result(re_hass)
|
||||
# pylint: disable=broad-except
|
||||
except Exception as exc:
|
||||
future.set_exception(exc)
|
||||
|
||||
# run task
|
||||
future = asyncio.Future()
|
||||
asyncio.Task(_async_init_from_config_dict(future), loop=hass.loop)
|
||||
hass.loop.run_until_complete(future)
|
||||
|
||||
return future.result()
|
||||
|
||||
|
||||
@asyncio.coroutine
|
||||
# pylint: disable=too-many-branches, too-many-statements, too-many-arguments
|
||||
def async_from_config_dict(config: Dict[str, Any],
|
||||
hass: core.HomeAssistant,
|
||||
config_dir: Optional[str]=None,
|
||||
enable_log: bool=True,
|
||||
verbose: bool=False,
|
||||
skip_pip: bool=False,
|
||||
log_rotate_days: Any=None) \
|
||||
-> Optional[core.HomeAssistant]:
|
||||
"""Try to configure Home Assistant from a config dict.
|
||||
|
||||
Dynamically loads required components and its dependencies.
|
||||
This method is a coroutine.
|
||||
"""
|
||||
core_config = config.get(core.DOMAIN, {})
|
||||
|
||||
try:
|
||||
conf_util.process_ha_core_config(hass, core_config)
|
||||
yield from conf_util.async_process_ha_core_config(hass, core_config)
|
||||
except vol.Invalid as ex:
|
||||
log_exception(ex, 'homeassistant', core_config, hass)
|
||||
async_log_exception(ex, 'homeassistant', core_config, hass)
|
||||
return None
|
||||
|
||||
conf_util.process_ha_config_upgrade(hass)
|
||||
yield from hass.loop.run_in_executor(
|
||||
None, conf_util.process_ha_config_upgrade, hass)
|
||||
|
||||
if enable_log:
|
||||
enable_logging(hass, verbose, log_rotate_days)
|
||||
@@ -279,7 +369,8 @@ def from_config_dict(config: Dict[str, Any],
|
||||
_LOGGER.warning('Skipping pip installation of required modules. '
|
||||
'This may cause issues.')
|
||||
|
||||
_ensure_loader_prepared(hass)
|
||||
if not loader.PREPARED:
|
||||
yield from hass.loop.run_in_executor(None, loader.prepare, hass)
|
||||
|
||||
# Make a copy because we are mutating it.
|
||||
# Convert it to defaultdict so components can always have config dict
|
||||
@@ -291,29 +382,25 @@ def from_config_dict(config: Dict[str, Any],
|
||||
components = set(key.split(' ')[0] for key in config.keys()
|
||||
if key != core.DOMAIN)
|
||||
|
||||
# Setup in a thread to avoid blocking
|
||||
def component_setup():
|
||||
"""Set up a component."""
|
||||
if not core_components.setup(hass, config):
|
||||
_LOGGER.error('Home Assistant core failed to initialize. '
|
||||
'Further initialization aborted.')
|
||||
return hass
|
||||
# setup components
|
||||
# pylint: disable=not-an-iterable
|
||||
res = yield from core_components.async_setup(hass, config)
|
||||
if not res:
|
||||
_LOGGER.error('Home Assistant core failed to initialize. '
|
||||
'Further initialization aborted.')
|
||||
return hass
|
||||
|
||||
persistent_notification.setup(hass, config)
|
||||
yield from persistent_notification.async_setup(hass, config)
|
||||
|
||||
_LOGGER.info('Home Assistant core initialized')
|
||||
_LOGGER.info('Home Assistant core initialized')
|
||||
|
||||
# Give event decorators access to HASS
|
||||
event_decorators.HASS = hass
|
||||
service.HASS = hass
|
||||
# Give event decorators access to HASS
|
||||
event_decorators.HASS = hass
|
||||
service.HASS = hass
|
||||
|
||||
# Setup the components
|
||||
for domain in loader.load_order_components(components):
|
||||
_setup_component(hass, domain, config)
|
||||
|
||||
hass.loop.run_until_complete(
|
||||
hass.loop.run_in_executor(None, component_setup)
|
||||
)
|
||||
# Setup the components
|
||||
for domain in loader.load_order_components(components):
|
||||
yield from _async_setup_component(hass, domain, config)
|
||||
|
||||
return hass
|
||||
|
||||
@@ -331,27 +418,62 @@ def from_config_file(config_path: str,
|
||||
if hass is None:
|
||||
hass = core.HomeAssistant()
|
||||
|
||||
@asyncio.coroutine
|
||||
def _async_init_from_config_file(future):
|
||||
try:
|
||||
re_hass = yield from async_from_config_file(
|
||||
config_path, hass, verbose, skip_pip, log_rotate_days)
|
||||
future.set_result(re_hass)
|
||||
# pylint: disable=broad-except
|
||||
except Exception as exc:
|
||||
future.set_exception(exc)
|
||||
|
||||
# run task
|
||||
future = asyncio.Future()
|
||||
asyncio.Task(_async_init_from_config_file(future), loop=hass.loop)
|
||||
hass.loop.run_until_complete(future)
|
||||
|
||||
return future.result()
|
||||
|
||||
|
||||
@asyncio.coroutine
|
||||
def async_from_config_file(config_path: str,
|
||||
hass: core.HomeAssistant,
|
||||
verbose: bool=False,
|
||||
skip_pip: bool=True,
|
||||
log_rotate_days: Any=None):
|
||||
"""Read the configuration file and try to start all the functionality.
|
||||
|
||||
Will add functionality to 'hass' parameter.
|
||||
This method is a coroutine.
|
||||
"""
|
||||
# Set config dir to directory holding config file
|
||||
config_dir = os.path.abspath(os.path.dirname(config_path))
|
||||
hass.config.config_dir = config_dir
|
||||
mount_local_lib_path(config_dir)
|
||||
yield from hass.loop.run_in_executor(
|
||||
None, mount_local_lib_path, config_dir)
|
||||
|
||||
enable_logging(hass, verbose, log_rotate_days)
|
||||
|
||||
try:
|
||||
config_dict = conf_util.load_yaml_config_file(config_path)
|
||||
config_dict = yield from hass.loop.run_in_executor(
|
||||
None, conf_util.load_yaml_config_file, config_path)
|
||||
except HomeAssistantError:
|
||||
return None
|
||||
finally:
|
||||
clear_secret_cache()
|
||||
|
||||
return from_config_dict(config_dict, hass, enable_log=False,
|
||||
skip_pip=skip_pip)
|
||||
hass = yield from async_from_config_dict(
|
||||
config_dict, hass, enable_log=False, skip_pip=skip_pip)
|
||||
return hass
|
||||
|
||||
|
||||
def enable_logging(hass: core.HomeAssistant, verbose: bool=False,
|
||||
log_rotate_days=None) -> None:
|
||||
"""Setup the logging."""
|
||||
"""Setup the logging.
|
||||
|
||||
Async friendly.
|
||||
"""
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
fmt = ("%(log_color)s%(asctime)s %(levelname)s (%(threadName)s) "
|
||||
"[%(name)s] %(message)s%(reset)s")
|
||||
@@ -407,44 +529,50 @@ def enable_logging(hass: core.HomeAssistant, verbose: bool=False,
|
||||
'Unable to setup error log %s (access denied)', err_log_path)
|
||||
|
||||
|
||||
def _ensure_loader_prepared(hass: core.HomeAssistant) -> None:
|
||||
"""Ensure Home Assistant loader is prepared."""
|
||||
if not loader.PREPARED:
|
||||
loader.prepare(hass)
|
||||
|
||||
|
||||
def log_exception(ex, domain, config, hass=None):
|
||||
def log_exception(ex, domain, config, hass):
|
||||
"""Generate log exception for config validation."""
|
||||
run_callback_threadsafe(
|
||||
hass.loop, async_log_exception, ex, domain, config, hass).result()
|
||||
|
||||
|
||||
@core.callback
|
||||
def async_log_exception(ex, domain, config, hass):
|
||||
"""Generate log exception for config validation.
|
||||
|
||||
Need to run in a async loop.
|
||||
"""
|
||||
message = 'Invalid config for [{}]: '.format(domain)
|
||||
if hass is not None:
|
||||
_PERSISTENT_VALIDATION.add(domain)
|
||||
message = ('The following platforms contain invalid configuration: ' +
|
||||
', '.join(list(_PERSISTENT_VALIDATION)) +
|
||||
' (please check your configuration)')
|
||||
persistent_notification.create(
|
||||
hass, message, 'Invalid config', 'invalid_config')
|
||||
_PERSISTENT_VALIDATION.add(domain)
|
||||
message = ('The following platforms contain invalid configuration: ' +
|
||||
', '.join(list(_PERSISTENT_VALIDATION)) +
|
||||
' (please check your configuration). ')
|
||||
persistent_notification.async_create(
|
||||
hass, message, 'Invalid config', 'invalid_config')
|
||||
|
||||
if 'extra keys not allowed' in ex.error_message:
|
||||
message += '[{}] is an invalid option for [{}]. Check: {}->{}.'\
|
||||
.format(ex.path[-1], domain, domain,
|
||||
'->'.join('%s' % m for m in ex.path))
|
||||
'->'.join(str(m) for m in ex.path))
|
||||
else:
|
||||
message += '{}.'.format(humanize_error(config, ex))
|
||||
|
||||
domain_config = config.get(domain, config)
|
||||
message += " (See {}:{})".format(
|
||||
message += " (See {}:{}). ".format(
|
||||
getattr(domain_config, '__config_file__', '?'),
|
||||
getattr(domain_config, '__line__', '?'))
|
||||
|
||||
if domain != 'homeassistant':
|
||||
message += (' Please check the docs at '
|
||||
message += ('Please check the docs at '
|
||||
'https://home-assistant.io/components/{}/'.format(domain))
|
||||
|
||||
_LOGGER.error(message)
|
||||
|
||||
|
||||
def mount_local_lib_path(config_dir: str) -> str:
|
||||
"""Add local library to Python Path."""
|
||||
"""Add local library to Python Path.
|
||||
|
||||
Async friendly.
|
||||
"""
|
||||
deps_dir = os.path.join(config_dir, 'deps')
|
||||
if deps_dir not in sys.path:
|
||||
sys.path.insert(0, os.path.join(config_dir, 'deps'))
|
||||
|
||||
Reference in new issue
Block a user