Restore states through a JSON store instead of recorder (#17270)

* Restore states through a JSON store

* Accept entity_id directly in restore state helper

* Keep states stored between runs for a limited time

* Remove warning
This commit is contained in:
Adam Mills authored and Paulus Schoutsen committed 2018-11-28 13:16:43 +01:00
1 parent a039c3209b
commit 5c3a4e3d10
46 files changed
+493 -422

No files matched your search

+18 -13
View File
@@ -114,8 +114,7 @@ def get_test_home_assistant():
# pylint: disable=protected-access
@asyncio.coroutine
def async_test_home_assistant(loop):
async def async_test_home_assistant(loop):
"""Return a Home Assistant object pointing at test config dir."""
hass = ha.HomeAssistant(loop)
hass.config.async_load = Mock()
@@ -168,13 +167,12 @@ def async_test_home_assistant(loop):
# Mock async_start
orig_start = hass.async_start
@asyncio.coroutine
def mock_async_start():
async def mock_async_start():
"""Start the mocking."""
# We only mock time during tests and we want to track tasks
with patch('homeassistant.core._async_create_timer'), \
patch.object(hass, 'async_stop_track_tasks'):
yield from orig_start()
await orig_start()
hass.async_start = mock_async_start
@@ -715,14 +713,20 @@ def init_recorder_component(hass, add_config=None):
def mock_restore_cache(hass, states):
"""Mock the DATA_RESTORE_CACHE."""
key = restore_state.DATA_RESTORE_CACHE
hass.data[key] = {
key = restore_state.DATA_RESTORE_STATE_TASK
data = restore_state.RestoreStateData(hass)
data.last_states = {
state.entity_id: state for state in states}
_LOGGER.debug('Restore cache: %s', hass.data[key])
assert len(hass.data[key]) == len(states), \
_LOGGER.debug('Restore cache: %s', data.last_states)
assert len(data.last_states) == len(states), \
"Duplicate entity_id? {}".format(states)
hass.state = ha.CoreState.starting
mock_component(hass, recorder.DOMAIN)
async def get_restore_state_data() -> restore_state.RestoreStateData:
return data
# Patch the singleton task in hass.data to return our new RestoreStateData
hass.data[key] = hass.async_create_task(get_restore_state_data())
class MockDependency:
@@ -846,9 +850,10 @@ def mock_storage(data=None):
def mock_write_data(store, path, data_to_write):
"""Mock version of write data."""
# To ensure that the data can be serialized
_LOGGER.info('Writing data to %s: %s', store.key, data_to_write)
data[store.key] = json.loads(json.dumps(data_to_write))
# To ensure that the data can be serialized
data[store.key] = json.loads(json.dumps(
data_to_write, cls=store._encoder))
with patch('homeassistant.helpers.storage.Store._async_load',
side_effect=mock_async_load, autospec=True), \
+1 -31
View File
@@ -6,10 +6,8 @@ from unittest.mock import patch
import requests
from aiohttp.hdrs import CONTENT_TYPE
from homeassistant import setup, const, core
import homeassistant.components as core_components
from homeassistant import setup, const
from homeassistant.components import emulated_hue, http
from homeassistant.util.async_ import run_coroutine_threadsafe
from tests.common import get_test_instance_port, get_test_home_assistant
@@ -20,29 +18,6 @@ BRIDGE_URL_BASE = 'http://127.0.0.1:{}'.format(BRIDGE_SERVER_PORT) + '{}'
JSON_HEADERS = {CONTENT_TYPE: const.CONTENT_TYPE_JSON}
def setup_hass_instance(emulated_hue_config):
"""Set up the Home Assistant instance to test."""
hass = get_test_home_assistant()
# We need to do this to get access to homeassistant/turn_(on,off)
run_coroutine_threadsafe(
core_components.async_setup(hass, {core.DOMAIN: {}}), hass.loop
).result()
setup.setup_component(
hass, http.DOMAIN,
{http.DOMAIN: {http.CONF_SERVER_PORT: HTTP_SERVER_PORT}})
setup.setup_component(hass, emulated_hue.DOMAIN, emulated_hue_config)
return hass
def start_hass_instance(hass):
"""Start the Home Assistant instance to test."""
hass.start()
class TestEmulatedHue(unittest.TestCase):
"""Test the emulated Hue component."""
@@ -53,11 +28,6 @@ class TestEmulatedHue(unittest.TestCase):
"""Set up the class."""
cls.hass = hass = get_test_home_assistant()
# We need to do this to get access to homeassistant/turn_(on,off)
run_coroutine_threadsafe(
core_components.async_setup(hass, {core.DOMAIN: {}}), hass.loop
).result()
setup.setup_component(
hass, http.DOMAIN,
{http.DOMAIN: {http.CONF_SERVER_PORT: HTTP_SERVER_PORT}})
+1 -1
View File
@@ -585,7 +585,7 @@ async def test_sending_mqtt_commands_and_optimistic(hass, mqtt_mock):
'effect': 'random',
'color_temp': 100,
'white_value': 50})
with patch('homeassistant.components.light.mqtt.schema_basic'
with patch('homeassistant.helpers.restore_state.RestoreEntity'
'.async_get_last_state',
return_value=mock_coro(fake_state)):
with assert_setup_component(1, light.DOMAIN):
+1 -1
View File
@@ -279,7 +279,7 @@ async def test_sending_mqtt_commands_and_optimistic(hass, mqtt_mock):
'color_temp': 100,
'white_value': 50})
with patch('homeassistant.components.light.mqtt.schema_json'
with patch('homeassistant.helpers.restore_state.RestoreEntity'
'.async_get_last_state',
return_value=mock_coro(fake_state)):
assert await async_setup_component(hass, light.DOMAIN, {
+1 -1
View File
@@ -245,7 +245,7 @@ async def test_optimistic(hass, mqtt_mock):
'color_temp': 100,
'white_value': 50})
with patch('homeassistant.components.light.mqtt.schema_template'
with patch('homeassistant.helpers.restore_state.RestoreEntity'
'.async_get_last_state',
return_value=mock_coro(fake_state)):
with assert_setup_component(1, light.DOMAIN):
+7 -10
View File
@@ -1,6 +1,5 @@
"""The tests for the Recorder component."""
# pylint: disable=protected-access
import asyncio
from unittest.mock import patch, call
import pytest
@@ -9,7 +8,7 @@ from sqlalchemy.pool import StaticPool
from homeassistant.bootstrap import async_setup_component
from homeassistant.components.recorder import (
wait_connection_ready, migration, const, models)
migration, const, models)
from tests.components.recorder import models_original
@@ -23,26 +22,24 @@ def create_engine_test(*args, **kwargs):
return engine
@asyncio.coroutine
def test_schema_update_calls(hass):
async def test_schema_update_calls(hass):
"""Test that schema migrations occur in correct order."""
with patch('sqlalchemy.create_engine', new=create_engine_test), \
patch('homeassistant.components.recorder.migration._apply_update') as \
update:
yield from async_setup_component(hass, 'recorder', {
await async_setup_component(hass, 'recorder', {
'recorder': {
'db_url': 'sqlite://'
}
})
yield from wait_connection_ready(hass)
await hass.async_block_till_done()
update.assert_has_calls([
call(hass.data[const.DATA_INSTANCE].engine, version+1, 0) for version
in range(0, models.SCHEMA_VERSION)])
@asyncio.coroutine
def test_schema_migrate(hass):
async def test_schema_migrate(hass):
"""Test the full schema migration logic.
We're just testing that the logic can execute successfully here without
@@ -52,12 +49,12 @@ def test_schema_migrate(hass):
with patch('sqlalchemy.create_engine', new=create_engine_test), \
patch('homeassistant.components.recorder.Recorder._setup_run') as \
setup_run:
yield from async_setup_component(hass, 'recorder', {
await async_setup_component(hass, 'recorder', {
'recorder': {
'db_url': 'sqlite://'
}
})
yield from wait_connection_ready(hass)
await hass.async_block_till_done()
assert setup_run.called
+2 -1
View File
@@ -57,7 +57,8 @@ async def test_sending_mqtt_commands_and_optimistic(hass, mock_publish):
"""Test the sending MQTT commands in optimistic mode."""
fake_state = ha.State('switch.test', 'on')
with patch('homeassistant.components.switch.mqtt.async_get_last_state',
with patch('homeassistant.helpers.restore_state.RestoreEntity'
'.async_get_last_state',
return_value=mock_coro(fake_state)):
assert await async_setup_component(hass, switch.DOMAIN, {
switch.DOMAIN: {
-1
View File
@@ -519,7 +519,6 @@ async def test_fetch_period_api(hass, hass_client):
"""Test the fetch period view for history."""
await hass.async_add_job(init_recorder_component, hass)
await async_setup_component(hass, 'history', {})
await hass.components.recorder.wait_connection_ready()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
client = await hass_client()
response = await client.get(
-2
View File
@@ -575,7 +575,6 @@ async def test_logbook_view(hass, aiohttp_client):
"""Test the logbook view."""
await hass.async_add_job(init_recorder_component, hass)
await async_setup_component(hass, 'logbook', {})
await hass.components.recorder.wait_connection_ready()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
client = await aiohttp_client(hass.http.app)
response = await client.get(
@@ -587,7 +586,6 @@ async def test_logbook_view_period_entity(hass, aiohttp_client):
"""Test the logbook view with period and entity."""
await hass.async_add_job(init_recorder_component, hass)
await async_setup_component(hass, 'logbook', {})
await hass.components.recorder.wait_connection_ready()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
entity_id_test = 'switch.test'
+161 -154
View File
@@ -1,60 +1,52 @@
"""The tests for the Restore component."""
import asyncio
from datetime import timedelta
from unittest.mock import patch, MagicMock
from datetime import datetime
from homeassistant.setup import setup_component
from homeassistant.const import EVENT_HOMEASSISTANT_START
from homeassistant.core import CoreState, split_entity_id, State
import homeassistant.util.dt as dt_util
from homeassistant.components import input_boolean, recorder
from homeassistant.core import CoreState, State
from homeassistant.exceptions import HomeAssistantError
from homeassistant.helpers.entity import Entity
from homeassistant.helpers.restore_state import (
async_get_last_state, DATA_RESTORE_CACHE)
from homeassistant.components.recorder.models import RecorderRuns, States
RestoreStateData, RestoreEntity, DATA_RESTORE_STATE_TASK)
from homeassistant.util import dt as dt_util
from tests.common import (
get_test_home_assistant, mock_coro, init_recorder_component,
mock_component)
from asynctest import patch
from tests.common import mock_coro
@asyncio.coroutine
def test_caching_data(hass):
async def test_caching_data(hass):
"""Test that we cache data."""
mock_component(hass, 'recorder')
hass.state = CoreState.starting
states = [
State('input_boolean.b0', 'on'),
State('input_boolean.b1', 'on'),
State('input_boolean.b2', 'on'),
]
with patch('homeassistant.helpers.restore_state.last_recorder_run',
return_value=MagicMock(end=dt_util.utcnow())), \
patch('homeassistant.helpers.restore_state.get_states',
return_value=states), \
patch('homeassistant.helpers.restore_state.wait_connection_ready',
return_value=mock_coro(True)):
state = yield from async_get_last_state(hass, 'input_boolean.b1')
data = await RestoreStateData.async_get_instance(hass)
await data.store.async_save([state.as_dict() for state in states])
assert DATA_RESTORE_CACHE in hass.data
assert hass.data[DATA_RESTORE_CACHE] == {st.entity_id: st for st in states}
# Emulate a fresh load
hass.data[DATA_RESTORE_STATE_TASK] = None
entity = RestoreEntity()
entity.hass = hass
entity.entity_id = 'input_boolean.b1'
# Mock that only b1 is present this run
with patch('homeassistant.helpers.restore_state.Store.async_save'
) as mock_write_data:
state = await entity.async_get_last_state()
assert state is not None
assert state.entity_id == 'input_boolean.b1'
assert state.state == 'on'
hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
yield from hass.async_block_till_done()
assert DATA_RESTORE_CACHE not in hass.data
assert mock_write_data.called
@asyncio.coroutine
def test_hass_running(hass):
"""Test that cache cannot be accessed while hass is running."""
mock_component(hass, 'recorder')
async def test_hass_starting(hass):
"""Test that we cache data."""
hass.state = CoreState.starting
states = [
State('input_boolean.b0', 'on'),
@@ -62,129 +54,144 @@ def test_hass_running(hass):
State('input_boolean.b2', 'on'),
]
with patch('homeassistant.helpers.restore_state.last_recorder_run',
return_value=MagicMock(end=dt_util.utcnow())), \
patch('homeassistant.helpers.restore_state.get_states',
return_value=states), \
patch('homeassistant.helpers.restore_state.wait_connection_ready',
return_value=mock_coro(True)):
state = yield from async_get_last_state(hass, 'input_boolean.b1')
assert state is None
data = await RestoreStateData.async_get_instance(hass)
await data.store.async_save([state.as_dict() for state in states])
# Emulate a fresh load
hass.data[DATA_RESTORE_STATE_TASK] = None
@asyncio.coroutine
def test_not_connected(hass):
"""Test that cache cannot be accessed if db connection times out."""
mock_component(hass, 'recorder')
hass.state = CoreState.starting
entity = RestoreEntity()
entity.hass = hass
entity.entity_id = 'input_boolean.b1'
states = [State('input_boolean.b1', 'on')]
# Mock that only b1 is present this run
states = [
State('input_boolean.b1', 'on'),
]
with patch('homeassistant.helpers.restore_state.Store.async_save'
) as mock_write_data, patch.object(
hass.states, 'async_all', return_value=states):
state = await entity.async_get_last_state()
with patch('homeassistant.helpers.restore_state.last_recorder_run',
return_value=MagicMock(end=dt_util.utcnow())), \
patch('homeassistant.helpers.restore_state.get_states',
return_value=states), \
patch('homeassistant.helpers.restore_state.wait_connection_ready',
return_value=mock_coro(False)):
state = yield from async_get_last_state(hass, 'input_boolean.b1')
assert state is None
@asyncio.coroutine
def test_no_last_run_found(hass):
"""Test that cache cannot be accessed if no last run found."""
mock_component(hass, 'recorder')
hass.state = CoreState.starting
states = [State('input_boolean.b1', 'on')]
with patch('homeassistant.helpers.restore_state.last_recorder_run',
return_value=None), \
patch('homeassistant.helpers.restore_state.get_states',
return_value=states), \
patch('homeassistant.helpers.restore_state.wait_connection_ready',
return_value=mock_coro(True)):
state = yield from async_get_last_state(hass, 'input_boolean.b1')
assert state is None
@asyncio.coroutine
def test_cache_timeout(hass):
"""Test that cache timeout returns none."""
mock_component(hass, 'recorder')
hass.state = CoreState.starting
states = [State('input_boolean.b1', 'on')]
@asyncio.coroutine
def timeout_coro():
raise asyncio.TimeoutError()
with patch('homeassistant.helpers.restore_state.last_recorder_run',
return_value=MagicMock(end=dt_util.utcnow())), \
patch('homeassistant.helpers.restore_state.get_states',
return_value=states), \
patch('homeassistant.helpers.restore_state.wait_connection_ready',
return_value=timeout_coro()):
state = yield from async_get_last_state(hass, 'input_boolean.b1')
assert state is None
def _add_data_in_last_run(hass, entities):
"""Add test data in the last recorder_run."""
# pylint: disable=protected-access
t_now = dt_util.utcnow() - timedelta(minutes=10)
t_min_1 = t_now - timedelta(minutes=20)
t_min_2 = t_now - timedelta(minutes=30)
with recorder.session_scope(hass=hass) as session:
session.add(RecorderRuns(
start=t_min_2,
end=t_now,
created=t_min_2
))
for entity_id, state in entities.items():
session.add(States(
entity_id=entity_id,
domain=split_entity_id(entity_id)[0],
state=state,
attributes='{}',
last_changed=t_min_1,
last_updated=t_min_1,
created=t_min_1))
def test_filling_the_cache():
"""Test filling the cache from the DB."""
test_entity_id1 = 'input_boolean.b1'
test_entity_id2 = 'input_boolean.b2'
hass = get_test_home_assistant()
hass.state = CoreState.starting
init_recorder_component(hass)
_add_data_in_last_run(hass, {
test_entity_id1: 'on',
test_entity_id2: 'off',
})
hass.block_till_done()
setup_component(hass, input_boolean.DOMAIN, {
input_boolean.DOMAIN: {
'b1': None,
'b2': None,
}})
hass.start()
state = hass.states.get('input_boolean.b1')
assert state
assert state is not None
assert state.entity_id == 'input_boolean.b1'
assert state.state == 'on'
state = hass.states.get('input_boolean.b2')
assert state
assert state.state == 'off'
# Assert that no data was written yet, since hass is still starting.
assert not mock_write_data.called
hass.stop()
# Finish hass startup
with patch('homeassistant.helpers.restore_state.Store.async_save'
) as mock_write_data:
hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
await hass.async_block_till_done()
# Assert that this session states were written
assert mock_write_data.called
async def test_dump_data(hass):
"""Test that we cache data."""
states = [
State('input_boolean.b0', 'on'),
State('input_boolean.b1', 'on'),
State('input_boolean.b2', 'on'),
]
entity = Entity()
entity.hass = hass
entity.entity_id = 'input_boolean.b0'
await entity.async_added_to_hass()
entity = RestoreEntity()
entity.hass = hass
entity.entity_id = 'input_boolean.b1'
await entity.async_added_to_hass()
data = await RestoreStateData.async_get_instance(hass)
data.last_states = {
'input_boolean.b0': State('input_boolean.b0', 'off'),
'input_boolean.b1': State('input_boolean.b1', 'off'),
'input_boolean.b2': State('input_boolean.b2', 'off'),
'input_boolean.b3': State('input_boolean.b3', 'off'),
'input_boolean.b4': State(
'input_boolean.b4', 'off', last_updated=datetime(
1985, 10, 26, 1, 22, tzinfo=dt_util.UTC)),
}
with patch('homeassistant.helpers.restore_state.Store.async_save'
) as mock_write_data, patch.object(
hass.states, 'async_all', return_value=states):
await data.async_dump_states()
assert mock_write_data.called
args = mock_write_data.mock_calls[0][1]
written_states = args[0]
# b0 should not be written, since it didn't extend RestoreEntity
# b1 should be written, since it is present in the current run
# b2 should not be written, since it is not registered with the helper
# b3 should be written, since it is still not expired
# b4 should not be written, since it is now expired
assert len(written_states) == 2
assert written_states[0]['entity_id'] == 'input_boolean.b1'
assert written_states[0]['state'] == 'on'
assert written_states[1]['entity_id'] == 'input_boolean.b3'
assert written_states[1]['state'] == 'off'
# Test that removed entities are not persisted
await entity.async_will_remove_from_hass()
with patch('homeassistant.helpers.restore_state.Store.async_save'
) as mock_write_data, patch.object(
hass.states, 'async_all', return_value=states):
await data.async_dump_states()
assert mock_write_data.called
args = mock_write_data.mock_calls[0][1]
written_states = args[0]
assert len(written_states) == 1
assert written_states[0]['entity_id'] == 'input_boolean.b3'
assert written_states[0]['state'] == 'off'
async def test_dump_error(hass):
"""Test that we cache data."""
states = [
State('input_boolean.b0', 'on'),
State('input_boolean.b1', 'on'),
State('input_boolean.b2', 'on'),
]
entity = Entity()
entity.hass = hass
entity.entity_id = 'input_boolean.b0'
await entity.async_added_to_hass()
entity = RestoreEntity()
entity.hass = hass
entity.entity_id = 'input_boolean.b1'
await entity.async_added_to_hass()
data = await RestoreStateData.async_get_instance(hass)
with patch('homeassistant.helpers.restore_state.Store.async_save',
return_value=mock_coro(exception=HomeAssistantError)
) as mock_write_data, patch.object(
hass.states, 'async_all', return_value=states):
await data.async_dump_states()
assert mock_write_data.called
async def test_load_error(hass):
"""Test that we cache data."""
entity = RestoreEntity()
entity.hass = hass
entity.entity_id = 'input_boolean.b1'
with patch('homeassistant.helpers.storage.Store.async_load',
return_value=mock_coro(exception=HomeAssistantError)):
state = await entity.async_get_last_state()
assert state is None
+17 -1
View File
@@ -1,7 +1,8 @@
"""Tests for the storage helper."""
import asyncio
from datetime import timedelta
from unittest.mock import patch
import json
from unittest.mock import patch, Mock
import pytest
@@ -31,6 +32,21 @@ async def test_loading(hass, store):
assert data == MOCK_DATA
async def test_custom_encoder(hass):
"""Test we can save and load data."""
class JSONEncoder(json.JSONEncoder):
"""Mock JSON encoder."""
def default(self, o):
"""Mock JSON encode method."""
return "9"
store = storage.Store(hass, MOCK_VERSION, MOCK_KEY, encoder=JSONEncoder)
await store.async_save(Mock())
data = await store.async_load()
assert data == "9"
async def test_loading_non_existing(hass, store):
"""Test we can save and load data."""
with patch('homeassistant.util.json.open', side_effect=FileNotFoundError):
+19 -2
View File
@@ -1,14 +1,17 @@
"""Test Home Assistant json utility functions."""
from json import JSONEncoder
import os
import unittest
import sys
from tempfile import mkdtemp
from homeassistant.util.json import (SerializationError,
load_json, save_json)
from homeassistant.util.json import (
SerializationError, load_json, save_json)
from homeassistant.exceptions import HomeAssistantError
import pytest
from unittest.mock import Mock
# Test data that can be saved as JSON
TEST_JSON_A = {"a": 1, "B": "two"}
TEST_JSON_B = {"a": "one", "B": 2}
@@ -74,3 +77,17 @@ class TestJSON(unittest.TestCase):
fh.write(TEST_BAD_SERIALIED)
with pytest.raises(HomeAssistantError):
load_json(fname)
def test_custom_encoder(self):
"""Test serializing with a custom encoder."""
class MockJSONEncoder(JSONEncoder):
"""Mock JSON encoder."""
def default(self, o):
"""Mock JSON encode method."""
return "9"
fname = self._path_for("test6")
save_json(fname, Mock(), encoder=MockJSONEncoder)
data = load_json(fname)
self.assertEqual(data, "9")