Add a commit interval setting to recorder (#32596)

* Add a commit interval setting to recorder

* Make the default every 1s instead of immediate

* See attached py-spy flamegraphs for why 1s

* This avoids disk thrashing during event storms

* Make Home Assistant significantly more responsive on busy systems

* remove debug

* Add commit forces for tests that expect commits to be immediate

* Add commit forces for tests that expect commits to be immediate

* make sure _trigger_db_commit is in the right place (all effective "wait_recording_done" calls)

* De-duplicate wait_recording_done code
This commit is contained in:
J. Nick Koston authored and GitHub committed 2020-03-09 17:43:26 -07:00
1 parent 8a46d93be4
commit 3a680bf7b7
5 files changed
+124 -63

No files matched your search

+8 -11
View File
@@ -14,6 +14,7 @@ from tests.common import (
init_recorder_component,
mock_state_change_event,
)
from tests.components.recorder.common import wait_recording_done
class TestComponentHistory(unittest.TestCase):
@@ -31,12 +32,7 @@ class TestComponentHistory(unittest.TestCase):
"""Initialize the recorder."""
init_recorder_component(self.hass)
self.hass.start()
self.wait_recording_done()
def wait_recording_done(self):
"""Block till recording is done."""
self.hass.block_till_done()
self.hass.data[recorder.DATA_INSTANCE].block_till_done()
wait_recording_done(self.hass)
def test_setup(self):
"""Test setup method of history."""
@@ -78,7 +74,7 @@ class TestComponentHistory(unittest.TestCase):
states.append(state)
self.wait_recording_done()
wait_recording_done(self.hass)
future = now + timedelta(seconds=1)
with patch(
@@ -93,7 +89,7 @@ class TestComponentHistory(unittest.TestCase):
mock_state_change_event(self.hass, state)
self.wait_recording_done()
wait_recording_done(self.hass)
# Get states returns everything before POINT
for state1, state2 in zip(
@@ -115,7 +111,7 @@ class TestComponentHistory(unittest.TestCase):
def set_state(state):
"""Set the state."""
self.hass.states.set(entity_id, state)
self.wait_recording_done()
wait_recording_done(self.hass)
return self.hass.states.get(entity_id)
start = dt_util.utcnow()
@@ -156,7 +152,7 @@ class TestComponentHistory(unittest.TestCase):
def set_state(state):
"""Set the state."""
self.hass.states.set(entity_id, state)
self.wait_recording_done()
wait_recording_done(self.hass)
return self.hass.states.get(entity_id)
start = dt_util.utcnow() - timedelta(minutes=2)
@@ -559,7 +555,7 @@ class TestComponentHistory(unittest.TestCase):
def set_state(entity_id, state, **kwargs):
"""Set the state."""
self.hass.states.set(entity_id, state, **kwargs)
self.wait_recording_done()
wait_recording_done(self.hass)
return self.hass.states.get(entity_id)
zero = dt_util.utcnow()
@@ -615,6 +611,7 @@ class TestComponentHistory(unittest.TestCase):
)
# state will be skipped since entity is hidden
set_state(therm, 22, attributes={"current_temperature": 21, "hidden": True})
return zero, four, states
+3
View File
@@ -1,6 +1,7 @@
"""The tests for the logbook component."""
# pylint: disable=protected-access,invalid-name
from datetime import datetime, timedelta
from functools import partial
import logging
import unittest
@@ -35,6 +36,7 @@ from homeassistant.setup import async_setup_component, setup_component
import homeassistant.util.dt as dt_util
from tests.common import get_test_home_assistant, init_recorder_component
from tests.components.recorder.common import trigger_db_commit
_LOGGER = logging.getLogger(__name__)
@@ -1288,6 +1290,7 @@ async def test_logbook_view_period_entity(hass, hass_client):
entity_id_second = "switch.second"
hass.states.async_set(entity_id_second, STATE_OFF)
hass.states.async_set(entity_id_second, STATE_ON)
await hass.async_add_job(partial(trigger_db_commit, hass))
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
+22
View File
@@ -0,0 +1,22 @@
"""Common test utils for working with recorder."""
from homeassistant.components import recorder
from homeassistant.util import dt as dt_util
from tests.common import fire_time_changed
DB_COMMIT_INTERVAL = 50
def wait_recording_done(hass):
"""Block till recording is done."""
trigger_db_commit(hass)
hass.block_till_done()
hass.data[recorder.DATA_INSTANCE].block_till_done()
def trigger_db_commit(hass):
"""Force the recorder to commit."""
for _ in range(DB_COMMIT_INTERVAL):
# We only commit on time change
fire_time_changed(hass, dt_util.utcnow())
+7 -7
View File
@@ -13,6 +13,8 @@ from homeassistant.const import MATCH_ALL
from homeassistant.core import callback
from homeassistant.setup import async_setup_component
from .common import wait_recording_done
from tests.common import get_test_home_assistant, init_recorder_component
@@ -37,8 +39,7 @@ class TestRecorder(unittest.TestCase):
self.hass.states.set(entity_id, state, attributes)
self.hass.block_till_done()
self.hass.data[DATA_INSTANCE].block_till_done()
wait_recording_done(self.hass)
with session_scope(hass=self.hass) as session:
db_states = list(session.query(States))
@@ -65,7 +66,7 @@ class TestRecorder(unittest.TestCase):
self.hass.bus.fire(event_type, event_data)
self.hass.block_till_done()
wait_recording_done(self.hass)
assert len(events) == 1
event = events[0]
@@ -109,8 +110,7 @@ def _add_entities(hass, entity_ids):
attributes = {"test_attr": 5, "test_attr_10": "nice"}
for idx, entity_id in enumerate(entity_ids):
hass.states.set(entity_id, "state{}".format(idx), attributes)
hass.block_till_done()
hass.data[DATA_INSTANCE].block_till_done()
wait_recording_done(hass)
with session_scope(hass=hass) as session:
return [st.to_native() for st in session.query(States)]
@@ -121,8 +121,7 @@ def _add_events(hass, events):
session.query(Events).delete(synchronize_session=False)
for event_type in events:
hass.bus.fire(event_type)
hass.block_till_done()
hass.data[DATA_INSTANCE].block_till_done()
wait_recording_done(hass)
with session_scope(hass=hass) as session:
return [ev.to_native() for ev in session.query(Events)]
@@ -201,6 +200,7 @@ def test_recorder_setup_failure():
hass,
keep_days=7,
purge_interval=2,
commit_interval=1,
uri="sqlite://",
db_max_retries=10,
db_retry_wait=3,