Add a commit interval setting to recorder (#32596)

* Add a commit interval setting to recorder

* Make the default every 1s instead of immediate

* See attached py-spy flamegraphs for why 1s

* This avoids disk thrashing during event storms

* Make Home Assistant significantly more responsive on busy systems

* remove debug

* Add commit forces for tests that expect commits to be immediate

* Add commit forces for tests that expect commits to be immediate

* make sure _trigger_db_commit is in the right place (all effective "wait_recording_done" calls)

* De-duplicate wait_recording_done code
This commit is contained in:
J. Nick Koston authored and GitHub committed 2020-03-09 17:43:26 -07:00
1 parent 8a46d93be4
commit 3a680bf7b7
5 files changed
+124 -63

No files matched your search

+84 -45
View File
@@ -68,6 +68,7 @@ CONF_DB_RETRY_WAIT = "db_retry_wait"
CONF_PURGE_KEEP_DAYS = "purge_keep_days"
CONF_PURGE_INTERVAL = "purge_interval"
CONF_EVENT_TYPES = "event_types"
CONF_COMMIT_INTERVAL = "commit_interval"
FILTER_SCHEMA = vol.Schema(
{
@@ -98,6 +99,9 @@ CONFIG_SCHEMA = vol.Schema(
vol.Coerce(int), vol.Range(min=0)
),
vol.Optional(CONF_DB_URL): cv.string,
vol.Optional(CONF_COMMIT_INTERVAL, default=1): vol.All(
vol.Coerce(int), vol.Range(min=0)
),
vol.Optional(
CONF_DB_MAX_RETRIES, default=DEFAULT_DB_MAX_RETRIES
): cv.positive_int,
@@ -141,6 +145,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
conf = config[DOMAIN]
keep_days = conf.get(CONF_PURGE_KEEP_DAYS)
purge_interval = conf.get(CONF_PURGE_INTERVAL)
commit_interval = conf[CONF_COMMIT_INTERVAL]
db_max_retries = conf[CONF_DB_MAX_RETRIES]
db_retry_wait = conf[CONF_DB_RETRY_WAIT]
@@ -154,6 +159,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
hass=hass,
keep_days=keep_days,
purge_interval=purge_interval,
commit_interval=commit_interval,
uri=db_url,
db_max_retries=db_max_retries,
db_retry_wait=db_retry_wait,
@@ -185,6 +191,7 @@ class Recorder(threading.Thread):
hass: HomeAssistant,
keep_days: int,
purge_interval: int,
commit_interval: int,
uri: str,
db_max_retries: int,
db_retry_wait: int,
@@ -197,6 +204,7 @@ class Recorder(threading.Thread):
self.hass = hass
self.keep_days = keep_days
self.purge_interval = purge_interval
self.commit_interval = commit_interval
self.queue: Any = queue.Queue()
self.recording_start = dt_util.utcnow()
self.db_url = uri
@@ -214,6 +222,8 @@ class Recorder(threading.Thread):
)
self.exclude_t = exclude.get(CONF_EVENT_TYPES, [])
self._timechanges_seen = 0
self.event_session = None
self.get_session = None
@callback
@@ -326,6 +336,10 @@ class Recorder(threading.Thread):
self.hass.helpers.event.track_point_in_time(async_purge, run)
self.event_session = self.get_session()
# Use a session for the event read loop
# with a commit every time the event time
# has changed. This reduces the disk io.
while True:
event = self.queue.get()
@@ -340,6 +354,11 @@ class Recorder(threading.Thread):
continue
if event.event_type == EVENT_TIME_CHANGED:
self.queue.task_done()
if self.commit_interval:
self._timechanges_seen += 1
if self.commit_interval >= self._timechanges_seen:
self._timechanges_seen = 0
self._commit_event_session_or_retry()
continue
if event.event_type in self.exclude_t:
self.queue.task_done()
@@ -351,55 +370,72 @@ class Recorder(threading.Thread):
self.queue.task_done()
continue
tries = 1
updated = False
while not updated and tries <= self.db_max_retries:
if tries != 1:
time.sleep(self.db_retry_wait)
try:
dbevent = Events.from_event(event)
self.event_session.add(dbevent)
self.event_session.flush()
except (TypeError, ValueError):
_LOGGER.warning("Event is not JSON serializable: %s", event)
if dbevent and event.event_type == EVENT_STATE_CHANGED:
try:
with session_scope(session=self.get_session()) as session:
try:
dbevent = Events.from_event(event)
session.add(dbevent)
session.flush()
except (TypeError, ValueError):
_LOGGER.warning("Event is not JSON serializable: %s", event)
if event.event_type == EVENT_STATE_CHANGED:
try:
dbstate = States.from_event(event)
dbstate.event_id = dbevent.event_id
session.add(dbstate)
except (TypeError, ValueError):
_LOGGER.warning(
"State is not JSON serializable: %s",
event.data.get("new_state"),
)
updated = True
except exc.OperationalError as err:
_LOGGER.error(
"Error in database connectivity: %s. "
"(retrying in %s seconds)",
err,
self.db_retry_wait,
dbstate = States.from_event(event)
dbstate.event_id = dbevent.event_id
self.event_session.add(dbstate)
except (TypeError, ValueError):
_LOGGER.warning(
"State is not JSON serializable: %s",
event.data.get("new_state"),
)
tries += 1
except exc.SQLAlchemyError:
updated = True
_LOGGER.exception("Error saving event: %s", event)
if not updated:
_LOGGER.error(
"Error in database update. Could not save "
"after %d tries. Giving up",
tries,
)
# If they do not have a commit interval
# than we commit right away
if not self.commit_interval:
self._commit_event_session_or_retry()
self.queue.task_done()
def _commit_event_session_or_retry(self):
tries = 1
while tries <= self.db_max_retries:
if tries != 1:
time.sleep(self.db_retry_wait)
try:
self._commit_event_session()
return
except exc.OperationalError as err:
_LOGGER.error(
"Error in database connectivity: %s. " "(retrying in %s seconds)",
err,
self.db_retry_wait,
)
tries += 1
except exc.SQLAlchemyError:
_LOGGER.exception("Error saving events")
return
_LOGGER.error(
"Error in database update. Could not save " "after %d tries. Giving up",
tries,
)
try:
self.event_session.close()
except exc.SQLAlchemyError:
_LOGGER.exception("Failed to close event session.")
self.event_session = self.get_session()
def _commit_event_session(self):
try:
self.event_session.commit()
except Exception as err:
_LOGGER.error("Error executing query: %s", err)
self.event_session.rollback()
raise
@callback
def event_listener(self, event):
"""Listen for new events and put them in the process queue."""
@@ -465,7 +501,10 @@ class Recorder(threading.Thread):
def _close_run(self):
"""Save end time for current run."""
with session_scope(session=self.get_session()) as session:
if self.event_session is not None:
self.run_info.end = dt_util.utcnow()
session.add(self.run_info)
self.event_session.add(self.run_info)
self._commit_event_session_or_retry()
self.event_session.close()
self.run_info = None