Prefilter more logbook events in sql (#36958)

* Prefilter more logbook events in sql

Prefilter sensor events in _keep_event before humanify

Cache static attribute lookup

Reduces logbook execution time by ~35%

* fix mocking in benchmark

* Update tests for logbook users
This commit is contained in:
J. Nick Koston authored and GitHub committed 2020-06-21 10:50:58 -07:00
1 parent 29f128eaad
commit 59e43ab6e4
7 files changed
+275 -108

No files matched your search

+90 -26
View File
@@ -18,11 +18,13 @@ from homeassistant.components.recorder.util import (
session_scope,
)
from homeassistant.const import (
ATTR_DEVICE_CLASS,
ATTR_DOMAIN,
ATTR_ENTITY_ID,
ATTR_FRIENDLY_NAME,
ATTR_HIDDEN,
ATTR_NAME,
ATTR_UNIT_OF_MEASUREMENT,
CONF_EXCLUDE,
CONF_INCLUDE,
EVENT_HOMEASSISTANT_START,
@@ -52,6 +54,8 @@ DOMAIN = "logbook"
GROUP_BY_MINUTES = 15
EMPTY_JSON_OBJECT = "{}"
CONFIG_SCHEMA = vol.Schema(
{
DOMAIN: vol.Schema(
@@ -194,7 +198,7 @@ class LogbookView(HomeAssistantView):
return await hass.async_add_job(json_events)
def humanify(hass, events, prev_states=None):
def humanify(hass, events, entity_attr_cache, prev_states=None):
"""Generate a converted list of events into Entry objects.
Will try to group events if possible:
@@ -257,24 +261,22 @@ def humanify(hass, events, prev_states=None):
prev_states[entity_id] = event.state
domain = event.domain
if domain in CONTINUOUS_DOMAINS:
if (
domain in CONTINUOUS_DOMAINS
and event != last_sensor_event[entity_id]
):
# Skip all but the last sensor state
if event != last_sensor_event[entity_id]:
continue
continue
# Don't show continuous sensor value changes in the logbook
if _get_attribute(hass, entity_id, event, "unit_of_measurement"):
continue
name = _get_attribute(
hass, entity_id, event, ATTR_FRIENDLY_NAME
name = entity_attr_cache.get(
entity_id, ATTR_FRIENDLY_NAME, event
) or split_entity_id(entity_id)[1].replace("_", " ")
yield {
"when": event.time_fired,
"name": name,
"message": _entry_message_from_event(
hass, entity_id, domain, event
hass, entity_id, domain, event, entity_attr_cache
),
"domain": domain,
"entity_id": entity_id,
@@ -375,12 +377,13 @@ def _generate_filter_from_config(config):
def _get_events(hass, config, start_day, end_day, entity_id=None):
"""Get events for a period of time."""
entities_filter = _generate_filter_from_config(config)
entity_attr_cache = EntityAttributeCache(hass)
def yield_events(query):
"""Yield Events that are not filtered away."""
for row in query.yield_per(1000):
event = LazyEventPartialState(row)
if _keep_event(hass, event, entities_filter):
if _keep_event(hass, event, entities_filter, entity_attr_cache):
yield event
with session_scope(hass=hass) as session:
@@ -409,6 +412,24 @@ def _get_events(hass, config, start_day, end_day, entity_id=None):
.order_by(Events.time_fired)
.outerjoin(States, (Events.event_id == States.event_id))
.outerjoin(old_state, (States.old_state_id == old_state.state_id))
# The below filter, removes state change events that do not have
# and old_state, new_state, or the old and
# new state are the same for v8 schema or later.
#
# If the events/states were stored before v8 schema, we relay on the
# prev_states dict to remove them.
#
# When all data is schema v8 or later, the check for EMPTY_JSON_OBJECT
# can be removed.
.filter(
(Events.event_type != EVENT_STATE_CHANGED)
| (Events.event_data != EMPTY_JSON_OBJECT)
| (
(States.state_id.isnot(None))
& (old_state.state_id.isnot(None))
& (States.state != old_state.state)
)
)
.filter(
Events.event_type.in_(ALL_EVENT_TYPES + list(hass.data.get(DOMAIN, {})))
)
@@ -429,18 +450,12 @@ def _get_events(hass, config, start_day, end_day, entity_id=None):
| (States.state_id.is_(None))
)
# When all data is schema v8 or later, prev_states can be removed
prev_states = {}
return list(humanify(hass, yield_events(query), prev_states))
return list(humanify(hass, yield_events(query), entity_attr_cache, prev_states))
def _get_attribute(hass, entity_id, event, attribute):
current_state = hass.states.get(entity_id)
if not current_state:
return event.attributes.get(attribute)
return current_state.attributes.get(attribute, None)
def _keep_event(hass, event, entities_filter):
def _keep_event(hass, event, entities_filter, entity_attr_cache):
if event.event_type == EVENT_STATE_CHANGED:
entity_id = event.entity_id
@@ -456,6 +471,11 @@ def _keep_event(hass, event, entities_filter):
if event.hidden:
return False
if event.domain in CONTINUOUS_DOMAINS and entity_attr_cache.get(
entity_id, ATTR_UNIT_OF_MEASUREMENT, event
):
# Don't show continuous sensor value changes in the logbook
return False
elif event.event_type == EVENT_LOGBOOK_ENTRY:
event_data = event.data
domain = event_data.get(ATTR_DOMAIN)
@@ -478,7 +498,7 @@ def _keep_event(hass, event, entities_filter):
return not entity_id or entities_filter(entity_id)
def _entry_message_from_event(hass, entity_id, domain, event):
def _entry_message_from_event(hass, entity_id, domain, event, entity_attr_cache):
"""Convert a state to a message for the logbook."""
# We pass domain in so we don't have to split entity_id again
state_state = event.state
@@ -494,7 +514,7 @@ def _entry_message_from_event(hass, entity_id, domain, event):
return "has set"
if domain == "binary_sensor":
device_class = _get_attribute(hass, entity_id, event, "device_class")
device_class = entity_attr_cache.get(entity_id, ATTR_DEVICE_CLASS, event)
if device_class == "battery":
if state_state == STATE_ON:
return "is low"
@@ -600,7 +620,10 @@ class LazyEventPartialState:
def attributes(self):
"""State attributes."""
if not self._attributes:
if self._row.attributes is None or self._row.attributes == "{}":
if (
self._row.attributes is None
or self._row.attributes == EMPTY_JSON_OBJECT
):
self._attributes = {}
else:
self._attributes = json.loads(self._row.attributes)
@@ -611,7 +634,7 @@ class LazyEventPartialState:
"""Event data."""
if not self._event_data:
if self._row.event_data == "{}":
if self._row.event_data == EMPTY_JSON_OBJECT:
self._event_data = {}
else:
self._event_data = json.loads(self._row.event_data)
@@ -634,9 +657,15 @@ class LazyEventPartialState:
@property
def has_old_and_new_state(self):
"""Check the json data to see if new_state and old_state is present without decoding."""
if self._row.event_data == "{}":
# Delete this check once all states are saved in the v8 schema
# format or later (they have the old_state_id column).
# New events in v8 schema format
if self._row.event_data == EMPTY_JSON_OBJECT:
return self._row.state_id is not None and self._row.old_state_id is not None
# Old events not in v8 schema format
return (
'"old_state": {' in self._row.event_data
and '"new_state": {' in self._row.event_data
@@ -648,3 +677,38 @@ class LazyEventPartialState:
if '"hidden":' in self._row.attributes:
return self.attributes.get(ATTR_HIDDEN, False)
return False
class EntityAttributeCache:
"""A cache to lookup static entity_id attribute.
This class should not be used to lookup attributes
that are expected to change state.
"""
def __init__(self, hass):
"""Init the cache."""
self._hass = hass
self._cache = {}
def get(self, entity_id, attribute, event):
"""Lookup an attribute for an entity or get it from the cache."""
if entity_id in self._cache:
if attribute in self._cache[entity_id]:
return self._cache[entity_id][attribute]
else:
self._cache[entity_id] = {}
current_state = self._hass.states.get(entity_id)
if current_state:
# Try the current state as its faster than decoding the
# attributes
self._cache[entity_id][attribute] = current_state.attributes.get(
attribute, None
)
else:
# If the entity has been removed, decode the attributes
# instead
self._cache[entity_id][attribute] = event.attributes.get(attribute)
return self._cache[entity_id][attribute]