Improve history api performance (#35822)
* Improve history api performance
A new option "minimal_response" reduces the amount of data
sent between the first and last history states to only the
"last_changed" and "state" fields.
Calling to_native is now avoided where possible and only
done at the end for rows that will be returned in the response.
When sending the `minimal_response` option, the history
api now returns a json response similar to the following
for an entity
Testing:
History API Response time for 1 day
Average of 10 runs with minimal_response
Before: 19.89s. (content length : 3427428)
After: 8.44s (content length: 592199)
```
[{
"attributes": {--TRUNCATED--},
"context": {--TRUNCATED--},
"entity_id": "binary_sensor.powerwall_status",
"last_changed": "2020-05-18T23:20:03.213000+00:00",
"last_updated": "2020-05-18T23:20:03.213000+00:00",
"state": "on"
},
...
{
"last_changed": "2020-05-19T00:41:08Z",
"state": "unavailable"
},
...
{
"attributes": {--TRUNCATED--},
"context": {--TRUNCATED--},
"entity_id": "binary_sensor.powerwall_status",
"last_changed": "2020-05-19T00:42:08.069698+00:00",
"last_updated": "2020-05-19T00:42:08.069698+00:00",
"state": "on"
}]
```
* Remove impossible state check
* Remove another impossible state check
* Update homeassistant/components/history/__init__.py
Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>
* Reorder to save some indent per review
* Make query response make sense with to_native=False
* Update test for 00:00 to Z change
* Update homeassistant/components/recorder/models.py
Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>
Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>
This commit is contained in:
1 parent
6a06d648d7
commit
0a6deeb49b
4 files changed
+194
-37
No files matched your search
@@ -1,10 +1,13 @@
|
||||
"""The tests the History component."""
|
||||
# pylint: disable=protected-access,invalid-name
|
||||
from datetime import timedelta
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from homeassistant.components import history, recorder
|
||||
from homeassistant.components.recorder.models import process_timestamp
|
||||
import homeassistant.core as ha
|
||||
from homeassistant.helpers.json import JSONEncoder
|
||||
from homeassistant.setup import async_setup_component, setup_component
|
||||
import homeassistant.util.dt as dt_util
|
||||
|
||||
@@ -197,6 +200,41 @@ class TestComponentHistory(unittest.TestCase):
|
||||
)
|
||||
assert states == hist
|
||||
|
||||
def test_get_significant_states_minimal_response(self):
|
||||
"""Test that only significant states are returned.
|
||||
|
||||
When minimal responses is set only the first and
|
||||
last states return a complete state.
|
||||
|
||||
We should get back every thermostat change that
|
||||
includes an attribute change, but only the state updates for
|
||||
media player (attribute changes are not significant and not returned).
|
||||
"""
|
||||
zero, four, states = self.record_states()
|
||||
hist = history.get_significant_states(
|
||||
self.hass, zero, four, filters=history.Filters(), minimal_response=True
|
||||
)
|
||||
|
||||
# The second media_player.test state is reduced
|
||||
# down to last_changed and state when minimal_response
|
||||
# is set. We use JSONEncoder to make sure that are
|
||||
# pre-encoded last_changed is always the same as what
|
||||
# will happen with encoding a native state
|
||||
input_state = states["media_player.test"][1]
|
||||
orig_last_changed = json.dumps(
|
||||
process_timestamp(input_state.last_changed.replace(microsecond=0)),
|
||||
cls=JSONEncoder,
|
||||
).replace('"', "")
|
||||
if orig_last_changed.endswith("+00:00"):
|
||||
orig_last_changed = f"{orig_last_changed[:-6]}{recorder.models.DB_TIMEZONE}"
|
||||
orig_state = input_state.state
|
||||
states["media_player.test"][1] = {
|
||||
"last_changed": orig_last_changed,
|
||||
"state": orig_state,
|
||||
}
|
||||
|
||||
assert states == hist
|
||||
|
||||
def test_get_significant_states_with_initial(self):
|
||||
"""Test that only significant states are returned.
|
||||
|
||||
@@ -252,6 +290,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
"""Test that only significant states are returned for one entity."""
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["thermostat.test"]
|
||||
del states["thermostat.test2"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
@@ -265,6 +304,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
"""Test that only significant states are returned for one entity."""
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["thermostat.test2"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
|
||||
@@ -286,6 +326,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test"]
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
|
||||
config = history.CONFIG_SCHEMA(
|
||||
{
|
||||
@@ -346,6 +387,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
"""
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["thermostat.test"]
|
||||
del states["thermostat.test2"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
@@ -372,6 +414,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test"]
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
|
||||
config = history.CONFIG_SCHEMA(
|
||||
{
|
||||
@@ -392,6 +435,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
"""
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["thermostat.test"]
|
||||
del states["thermostat.test2"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
@@ -414,6 +458,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
"""
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
|
||||
config = history.CONFIG_SCHEMA(
|
||||
@@ -438,6 +483,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test"]
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["thermostat.test"]
|
||||
del states["thermostat.test2"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
@@ -462,6 +508,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
zero, four, states = self.record_states()
|
||||
del states["media_player.test"]
|
||||
del states["media_player.test2"]
|
||||
del states["media_player.test3"]
|
||||
del states["thermostat.test"]
|
||||
del states["thermostat.test2"]
|
||||
del states["script.can_cancel_this_one"]
|
||||
@@ -607,6 +654,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
self.init_recorder()
|
||||
mp = "media_player.test"
|
||||
mp2 = "media_player.test2"
|
||||
mp3 = "media_player.test3"
|
||||
therm = "thermostat.test"
|
||||
therm2 = "thermostat.test2"
|
||||
zone = "zone.home"
|
||||
@@ -625,7 +673,7 @@ class TestComponentHistory(unittest.TestCase):
|
||||
three = two + timedelta(seconds=1)
|
||||
four = three + timedelta(seconds=1)
|
||||
|
||||
states = {therm: [], therm2: [], mp: [], mp2: [], script_c: []}
|
||||
states = {therm: [], therm2: [], mp: [], mp2: [], mp3: [], script_c: []}
|
||||
with patch(
|
||||
"homeassistant.components.recorder.dt_util.utcnow", return_value=one
|
||||
):
|
||||
@@ -638,6 +686,9 @@ class TestComponentHistory(unittest.TestCase):
|
||||
states[mp2].append(
|
||||
set_state(mp2, "YouTube", attributes={"media_title": str(sentinel.mt2)})
|
||||
)
|
||||
states[mp3].append(
|
||||
set_state(mp3, "idle", attributes={"media_title": str(sentinel.mt1)})
|
||||
)
|
||||
states[therm].append(
|
||||
set_state(therm, 20, attributes={"current_temperature": 19.5})
|
||||
)
|
||||
@@ -647,6 +698,12 @@ class TestComponentHistory(unittest.TestCase):
|
||||
):
|
||||
# This state will be skipped only different in time
|
||||
set_state(mp, "YouTube", attributes={"media_title": str(sentinel.mt3)})
|
||||
# This state will be skipped as it hidden
|
||||
set_state(
|
||||
mp3,
|
||||
"Apple TV",
|
||||
attributes={"media_title": str(sentinel.mt2), "hidden": True},
|
||||
)
|
||||
# This state will be skipped because domain blacklisted
|
||||
set_state(zone, "zoning")
|
||||
set_state(script_nc, "off")
|
||||
@@ -666,6 +723,9 @@ class TestComponentHistory(unittest.TestCase):
|
||||
states[mp].append(
|
||||
set_state(mp, "Netflix", attributes={"media_title": str(sentinel.mt4)})
|
||||
)
|
||||
states[mp3].append(
|
||||
set_state(mp3, "Netflix", attributes={"media_title": str(sentinel.mt3)})
|
||||
)
|
||||
# Attributes changed even though state is the same
|
||||
states[therm].append(
|
||||
set_state(therm, 21, attributes={"current_temperature": 20})
|
||||
@@ -686,6 +746,30 @@ async def test_fetch_period_api(hass, hass_client):
|
||||
assert response.status == 200
|
||||
|
||||
|
||||
async def test_fetch_period_api_with_use_include_order(hass, hass_client):
|
||||
"""Test the fetch period view for history with include order."""
|
||||
await hass.async_add_executor_job(init_recorder_component, hass)
|
||||
await async_setup_component(
|
||||
hass, "history", {history.DOMAIN: {history.CONF_ORDER: True}}
|
||||
)
|
||||
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
|
||||
client = await hass_client()
|
||||
response = await client.get(f"/api/history/period/{dt_util.utcnow().isoformat()}")
|
||||
assert response.status == 200
|
||||
|
||||
|
||||
async def test_fetch_period_api_with_minimal_response(hass, hass_client):
|
||||
"""Test the fetch period view for history with minimal_response."""
|
||||
await hass.async_add_executor_job(init_recorder_component, hass)
|
||||
await async_setup_component(hass, "history", {})
|
||||
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
|
||||
client = await hass_client()
|
||||
response = await client.get(
|
||||
f"/api/history/period/{dt_util.utcnow().isoformat()}?minimal_response"
|
||||
)
|
||||
assert response.status == 200
|
||||
|
||||
|
||||
async def test_fetch_period_api_with_include_order(hass, hass_client):
|
||||
"""Test the fetch period view for history."""
|
||||
await hass.async_add_executor_job(init_recorder_component, hass)
|
||||
|
||||
Reference in new issue
Block a user