Improve history api performance (#35822)

* Improve history api performance

A new option "minimal_response" reduces the amount of data
sent between the first and last history states to only the
"last_changed" and "state" fields.

Calling to_native is now avoided where possible and only
done at the end for rows that will be returned in the response.

When sending the `minimal_response` option, the history
api now returns a json response similar to the following
for an entity

Testing:

History API Response time for 1 day
Average of 10 runs with minimal_response

Before: 19.89s. (content length : 3427428)
After: 8.44s (content length: 592199)

```
[{
	"attributes": {--TRUNCATED--},
	"context": {--TRUNCATED--},
	"entity_id": "binary_sensor.powerwall_status",
	"last_changed": "2020-05-18T23:20:03.213000+00:00",
	"last_updated": "2020-05-18T23:20:03.213000+00:00",
	"state": "on"
},
...
{
	"last_changed": "2020-05-19T00:41:08Z",
	"state": "unavailable"
},
...
{
	"attributes": {--TRUNCATED--},
	"context": {--TRUNCATED--},
	"entity_id": "binary_sensor.powerwall_status",
	"last_changed": "2020-05-19T00:42:08.069698+00:00",
	"last_updated": "2020-05-19T00:42:08.069698+00:00",
	"state": "on"
}]
```

* Remove impossible state check

* Remove another impossible state check

* Update homeassistant/components/history/__init__.py

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>

* Reorder to save some indent per review

* Make query response make sense with to_native=False

* Update test for 00:00 to Z change

* Update homeassistant/components/recorder/models.py

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>
This commit is contained in:
J. Nick KostonandPaulus Schoutsen authored and GitHub committed 2020-05-26 21:53:56 -05:00
1 parent 6a06d648d7
commit 0a6deeb49b
4 files changed
+194 -37

No files matched your search

+85 -1
View File
@@ -1,10 +1,13 @@
"""The tests the History component."""
# pylint: disable=protected-access,invalid-name
from datetime import timedelta
import json
import unittest
from homeassistant.components import history, recorder
from homeassistant.components.recorder.models import process_timestamp
import homeassistant.core as ha
from homeassistant.helpers.json import JSONEncoder
from homeassistant.setup import async_setup_component, setup_component
import homeassistant.util.dt as dt_util
@@ -197,6 +200,41 @@ class TestComponentHistory(unittest.TestCase):
)
assert states == hist
def test_get_significant_states_minimal_response(self):
"""Test that only significant states are returned.
When minimal responses is set only the first and
last states return a complete state.
We should get back every thermostat change that
includes an attribute change, but only the state updates for
media player (attribute changes are not significant and not returned).
"""
zero, four, states = self.record_states()
hist = history.get_significant_states(
self.hass, zero, four, filters=history.Filters(), minimal_response=True
)
# The second media_player.test state is reduced
# down to last_changed and state when minimal_response
# is set. We use JSONEncoder to make sure that are
# pre-encoded last_changed is always the same as what
# will happen with encoding a native state
input_state = states["media_player.test"][1]
orig_last_changed = json.dumps(
process_timestamp(input_state.last_changed.replace(microsecond=0)),
cls=JSONEncoder,
).replace('"', "")
if orig_last_changed.endswith("+00:00"):
orig_last_changed = f"{orig_last_changed[:-6]}{recorder.models.DB_TIMEZONE}"
orig_state = input_state.state
states["media_player.test"][1] = {
"last_changed": orig_last_changed,
"state": orig_state,
}
assert states == hist
def test_get_significant_states_with_initial(self):
"""Test that only significant states are returned.
@@ -252,6 +290,7 @@ class TestComponentHistory(unittest.TestCase):
"""Test that only significant states are returned for one entity."""
zero, four, states = self.record_states()
del states["media_player.test2"]
del states["media_player.test3"]
del states["thermostat.test"]
del states["thermostat.test2"]
del states["script.can_cancel_this_one"]
@@ -265,6 +304,7 @@ class TestComponentHistory(unittest.TestCase):
"""Test that only significant states are returned for one entity."""
zero, four, states = self.record_states()
del states["media_player.test2"]
del states["media_player.test3"]
del states["thermostat.test2"]
del states["script.can_cancel_this_one"]
@@ -286,6 +326,7 @@ class TestComponentHistory(unittest.TestCase):
zero, four, states = self.record_states()
del states["media_player.test"]
del states["media_player.test2"]
del states["media_player.test3"]
config = history.CONFIG_SCHEMA(
{
@@ -346,6 +387,7 @@ class TestComponentHistory(unittest.TestCase):
"""
zero, four, states = self.record_states()
del states["media_player.test2"]
del states["media_player.test3"]
del states["thermostat.test"]
del states["thermostat.test2"]
del states["script.can_cancel_this_one"]
@@ -372,6 +414,7 @@ class TestComponentHistory(unittest.TestCase):
zero, four, states = self.record_states()
del states["media_player.test"]
del states["media_player.test2"]
del states["media_player.test3"]
config = history.CONFIG_SCHEMA(
{
@@ -392,6 +435,7 @@ class TestComponentHistory(unittest.TestCase):
"""
zero, four, states = self.record_states()
del states["media_player.test2"]
del states["media_player.test3"]
del states["thermostat.test"]
del states["thermostat.test2"]
del states["script.can_cancel_this_one"]
@@ -414,6 +458,7 @@ class TestComponentHistory(unittest.TestCase):
"""
zero, four, states = self.record_states()
del states["media_player.test2"]
del states["media_player.test3"]
del states["script.can_cancel_this_one"]
config = history.CONFIG_SCHEMA(
@@ -438,6 +483,7 @@ class TestComponentHistory(unittest.TestCase):
zero, four, states = self.record_states()
del states["media_player.test"]
del states["media_player.test2"]
del states["media_player.test3"]
del states["thermostat.test"]
del states["thermostat.test2"]
del states["script.can_cancel_this_one"]
@@ -462,6 +508,7 @@ class TestComponentHistory(unittest.TestCase):
zero, four, states = self.record_states()
del states["media_player.test"]
del states["media_player.test2"]
del states["media_player.test3"]
del states["thermostat.test"]
del states["thermostat.test2"]
del states["script.can_cancel_this_one"]
@@ -607,6 +654,7 @@ class TestComponentHistory(unittest.TestCase):
self.init_recorder()
mp = "media_player.test"
mp2 = "media_player.test2"
mp3 = "media_player.test3"
therm = "thermostat.test"
therm2 = "thermostat.test2"
zone = "zone.home"
@@ -625,7 +673,7 @@ class TestComponentHistory(unittest.TestCase):
three = two + timedelta(seconds=1)
four = three + timedelta(seconds=1)
states = {therm: [], therm2: [], mp: [], mp2: [], script_c: []}
states = {therm: [], therm2: [], mp: [], mp2: [], mp3: [], script_c: []}
with patch(
"homeassistant.components.recorder.dt_util.utcnow", return_value=one
):
@@ -638,6 +686,9 @@ class TestComponentHistory(unittest.TestCase):
states[mp2].append(
set_state(mp2, "YouTube", attributes={"media_title": str(sentinel.mt2)})
)
states[mp3].append(
set_state(mp3, "idle", attributes={"media_title": str(sentinel.mt1)})
)
states[therm].append(
set_state(therm, 20, attributes={"current_temperature": 19.5})
)
@@ -647,6 +698,12 @@ class TestComponentHistory(unittest.TestCase):
):
# This state will be skipped only different in time
set_state(mp, "YouTube", attributes={"media_title": str(sentinel.mt3)})
# This state will be skipped as it hidden
set_state(
mp3,
"Apple TV",
attributes={"media_title": str(sentinel.mt2), "hidden": True},
)
# This state will be skipped because domain blacklisted
set_state(zone, "zoning")
set_state(script_nc, "off")
@@ -666,6 +723,9 @@ class TestComponentHistory(unittest.TestCase):
states[mp].append(
set_state(mp, "Netflix", attributes={"media_title": str(sentinel.mt4)})
)
states[mp3].append(
set_state(mp3, "Netflix", attributes={"media_title": str(sentinel.mt3)})
)
# Attributes changed even though state is the same
states[therm].append(
set_state(therm, 21, attributes={"current_temperature": 20})
@@ -686,6 +746,30 @@ async def test_fetch_period_api(hass, hass_client):
assert response.status == 200
async def test_fetch_period_api_with_use_include_order(hass, hass_client):
"""Test the fetch period view for history with include order."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(
hass, "history", {history.DOMAIN: {history.CONF_ORDER: True}}
)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
client = await hass_client()
response = await client.get(f"/api/history/period/{dt_util.utcnow().isoformat()}")
assert response.status == 200
async def test_fetch_period_api_with_minimal_response(hass, hass_client):
"""Test the fetch period view for history with minimal_response."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "history", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
client = await hass_client()
response = await client.get(
f"/api/history/period/{dt_util.utcnow().isoformat()}?minimal_response"
)
assert response.status == 200
async def test_fetch_period_api_with_include_order(hass, hass_client):
"""Test the fetch period view for history."""
await hass.async_add_executor_job(init_recorder_component, hass)