Add audio to stream (#38846)

* Add audio to stream component

* Use container options to do most fmp4 formatting

* Add test for treatment of different audio inputs

* Add test for treatment of different audio inputs

* pcm_mulaw frames should be s16

* Use seek to get BytesIO length

* Remove unused utcnow

* Remove peek_next_audio_pts

* only demux audio and video packets - ignoring data and subtitle streams

Co-authored-by: Jason Hunter <hunterjm@gmail.com>
This commit is contained in:
uvjustinandJason Hunter authored and GitHub committed 2020-08-19 23:18:54 -04:00
1 parent b0974d89a6
commit dc2d0b9297
8 files changed
+287 -196

No files matched your search

+48 -3
View File
@@ -1,5 +1,7 @@
"""Collection of test helpers."""
from fractions import Fraction
import io
import logging
import av
import numpy as np
@@ -7,27 +9,59 @@ import numpy as np
from homeassistant.components.stream import Stream
from homeassistant.components.stream.const import ATTR_STREAMS, DOMAIN
_LOGGER = logging.getLogger(__name__)
def generate_h264_video():
AUDIO_SAMPLE_RATE = 8000
def generate_h264_video(container_format="mp4", audio_codec=None):
"""
Generate a test video.
See: http://docs.mikeboers.com/pyav/develop/cookbook/numpy.html
"""
def generate_audio_frame(pcm_mulaw=False):
"""Generate a blank audio frame."""
if pcm_mulaw:
audio_frame = av.AudioFrame(format="s16", layout="mono", samples=1)
audio_bytes = b"\x00\x00"
else:
audio_frame = av.AudioFrame(format="dbl", layout="mono", samples=1024)
audio_bytes = b"\x00\x00\x00\x00\x00\x00\x00\x00" * 1024
audio_frame.planes[0].update(audio_bytes)
audio_frame.sample_rate = AUDIO_SAMPLE_RATE
audio_frame.time_base = Fraction(1, AUDIO_SAMPLE_RATE)
return audio_frame
duration = 5
fps = 24
total_frames = duration * fps
output = io.BytesIO()
output.name = "test.mp4"
container = av.open(output, mode="w")
output.name = "test.mov" if container_format == "mov" else "test.mp4"
container = av.open(output, mode="w", format=container_format)
stream = container.add_stream("libx264", rate=fps)
stream.width = 480
stream.height = 320
stream.pix_fmt = "yuv420p"
a_packet = None
last_a_dts = -1
if audio_codec is not None:
if audio_codec == "empty": # empty we add a stream but don't mux any audio
astream = container.add_stream("aac", AUDIO_SAMPLE_RATE)
else:
astream = container.add_stream(audio_codec, AUDIO_SAMPLE_RATE)
# Need to do it multiple times for some reason
while not a_packet:
a_packets = astream.encode(
generate_audio_frame(pcm_mulaw=audio_codec == "pcm_mulaw")
)
if a_packets:
a_packet = a_packets[0]
for frame_i in range(total_frames):
img = np.empty((480, 320, 3))
@@ -42,6 +76,17 @@ def generate_h264_video():
for packet in stream.encode(frame):
container.mux(packet)
if a_packet is not None:
a_packet.pts = int(frame_i / (fps * a_packet.time_base))
while a_packet.pts * a_packet.time_base * fps < frame_i + 1:
a_packet.dts = a_packet.pts
if (
a_packet.dts > last_a_dts
): # avoid writing same dts twice in case of rounding
container.mux(a_packet)
last_a_dts = a_packet.dts
a_packet.pts += a_packet.duration
# Flush stream
for packet in stream.encode():
container.mux(packet)
+40 -1
View File
@@ -2,6 +2,7 @@
from datetime import timedelta
from io import BytesIO
import av
import pytest
from homeassistant.components.stream.core import Segment
@@ -75,7 +76,45 @@ async def test_recorder_save():
output.name = "test.mp4"
# Run
recorder_save_worker(output, [Segment(1, source, 4, (360000, 176400))])
recorder_save_worker(output, [Segment(1, source, 4)], "mp4")
# Assert
assert output.getvalue()
@pytest.mark.skip("Flaky in CI")
async def test_record_stream_audio(hass, hass_client):
"""
Test treatment of different audio inputs.
Record stream output should have an audio channel when input has
a valid codec and audio packets and no audio channel otherwise.
"""
await async_setup_component(hass, "stream", {"stream": {}})
for a_codec, expected_audio_streams in (
("aac", 1), # aac is a valid mp4 codec
("pcm_mulaw", 0), # G.711 is not a valid mp4 codec
("empty", 0), # audio stream with no packets
(None, 0), # no audio stream
):
with patch("homeassistant.components.stream.recorder.recorder_save_worker"):
# Setup demo track
source = generate_h264_video(
container_format="mov", audio_codec=a_codec
) # mov can store PCM
stream = preload_stream(hass, source)
recorder = stream.add_provider("recorder")
stream.start()
while True:
segment = await recorder.recv()
if not segment:
break
last_segment = segment
result = av.open(last_segment.segment, "r", format="mp4")
assert len(result.streams.audio) == expected_audio_streams
result.close()
stream.stop()