Add audio to stream (#38846)
* Add audio to stream component * Use container options to do most fmp4 formatting * Add test for treatment of different audio inputs * Add test for treatment of different audio inputs * pcm_mulaw frames should be s16 * Use seek to get BytesIO length * Remove unused utcnow * Remove peek_next_audio_pts * only demux audio and video packets - ignoring data and subtitle streams Co-authored-by: Jason Hunter <hunterjm@gmail.com>
This commit is contained in:
1 parent
b0974d89a6
commit
dc2d0b9297
8 files changed
+287
-196
No files matched your search
@@ -1,5 +1,7 @@
|
||||
"""Collection of test helpers."""
|
||||
from fractions import Fraction
|
||||
import io
|
||||
import logging
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
@@ -7,27 +9,59 @@ import numpy as np
|
||||
from homeassistant.components.stream import Stream
|
||||
from homeassistant.components.stream.const import ATTR_STREAMS, DOMAIN
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
def generate_h264_video():
|
||||
AUDIO_SAMPLE_RATE = 8000
|
||||
|
||||
|
||||
def generate_h264_video(container_format="mp4", audio_codec=None):
|
||||
"""
|
||||
Generate a test video.
|
||||
|
||||
See: http://docs.mikeboers.com/pyav/develop/cookbook/numpy.html
|
||||
"""
|
||||
|
||||
def generate_audio_frame(pcm_mulaw=False):
|
||||
"""Generate a blank audio frame."""
|
||||
if pcm_mulaw:
|
||||
audio_frame = av.AudioFrame(format="s16", layout="mono", samples=1)
|
||||
audio_bytes = b"\x00\x00"
|
||||
else:
|
||||
audio_frame = av.AudioFrame(format="dbl", layout="mono", samples=1024)
|
||||
audio_bytes = b"\x00\x00\x00\x00\x00\x00\x00\x00" * 1024
|
||||
audio_frame.planes[0].update(audio_bytes)
|
||||
audio_frame.sample_rate = AUDIO_SAMPLE_RATE
|
||||
audio_frame.time_base = Fraction(1, AUDIO_SAMPLE_RATE)
|
||||
return audio_frame
|
||||
|
||||
duration = 5
|
||||
fps = 24
|
||||
total_frames = duration * fps
|
||||
|
||||
output = io.BytesIO()
|
||||
output.name = "test.mp4"
|
||||
container = av.open(output, mode="w")
|
||||
output.name = "test.mov" if container_format == "mov" else "test.mp4"
|
||||
container = av.open(output, mode="w", format=container_format)
|
||||
|
||||
stream = container.add_stream("libx264", rate=fps)
|
||||
stream.width = 480
|
||||
stream.height = 320
|
||||
stream.pix_fmt = "yuv420p"
|
||||
|
||||
a_packet = None
|
||||
last_a_dts = -1
|
||||
if audio_codec is not None:
|
||||
if audio_codec == "empty": # empty we add a stream but don't mux any audio
|
||||
astream = container.add_stream("aac", AUDIO_SAMPLE_RATE)
|
||||
else:
|
||||
astream = container.add_stream(audio_codec, AUDIO_SAMPLE_RATE)
|
||||
# Need to do it multiple times for some reason
|
||||
while not a_packet:
|
||||
a_packets = astream.encode(
|
||||
generate_audio_frame(pcm_mulaw=audio_codec == "pcm_mulaw")
|
||||
)
|
||||
if a_packets:
|
||||
a_packet = a_packets[0]
|
||||
|
||||
for frame_i in range(total_frames):
|
||||
|
||||
img = np.empty((480, 320, 3))
|
||||
@@ -42,6 +76,17 @@ def generate_h264_video():
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
|
||||
if a_packet is not None:
|
||||
a_packet.pts = int(frame_i / (fps * a_packet.time_base))
|
||||
while a_packet.pts * a_packet.time_base * fps < frame_i + 1:
|
||||
a_packet.dts = a_packet.pts
|
||||
if (
|
||||
a_packet.dts > last_a_dts
|
||||
): # avoid writing same dts twice in case of rounding
|
||||
container.mux(a_packet)
|
||||
last_a_dts = a_packet.dts
|
||||
a_packet.pts += a_packet.duration
|
||||
|
||||
# Flush stream
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from datetime import timedelta
|
||||
from io import BytesIO
|
||||
|
||||
import av
|
||||
import pytest
|
||||
|
||||
from homeassistant.components.stream.core import Segment
|
||||
@@ -75,7 +76,45 @@ async def test_recorder_save():
|
||||
output.name = "test.mp4"
|
||||
|
||||
# Run
|
||||
recorder_save_worker(output, [Segment(1, source, 4, (360000, 176400))])
|
||||
recorder_save_worker(output, [Segment(1, source, 4)], "mp4")
|
||||
|
||||
# Assert
|
||||
assert output.getvalue()
|
||||
|
||||
|
||||
@pytest.mark.skip("Flaky in CI")
|
||||
async def test_record_stream_audio(hass, hass_client):
|
||||
"""
|
||||
Test treatment of different audio inputs.
|
||||
|
||||
Record stream output should have an audio channel when input has
|
||||
a valid codec and audio packets and no audio channel otherwise.
|
||||
"""
|
||||
await async_setup_component(hass, "stream", {"stream": {}})
|
||||
|
||||
for a_codec, expected_audio_streams in (
|
||||
("aac", 1), # aac is a valid mp4 codec
|
||||
("pcm_mulaw", 0), # G.711 is not a valid mp4 codec
|
||||
("empty", 0), # audio stream with no packets
|
||||
(None, 0), # no audio stream
|
||||
):
|
||||
with patch("homeassistant.components.stream.recorder.recorder_save_worker"):
|
||||
# Setup demo track
|
||||
source = generate_h264_video(
|
||||
container_format="mov", audio_codec=a_codec
|
||||
) # mov can store PCM
|
||||
stream = preload_stream(hass, source)
|
||||
recorder = stream.add_provider("recorder")
|
||||
stream.start()
|
||||
|
||||
while True:
|
||||
segment = await recorder.recv()
|
||||
if not segment:
|
||||
break
|
||||
last_segment = segment
|
||||
|
||||
result = av.open(last_segment.segment, "r", format="mp4")
|
||||
|
||||
assert len(result.streams.audio) == expected_audio_streams
|
||||
result.close()
|
||||
stream.stop()
|
||||
Reference in new issue
Block a user