Add audio to stream (#38846)
* Add audio to stream component * Use container options to do most fmp4 formatting * Add test for treatment of different audio inputs * Add test for treatment of different audio inputs * pcm_mulaw frames should be s16 * Use seek to get BytesIO length * Remove unused utcnow * Remove peek_next_audio_pts * only demux audio and video packets - ignoring data and subtitle streams Co-authored-by: Jason Hunter <hunterjm@gmail.com>
This commit is contained in:
1 parent
b0974d89a6
commit
dc2d0b9297
8 files changed
+287
-196
No files matched your search
@@ -1,54 +1,37 @@
|
||||
"""Provides the worker thread needed for processing streams."""
|
||||
from fractions import Fraction
|
||||
from collections import deque
|
||||
import io
|
||||
import logging
|
||||
|
||||
import av
|
||||
|
||||
from .const import AUDIO_SAMPLE_RATE, MIN_SEGMENT_DURATION
|
||||
from .const import MIN_SEGMENT_DURATION, PACKETS_TO_WAIT_FOR_AUDIO
|
||||
from .core import Segment, StreamBuffer
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_audio_frame():
|
||||
"""Generate a blank audio frame."""
|
||||
|
||||
audio_frame = av.AudioFrame(format="dbl", layout="mono", samples=1024)
|
||||
# audio_bytes = b''.join(b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
# for i in range(0, 1024))
|
||||
audio_bytes = b"\x00\x00\x00\x00\x00\x00\x00\x00" * 1024
|
||||
audio_frame.planes[0].update(audio_bytes)
|
||||
audio_frame.sample_rate = AUDIO_SAMPLE_RATE
|
||||
audio_frame.time_base = Fraction(1, AUDIO_SAMPLE_RATE)
|
||||
return audio_frame
|
||||
|
||||
|
||||
def create_stream_buffer(stream_output, video_stream, audio_frame):
|
||||
def create_stream_buffer(stream_output, video_stream, audio_stream, sequence):
|
||||
"""Create a new StreamBuffer."""
|
||||
|
||||
a_packet = None
|
||||
segment = io.BytesIO()
|
||||
container_options = (
|
||||
stream_output.container_options(sequence)
|
||||
if stream_output.container_options
|
||||
else {}
|
||||
)
|
||||
output = av.open(
|
||||
segment,
|
||||
mode="w",
|
||||
format=stream_output.format,
|
||||
container_options={
|
||||
"video_track_timescale": str(int(1 / video_stream.time_base)),
|
||||
**(stream_output.container_options or {}),
|
||||
},
|
||||
container_options=container_options,
|
||||
)
|
||||
vstream = output.add_stream(template=video_stream)
|
||||
# Check if audio is requested
|
||||
astream = None
|
||||
if stream_output.audio_codec:
|
||||
astream = output.add_stream(stream_output.audio_codec, AUDIO_SAMPLE_RATE)
|
||||
# Need to do it multiple times for some reason
|
||||
while not a_packet:
|
||||
a_packets = astream.encode(audio_frame)
|
||||
if a_packets:
|
||||
a_packet = a_packets[0]
|
||||
return (a_packet, StreamBuffer(segment, output, vstream, astream))
|
||||
if audio_stream and audio_stream.name in stream_output.audio_codecs:
|
||||
astream = output.add_stream(template=audio_stream)
|
||||
return StreamBuffer(segment, output, vstream, astream)
|
||||
|
||||
|
||||
def stream_worker(hass, stream, quit_event):
|
||||
@@ -60,32 +43,133 @@ def stream_worker(hass, stream, quit_event):
|
||||
except (KeyError, IndexError):
|
||||
_LOGGER.error("Stream has no video")
|
||||
return
|
||||
try:
|
||||
audio_stream = container.streams.audio[0]
|
||||
except (KeyError, IndexError):
|
||||
audio_stream = None
|
||||
|
||||
audio_frame = generate_audio_frame()
|
||||
|
||||
first_packet = True
|
||||
# Holds the buffers for each stream provider
|
||||
outputs = {}
|
||||
# Keep track of the number of segments we've processed
|
||||
sequence = 1
|
||||
# Holds the generated silence that needs to be muxed into the output
|
||||
audio_packets = {}
|
||||
# The presentation timestamp of the first video packet we receive
|
||||
first_pts = 0
|
||||
# The decoder timestamp of the latest packet we processed
|
||||
# The presentation timestamps of the first packet in each stream we receive
|
||||
# Use to adjust before muxing or outputting, but we don't adjust internally
|
||||
first_pts = {}
|
||||
# The decoder timestamps of the latest packet in each stream we processed
|
||||
last_dts = None
|
||||
# Keep track of consecutive packets without a dts to detect end of stream.
|
||||
last_packet_was_without_dts = False
|
||||
# The pts at the beginning of the segment
|
||||
segment_start_v_pts = 0
|
||||
segment_start_a_pts = 0
|
||||
# Holds the buffers for each stream provider
|
||||
outputs = None
|
||||
# Keep track of the number of segments we've processed
|
||||
sequence = 0
|
||||
# The video pts at the beginning of the segment
|
||||
segment_start_pts = None
|
||||
# Because of problems 1 and 2 below, we need to store the first few packets and replay them
|
||||
initial_packets = deque()
|
||||
|
||||
# Have to work around two problems with RTSP feeds in ffmpeg
|
||||
# 1 - first frame has bad pts/dts https://trac.ffmpeg.org/ticket/5018
|
||||
# 2 - seeking can be problematic https://trac.ffmpeg.org/ticket/7815
|
||||
|
||||
def peek_first_pts():
|
||||
nonlocal first_pts, audio_stream
|
||||
|
||||
def empty_stream_dict():
|
||||
return {
|
||||
video_stream: None,
|
||||
**({audio_stream: None} if audio_stream else {}),
|
||||
}
|
||||
|
||||
try:
|
||||
first_packet = empty_stream_dict()
|
||||
first_pts = empty_stream_dict()
|
||||
# Get to first video keyframe
|
||||
while first_packet[video_stream] is None:
|
||||
packet = next(container.demux())
|
||||
if packet.stream == video_stream and packet.is_keyframe:
|
||||
first_packet[video_stream] = packet
|
||||
initial_packets.append(packet)
|
||||
# Get first_pts from subsequent frame to first keyframe
|
||||
while any(
|
||||
[pts is None for pts in {**first_packet, **first_pts}.values()]
|
||||
) and (len(initial_packets) < PACKETS_TO_WAIT_FOR_AUDIO):
|
||||
packet = next(container.demux((video_stream, audio_stream)))
|
||||
if (
|
||||
first_packet[packet.stream] is None
|
||||
): # actually video already found above so only for audio
|
||||
if packet.is_keyframe:
|
||||
first_packet[packet.stream] = packet
|
||||
else: # Discard leading non-keyframes
|
||||
continue
|
||||
else: # This is the second frame to calculate first_pts from
|
||||
if first_pts[packet.stream] is None:
|
||||
first_pts[packet.stream] = packet.dts - packet.duration
|
||||
first_packet[packet.stream].pts = first_pts[packet.stream]
|
||||
first_packet[packet.stream].dts = first_pts[packet.stream]
|
||||
initial_packets.append(packet)
|
||||
if audio_stream and first_packet[audio_stream] is None:
|
||||
_LOGGER.warning(
|
||||
"Audio stream not found"
|
||||
) # Some streams declare an audio stream and never send any packets
|
||||
del first_pts[audio_stream]
|
||||
audio_stream = None
|
||||
|
||||
except (av.AVError, StopIteration) as ex:
|
||||
# End of stream, clear listeners and stop thread
|
||||
for fmt, _ in outputs.items():
|
||||
hass.loop.call_soon_threadsafe(stream.outputs[fmt].put, None)
|
||||
_LOGGER.error(
|
||||
"Error demuxing stream while finding first packet: %s", str(ex)
|
||||
)
|
||||
quit_event.set()
|
||||
|
||||
def initialize_segment(video_pts):
|
||||
"""Reset some variables and initialize outputs for each segment."""
|
||||
nonlocal outputs, sequence, segment_start_pts
|
||||
# Clear outputs and increment sequence
|
||||
outputs = {}
|
||||
sequence += 1
|
||||
segment_start_pts = video_pts
|
||||
for stream_output in stream.outputs.values():
|
||||
if video_stream.name not in stream_output.video_codecs:
|
||||
continue
|
||||
buffer = create_stream_buffer(
|
||||
stream_output, video_stream, audio_stream, sequence
|
||||
)
|
||||
outputs[stream_output.name] = (
|
||||
buffer,
|
||||
{video_stream: buffer.vstream, audio_stream: buffer.astream},
|
||||
)
|
||||
|
||||
def mux_video_packet(packet):
|
||||
# adjust pts and dts before muxing
|
||||
packet.pts -= first_pts[video_stream]
|
||||
packet.dts -= first_pts[video_stream]
|
||||
# mux packets to each buffer
|
||||
for buffer, output_streams in outputs.values():
|
||||
# Assign the packet to the new stream & mux
|
||||
packet.stream = output_streams[video_stream]
|
||||
buffer.output.mux(packet)
|
||||
|
||||
def mux_audio_packet(packet):
|
||||
# almost the same as muxing video but add extra check
|
||||
# adjust pts and dts before muxing
|
||||
packet.pts -= first_pts[audio_stream]
|
||||
packet.dts -= first_pts[audio_stream]
|
||||
for buffer, output_streams in outputs.values():
|
||||
# Assign the packet to the new stream & mux
|
||||
if output_streams.get(audio_stream):
|
||||
packet.stream = output_streams[audio_stream]
|
||||
buffer.output.mux(packet)
|
||||
|
||||
peek_first_pts()
|
||||
last_dts = {k: v - 1 for k, v in first_pts.items()}
|
||||
initialize_segment(first_pts[video_stream])
|
||||
|
||||
while not quit_event.is_set():
|
||||
try:
|
||||
packet = next(container.demux(video_stream))
|
||||
if len(initial_packets) > 0:
|
||||
packet = initial_packets.popleft()
|
||||
else:
|
||||
packet = next(container.demux((video_stream, audio_stream)))
|
||||
if packet.dts is None:
|
||||
if first_packet:
|
||||
continue
|
||||
_LOGGER.error("Stream packet without dts detected, skipping...")
|
||||
# Allow a single packet without dts before terminating the stream.
|
||||
if last_packet_was_without_dts:
|
||||
@@ -101,101 +185,35 @@ def stream_worker(hass, stream, quit_event):
|
||||
_LOGGER.error("Error demuxing stream: %s", str(ex))
|
||||
break
|
||||
|
||||
# Skip non monotonically increasing dts in feed
|
||||
if not first_packet and last_dts >= packet.dts:
|
||||
# Discard packet if dts is not monotonic
|
||||
if packet.dts <= last_dts[packet.stream]:
|
||||
continue
|
||||
last_dts = packet.dts
|
||||
|
||||
# Reset timestamps from a 0 time base for this stream
|
||||
packet.dts -= first_pts
|
||||
packet.pts -= first_pts
|
||||
# Check for end of segment
|
||||
if packet.stream == video_stream and packet.is_keyframe:
|
||||
segment_duration = (packet.pts - segment_start_pts) * packet.time_base
|
||||
if segment_duration >= MIN_SEGMENT_DURATION:
|
||||
# Save segment to outputs
|
||||
for fmt, (buffer, _) in outputs.items():
|
||||
buffer.output.close()
|
||||
if stream.outputs.get(fmt):
|
||||
hass.loop.call_soon_threadsafe(
|
||||
stream.outputs[fmt].put,
|
||||
Segment(sequence, buffer.segment, segment_duration,),
|
||||
)
|
||||
|
||||
# Reset segment on keyframe after we reach desired segment duration
|
||||
if (
|
||||
packet.is_keyframe
|
||||
and (packet.pts - segment_start_v_pts) * packet.time_base
|
||||
>= MIN_SEGMENT_DURATION
|
||||
):
|
||||
# Calculate the segment duration by multiplying the difference of the next and the current
|
||||
# keyframe presentation timestamps by the time base, which gets us total seconds.
|
||||
segment_duration = (packet.pts - segment_start_v_pts) * packet.time_base
|
||||
# Save segment to outputs
|
||||
for fmt, buffer in outputs.items():
|
||||
buffer.output.close()
|
||||
del audio_packets[buffer.astream]
|
||||
if stream.outputs.get(fmt):
|
||||
hass.loop.call_soon_threadsafe(
|
||||
stream.outputs[fmt].put,
|
||||
Segment(
|
||||
sequence,
|
||||
buffer.segment,
|
||||
segment_duration,
|
||||
(segment_start_v_pts, segment_start_a_pts),
|
||||
),
|
||||
)
|
||||
# Reinitialize
|
||||
initialize_segment(packet.pts)
|
||||
|
||||
# Clear outputs and increment sequence
|
||||
outputs = {}
|
||||
if not first_packet:
|
||||
sequence += 1
|
||||
segment_start_v_pts = packet.pts
|
||||
segment_start_a_pts = int(
|
||||
packet.pts * packet.time_base * AUDIO_SAMPLE_RATE
|
||||
)
|
||||
|
||||
# Initialize outputs
|
||||
for stream_output in stream.outputs.values():
|
||||
if video_stream.name not in stream_output.video_codecs:
|
||||
continue
|
||||
|
||||
a_packet, buffer = create_stream_buffer(
|
||||
stream_output, video_stream, audio_frame
|
||||
)
|
||||
audio_packets[buffer.astream] = a_packet
|
||||
outputs[stream_output.name] = buffer
|
||||
|
||||
# First video packet tends to have a weird dts/pts
|
||||
if first_packet:
|
||||
# If we are attaching to a live stream that does not reset
|
||||
# timestamps for us, we need to do it ourselves by recording
|
||||
# the first presentation timestamp and subtracting it from
|
||||
# subsequent packets we receive.
|
||||
if (packet.pts * packet.time_base) > 1:
|
||||
first_pts = packet.pts
|
||||
packet.dts = 0
|
||||
packet.pts = 0
|
||||
first_packet = False
|
||||
|
||||
# Store packets on each output
|
||||
for buffer in outputs.values():
|
||||
# Check if the format requires audio
|
||||
if audio_packets.get(buffer.astream):
|
||||
a_packet = audio_packets[buffer.astream]
|
||||
a_time_base = a_packet.time_base
|
||||
|
||||
# Determine video start timestamp and duration
|
||||
video_start = packet.pts * packet.time_base
|
||||
video_duration = packet.duration * packet.time_base
|
||||
|
||||
if packet.is_keyframe:
|
||||
# Set first audio packet in sequence to equal video pts
|
||||
a_packet.pts = int(video_start / a_time_base)
|
||||
a_packet.dts = int(video_start / a_time_base)
|
||||
|
||||
# Determine target end timestamp for audio
|
||||
target_pts = int((video_start + video_duration) / a_time_base)
|
||||
while a_packet.pts < target_pts:
|
||||
# Mux audio packet and adjust points until target hit
|
||||
buffer.output.mux(a_packet)
|
||||
a_packet.pts += a_packet.duration
|
||||
a_packet.dts += a_packet.duration
|
||||
audio_packets[buffer.astream] = a_packet
|
||||
|
||||
# Assign the video packet to the new stream & mux
|
||||
packet.stream = buffer.vstream
|
||||
buffer.output.mux(packet)
|
||||
# Update last_dts processed
|
||||
last_dts[packet.stream] = packet.dts
|
||||
# mux video packets immediately, save audio packets to be muxed all at once
|
||||
if packet.stream == video_stream:
|
||||
mux_video_packet(packet) # mutates packet timestamps
|
||||
else:
|
||||
mux_audio_packet(packet) # mutates packet timestamps
|
||||
|
||||
# Close stream
|
||||
for buffer in outputs.values():
|
||||
for buffer, _ in outputs.values():
|
||||
buffer.output.close()
|
||||
container.close()
|
||||
Reference in new issue
Block a user