Add audio to stream (#38846)

* Add audio to stream component

* Use container options to do most fmp4 formatting

* Add test for treatment of different audio inputs

* Add test for treatment of different audio inputs

* pcm_mulaw frames should be s16

* Use seek to get BytesIO length

* Remove unused utcnow

* Remove peek_next_audio_pts

* only demux audio and video packets - ignoring data and subtitle streams

Co-authored-by: Jason Hunter <hunterjm@gmail.com>
This commit is contained in:
uvjustinandJason Hunter authored and GitHub committed 2020-08-19 23:18:54 -04:00
1 parent b0974d89a6
commit dc2d0b9297
8 files changed
+287 -196

No files matched your search

+155 -137
View File
@@ -1,54 +1,37 @@
"""Provides the worker thread needed for processing streams."""
from fractions import Fraction
from collections import deque
import io
import logging
import av
from .const import AUDIO_SAMPLE_RATE, MIN_SEGMENT_DURATION
from .const import MIN_SEGMENT_DURATION, PACKETS_TO_WAIT_FOR_AUDIO
from .core import Segment, StreamBuffer
_LOGGER = logging.getLogger(__name__)
def generate_audio_frame():
"""Generate a blank audio frame."""
audio_frame = av.AudioFrame(format="dbl", layout="mono", samples=1024)
# audio_bytes = b''.join(b'\x00\x00\x00\x00\x00\x00\x00\x00'
# for i in range(0, 1024))
audio_bytes = b"\x00\x00\x00\x00\x00\x00\x00\x00" * 1024
audio_frame.planes[0].update(audio_bytes)
audio_frame.sample_rate = AUDIO_SAMPLE_RATE
audio_frame.time_base = Fraction(1, AUDIO_SAMPLE_RATE)
return audio_frame
def create_stream_buffer(stream_output, video_stream, audio_frame):
def create_stream_buffer(stream_output, video_stream, audio_stream, sequence):
"""Create a new StreamBuffer."""
a_packet = None
segment = io.BytesIO()
container_options = (
stream_output.container_options(sequence)
if stream_output.container_options
else {}
)
output = av.open(
segment,
mode="w",
format=stream_output.format,
container_options={
"video_track_timescale": str(int(1 / video_stream.time_base)),
**(stream_output.container_options or {}),
},
container_options=container_options,
)
vstream = output.add_stream(template=video_stream)
# Check if audio is requested
astream = None
if stream_output.audio_codec:
astream = output.add_stream(stream_output.audio_codec, AUDIO_SAMPLE_RATE)
# Need to do it multiple times for some reason
while not a_packet:
a_packets = astream.encode(audio_frame)
if a_packets:
a_packet = a_packets[0]
return (a_packet, StreamBuffer(segment, output, vstream, astream))
if audio_stream and audio_stream.name in stream_output.audio_codecs:
astream = output.add_stream(template=audio_stream)
return StreamBuffer(segment, output, vstream, astream)
def stream_worker(hass, stream, quit_event):
@@ -60,32 +43,133 @@ def stream_worker(hass, stream, quit_event):
except (KeyError, IndexError):
_LOGGER.error("Stream has no video")
return
try:
audio_stream = container.streams.audio[0]
except (KeyError, IndexError):
audio_stream = None
audio_frame = generate_audio_frame()
first_packet = True
# Holds the buffers for each stream provider
outputs = {}
# Keep track of the number of segments we've processed
sequence = 1
# Holds the generated silence that needs to be muxed into the output
audio_packets = {}
# The presentation timestamp of the first video packet we receive
first_pts = 0
# The decoder timestamp of the latest packet we processed
# The presentation timestamps of the first packet in each stream we receive
# Use to adjust before muxing or outputting, but we don't adjust internally
first_pts = {}
# The decoder timestamps of the latest packet in each stream we processed
last_dts = None
# Keep track of consecutive packets without a dts to detect end of stream.
last_packet_was_without_dts = False
# The pts at the beginning of the segment
segment_start_v_pts = 0
segment_start_a_pts = 0
# Holds the buffers for each stream provider
outputs = None
# Keep track of the number of segments we've processed
sequence = 0
# The video pts at the beginning of the segment
segment_start_pts = None
# Because of problems 1 and 2 below, we need to store the first few packets and replay them
initial_packets = deque()
# Have to work around two problems with RTSP feeds in ffmpeg
# 1 - first frame has bad pts/dts https://trac.ffmpeg.org/ticket/5018
# 2 - seeking can be problematic https://trac.ffmpeg.org/ticket/7815
def peek_first_pts():
nonlocal first_pts, audio_stream
def empty_stream_dict():
return {
video_stream: None,
**({audio_stream: None} if audio_stream else {}),
}
try:
first_packet = empty_stream_dict()
first_pts = empty_stream_dict()
# Get to first video keyframe
while first_packet[video_stream] is None:
packet = next(container.demux())
if packet.stream == video_stream and packet.is_keyframe:
first_packet[video_stream] = packet
initial_packets.append(packet)
# Get first_pts from subsequent frame to first keyframe
while any(
[pts is None for pts in {**first_packet, **first_pts}.values()]
) and (len(initial_packets) < PACKETS_TO_WAIT_FOR_AUDIO):
packet = next(container.demux((video_stream, audio_stream)))
if (
first_packet[packet.stream] is None
): # actually video already found above so only for audio
if packet.is_keyframe:
first_packet[packet.stream] = packet
else: # Discard leading non-keyframes
continue
else: # This is the second frame to calculate first_pts from
if first_pts[packet.stream] is None:
first_pts[packet.stream] = packet.dts - packet.duration
first_packet[packet.stream].pts = first_pts[packet.stream]
first_packet[packet.stream].dts = first_pts[packet.stream]
initial_packets.append(packet)
if audio_stream and first_packet[audio_stream] is None:
_LOGGER.warning(
"Audio stream not found"
) # Some streams declare an audio stream and never send any packets
del first_pts[audio_stream]
audio_stream = None
except (av.AVError, StopIteration) as ex:
# End of stream, clear listeners and stop thread
for fmt, _ in outputs.items():
hass.loop.call_soon_threadsafe(stream.outputs[fmt].put, None)
_LOGGER.error(
"Error demuxing stream while finding first packet: %s", str(ex)
)
quit_event.set()
def initialize_segment(video_pts):
"""Reset some variables and initialize outputs for each segment."""
nonlocal outputs, sequence, segment_start_pts
# Clear outputs and increment sequence
outputs = {}
sequence += 1
segment_start_pts = video_pts
for stream_output in stream.outputs.values():
if video_stream.name not in stream_output.video_codecs:
continue
buffer = create_stream_buffer(
stream_output, video_stream, audio_stream, sequence
)
outputs[stream_output.name] = (
buffer,
{video_stream: buffer.vstream, audio_stream: buffer.astream},
)
def mux_video_packet(packet):
# adjust pts and dts before muxing
packet.pts -= first_pts[video_stream]
packet.dts -= first_pts[video_stream]
# mux packets to each buffer
for buffer, output_streams in outputs.values():
# Assign the packet to the new stream & mux
packet.stream = output_streams[video_stream]
buffer.output.mux(packet)
def mux_audio_packet(packet):
# almost the same as muxing video but add extra check
# adjust pts and dts before muxing
packet.pts -= first_pts[audio_stream]
packet.dts -= first_pts[audio_stream]
for buffer, output_streams in outputs.values():
# Assign the packet to the new stream & mux
if output_streams.get(audio_stream):
packet.stream = output_streams[audio_stream]
buffer.output.mux(packet)
peek_first_pts()
last_dts = {k: v - 1 for k, v in first_pts.items()}
initialize_segment(first_pts[video_stream])
while not quit_event.is_set():
try:
packet = next(container.demux(video_stream))
if len(initial_packets) > 0:
packet = initial_packets.popleft()
else:
packet = next(container.demux((video_stream, audio_stream)))
if packet.dts is None:
if first_packet:
continue
_LOGGER.error("Stream packet without dts detected, skipping...")
# Allow a single packet without dts before terminating the stream.
if last_packet_was_without_dts:
@@ -101,101 +185,35 @@ def stream_worker(hass, stream, quit_event):
_LOGGER.error("Error demuxing stream: %s", str(ex))
break
# Skip non monotonically increasing dts in feed
if not first_packet and last_dts >= packet.dts:
# Discard packet if dts is not monotonic
if packet.dts <= last_dts[packet.stream]:
continue
last_dts = packet.dts
# Reset timestamps from a 0 time base for this stream
packet.dts -= first_pts
packet.pts -= first_pts
# Check for end of segment
if packet.stream == video_stream and packet.is_keyframe:
segment_duration = (packet.pts - segment_start_pts) * packet.time_base
if segment_duration >= MIN_SEGMENT_DURATION:
# Save segment to outputs
for fmt, (buffer, _) in outputs.items():
buffer.output.close()
if stream.outputs.get(fmt):
hass.loop.call_soon_threadsafe(
stream.outputs[fmt].put,
Segment(sequence, buffer.segment, segment_duration,),
)
# Reset segment on keyframe after we reach desired segment duration
if (
packet.is_keyframe
and (packet.pts - segment_start_v_pts) * packet.time_base
>= MIN_SEGMENT_DURATION
):
# Calculate the segment duration by multiplying the difference of the next and the current
# keyframe presentation timestamps by the time base, which gets us total seconds.
segment_duration = (packet.pts - segment_start_v_pts) * packet.time_base
# Save segment to outputs
for fmt, buffer in outputs.items():
buffer.output.close()
del audio_packets[buffer.astream]
if stream.outputs.get(fmt):
hass.loop.call_soon_threadsafe(
stream.outputs[fmt].put,
Segment(
sequence,
buffer.segment,
segment_duration,
(segment_start_v_pts, segment_start_a_pts),
),
)
# Reinitialize
initialize_segment(packet.pts)
# Clear outputs and increment sequence
outputs = {}
if not first_packet:
sequence += 1
segment_start_v_pts = packet.pts
segment_start_a_pts = int(
packet.pts * packet.time_base * AUDIO_SAMPLE_RATE
)
# Initialize outputs
for stream_output in stream.outputs.values():
if video_stream.name not in stream_output.video_codecs:
continue
a_packet, buffer = create_stream_buffer(
stream_output, video_stream, audio_frame
)
audio_packets[buffer.astream] = a_packet
outputs[stream_output.name] = buffer
# First video packet tends to have a weird dts/pts
if first_packet:
# If we are attaching to a live stream that does not reset
# timestamps for us, we need to do it ourselves by recording
# the first presentation timestamp and subtracting it from
# subsequent packets we receive.
if (packet.pts * packet.time_base) > 1:
first_pts = packet.pts
packet.dts = 0
packet.pts = 0
first_packet = False
# Store packets on each output
for buffer in outputs.values():
# Check if the format requires audio
if audio_packets.get(buffer.astream):
a_packet = audio_packets[buffer.astream]
a_time_base = a_packet.time_base
# Determine video start timestamp and duration
video_start = packet.pts * packet.time_base
video_duration = packet.duration * packet.time_base
if packet.is_keyframe:
# Set first audio packet in sequence to equal video pts
a_packet.pts = int(video_start / a_time_base)
a_packet.dts = int(video_start / a_time_base)
# Determine target end timestamp for audio
target_pts = int((video_start + video_duration) / a_time_base)
while a_packet.pts < target_pts:
# Mux audio packet and adjust points until target hit
buffer.output.mux(a_packet)
a_packet.pts += a_packet.duration
a_packet.dts += a_packet.duration
audio_packets[buffer.astream] = a_packet
# Assign the video packet to the new stream & mux
packet.stream = buffer.vstream
buffer.output.mux(packet)
# Update last_dts processed
last_dts[packet.stream] = packet.dts
# mux video packets immediately, save audio packets to be muxed all at once
if packet.stream == video_stream:
mux_video_packet(packet) # mutates packet timestamps
else:
mux_audio_packet(packet) # mutates packet timestamps
# Close stream
for buffer in outputs.values():
for buffer, _ in outputs.values():
buffer.output.close()
container.close()