diff --git a/.latexmkrc b/.latexmkrc index 1df49e7..5ddd588 100644 --- a/.latexmkrc +++ b/.latexmkrc @@ -19,9 +19,9 @@ $force_mode = 0; # Define all output directories to be "tmp" # All build files will now be placed in this directory -$aux_dir = "tmp"; -$tmpdir = "tmp"; -$out_dir = 'tmp'; +#$aux_dir = "tmp"; +#$tmpdir = "tmp"; +#$out_dir = 'tmp'; add_cus_dep('glo', 'gls', 0, 'makeglossaries'); add_cus_dep('acn', 'acr', 0, 'makeglossaries'); diff --git a/000_Preamble/abstract.tex b/000_Preamble/abstract.tex index 7a541f7..38caf22 100755 --- a/000_Preamble/abstract.tex +++ b/000_Preamble/abstract.tex @@ -19,9 +19,9 @@ % % Further research will be pursed to improve the error robustness of the \ac{hevc} standard and develop novel concealment methods using combined info between the encoder and decoder side. -The increasing diversity of services and the demand for ever higher quality multimedia at wider resolutions, which leads to the growing popularity of Ultra-HD (UHD) format, are pushing technologies forward and opening new research directions. The recent developments in video coding techniques have resulted in a novel standard called \ac{hevc} that is the next breakthrough in multimedia compression after the globally adopted MPEG-4/\ac{h264} standard. However, the significantly improved levels of compression efficiency achieved by the \ac{hevc} standard lead to an inherent reduction in the error resilience performance under error prone conditions. It should be aimed at ensuring an adequate level of protection against network and transmission induced impairments (e.g., packet losses), without sacrificing the compression efficiency thanks to the novel block level operations done by \ac{hevc}. - -My research work takes the \ac{hevc} standard to its core and builds new techniques on top of it to enhance its error resilience performance, and develop new processing techniques to improve decoder side error concealment performance. The first scope of this research work is to make an extensive analysis of various error resilience techniques developed for the \ac{hevc} standard, but also for the previous video coding standards (most notably \ac{h264}); identify their shortcomings and address those shortcomings. Up to date work has revealed that the \ac{mv} coding technique lacks robustness, which leads to quality degradation due to error propagation in lossy transmission environments. It has also been seen that the temporal dependency of motion information is comparatively higher than that in the \ac{h264} standard. Based on this evidence, a method was developed to make intelligent use of temporal \ac{mv} candidates during the motion estimation process, in order to decrease the temporal dependency, and improve the error resiliency without penalising the rate-distortion performance. The simulation results have shown that the proposed method improves the error resilience under tested conditions, compared to the reference method that always enables temporal \acs{mv} candidates. Another proposed method has exploited the prediction dependency of \glspl{mv}in order to select the most relevant ones for redundant coding with reduced overhead. Simulation results have shown that the proposed \gls{mv} selection method can effectively identify the most relevant motion fields, achieving improved error robustness with a significantly reduced redundancy overhead. - -The second scope of this research work is to develop new processing techniques for the concealment of the coding elements lost during the transmission and delivery system, using combined information between the encoder and decoder sides. Error concealment techniques mitigate the detrimental effects of network impairments on the perceptual multimedia quality and the performance these techniques determine the quality of a system directly. The third and the ultimate aim of this research work is to strike a balance between the designed error resilience and error concealment techniques so as to meet a set of QoS/QoE requirements associated with current and future multimedia-based services (e.g., real-time multi-party video conferencing, Ultra-HD video on demand, etc.), such as bandwidth and delay. It is envisaged to produce a set of configuration profiles for chosen services based around the developed tools for \ac{hevc}. +%The increasing diversity of services and the demand for ever higher quality multimedia at wider resolutions, which leads to the growing popularity of Ultra-HD (UHD) format, are pushing technologies forward and opening new research directions. The recent developments in video coding techniques have resulted in a novel standard called \ac{hevc} that is the next breakthrough in multimedia compression after the globally adopted MPEG-4/\ac{h264} standard. However, the significantly improved levels of compression efficiency achieved by the \ac{hevc} standard lead to an inherent reduction in the error resilience performance under error prone conditions. It should be aimed at ensuring an adequate level of protection against network and transmission induced impairments (e.g., packet losses), without sacrificing the compression efficiency thanks to the novel block level operations done by \ac{hevc}. +% +%My research work takes the \ac{hevc} standard to its core and builds new techniques on top of it to enhance its error resilience performance, and develop new processing techniques to improve decoder side error concealment performance. The first scope of this research work is to make an extensive analysis of various error resilience techniques developed for the \ac{hevc} standard, but also for the previous video coding standards (most notably \ac{h264}); identify their shortcomings and address those shortcomings. Up to date work has revealed that the \ac{mv} coding technique lacks robustness, which leads to quality degradation due to error propagation in lossy transmission environments. It has also been seen that the temporal dependency of motion information is comparatively higher than that in the \ac{h264} standard. Based on this evidence, a method was developed to make intelligent use of temporal \ac{mv} candidates during the motion estimation process, in order to decrease the temporal dependency, and improve the error resiliency without penalising the rate-distortion performance. The simulation results have shown that the proposed method improves the error resilience under tested conditions, compared to the reference method that always enables temporal \acs{mv} candidates. Another proposed method has exploited the prediction dependency of \glspl{mv}in order to select the most relevant ones for redundant coding with reduced overhead. Simulation results have shown that the proposed \gls{mv} selection method can effectively identify the most relevant motion fields, achieving improved error robustness with a significantly reduced redundancy overhead. +% +%The second scope of this research work is to develop new processing techniques for the concealment of the coding elements lost during the transmission and delivery system, using combined information between the encoder and decoder sides. Error concealment techniques mitigate the detrimental effects of network impairments on the perceptual multimedia quality and the performance these techniques determine the quality of a system directly. The third and the ultimate aim of this research work is to strike a balance between the designed error resilience and error concealment techniques so as to meet a set of QoS/QoE requirements associated with current and future multimedia-based services (e.g., real-time multi-party video conferencing, Ultra-HD video on demand, etc.), such as bandwidth and delay. It is envisaged to produce a set of configuration profiles for chosen services based around the developed tools for \ac{hevc}. diff --git a/022_ErrorResilience/RobustTransmissionReview.tex b/022_ErrorResilience/RobustTransmissionReview.tex index 8384a6a..cc40ccb 100755 --- a/022_ErrorResilience/RobustTransmissionReview.tex +++ b/022_ErrorResilience/RobustTransmissionReview.tex @@ -182,6 +182,7 @@ The simulation results reveal an increasing on the achieved video quality when h \subsection{Leaky-predictions} +\label{SECTION2:LeakyPred} Directly using reconstructed past frames as reference frames for predictive coding, may result in substantial error propagation whenever the reference frames are either lost or reconstructed from other distorted frames. To reduce error propagation one can employ leaky predictions, which scales down reconstructed frames before being used as reference~\cite{LeakyPred2009,ROILeakyPred2009}. diff --git a/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex b/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex index 4ed0724..c7dc06d 100644 --- a/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex +++ b/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex @@ -77,9 +77,12 @@ The spatial and temporal complexity of these video sequences was measured by usi The values of \ac{si} and \ac{ti} calculated for the test sequences are shown in the Table~\ref{TableTestSequences} (see second column). For the simulation carried out in this study the HM reference software, version $16.2$, was used \cite{HEVCReferenceSoftware162}. -The sequences were encoded with an \ac{idr} period of $16$ frames and all the coding modes enabled, following two recommended configurations: Low-Delay (P-frames with 4 references frames) and Random-Access (B-frames with 2 reference frames in each list)~\cite{HEVCCommonTests2012}. -In order to achieve high quality images for all coded sequences (i.e., \ac{psnr} around 40 dB) different bitrates were used, as listed in Table~\ref{TableTestSequences}. +The sequences were encoded with an \ac{idr} period of $16$ frames and all the coding modes enabled, following two recommended configurations: \ac{ld}, i.e., P-frames with 4 references frames, and \ac{ra}, i.e., B-frames with 2 reference frames in each list~\cite{HEVCCommonTests2012}. +These prediction structures cover the use of P-frames and hierarchical B-frames~\cite{H264ScalableOverviewSep2007}, which are commonly used in HEVC streaming~\cite{RealTimeHEVC2016}. +In order to achieve high quality images for all coded sequences (i.e., \ac{psnr} around 40 dB) different bitrates were used, as listed in Table~\ref{TableTestSequences}. The filtering operations were disabled at slices boundaries in order to keep the slices self-contained. Moreover, the ratio of inter-coded blocks of each coded sequence using the default coding parameters is also shown in the third column of the table to illustrate the predictions distribution. + + %The following bitrates were selected to achieve high quality (i.e., around 40 dB): 1.0 Mbits/s (Kendo), 1.5 Mbits/s (Book Arrival), 4.0 Mbits/s (BQSquare, Park Scene and Race Horses) and 12.0 Mbits/s (People on Street). % At the decoder side, error concealment is based on motion compensation from previously decoded frames. % The motion field of a missing frame is obtained applying \ac{mv} extrapolation from the two closest neighbours. @@ -100,7 +103,7 @@ In the first set of experiments the impact of the maximum \ac{cu} size in the de Table~\ref{Table3:Quality_BlockSize} shows the average \ac{psnr} obtained for the three test conditions and for four different packet loss ratios. Results for the error-free scenario shows a consistent quality increase as the maximum \ac{cu} size increases, up to a maximum of 1.41 dB for the Tennis sequence when comparing the maximum \ac{cu} size of $64\times64$ with $16\times16$. This confirms the principle used by the \ac{hevc} standard, that allowing larger coding units lead to higher coding efficiency, achieving a maximum of 1.41~dB for the Tennis sequence which has the higher \ac{ti} parameter (see Table~\ref{TableTestSequences}). Moreover, it is also noticeable that sequences with higher amount of inter-coded blocks take less advantage of the larger \acs{cu}, by the results of the BQSquare sequence (i.e., highest ratio of inter-coded blocks and lowest quality gain in the error-free case). Overall, comparing the results for the error-free and data loss cases it is noticeable that similar quality gains are obtained, which are achieved due to the superior coding efficiency of larger \acs{cu}. In most of the cases the quality gains slightly decrease for increasing \acs{plr}. -Therefore, one may conclude that the error robustness of \ac{hevc} streams is not affected by limiting the maximum \ac{cu} size. \HERE +Therefore, one may conclude that the error robustness of \ac{hevc} streams is not affected by limiting the maximum \ac{cu} size. \todo{Complete discussion of partition unit size} %Results for the case of data loss indicate that using larger coding units leads to higher video quality. @@ -168,7 +171,7 @@ Figure~\ref{Figure3:QualityvsIntra} shows the average decoded video quality (\ac Results in the figure indicate that as the distance between intra frames increases, the decoded video quality significantly decreases. The quality decreasing is especially noticeable for intra periods higher than 32 frames, indicating that temporal distances greater than approximately one second are not suitable for video streaming under error-prone conditions. When a high temporal distance between intra frames is used, the quality decreasing is more noticeable in the People On Street and BQSquare sequences, which have higher spatial complexity (see values for \ac{si}), leading to a reduction of 13 dB and 14 dB, respectively. As spatial complexity increases the encoder rely more on temporal predictions making the bitstream more prone to error propagation. -\HERE +\todo{Complete discussion of intra refreshing} One should also note from the results of Figure~\ref{Figure3:QualityvsIntra} that using a small distance between intra frames is not always more efficient. Since intra frames require higher amount of bits, the overall coding efficiency is reduced, resulting in an overall quality that decreases even in data loss conditions. @@ -193,7 +196,8 @@ In this sub-section the amount of bits per slice is evaluated, in order to evalu Figure~\ref{Figure3:SlicePartitioning} shows the average \ac{psnr} for three difference slice partitioning modes and different \acs{plr}. The first mode uses a single slice per frame, corresponding to the case where slice partitioning is disabled. -The two remaining are characterised by a different amount of bits per slice, namely 1200 and 2400 bytes per slice, respectively. In these two cases each frame has a variable number of slices. \HERE +The two remaining are characterised by a different amount of bits per slice, namely 1200 and 2400 bytes per slice, respectively. In these two cases each frame has a variable number of slices. +\todo{Complete discussion of slice partitioning} Results reveals a clear impact of the slice partitioning on the error robustness of the \ac{hevc}, as the quality degradation as the \ac{plr} increases varies for each scheme. Firstly it is clearly notice the disadvantage of using a single slice per frame as it leads to the lower quality when comparing the test cases studied. This is due to the lack of refresh points within a frame, resulting in loss events that affects the whole frame which will propagate to the subsequent inter predicted frames. @@ -258,14 +262,14 @@ Figure~\ref{Figure3:PSNRvsFrameErrorFreeConc} illustrates the error propagation, The results show that when the \ac{tmvp} mode is enabled, the loss of Frame \#6 leads to significant reduction in the reconstruction quality of the subsequent frames. % This is true even for sequences with low motion activity as per Table~\ref{tab:sequences}. On the contrary, when the temporal \ac{mv} candidate is disabled, the motion information on subsequent frames is not affected by the loss of a single frame and significantly higher quality is achieved. This is the effect of breaking the temporal dependencies between \acs{mv}, which prevents erroneous \acs{mv} to propagate throughout the set of dependent frames. -%This is more noticeable in the Low-Delay configuration because more error propagation occurs in P-frames and several frames are affected until an intra frame is decoded (refresh point). +%This is more noticeable in the \ac{ld} configuration because more error propagation occurs in P-frames and several frames are affected until an intra frame is decoded (refresh point). \begin{figure}[!t] \centering \renewcommand{\FIGSIZE}{0.76\textwidth} % \includegraphics[width=\FIGSIZE,page=2]{\HEVCRobustnessEvaluation/PSNRvsFrameSingleFrameTMVP} \input{./031_ErrorResilienceEvaluation/QualityEvaluation/PSNRvsFrameSingleFrameTMVP} - \caption{Error propagation for Kendo sequence when Frame~\#6 is lost for the Low-Delay configuration.} + \caption{Error propagation for Kendo sequence when Frame~\#6 is lost for the \ac{ld} configuration.} \label{Figure3:PSNRvsFrameErrorFreeConc} \end{figure} @@ -273,11 +277,11 @@ On the contrary, when the temporal \ac{mv} candidate is disabled, the motion inf In order to further evaluate the impact of the temporal \ac{mv} dependencies on the error propagation, the \ac{hevc} streams were subject to random packet losses. Table~\ref{Table3:AvgPSNRWithoutTMVP} presents the average quality results, measured using the \ac{psnr}. As expected, the results indicate that in error-free conditions higher quality is achieved by enabling the \ac{tmvp} candidate, since higher compression efficiency is obtained because more \ac{mv} predictors are used. -This is true for all both test conditions, \emph{i.e.}, both Low-Delay and Random-Access. -However, in the presence of errors, the video quality significantly decreases due to low error robustness when temporal \ac{mv} candidates are used. Alternatively, average quality up to 5.17 dB of \ac{psnr} when the temporal \ac{mv} predictors are disabled is achieved in the Low-Delay configuration. +This is true for all both test conditions, \emph{i.e.}, both \ac{ld} and \ac{ra}. +However, in the presence of errors, the video quality significantly decreases due to low error robustness when temporal \ac{mv} candidates are used. Alternatively, average quality up to 5.17 dB of \ac{psnr} when the temporal \ac{mv} predictors are disabled is achieved in the \ac{ld} configuration. The results also show that the negative impact of using temporal \ac{mv} dependencies is smaller when B-frames are used. For the Random-Access, the average gain of disabling the \ac{tmvp} is 0.78 dB. %This also confirms the previous results of Figure~\ref{Figure3:PSNRvsFrameErrorFreeConc}. -A higher negative impact is obtained when only P-frames are used (Low-Delay), since subsequent frames are temporally closer, leading to more accurate temporal \ac{mv} candidates. +A higher negative impact is obtained when only P-frames are used (\ac{ld} configuration), since subsequent frames are temporally closer, leading to more accurate temporal \ac{mv} candidates. Moreover, the results also indicate that for higher loss ratios the negative impact of using the \ac{tmvp} does not increase as both configuration are subject to high quality degradation. \begin{table}[!t] @@ -383,7 +387,7 @@ However, reducing the temporal dependencies of the \acs{mv} using the proposed m \begin{table}[tbp] \centering -\caption{Bjontegaard's average \ac{psnr} (dB) differences for all sequences tested at different frame loss conditions.} \label{Table3:AverageResultsProposed} +\caption{BD-PSNR (dB) differences for all sequences tested at different frame loss conditions.} \label{Table3:AverageResultsProposed} \input{031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNR.tex} \end{table} diff --git a/041_ImprovedErrorResilience/Figures/MVDependencies.tikz b/041_ImprovedErrorResilience/Figures/MVDependencies.tikz index efaa9d3..9372167 100644 --- a/041_ImprovedErrorResilience/Figures/MVDependencies.tikz +++ b/041_ImprovedErrorResilience/Figures/MVDependencies.tikz @@ -2,12 +2,13 @@ \def\MVDEPB{\mbox{(b)}\xspace} \def\MVDEPC{\mbox{(c)}\xspace} -\newcommand{\scale}{1} -\newcommand{\xfshift}{4.5cm} +\newcommand{\scale}{.8} +\newcommand{\xfshift}{5cm} \begin{tikzpicture}[scale=\scale] \tikzstyle{block}=[thick] \tikzstyle{refmv}=[-latex,very thick,scale=1] \tikzstyle{MvLabel}=[right, font=\footnotesize] +\tikzstyle{DepsLabel}=[font=\footnotesize, fill=white, inner sep=0] \newcommand{\DrawFrameMVDependency} { \newcommand{\fsize}{4cm} @@ -41,8 +42,8 @@ \draw[refmv] (T0B22)+(0.25,-0.5) -- ++(-0.25,-0.25); \draw[refmv] (T0B3)+(0.5,-0.5) -- ++(-0.25,-0.5); \draw[refmv] (T0B6)+(0.5,-0.5) -- ++(-0.5,-0.5); - \node[font=\scriptsize] () at (1.35,2.75) {\MVDEPA}; - \node[font=\scriptsize] () at (2.35,0.75) {\MVDEPB}; + \node[DepsLabel] () at (1.2,2.8) {\MVDEPA}; + \node[DepsLabel] () at (2.2,0.8) {\MVDEPB}; \node at (4cm/2,4.7cm) { $f_{0}$ }; \end{scope} % Frame t-1 @@ -75,7 +76,7 @@ \draw[refmv] (T0B6)+(0.75,-0.25) -- ++(0.25,-0.25); \coordinate (T0B7) at (2.5,0.5); \draw[block] (T0B7) rectangle ++(0.5,-0.5); - \node[font=\scriptsize] () at (1.35,2.75) {\MVDEPC}; + \node[DepsLabel] () at (1.2,2.8) {\MVDEPC}; \node at (4cm/2,4.7cm) { $f_{-1}$ }; \end{scope} % Frame t-2 diff --git a/041_ImprovedErrorResilience/ImprovedErrorResilience.tex b/041_ImprovedErrorResilience/ImprovedErrorResilience.tex index 3eee35e..655b487 100644 --- a/041_ImprovedErrorResilience/ImprovedErrorResilience.tex +++ b/041_ImprovedErrorResilience/ImprovedErrorResilience.tex @@ -16,48 +16,67 @@ \glsreset{mv} \glsreset{rd} -The increased compression ratios achieved by the \ac{hevc} standard lead to reduced robustness of coded streams, with increased susceptibility to network errors and consequent video quality degradation, as shown in Chapter~\ref{Chapter:HEVCErrorResilience}. -Therefore, this chapter proposes a two-stage approach method to improve the error robustness of \ac{hevc} streaming, by reducing temporal error propagation in case of frame loss. -The prediction mismatch that occurs at the decoder after frame loss is reduced through the following two stages: (i) at the encoding stage, the reference pictures are dynamically selected based on constraining conditions and Lagrangian optimisation, which distributes the use of reference pictures, by reducing the number of \acs{pu} that depend on a single reference; (ii) at the streaming stage, a \ac{mv} prioritisation algorithm, based on spatial dependencies, selects an optimal sub-set of \acs{mv} to be transmitted, redundantly, as side information to reduce mismatched \ac{mv} predictions at the decoder. +\glsreset{ra} +\glsreset{ld} -This chapter is organized as follows. Section~\ref{Section4:Problem} presents the problem formulation and motivation for the method proposed in this chapter. Section~\ref{Section4:Introduction} provides an overview of the proposed method, describing the functional blocks that compose the two-stage approach for robust coding and streaming. Section~\ref{Section4:RefPicSel} describes the encoding stage of the proposed method, that is used to selected the optimal reference picture for inter-prediction. Section~\ref{Section4:MVSelection} describes the streaming stage used to select a sub-set of \acs{mv} to reduce the mismatch motion predictions. Section~\ref{Section4:Performance} provides a performance evaluation of the proposed method, illustrating the advantages of each stage and how they are combined together to achieve superior error robustness in \ac{hevc} standard. Finally, Section~\ref{Section4:Summary} concludes this chapter. +%The increased compression ratios achieved by the \ac{hevc} standard lead to reduced robustness of coded streams, with increased susceptibility to network errors and consequent video quality degradation, as shown in Chapter~\ref{Chapter:HEVCErrorResilience}. +Therefore, this chapter proposes a coding method, based on a two-stage approach, to improve the error robustness of \ac{hevc} streaming, by reducing temporal error propagation in case of frame loss. +This is accomplished by reducing the prediction mismatch that occurs at the decoder after frame loss through the following two stages: (i) at the encoding stage, the reference pictures are dynamically selected based on constraining conditions and Lagrangian optimisation, which distributes the use of reference pictures throughout each frame, by reducing the number of \acs{pu} that depend on a single reference; (ii) at the streaming stage, a \ac{mv} prioritisation algorithm, based on spatial dependencies, selects an optimal sub-set of \acs{mv} to be transmitted, redundantly, as side information to reduce mismatched \ac{mv} predictions at the decoder. + +This chapter is organized as follows. Section~\ref{Section4:Problem} presents the problem definition and motivation for the method proposed in this chapter. Section~\ref{Section4:Architecture} provides an overview of the proposed method, describing the functional blocks that compose the two-stage approach for robust coding and streaming. Section~\ref{Section4:RefPicSel} describes the encoding stage of the proposed method, that is used to select the optimal reference picture for inter-prediction. Section~\ref{Section4:MVSelection} describes the streaming stage used to select a sub-set of \acs{mv} to reduce the mismatch motion predictions. Section~\ref{Section4:Performance} provides a performance evaluation of the proposed method, illustrating the advantages of each stage and how they are combined to achieve superior error robustness in \ac{hevc} standard. Finally, Section~\ref{Section4:Summary} concludes the chapter. \clearpage %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\section{Problem formulation} +\section{Problem definition} \label{Section4:Problem} -The error robustness characteristics of \ac{hevc} analysed in Chapter~\ref{Chapter:HEVCErrorResilience} confirmed that, in general, \ac{hevc} presents poor error resilience performance. Thus, \ac{hevc} bitstreams subject to transmission losses result in significant degradation of both objective and subjective quality. This is mostly due to the strong decoding dependencies imposed by the highly complex prediction modes of \ac{hevc}, which are selected according to a \ac{rd} optimisation criteria, assuming error-free transmission~\cite{ROPE2007}. Therefore, high compression efficiency is achieved, but resilience to transmission errors is penalised. This compromises the error robustness of video decoders which leads to poor performance under error conditions. Moreover, the use of advanced \ac{mv} prediction in \ac{hevc} \cite{MotionCodingHEVC2011} also contributes to increase the spatial and temporal dependencies to a great extent. Although the spatial MV candidates are used more often, temporally predicted \acs{mv} are responsible for a greater impact on error propagation. +As mentioned before \ac{hevc} bitstreams subject to transmission losses result in significant degradation of both objective and subjective quality. +This is mostly due to the strong decoding dependencies imposed by the highly complex prediction modes, which are selected according to a \ac{rd} optimisation criteria, assuming error-free transmission~\cite{ROPE2007}. +Therefore, high compression efficiency is achieved, but resilience to transmission errors is penalised. +\begin{figure}[tb] +\centering \input{./041_ImprovedErrorResilience/Results/AverageRefPicUsage.tex} +\caption{Average usage of the reference frame 1 ($r_1$) for different sequences.} +\label{FIG4:ReferenceUsageRatio} +\end{figure} -As mentioned before the video decoder may rely of a distributed use of references frames to achieve higher video quality, by experienced lower error propagation and by optimising the error concealment algorithms to extract information from correctly decoded reference frames. In order to evaluate how the reference \ac{rd} optimisation presented in the reference \ac{hevc} encoder different test sequences were encoded using a configuration with two reference frames, referred to as $r_1$ and $r_2$. Figure~\ref{FIG4:ReferenceUsageRatio} illustrates the average usage ratio of reference $r_1$ for different sequences. The remaining corresponds to those cases where reference $r_2$ were selected. -Results in Figure~\ref{FIG4:ReferenceUsageRatio} shows that the reference $r_1$ is selected in most of the cases when two references are available. This is mainly true for sequences with higher motion. For those cases with lower motion but high spatial complexity, e.g., BQSquare sequence (see \ac{ti} in Table~\ref{TableTestSequences}), both frames are selected for temporal prediction, with $r_1$ being used for 58\% of cases and $r_2$ for 42\%. This reveals that even a configuration with two references the encoder will mainly use the temporally closer one as the reference for inter-frame predictions. This will lead to a higher number of mismatch predictions in case of frame loss and consequently to higher error propagation. +Highly efficient video encoders support the use of multiple reference frames for inter-frame prediction, which are selected based on a \ac{rd} cost which may lead to an unbalance distribution. +In order to show evaluate the unbalanced is the reference frame selection attained by the \ac{rd} implemented in the reference \ac{hevc} encoder, different test sequences are encoded using two reference frames, referred to as $r_1$ and $r_2$, and its selection ratios are analysed. In this study, reference $r_1$ corresponds to the closest neighbouring frame and reference $r_2$ the second closest. +Figure~\ref{FIG4:ReferenceUsageRatio} illustrates the average usage percentage of reference $r_1$ for different sequences. +The remaining corresponds to those cases where reference $r_2$ were selected. +These results show that, in general, reference $r_1$ is selected more often than $r_2$. +This is mainly true for sequences with higher motion. For those sequences with lower motion but high spatial complexity, e.g., BQSquare (see \ac{si} in Table~\ref{TableTestSequences}), both reference frames equally (approximately) selected for temporal predictions, with $r_1$ being used for 58\% of cases and $r_2$ for 42\%. +This reveals that even with a configuration with two references the encoder will mainly use the temporally closer one as the reference for inter-frame predictions. +Therefore, if such reference frame is lost, a higher number of mismatched predictions will occur, inevitabely leading to higher error propagation. -As mentioned in Chapter~\ref{Chapter:HEVCErrorResilience}, another technique that contributes to increased error propagation is the use of advanced \ac{mv} prediction in \ac{hevc} \cite{MotionCodingHEVC2011}. This contributes to increase the spatial and temporal dependencies to a great extent. Although the spatial \ac{mv} candidates are used more often, temporally predicted \acs{mv} are responsible for a greater impact on error propagation, as they propagate errors in the \ac{mv} decoding to across different time instances. -In general, increased error robustness in video streaming can be jointly achieved at two different stages: (i) while encoding live video, by selecting robust coding modes and (ii) after encoding, by adding robust features to the compressed streams. While in the former this is accomplished by optimising coding parameters and decisions to increase robustness, in the latter some type of compressed domain processing must be used to increase error robustness of pre-encoded streams, e.g., error resilient transcoding. Inevitably, in both cases this is achieved at the expense of some loss in coding efficiency, therefore the challenge is to find an optimum trade-off between robustness and bitrate overhead. In this chapter a two-fold approach is used to overcome the two problems mentioned above. +Another technique that contributes to increased error propagation is the use of predictions in \ac{mv} coding~\cite{MotionCodingHEVC2011}, which contributes to increase the spatial and temporal dependencies. +Although the spatial \ac{mv} candidates are used more often, temporally predicted \acs{mv} are responsible for a greater impact on error propagation, as they propagate errors in the \ac{mv} decoding across different time instances. + +To overcome these issues, in the next sections a two-fold approach is described to increase the error robustness in video streaming by jointly considering two different stages: (i) while encoding live video, by selecting robust coding modes and (ii) after encoding, by adding robust features to the compressed streams. While in the former this is accomplished by optimising coding parameters and decisions to increase robustness, in the latter some type of compressed domain processing is used to increase error robustness of pre-encoded streams, e.g., error resilient transcoding. Inevitably, in both cases this is achieved at the expense of some loss in coding efficiency, therefore the challenge is to find an optimum trade-off between robustness and bitrate overhead. In this chapter a two-fold approach is used to overcome the two problems mentioned above. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\section{Proposed architecture} -\label{Section4:Introduction} +\section{Proposed two-stage architecture} +\label{Section4:Architecture} -The goal of the proposed method is to reduce the error propagation in decoded video in case of frame loss. This is accomplished by constraining inter-frame prediction dependencies, in order to minimise the distortion $D_t$ given by +The goal of the proposed method is to reduce the error propagation in decoded video in case of frame loss. +The underlying idea is to constrain inter-frame prediction dependencies, in order to minimise the distortion $D_t$ given by \begin{equation} D_t = \tilde f_t - \hat f_t, \text{when } \tilde f_{t-1} \neq \hat f_{t-1}, \label{eq1} \end{equation} where $\hat f_t$ and $\tilde f_t$ are the encoded and decoded frames, respectively, at instant $t$. -Figure \ref{fig:proposedMethod} illustrates the block diagram of the two-stage method proposed in this work. -At the encoding stage, a reference frame selection mechanism is used for each \ac{pu} within a coding tree unit (\ac{ctu}) forcing the use of different reference frames in adjacent \acs{ctu}. The decision criterion contributes to reduce the amount of predictions from a single reference frame, and forces to interleave different reference frames along all encoded \acs{pu}, based on a \ac{rd} optimised criterion. -At the streaming stage, the coded stream is parsed to extract the \ac{mv} information, which allows ranking \acs{mv} according to a relevance criterion. Then, a sub-set of these \acs{mv} is selected to be transmitted as side information. The amount of selected \acs{mv} is not fixed and can be optimised for different content and end-network characteristics. -Note that the streaming stage is an independent and complementary subsystem to the encoder. Thus, it is able to operate at network nodes, including the streaming server on pre-encoded streams. +Figure \ref{fig:proposedMethod} illustrates the block diagram of the two-stage architecture proposed in this work. +At the encoding stage, a reference frame selection mechanism is used for each \ac{pu} within a coding tree unit (\ac{ctu}) forcing the use of different reference frames in adjacent \acs{ctu}. The decision criterion contributes to reduce the amount of predictions from a single reference frame, and forces to interleave different reference frames along all encoded \acs{pu}, based on \ac{rd} optimisation. +At the streaming stage, the coded stream is parsed to extract the \ac{mv} information, which allows ranking \acs{mv} according to a relevance criterion. Then, a sub-set of these \acs{mv} is selected to be transmitted as side information. The amount of selected \acs{mv} is not fixed and can be optimised for different content and network characteristics. +Note that the streaming stage is an independent and complementary subsystem to the encoder. Thus, it is able to operate at network nodes, including streaming servers delivering pre-encoded streams. \begin{figure}[tb] \centering @@ -76,16 +95,25 @@ This is equivalent to reducing the error propagation by using two techniques: (i \label{Section4:RefPicSel} -The reference picture selection mechanism attempts to uniformly distribute the use of different reference pictures across the \ac{ctu} of a coded video frame. In case of frame loss, the error propagation in the decoder is reduced because the number of \acs{ctu} not predicted from the lost frame decreases and thus error concealment benefits from more data correctly decoded. Note that standard compliance is maintained. -The underlying principle of this method is to minimise the probability of using the same reference picture for neighbouring \acs{ctu}, which tends to result in a checkerboard structure, where a different reference frame is used for predicting every other \ac{ctu} within the current frame. -For instance, in a coding structure with two reference frames, as shown in Figure~\ref{fig:checkerboardRefSelection}, the reference frames $r_{1}$ and $r_{2}$ are used to predict the same number of \acs{ctu} in current frame~($C$). +The reference picture selection mechanism attempts to uniformly distribute the use of different reference pictures across the \ac{ctu} of a coded video frame. +In case of frame loss, the error propagation in the decoder is reduced because the number of \acs{ctu} predicted from the lost frame decreases and thus error concealment benefits as more data is correctly decoded from other frames. +The underlying principle of this method is to minimise the probability of using the same reference picture for neighbouring \acs{ctu}. + +%For instance, +In a coding structure with two reference frames, as shown in Figure~\ref{fig:checkerboardRefSelection}, the reference frames $r_{1}$ and $r_{2}$ are used to predict the same number of \acs{ctu} in current frame~($C$), resulting in a checkerboard structure, where a different reference frame is used for predicting every other \ac{ctu} within the current frame. If each reference frame is used to predict half of the pixels in the current frame, when any of these references is lost, the subsequent predicted frame is only partially distorted by mismatched predictions. This is because only half of its predictions depend on the missing reference frame, which limits the overall error propagation. -Moreover, since the motion information of the predicted frame is encoded with reference not only to the missing frame but also to the past ones, those \acs{mv} that cross over a missing reference frame, pointing to the previous ones, are still useful for reconstructing correct \acs{mv} and also for error concealment. - +Moreover, since the motion information of the predicted frame is encoded with not only reference to the missing frame but also to the past ones, those \acs{mv} that cross over a missing reference frame, pointing to the previous ones, are still useful for reconstructing correct \acs{mv} and also for error concealment. In the limit, a straightforward implementation of the previous example leads to a simple interleaved uniform distribution of the reference frames over all \acs{ctu} in the current frame, forcing the checkerboard structure shown in Figure \ref{fig:checkerboardRefSelection}. In the case of two reference frames, 50\% of the \acs{ctu} would use one reference frame, while the other 50\% would use the other reference. However, such solution would not take into consideration coding efficiency, since the selection process of the reference frame would not be \ac{rd} optimised for each \ac{pu} within a \ac{ctu}. +\begin{figure}[tb] + \centering + \input{\ImprovedErrorResilience/Figures/RefPicSelection.tikz} + \caption{Example of the reference picture selection applied to the frame $C$ based on the checkerboard at \ac{ctu} level (\acs{pu} represented by small partitions).} + \label{fig:checkerboardRefSelection} +\end{figure} + Therefore, a more efficient approach to improve error robustness was devised to dynamically choose the coding mode for each \ac{pu}, that ensures a balanced use of all possible reference frames $F=\{r_1,r_2,...r_S\}$ across the \acs{ctu} of the current frame $C$, also optimising the \ac{rd} cost. This leads to the overall goal of finding the optimum R-D coding mode for each \ac{pu}, as the one which tends to minimise the standard deviation of the reference frames use count $\mathbf{N}_C(r_i)$, considering all encoded \acs{pu} of the current frame $C$, as follows: \begin{equation} @@ -94,12 +122,7 @@ This leads to the overall goal of finding the optimum R-D coding mode for each \ \end{equation} In this case $\sigma_C$ measures the deviation from the checkerboard solution, where $\sigma_C = 0$, since all possible reference frames $r_i\in F$ are uniformly used throughout the \acs{pu} of $C$. -\begin{figure}[t] -\centering -\input{\ImprovedErrorResilience/Figures/RefPicSelection.tikz} -\caption{Example of the reference picture selection applied to the frame $C$ based on the checkerboard at \ac{ctu} level (\acs{pu} represented by small partitions).} -\label{fig:checkerboardRefSelection} -\end{figure} + To attain the goal defined above, the best coding mode for each \ac{pu} is chosen through a Lagrangian \ac{rd} optimisation that tends to minimise $\sigma_C$. This is done by penalising (or benefiting) the \ac{rd} cost of those coding modes which intend to use reference frames that have already been used more (or less) frequently than their average use in previous \acs{pu} of the current frame. @@ -160,29 +183,36 @@ For instance, for $\gamma=5$, when a reference frame $r_\phi$ is used to predict \FIGSVSPACE \end{figure*} -Figure \ref{fig:RefPicSelectionExample} illustrates the distribution of reference frames achieved by the proposed method across different \acs{pu}. This result is compared against a fixed checkerboard approach (\CHKB) and the reference \ac{hevc} (\REF). The dark-gray and light-gray squares indicate the reference frame in use, either $r_1$ or $r_2$, respectively. The white squares indicate intra-coded \acs{pu}. Below each method, the percentages of the image areas predicted by each reference frame are shown. As shown in this figure, the proposed method is able to distribute the use of the reference frames closely to the \CHKB method, i.e., $r_2$ is used as reference for approximately the same amount of \acs{pu} as $r_1$ (see use percentages of the predicted image). However, the predicted regions in the proposed method do not exhibit the same regular pattern as in \CHKB method. +Figure~\ref{fig:RefPicSelectionExample} illustrates the distribution of reference frames achieved by the proposed method across different \acs{pu}. This result is compared against a fixed checkerboard approach (\CHKB) and the reference \ac{hevc} (\REF). The dark-gray and light-gray squares indicate the reference frame in use, either $r_1$ or $r_2$, respectively. The white squares indicate intra-coded \acs{pu}. Below each method, the percentages of the image areas predicted by each reference frame are shown. As shown in this figure, the proposed method is able to distribute the use of the reference frames closely to the \CHKB method, i.e., $r_2$ is used as reference for approximately the same amount of \acs{pu} as $r_1$ (see use percentages of the predicted image). However, the predicted regions in the proposed method do not exhibit the same regular pattern as in \CHKB method. +It is noticeable in Figure~\ref{fig:RefPicSelectionExample} that for several \acs{ctu} the \PROP method is able to keep the same partitions and reference frames when compared to the optimal encoding (\REF). This can be observed, for example in the small partitions on the left, where the \PROP method uses the same reference frame as \REF. Thus, the proposed method is expected to achieve better coding efficiency than \CHKB, since more optimal coding modes may be chosen. -It is noticeable in the figure that for several \acs{ctu} the \PROP method is able to keep the same partitions and reference frames when compared to the optimal encoding (\REF). This can be observed, for example in the small partitions on the left, where the \PROP method uses the same reference frame as \REF.Thus, the proposed method is expected to achieve better coding efficiency than \CHKB, since more optimal coding modes may be chosen. - -Summarising, the underlying principle of Equations~(\ref{eq:Lagrangian})~and~(\ref{eq:KFactor}) is that the cost of using a given reference frame increases with the number of \acs{pu} using it for prediction. However, the reference frames are not hard-selected without taking into account the R-D cost. In general, it is more difficult to reach a uniform use of reference frames in complex sequences than simpler ones. -Note that standard compatibility is maintained, since the proposed optimisation method does not require any syntax change. Moreover, as the proposed approach is applied at the Lagrangian computation already present in the \ac{hevc} optimisation, the complexity of the encoder is not significantly affected. +% TID aqui? +Summarising, by using Equations~(\ref{eq:Lagrangian})~and~(\ref{eq:KFactor}), the cost of using a given reference frame increases as the number of \acs{pu} using it for prediction increases. +However, the reference frames are not hard-selected without taking into account the R-D cost. +In general, it is more difficult to reach a uniform use of reference frames in complex sequences than in simpler ones. +Note that standard compatibility is maintained, since the proposed optimisation method does not require any syntax change. Moreover, since the proposed approach is employed at the Lagrangian optimisation already implemented in the \ac{hevc} reference software, the complexity of the encoder is not significantly affected. \section{Streaming stage: \ac{mv} selection} \label{Section4:MVSelection} -Since \ac{hevc} uses differential and predictive \ac{mv} coding, whenever a \ac{mv} is lost the subsequent ones will also be affected, until a refresh point is reached to break the dependency chain. This increases the error propagation whenever the temporal \ac{mv} candidate is used in the Merge Mode. -To increase the number of possible refresh points and improve error robustness, a reduced number of \acs{mv} are selected to be transmitted as side information. The selection of these redundant \acs{mv} is based on a trade-off that attempts to maximise the image area covered by such \acs{mv} using the lowest number of bits to encode them. +\glsreset{tmvp} + +Since \ac{hevc} uses differential and predictive \ac{mv} coding, whenever a \ac{mv} is lost the subsequent ones will also be affected, until a refresh point (i.e., \ac{mv} coded on its own, without using \ac{mv} prediction) is reached to break the dependency chain. +Thus, the amount of error propagation strongly depends on whether the \ac{tmvp} is used. +In this streaming stage, to increase the number of possible refresh points and improve error robustness, a small number of \acs{mv} are selected to be transmitted as side information. The selection of these redundant \acs{mv} is based on a trade-off between the image area covered by such \acs{mv} and the number of bits to encode them. +The best tradeoff corresponds to the \acs{mv} that cover the largest image area using the lowest amount of encoding bits. \def\MVDEPA{\mbox{(a)}\xspace} \def\MVDEPB{\mbox{(b)}\xspace} \def\MVDEPC{\mbox{(c)}\xspace} Figure~\ref{fig:mvdependencies} illustrates how \acs{mv} have different importance in error propagation. -In this figure, each block corresponds to a \ac{pu}, which has an associated \ac{mv}. The arrows point to the \ac{pu} containing the \ac{mv} used as predictor. In the current frame $f_0$, two \acs{pu} have temporally dependent \acs{mv}. -While the \ac{mv} of \ac{pu} \MVDEPA is used to predict a total of four other \acs{mv}, the \ac{mv} of \ac{pu} \MVDEPB is only used to predict one single \ac{mv}. Thus, in case of data loss, the \ac{mv} of \ac{pu} \MVDEPA has more impact on error propagation and quality degradation than the \ac{mv} of \ac{pu} \MVDEPB, because of its higher number of dependent \acs{mv}. -For the this reason, in the \ac{mv} selection process, the proposed method assigns higher importance to the \ac{mv} of \ac{pu} \MVDEPA than \ac{pu} \MVDEPB. Moreover in Figure~\ref{fig:mvdependencies}, it can be observed that although the number of \acs{mv} predicted from the \acs{mv} of \ac{pu} \MVDEPC is higher than \ac{pu} \MVDEPA (6 and 4 \acs{mv}, respectively), -the image area dependent from the former is smaller that the latter. This is due to the variable size of \acs{pu}. +In this figure, each block corresponds to a \ac{pu}, which has an associated \ac{mv}. The arrows point to the \ac{pu} containing the \ac{mv} used as predictor. In the current frame ($f_0$), two \acs{pu} have temporally dependent \acs{mv}. +While the \ac{mv} of \ac{pu} \MVDEPA is used to predict a total of four other \acs{mv}, the \ac{mv} of \ac{pu} \MVDEPB is only used to predict one single \ac{mv}. +Thus, in case of data loss (e.g., frame $f_{-1}$), the \ac{mv} of \ac{pu} \MVDEPA has more impact on error propagation to the frame $f_{0}$ and consequently quality degradation than the \ac{mv} of \ac{pu} \MVDEPB, because of its higher number of dependent \acs{mv}. +For this reason, in the \ac{mv} selection process, the proposed method assigns higher importance to the \ac{mv} of \ac{pu} \MVDEPA than \ac{pu} \MVDEPB. Moreover, in Figure~\ref{fig:mvdependencies}, it can be observed that although the number of \acs{mv} predicted from the \acs{mv} of \ac{pu} \MVDEPC is higher than \ac{pu} \MVDEPA (6 and 4 \acs{mv}, respectively), +the image area that is dependent from the former is smaller that the latter. This is due to the variable size of \acs{pu}. Therefore, the image area covered by all dependent \acs{pu} is separately considered in the \ac{mv} selection process. \begin{figure}[tb] @@ -192,8 +222,7 @@ Therefore, the image area covered by all dependent \acs{pu} is separately consi \label{fig:mvdependencies} \end{figure} - -Based on the evidence mentioned above, the best \acs{mv} to be encoded redundantly as side information for each frame are selected through the following procedure: +Based on the evidence mentioned above, the best \acs{mv} to be encoded as side information for each frame are selected through the following procedure: \begin{enumerate} \item the \acs{mv} with temporal dependencies are firstly selected as the most important set: $V$ = \{\emph{\ac{mv}: \ac{mv} is temporally dependent from others}\}. @@ -203,101 +232,93 @@ the $MVs \in V$ are ranked according to the number of spatial dependencies assoc the optimal sub-set $U \subset V$ is found as the best trade-off between the image area covered by the dependent \acs{pu} and bitrate overhead, i.e., the one that maximises the cost function in Equation (\ref{eq:MVSelectOpt}). \end{enumerate} -Considering $V$ the ordered set of \acs{mv} encoded for the current frame, i.e., $V=\{MV_1, MV_2, \cdots MV_s\}$ and $U \subset V$ always starting in the first vector, i.e., \mbox{$U=\{MV_1, MV_2, \cdots MV_n\}$} with $n\leq s$, the best set of redundant \acs{mv} ($U^*)$ is selected based on the optimisation procedure given by Equation (\ref{eq:MVSelectOpt}). Note that any value of $n$ corresponds to a unique subset $U$. +Considering $V$ the ordered set of \acs{mv} encoded for the current frame, i.e., $V=\{MV_1, MV_2, \cdots MV_s\}$ and $U \subset V$ always starting in the first vector, i.e., \mbox{$U_n=\{MV_1, MV_2, \cdots MV_n\}$} with $n\leq s$, the best set of \acs{mv} ($U_n^*)$ is selected based on the optimisation procedure given by Equation (\ref{eq:MVSelectOpt}). Note that any value of $n$ corresponds to a unique subset $U$. \begin{equation}\label{eq:MVSelectOpt} \newcommand{\argmax}{\operatornamewithlimits{arg\ max}} -n^* = \argmax_{n\leq s} \left\{ \sum_{i=1}^n A_{D}(i) - \alpha \frac{t_C-t_I}{T_{IDR}} R_U \right\}, +U_n^* = \argmax_{n\leq s} \left\{ \sum_{i=1}^n A_{D}(i) - \alpha \frac{t_C-t_I}{T_{I}} R_U \right\}, \end{equation} where $R_U$ is the amount of bits required to encode the $n$ \acs{mv} of $U$ and $A_{D}(i)$ is the total area of the \acs{pu} whose \acs{mv} are dependently encoded from $\ac{mv}_i$: \begin{equation} A_{D}(i) = \sum_{j=1}^{DV_i} ( w_j\times h_j ),\hspace{2mm} i = 1, 2 \cdots s, \end{equation} where $DV_i$ is the number of \acs{mv} dependent from $MV_i$ and $w_j$ and $h_j$ are the width and height of the corresponding \ac{pu}. -The weighting factor used to control the amount of redundancy $R_U$ is used to increase the cost of the overhead rate as the duration of error propagation decreases. -It is given by the ratio between the distance from the current frame ($t_c$) to the next \ac{idr} ($t_I$) and the \ac{idr} period ($T_{IDR}$), combined with the $\alpha$ parameter. Therefore, for higher values of $\alpha$, less redundant \acs{mv} are selected. - -As pointed out before, this streaming stage may be implemented at any point of the network, to increase robustness without fully decoding the stream. Note that the parsing operation represented in Figure~\ref{fig:proposedMethod} does not decode the video frames, introducing only one frame delay because the \ac{mv} selection is performed on a frame-by-frame basis. +A weighting factor is used to control the amount of bitrate overhead ($R_U$), by increasing the cost of the number of encoded bits based on the duration of the error propagation. +It is given by the ratio between the distance from the current frame ($t_c$) to the next refresh frame ($t_I$) and the \ac{idr} period ($T_{I}$), combined with the control parameter $\alpha$. +This weight varies in the range $\left[0;1 \right]$, achieving the lowest value for intra-coded frames and the maximum for the frame located immediately before an intra-coded one. +In the proposed approach, for higher values of $\alpha$, less redundant \acs{mv} are selected. +As pointed out before, this streaming stage operates on coded data and may be implemented at any point of the network, to increase robustness without fully decoding the stream. Note that the parsing operation represented in Figure~\ref{fig:proposedMethod} does not include decoding of the video frames. Furthermore, only one frame delay is introduced because the \ac{mv} selection process is performed on a frame-by-frame basis. To maintain compatibility with the \ac{hevc} standard, the redundant motion information is transmitted using the \ac{sei} \ac{nal} units \cite{HEVCHighLevelReferenceDez2012}. Thus, this information can be multiplexed into the coded bitstream without affecting the compressed video. -The subset of selected \acs{mv} is independently encoded, ensuring that no reference information is needed in the decoder to properly decode the redundant \acs{mv}. In this work arithmetic coding~\cite{ArithmeticCodingJun1994} was used for the redundant \acs{mv}, but any other entropy coding method can be used. +The subset of selected \acs{mv} is independently encoded, ensuring that no reference information is needed in the decoder to properly decode the redundant \acs{mv}. In this work arithmetic coding~\cite{ArithmeticCodingJun1994} was used for the side-information, but any other entropy coding method can be used. \section{Performance evaluation} \label{Section4:Performance} -In this section the robustness of the proposed method is evaluated and discussed for two cases: (i) only using the encoding stage and (ii) using the streaming stage in addition to the encoding stage. In the former case, identified as ``\PROP'', only the dynamic reference picture selection method described in Section~\ref{Section4:RefPicSel} is evaluated. In the latter case, identified as ``\DYN'', besides the previous method of reference picture selection, redundant \acs{mv} are encoded as described in Section~\ref{Section4:MVSelection}, resulting in a combination of both. +In this section the robustness of the proposed two-stage architecture is evaluated and discussed for two cases: (i) only using the encoding stage and (ii) using the streaming stage in addition to the encoding stage. In the former case, identified as ``\PROP'', only the dynamic reference picture selection method described in Section~\ref{Section4:RefPicSel} is evaluated. In the latter case, identified as ``\DYN'', besides the previous method of reference picture selection, redundant \acs{mv} are encoded as described in Section~\ref{Section4:MVSelection}, resulting in a combination of both stages. -As references for comparison, three other methods are used: (i) a fixed checkerboard approach (\CHKB), (ii) the reference picture generation method proposed in \cite{ROPERefGenJan2010}, identified as ``\AVG'', and (iii) long-term reference frames (\LONG). The \LONG method uses only the key frames as reference, i.e., in the Random-Access configuration only one out of eight frames is used as reference frame, allowing a more robust transmission, as the reference frames are less likely to be hit by errors. +As references for comparison, three other methods are used: (i) a fixed checkerboard approach (\CHKB), (ii) the reference picture generation method based on leaky predictions proposed in \cite{ROPERefGenJan2010}, identified as ``\AVG'', and (iii) long-term reference frames (\LONG). The \LONG method only uses the key frames as reference, i.e., in the \ac{ra} configuration only one out of eight frames is used as reference for prediction, allowing a more robust transmission, as the reference frames are less likely to be hit by errors. + +The experimental setup used in this section is the same as the one previously described in Section~\ref{Section3:ErrorResilienceEvaluation}. +Two test cases were considered for simulation: (i) in \emph{\LOSTALL} all frames may be equally affected by packet loss, including \ac{idr} frames; (ii) in \emph{\KEEPINTRA} \ac{idr} frames are assumed to be prioritised in the network and delivered error-free, or using some combined approach as recently proposed in \cite{ErrorResilienceHeterogenousNetworks2016}. +At the decoder side, error concealment is based on motion compensation from previously decoded frames. The motion field of a missing frame is obtained applying \ac{mv} extrapolation from the two closest neighbours. Afterwards, the final \acs{mv} are selected based on the residual energy of their original prediction~\cite{HEVCMCResidueDec2013}. +The evaluation of the encoding-stage is carried out separately from the streaming-stage. + + + +%This section presents an evaluation study about the error robustness of HEVC using different sequences and coding configurations. Table~\ref{tab:sequences} presents a summary of the main characteristics of the video sequences used in this work. Each sequence comprises 240 frames and the whole set covers different types of motion and texture complexity. This table also shows the spatial information (SI) and the temporal information (TI), as defined in \cite{ITUTP9111998}. The experimental results for this study were obtained by using the HM reference software, version $16.2$. +%The sequences were encoded with all coding modes enabled and following two recommended prediction structures: Low-Delay (P-frames with 4 references frames) and \ac{ra} (B-frames with 2 reference frames in each list)~\cite{HEVCCommonTests2012}. These prediction structures cover the use of P-frames and hierarchical B-frames~\cite{H264ScalableOverviewSep2007}, which are commonly used in HEVC streaming~\cite{RealTimeHEVC2016}. +%To simulate a broadcast transmission an \ac{idr} period of $32$ frames was used and the filtering operations were disabled at slices boundaries in order to keep the slices self-contained. The remaining configurations have been kept at their default values as given in the common test conditions~\cite{HEVCCommonTests2012}. +%In order to achieve high quality images for all coded sequences (i.e., PSNR around 40 dB) different bitrates were used. +%At the decoder side, error concealment is based on motion compensation from previously decoded frames. The motion field of a missing frame is obtained applying MV extrapolation from the two closest neighbours. Afterwards, the final MVs are selected based on the residual energy of their original prediction~\cite{HEVCMCResidueDec2013}. +%To simulate the lossy transmission environment, each frame is encoded into several slices, each one packetized into one NAL unit, which are then transmitted as the payload of independent packets. +% +%In our experiments a whole packet is lost whenever an error occurs, originating a lost slice. Random packet loss was simulated using a two-state Markov model with the following average burst length 1.24, 1.47, 1.83 and 2.05, and a maximum burst length of 2, 3, 5 and 7, for packet loss ratios (PLR) of 1\%, 3\%, 5\%, 10\%, respectively. -The experimental setup used in this section is the same as the one previously described in Section~\ref{Section3:ErrorResilienceEvaluation}. The evaluation of the encoding-stage is carried out separately from the streaming-stage. \subsection{Encoding stage} -The performance of the encoding stage is evaluated on its own in order to consider only the improvements achieved by the dynamic reference frame selection method, when reference frames were reconstructed with distortion (e.g. as a result of non-perfect error concealment). -Since the encoding stage does not deal with errors in temporal \ac{mv} predictions (this is handled by the streaming-stage of the proposed method), in this evaluation the temporal \ac{mv} predictions were not used. This is necessary to avoid masking the results with the errors caused by wrong \ac{mv} predictions. - -The performance of the proposed encoding stage (\PROP) was evaluated using $\gamma = 5$. This allows to increase the Lagrangian cost by about $50\%$, when a reference frame is used for three or more \acs{ctu} in comparison to the usage of the other ones (see Figure~\ref{fig:weightfct}). Further results for a wide range of $\gamma$ values are also presented. - -\subsubsection{Performance under error-free conditions} - -Table~\ref{tab:bd_error_free} shows the Bjontegaard's Delta PSNR (BD-PSNR) values compared to the reference \ac{hevc} encoder for different coding configurations in error-free decoding. Four QPs were used from the common test conditions: 22, 27, 32, 37. -The aim is to evaluate the \ac{rd} penalty incurred by the proposed method in comparison with \CHKB, as well as the other reference methods. -The use of long-term reference pictures (\LONG) is only tested for the case of Random-Access, where key frames are available, while method \cite{ROPERefGenJan2010} is tested for the Low-Delay configuration. -The results in Table~\ref{tab:bd_error_free} indicate a small loss of coding efficiency when the proposed reference picture selection approach is used under error-free conditions. As expected, this is due to the sub-optimal \ac{rd} optimisation of the proposed method, because the cost function also includes the cost associated to the choice of the reference frame, in addition to the rate and distortion. - -\begin{table}[t] -\centering -\newcommand{\resultswidth}{1.5cm} -\caption{BD-PSNR in error-free conditions.} -\TABLESVSPACE -\footnotesize -\label{tab:bd_error_free} -\begin{tabular}{p{3cm}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}} -\toprule -\multirow{2}{*}{\centering\textbf{Sequence}} & \multicolumn{3}{c|}{\textbf{Low-Delay}} & \multicolumn{3}{c}{\textbf{Random-Access}} \\ -& \AVG & \CHKB & \PROP & \LONG & \CHKB & \PROP \\ \hline -Basketball Drill & -1.14 & -1.15 & -0.51 & -0.85 & -0.29 & -0.22 \\ \hline -Kendo & -2.10 & -1.39 & -0.99 & -0.91 & -0.39 & -0.28 \\ \hline -Park Scene & -1.79 & -1.38 & -0.81 & -0.68 & -0.38 & -0.28 \\ \hline -Traffic & -1.91 & -1.21 & -0.71 & -0.55 & -0.36 & -0.21 \\ \hline -Bosphorus & -0.97 & -0.95 & -1.10 & -0.64 & -0.20 & -0.20 \\ \hline -\textbf{Average} & \textbf{-1.58} & \textbf{-1.22} & \textbf{-0.82} & \textbf{-0.73} & \textbf{-0.32} & \textbf{-0.24} \\ -\bottomrule -\end{tabular} -\end{table} - -While the fixed checkerboard approach leads to an average quality reduction of $1.22$~dB for the Low-Delay ($0.32$~dB for Random-Access), the proposed method only loses $0.82$~dB and $0.24$~dB in the Low-Delay and Random-Access configurations, respectively. In comparison to \LONG and the method in~\cite{ROPERefGenJan2010}, the proposed method is still more efficient, consistently achieving better results. -Overall, the loss in the quality compared to the reference \ac{hevc} is considered acceptable, given the increase in robustness obtained in lossy transmission, as discussed in the following sections. - -%\subsubsection{Reference picture selection results} +The performance of the encoding stage is evaluated on its own in order to only determine the improvements achieved by the dynamic reference frame selection method. +In this case, the reference frames were reconstructed with distortion, as a result of non-perfect error concealment. +Since the encoding stage does not deal with errors in temporal \ac{mv} predictions (this is handled by the streaming-stage), in this evaluation the temporal \ac{mv} predictor is disabled. +This is necessary to avoid masking the results with the errors caused by wrong \ac{mv} predictions. +The performance of the encoding stage (\PROP) was evaluated using $\gamma = 5$. +This value was chosen to allow increasing the Lagrangian cost by about $50\%$, when a reference frame is used for prediction of more than three \acs{ctu} in comparison to the usage of the other reference frames (see Figure~\ref{fig:weightfct}). Nevertheless, further results for a wide range of $\gamma$ values are also presented. +%\HERE \subsubsection{Quality evaluation under error-prone conditions} -The robustness achieved by the proposed method was evaluated by comparison of the error propagation resulting from a single loss event in the Low-Delay configuration. - -Figure~\ref{fig:RecoveredExample} shows the visual impact of error propagation on a region of the decoded Frame \#10 when Frame \#9 is affected by errors, for Basketball Drill and Race Horses sequences (the PSNR corresponds to the entire frame). As shown in this figure, the proposed method is able to reduce the impact of mismatch decoding in comparison with the method~in~\cite{ROPERefGenJan2010}, revealing the higher efficiency achieving by distributing the use of the references frames. - \begin{figure}[tb] \centering \footnotesize \newcommand{\CurrFigSize}{0.23\columnwidth} - \begin{tabular}{y{.23\columnwidth}y{.23\columnwidth}|y{.23\columnwidth}y{.23\columnwidth}} - \multicolumn{2}{c}{Basketball Drill} & \multicolumn{2}{|c}{Race Horses} \\ - \AVG & \PROP & \AVG & \PROP \\ - \includegraphics[width=\CurrFigSize]{Figure_6_1} & - \includegraphics[width=\CurrFigSize]{Figure_6_2} & - \includegraphics[width=\CurrFigSize]{Figure_6_3} & - \includegraphics[width=\CurrFigSize]{Figure_6_4} \\ - PSNR = 31.93 dB & PSNR = 33.83 dB & PSNR = 24.12 dB & PSNR = 31.93 dB \\ - \end{tabular} + \resizebox{\textwidth}{!}{ + \begin{tabular}{y{.23\columnwidth}y{.23\columnwidth}|y{.23\columnwidth}y{.23\columnwidth}} + \multicolumn{2}{c|}{Basketball Drill} & \multicolumn{2}{c}{Race Horses} \\ + \AVG & \PROP & \AVG & \PROP \\ + \includegraphics[width=\CurrFigSize]{Figure_6_1} & + \includegraphics[width=\CurrFigSize]{Figure_6_2} & + \includegraphics[width=\CurrFigSize]{Figure_6_3} & + \includegraphics[width=\CurrFigSize]{Figure_6_4} \\ + PSNR = 31.93 dB & PSNR = 33.83 dB & PSNR = 24.12 dB & PSNR = 31.93 dB \\ + \end{tabular} + } \caption{Zoom of recovered Frame \#10 after a loss event at Frame \#9 for Basketball Drill and Race Horses sequence.} \label{fig:RecoveredExample} \end{figure} +The error robustness achieved by the encoding-stage was firstly evaluated in regards to the error propagation resulting from a single loss event in the \ac{ld} configuration. +Figure~\ref{fig:RecoveredExample} shows the visual impact of error propagation on a region of the decoded Frame \#10 when Frame \#9 is affected by errors, for Basketball Drill and Race Horses sequences (the \ac{psnr} corresponds to the entire frame). +The proposed method is compared with the method~in~\cite{ROPERefGenJan2010} as both aim at reducing the error propagation. +The method~in~\cite{ROPERefGenJan2010} uses filtered frames as reference for predictions, therefore, in case of errors, the effects of mismatch predictions are significantly reduced by the filtering operation (see Sub-Section~\ref{SECTION2:LeakyPred}). +As shown in this figure, the proposed method is able to reduce the impact of mismatch decoding in comparison with the method~in~\cite{ROPERefGenJan2010}. Although both methods still reveal some predictions mismatch leading to inaccurate reconstructed of the objects contours, the proposed method is able to achieve a smoother reconstruction, which leads to higher quality in terms of \ac{psnr}. This is because, in frame \#10, the predictions depending on the missing frame \#9 are not used throughout the entire frame, thus, approximately half of the predictions are not affected by errors. In contrast, the method~in~\cite{ROPERefGenJan2010} requires several filtering operations to be applied (i.e, different reference frames need to be filtered) in order to reduce the effect of mismatch predictions. +Summarising, the reconstructed frames illustrated in Figure~\ref{fig:RecoveredExample} reveal the higher efficiency achieved by distributing the use of the references frames. + + \begin{figure}[t] \renewcommand{\FIGSIZE}{0.8\columnwidth} \centering @@ -308,26 +329,29 @@ Figure~\ref{fig:RecoveredExample} shows the visual impact of error propagation o \end{figure} -\input{\ImprovedErrorResilience/Results/TableResultsRandomLossRefPicSelection.tex} - -Figure~\ref{fig:PSNRvsFrameForRefPicSel}~shows the PSNR over a GOP with an \ac{idr} period of 32 frames, encoded at the same constant bitrate. +Figure~\ref{fig:PSNRvsFrameForRefPicSel}~shows the \ac{psnr} over a \ac{gop} with an \ac{idr} period of 32 frames, using the same constant bitrate for all streams. This figure shows that all methods are able to outperform the reference \ac{hevc} (\REF), gaining approximately $3$~dB in reconstruction quality. -Since the proposed method is more efficient than the others for error-free transmission (see~Table~\ref{tab:bd_error_free}), and it also provides better error robustness, it can be considered as an efficient solution to better deal with transmission errors. -The similar quality levels achieved by \CHKB and \PROP reveal that dynamic selection of the reference frames is more robust because a better trade-off between coding efficiency and error resilience is attained. +The similar quality levels achieved by \CHKB and \PROP reveal that the dynamic selection of the reference frames used in \PROP is more robust because a better trade-off between coding efficiency and error resilience is attained. +Moreover, results in Figure~\ref{fig:PSNRvsFrameForRefPicSel} are in line with the reconstruction quality shown in Figure~\ref{fig:RecoveredExample}, confirming that the proposed method is able to outperform the reference frame generation technique in~\cite{ROPERefGenJan2010}. +%Since the proposed method is more efficient than the others for error-free transmission (see~Table~\ref{tab:bd_error_free}), and it also provides better error robustness, it can be considered as an efficient solution to better deal with transmission errors. Overall, the method proposed for the encoding stage is able to increase the error robustness of \ac{hevc} without severely compromising the coding efficiency. -Further tests were run to evaluate the effectiveness of the proposed method under various packet loss rates. -For each test condition, 50 trials were performed and the average quality obtained across all trials for both coding configurations are shown in Table~\ref{tab:AverageQualityCHKBEXPLowDelay}~and~\ref{tab:AverageQualityCHKBEXPRandomAccess}, for the Low-Delay and Random-Access, respectively. -In both tables the absolute PSNR value is shown for the reference \ac{hevc} case (\REF), while the PSNR difference is presented for the other cases. -Results show that the proposed method is able to outperform other methods, by increasing the robustness of \ac{hevc}. An average gain up to $1.47$~dB is obtained for the Low-Delay configuration (see Table~\ref{tab:AverageQualityCHKBEXPLowDelay}), in comparison to the \REF, while the fixed checkerboard pattern approach (\CHKB) only achieves up to $1.33$~dB on average. -In \cite{ROPERefGenJan2010}, the use of interpolated references in the encoding loop reveals to reduce error propagation, at a cost of high degradation of the coding efficiency in the error-free environment, thus the overall performance is lower (see results for ``\AVG''). +\input{./041_ImprovedErrorResilience/Results/TableResultsRandomLossRefPicSelection.tex} +Further tests were run to evaluate the effectiveness of the proposed method under various packet loss rates for both test cases, i.e., with and without \ac{idr} frame loss, \LOSTALL and \KEEPINTRA, respectively. +The use of long-term reference pictures (\LONG) is only tested for the \ac{ra} configuration, where key frames are available, while method \cite{ROPERefGenJan2010} is tested for the \ac{ld} configuration. +For each test condition, 50 trials were performed and the average quality (\ac{psnr}) obtained across all trials for both coding configurations are shown in Table~\ref{tab:AverageQualityCHKBEXPLowDelay}~and~\ref{tab:AverageQualityCHKBEXPRandomAccess}, for the \ac{ld} and \ac{ra}, respectively. +In both tables the absolute \ac{psnr} is shown for the reference \ac{hevc} case (\REF), while the \ac{psnr} difference is presented for the other cases. +Results show that the proposed method is able to outperform other methods for the same loss conditions. +An average quality gain up to $2.26$~dB is obtained for the \ac{ld} configuration (see Table~\ref{tab:AverageQualityCHKBEXPLowDelay}), in comparison to the \REF, while the fixed checkerboard pattern approach (\CHKB) only achieves up to $2.15$~dB. +In \cite{ROPERefGenJan2010}, the use of interpolated references in the encoding loop helps to reduce error propagation, at a cost of poor predictions, thus degrading the coding efficiency. This is the main reason that justifies a lower overall performance of the method proposed in \cite{ROPERefGenJan2010} (see results for ``\AVG''). +The proposed method is also able to outperform the \ac{hevc} reference software when an hierarchical coding structure is used (see results for \ac{ra} configuration in Table~\ref{tab:AverageQualityCHKBEXPRandomAccess}), which implicitly limits the error propagation. +In comparison to the reference \ac{hevc}, quality improvements up to $1.42$~dB at PLR=$10\%$ are obtained for Kendo sequence.%, while the average gain across all test sequences is 1~dB. + +When comparing results for both test cases, it is noticeable that the proposed method is able to outperform the reference methods both when \ac{idr} frame loss is allowed or not (\LOSTALL and \KEEPINTRA, respectively). As the \ac{plr} increases, the gains of the proposed method decrease in comparison with the case where the \ac{idr} frames are delivered error-free (i.e., case \KEEPINTRA), due to the higher error propagation of the missing \ac{idr} frame. Moreover, the quality gains are lower in the \LOSTALL case when comparing with \KEEPINTRA for the same \ac{plr}, for both the \ac{ld} and \ac{ra}. +Taking into account the above results and the spatio-temporal information shown in Table~\ref{TableTestSequences}, one can observe that the performance of the proposed method is lower for sequences with high spatial details and low temporal activity (e.g., BQSquare and Park Scene), which indicates higher effectiveness for video content with higher motion activity. +%\HIGHLIGHT{complete with analysis of both cases} -The proposed method is also able to outperform the \ac{hevc} standard when an hierarchical coding structure is used (see results for Random-Access configuration in Table~\ref{tab:AverageQualityCHKBEXPRandomAccess}), which inherently limits the error propagation. -In comparison to the reference \ac{hevc}, quality improvements up to $1.42$~dB at PLR=$10\%$ are obtained (Kendo), while the average gain across all test sequences is 1~dB. -When comparing results for both cases, it is noticeable that the proposed method is able to outperform the reference methods both when \ac{idr} frame loss is allowed or not (case 1 and 2, respectively). -As the \ac{plr} increases the gains of the proposed method decreases when comparing with the case where the \ac{idr} frames delivered error-free (i.e., case 1), due to the higher error propagation of the missing \ac{idr} frame. -Taking into account the above results and the spatio-temporal information shown in Table~\ref{TableTestSequences}, one can observe that the performance of the proposed method is lower for sequences with high spatial details and low temporal activity (see results for BQSquare and Park Scene), which indicates higher effectiveness for video sequences with higher motion activity. \begin{figure}[!b] \renewcommand{\FIGSIZE}{0.8\columnwidth} @@ -335,7 +359,7 @@ Taking into account the above results and the spatio-temporal information shown %\includegraphics[width=\FIGSIZE,page=1]{\ImprovedErrorResilience/Results/PSNRversusGama} %\includegraphics[width=\FIGSIZE,page=2]{\ImprovedErrorResilience/Results/PSNRversusGama} \input{./041_ImprovedErrorResilience/Results/PSNRversusGama.tex} - \caption{Influence of the $\gamma$ parameter in the average quality gains, comparing with \REF, for different packet loss ratios.} + \caption{Influence of the $\gamma$ parameter in the average quality gains, comparing with \REF, for different \acs{plr}.} \label{fig:QualityVsGamma} \end{figure} @@ -343,20 +367,66 @@ Taking into account the above results and the spatio-temporal information shown -The influence of the $\gamma$ parameter used in Equation (\ref{eq:KFactor}) was also studied. -Figure \ref{fig:QualityVsGamma} shows the average quality gains in comparison to the reference \ac{hevc} (\REF) across all test sequences, when using different values of $\gamma$ in the proposed method (\PROP). -For the error-free case, the results in this figure show that increasing $\gamma$ leads to a decrease in the coding efficiency, especially for the Low-Delay configuration. The reason is that higher values of $\gamma$ lead to higher values of $W(r_{\phi})$, therefore the reference picture selection is more constrained by the exponential weights. Since $T_{ID}$ is constant for all frames in the Low-Delay, higher weights are used for all frames ($T_{ID}$ is used to reduce the weights in case of hierarchical coding) and the encoder is not able to select the best reference frame in terms of R-D optimisation. -In the case where packet loss occurs, higher values of $\gamma$ lead to higher error resilience. Since higher weights used in the \ac{rd} cost, the encoder is constrained to distribute the use of the reference frames, reducing the error propagation. +The influence of the $\gamma$ parameter used in Equation (\ref{eq:KFactor}) on the video quality was also evaluated, in order to check how the cost constrains affect the performance of the proposed method. +Figure \ref{fig:QualityVsGamma} shows the average quality gains of the proposed method in comparison to the reference \ac{hevc} (\REF) across all test sequences, for different values of $\gamma$. +For the error-free case (PLR=0\%), the results in this figure show that increasing $\gamma$ leads to a decrease in the coding efficiency, especially in the \ac{ld} configuration. +The reason is that higher values of $\gamma$ lead to higher values of $W(r_{\phi})$, therefore the reference picture selection cost is severely constrained by the exponential weights. +Since $T_{ID}$ is constant for all frames in the \ac{ld}, higher weights are used for all frames ($T_{ID}$ is used to reduce the weights in case of hierarchical coding) and the encoder is not able to select the best reference frame in \c{rd} sense. +In the case where packet loss occurs, higher values of $\gamma$ lead to improved error resilience. +Since higher weights are used in the \ac{rd} cost, the encoder is constrained to distribute the use of the reference frames of different \acs{pu}, reducing the error propagation. For very high values (e.g., $\gamma=1000$) the encoder is forced to uniformly select the reference frames, resulting in a fixed checkerboard pattern. -Overall, using the $\gamma$ parameter in the proposed method, the impact of the reference frame, selected by the \ac{rd} optimisation, can be controlled, and thus a better trade-off between higher error resilience and loss of coding efficiency can be achieved. +Overall, the $\gamma$ parameter is able to control the reference frame selection constrains, +%, which are selected by a modified \ac{rd} optimisation, , +thus achieving better trade-off between improved error resilience and loss of coding efficiency. + + +\subsubsection{BD-PSNR under error-free conditions} + +This sub-section evaluates the \ac{rd} penalty incurred by the proposed method in comparison with \CHKB, as well as the other reference methods. +Table~\ref{tab:bd_error_free} shows the Bjontegaard's Delta PSNR (BD-PSNR) values compared to the reference \ac{hevc} encoder for different coding configurations in error-free decoding. Four \acs{qp} were used from the common test conditions: 22, 27, 32, 37. + +\begin{table}[tb] + \centering + \newcommand{\resultswidth}{1.5cm} + \caption{BD-PSNR (dB) in error-free conditions.} + \TABLESVSPACE + \footnotesize + \label{tab:bd_error_free} + \begin{tabular}{p{3cm}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}} + \toprule + \multirow{2}{*}{\centering\textbf{Sequence}} & \multicolumn{3}{c|}{\textbf{Low-Delay}} & \multicolumn{3}{c}{\textbf{Random-Access}} \\ + & \AVG & \CHKB & \PROP & \LONG & \CHKB & \PROP \\ \hline + Basketball Drill & -1.14 & -1.15 & -0.51 & -0.85 & -0.29 & -0.22 \\ \hline + Kendo & -2.10 & -1.39 & -0.99 & -0.91 & -0.39 & -0.28 \\ \hline + Park Scene & -1.79 & -1.38 & -0.81 & -0.68 & -0.38 & -0.28 \\ \hline + Traffic & -1.91 & -1.21 & -0.71 & -0.55 & -0.36 & -0.21 \\ \hline + Bosphorus & -0.97 & -0.95 & -1.10 & -0.64 & -0.20 & -0.20 \\ \hline + \textbf{Average} & \textbf{-1.58} & \textbf{-1.22} & \textbf{-0.82} & \textbf{-0.73} & \textbf{-0.32} & \textbf{-0.24} \\ + \bottomrule + \end{tabular} +\end{table} + +The results in Table~\ref{tab:bd_error_free} indicate a small loss of coding efficiency of the proposed method, as well as, the reference methods tested. +The fixed checkerboard approach leads to an average quality reduction of $1.22$~dB for the \ac{ld} ($0.32$~dB for \ac{ra}), while the proposed method only loses $0.82$~dB and $0.24$~dB in the \ac{ld} and \ac{ra} configurations, respectively. This results clearly shows that dynamically selecting the reference frames is more efficient that using a fixed pattern, confirming the previous results for error-prone conditions. +Moreover, in comparison to \LONG and the method in~\cite{ROPERefGenJan2010}, the proposed method is still more efficient, consistently achieving better results. Comparing, the results for both error-free and error-prone conditions, one may conclude that the proposed method not only achieves higher quality gains, but also does reduces the impact on coding efficiency. +Overall, the loss in the quality compared to the reference \ac{hevc} is considered acceptable, given the increase in robustness obtained in lossy transmission demonstrated in the previous sub-sections. + +%As expected, this is due to the sub-optimal \ac{rd} optimisation of the proposed method, because the cost function also includes the cost associated to the choice of the reference frame, in addition to the rate and distortion. + \subsubsection{Complexity overhead} -The implementation of the proposed method has slightly increased the complexity of the standard non-robust encoder. Therefore, the computational complexity has also been considered as a performance metric in this research, which is measured as the encoding time in a controlled hardware platform. The reference used for comparison with other methods is the \ac{hevc} reference software encoding time, running on the same platform with the same coding configuration. -The relative complexity increase of the robust encoder using \PROP, \CHKB and the method presented in \cite{ROPERefGenJan2010} is shown in Table~\ref{tab:cpu_time}. +The implementation of this method slightly increases the complexity of a standard non-robust encoder. Therefore, the computational complexity has also been considered as a performance metric in this research, which is measured as the encoding time in a controlled hardware platform. The reference used for comparison with other methods is the \ac{hevc} reference software encoding time, running on the same platform with the same coding configuration. +The relative complexity increase of the robust encoder using \PROP, \CHKB and the method presented in \cite{ROPERefGenJan2010} is shown in Table~\ref{tab:cpu_time}. -\begin{table}[b] + +As shown in this table, the method presented in \cite{ROPERefGenJan2010} is the most time consuming because it requires post-processing of the encoded frames in order to interpolate new references. +The proposed scheme, using a reference picture selection, presents a lower complexity increase than \cite{ROPERefGenJan2010}, while the fixed checkerboard scheme (\CHKB) presents a slightly lower complexity \PROP method. +This is due to the fact that in the \CHKB method there is no \ac{rd} optimisation process for the reference frame selection (fixed pattern is used), thus reducing the overall computational complexity. + + +\begin{table}[tb] \newcommand{\resultswidth}{2.2cm} \centering \caption{Relative CPU times of tested methods.}\TABLESVSPACE @@ -374,35 +444,20 @@ The relative complexity increase of the robust encoder using \PROP, \CHKB and th \end{tabular} \end{table} - -As shown in this table, the method presented in \cite{ROPERefGenJan2010} is the most time consuming because it requires post-processing of the encoded frames in order to interpolate new references. -The proposed scheme, using a reference picture selection, presents a lower complexity increase than \cite{ROPERefGenJan2010}, while the fixed checkerboard scheme (\CHKB) presents a slightly lower complexity \PROP method. - \subsection{Streaming stage} +In this section the coding performance that results from transmitting redundant \ac{mv} (streaming stage) in combination with the reference picture selection scheme (encoding stage) is presented. +In these experimental tests the temporal \ac{mv} candidates are enabled and \acs{mv} are dynamically added as side-information into the stream, as described in Section \ref{Section4:MVSelection}. +Different values of $\alpha$ are used to achieve different overhead ratios. All streams were encoded with the same total bitrate, including the redundant bits, in order to make a fair quality comparison. - -In this section the coding performance that results from using redundant \ac{mv} (streaming stage) in combination with the reference picture selection scheme (encoding stage) is presented. In the experimental tests the temporal \ac{mv} candidates are enabled and redundant \acs{mv} are added into the stream, as described in Section \ref{Section4:MVSelection}. This is referred to as ``\DYN'', and compared to cases where a fixed amount of redundant \acs{mv} per frame are transmitted (i.e., the most important 10\%, 30\% and 50\% $\ac{mv} \in V$). Different $\alpha$ values are used for different redundancy ratios. All streams were encoded with the same total bitrate, including the redundant bits, in order to make a fair quality comparison. - - - - -% \begin{figure}[tbp] -% \centering -% \newcommand{\CurrFigSize}{0.45\columnwidth} -% -% \caption{Decoded video quality for different amounts of redundancy, selected using a fixed approach and the proposed method (Continuation).} -% \label{fig:QualityVsRedundancy\ac{mv}SelectionContinuation} -% \end{figure} - \subsubsection{Dynamic MV selection performance} \begin{figure}[!b] \centering \newcommand{\CurrFigSize}{0.49\columnwidth} -% \includegraphics[width=\CurrFigSize,page=1]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} -% \includegraphics[width=\CurrFigSize,page=2]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} + \includegraphics[width=\CurrFigSize,page=1]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} + \includegraphics[width=\CurrFigSize,page=2]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} \includegraphics[width=\CurrFigSize,page=3]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} \includegraphics[width=\CurrFigSize,page=4]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} \includegraphics[width=\CurrFigSize,page=5]{\ImprovedErrorResilience/Results/PSNRvsRedundancy.pdf} @@ -411,15 +466,30 @@ In this section the coding performance that results from using redundant \ac{mv} \label{fig:QualityVsRedundancyMVSelection} \end{figure} +Firstly the quality obtained by the proposed method, which dynamically selecting the amount of redundant \acs{mv}, is compared to cases where a fixed amount of \acs{mv} per frame are transmitted (i.e., the most important 10\%, 30\% and 50\% $\ac{mv} \in V$). This cased are referred to as "\DYN" and "\FIXED", respectively. + +%In the case where variable redundancy is allowed, \acs{mv} are selected using the proposed dynamic method (Dyn) and a fixed approach (Fixed). +Figure~\ref{fig:QualityVsRedundancyMVSelection} illustrates the relation between the quality obtained (\ac{psnr}) and the amount of redundancy used. +The results indicate that for approximately the same amount of redundant bits are used in both cases, however by using the proposed method to dynamically select the \acs{mv} to be used as side-information (\DYN) is able to outperform the fixed case for all test sequences. +Results also show that for both approaches the decoded video quality increases as the redundancy of the side-information increases, since a higher number of mismatch \ac{mv} predictions are able to be recovered. + +It is noticeable in Figure~\ref{fig:QualityVsRedundancyMVSelection} that for higher \acs{plr}, the proposed method is able to achieve higher gains. This reveals that for increasing data loss it is more important optimise the amount of \acs{mv} selected per frame in order to use more \acs{mv} in those frames which have a higher impact of the error propagation. +For instance, the proposed method is able to gain up to $1.5$~dB for the BQSquare sequence for a PLR=10\$. +For lower \acs{plr}, since fewer frames are affected by data losses, the use of a fixed approach achieves similar performance as the proposed method (\DYN). -In the case where variable redundancy is allowed, \acs{mv} are selected using the proposed dynamic method (Dyn) and a fixed approach (Fixed). -Figure~\ref{fig:QualityVsRedundancyMVSelection} illustrates the relation between the quality obtained and the amount of redundancy. -The results indicate that for approximately the same amount of redundant bits, dynamic selection of \acs{mv} lead to higher performance in all test sequences. For instance, the proposed method is able to gain up to $1.5$~dB for the BQSquare sequence. Moreover, the results are consistent for different \ac{plr}. \subsubsection{Quality evaluation} +Table~\ref{tab:AverageQualityRedundancies} presents the average \ac{psnr} of decoded video for different \ac{plr} (columns~4~to~8) and for various percentages of \ac{mv} redundancy (third column). The absolute quality values are shown for the reference case (\NOREDUND) and \ac{psnr} gains are shown for thee different levels of redundancy introduced by the proposed method (\DYN). +This study also conducted for the two different cases, with and without \ac{idr} loss. + +It can be confirmed from the results of Table~\ref{tab:AverageQualityRedundancies} that, for each sequence, as $\alpha$ decreases more redundant \acs{mv} are used leading to higher overhead rations. This results in higher objective video quality for all tested sequences. +This confirms that redundant \acs{mv} are able to minimise the error propagation, which is accomplished at a cost of an acceptable increase in the bitrate, since only a sub-set of \acs{mv} is transmitted. +The use of redundant \acs{mv} yields better video quality for \ac{plr} as low as $1\%$, achieving higher quality gains when \ac{idr} frames are not lost (\KEEPINTRA). This is because of the higher quality degradation when \ac{idr} frames are lost, which is not due to mismatched \ac{mv} predictions and thus cannot be reduced by transmitting extra \acs{mv} as side-information. +The use the proposed method achieves higher quality for sequences with higher motion activity and which makes use of inter-prediction more often, i.e., BQSquare and Traffic sequences (see inter ratio in Table~\ref{TableTestSequences}), leading to a maximum gain of $3.93$~dB for \LOSTALL ($7.82$~dB for \KEEPINTRA). + \begin{table}[!t] \centering \caption{Average quality using redundant \ac{mv} selected by the proposed approach.} @@ -429,23 +499,17 @@ The results indicate that for approximately the same amount of redundant bits, d \input{\ImprovedErrorResilience/Results/TableResultsRandomLossMvRedundancies.tex} \end{table} - -Table~\ref{tab:AverageQualityRedundancies} presents the average PSNR of decoded video for different \ac{plr} (columns~4~to~8) and for various percentages of \ac{mv} redundancy (third column). -It can be confirmed from the results that, for each sequence, as $\alpha$ decreases more redundant \acs{mv} are used, resulting in higher objective image quality for all tested sequences. -This indicates that redundant \acs{mv} are able to minimise the error propagation. This is accomplished with an acceptable increase in the bitrate, since only a sub-set of \acs{mv} is transmitted. -The use of redundant \acs{mv} reveals better results for \ac{plr} as low as $1\%$, achieving a maximum gain of $3.93$~dB for \LOSTALL ($7.82$~dB for \KEEPINTRA) for the BQSquare sequence at PLR=$5\%$. - -In summary, the proposed method is able to select the most relevant motion \ac{mv} information and when combined with a reference picture selection scheme the problem of mismatch \ac{mv} predictions is mitigated. -Moreover, the dynamic approach devised to select the best \acs{mv} also contribute to increase the overall performance. +In summary, the proposed method is able to select the most relevant motion \ac{mv} information and when combined with a reference picture selection scheme the problem of mismatch \ac{mv} predictions is mitigated. +Moreover, the dynamic approach devised to select the best \acs{mv} also contributes to increase the overall performance. Thus, using the proposed two-stage approach to recover temporal predictions consistently improves the error robustness of \ac{hevc}. \section{Summary} \label{Section4:Summary} -In this chapter a two-stage approach is proposed to increase the error robustness of \ac{hevc} streaming and reduce the error propagation in case of packet loss. +In this chapter a two-stage approach was proposed to increase the error robustness of \ac{hevc} streaming and reduce the error propagation in case of packet loss. A constrained coding approach was devised to select reference frames and dynamically distribute temporal dependencies. This is jointly used with a controlled amount of side information in coded streams, comprising a small set of the most relevant \acs{mv}, to minimise \ac{mv} mismatch at the decoder in the presence of frame loss. -As can be concluded from the results, this method contributes to reduce the temporal dependencies with consistent gains for different \ac{plr} and coding configurations, only incurring in a small drop of coding efficiency. +As can be concluded from the results, this method contributes to reduce the temporal dependencies with consistent quality gains for different \ac{plr} and coding configurations, only incurring in a small drop of coding efficiency. Overall, the proposed approach is an effective method for coping with video transmission over error-prone networks. diff --git a/041_ImprovedErrorResilience/Results/AverageRefPicUsage.tex b/041_ImprovedErrorResilience/Results/AverageRefPicUsage.tex index fe2ce54..8b12970 100644 --- a/041_ImprovedErrorResilience/Results/AverageRefPicUsage.tex +++ b/041_ImprovedErrorResilience/Results/AverageRefPicUsage.tex @@ -1,5 +1,3 @@ -\begin{figure}[tb] -\centering \newcommand{\pltw}{0.85\columnwidth} \newcommand{\plth}{0.25\textheight} \begin{gnuplot}[terminal=epslatex,terminaloptions={font ",\PlotFont" size \convertlen{\pltw},\convertlen{\plth}}] @@ -32,6 +30,3 @@ plot '-' using ($3*100):xtic(2) ls 20 6 "\\TitleFont Tennis" 0.865467 e \end{gnuplot} -\caption{Average usage of the reference frame 1 ($r_1$) for different sequences.} -\label{FIG4:ReferenceUsageRatio} -\end{figure} diff --git a/041_ImprovedErrorResilience/Results/TableResultsRandomLossMvRedundancies.tex b/041_ImprovedErrorResilience/Results/TableResultsRandomLossMvRedundancies.tex index 0986926..2b718a9 100644 --- a/041_ImprovedErrorResilience/Results/TableResultsRandomLossMvRedundancies.tex +++ b/041_ImprovedErrorResilience/Results/TableResultsRandomLossMvRedundancies.tex @@ -5,7 +5,7 @@ \footnotesize \begin{tabular}{x{2.0cm}|x{3.2cm}|x{1.8cm}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}} \toprule -\multirow{3}{*}{\centering\textbf{Sequence}} & \multirow{3}{*}{\vbox{\centering\textbf{Method}}} & \multirow{3}{*}{\vbox{\centering\textbf{Overhead ratio ($\%$)}}} & \multicolumn{5}{c}{\textbf{Packet loss rate}} \\ +\multirow{3}{*}{\centering\textbf{Sequence}} & \multirow{3}{*}{\vbox{\centering\textbf{Method}}} & \multirow{3}{*}{\vbox{\centering\textbf{Overhead ratio ($\%$)}}} & \multicolumn{5}{c}{\textbf{\TABLEPLRLABEL}} \\ & & & \multicolumn{2}{c|}{\textbf{\LOSTALL}} & \multicolumn{3}{c}{\textbf{\KEEPINTRA}} \\ & & & \textbf{1\%} & \textbf{5\%} & \textbf{1\%} & \textbf{5\%} & \textbf{10\%} \\ \hline diff --git a/041_ImprovedErrorResilience/Results/TableResultsRandomLossRefPicSelection.tex b/041_ImprovedErrorResilience/Results/TableResultsRandomLossRefPicSelection.tex index b2710df..dc59616 100644 --- a/041_ImprovedErrorResilience/Results/TableResultsRandomLossRefPicSelection.tex +++ b/041_ImprovedErrorResilience/Results/TableResultsRandomLossRefPicSelection.tex @@ -10,7 +10,7 @@ \resizebox{\columnwidth}{!}{ \begin{tabular}{p{2.2cm}|x{2.2cm}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}} \toprule -\multirow{3}{*}{\centering\textbf{Sequence}} & \multirow{3}{*}{\vbox{\centering\textbf{Method}}} & \multicolumn{6}{c}{\textbf{Packet loss rate}} \\ +\multirow{3}{*}{\centering\textbf{Sequence}} & \multirow{3}{*}{\vbox{\centering\textbf{Method}}} & \multicolumn{6}{c}{\textbf{\TABLEPLRLABEL}} \\ & & \multicolumn{2}{c|}{\textbf{\LOSTALL}} & \multicolumn{4}{c}{\textbf{\KEEPINTRA}} \\ & & \textbf{1\%} & \textbf{5\%} & \textbf{1\%} & \textbf{3\%} & \textbf{5\%} & \textbf{10\%} \\ \hline @@ -102,7 +102,7 @@ \resizebox{\columnwidth}{!}{ \begin{tabular}{p{2.2cm}|x{2.2cm}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}|x{\resultswidth}} \toprule -\multirow{3}{*}{\centering\textbf{Sequence}} & \multirow{3}{*}{\vbox{\centering\textbf{Method}}} & \multicolumn{6}{c}{\textbf{Packet loss rate}} \\ +\multirow{3}{*}{\centering\textbf{Sequence}} & \multirow{3}{*}{\vbox{\centering\textbf{Method}}} & \multicolumn{6}{c}{\textbf{\TABLEPLRLABEL}} \\ & & \multicolumn{2}{c|}{\textbf{\LOSTALL}} & \multicolumn{4}{c}{\textbf{\KEEPINTRA}} \\ & & \textbf{1\%} & \textbf{5\%} & \textbf{1\%} & \textbf{3\%} & \textbf{5\%} & \textbf{10\%} \\ \hline diff --git a/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex b/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex index 2d4b37e..8f2dc6a 100644 --- a/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex +++ b/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex @@ -4,7 +4,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % CHAPTER %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\chapter{Error Concealment-Aware for Robust Video Decoding} +\chapter{Error Concealment-Aware Video Encoding} \label{Chapter:ConcealmentAwareTechniques} %\vspace{-6mm} %\clearpage @@ -15,15 +15,18 @@ \section{Introduction} \glsreset{ec} -\glsreset{rd} +\glsreset{ra} +\glsreset{ld} +%\glsreset{rd} -In this chapter an error concealment-aware encoding scheme is proposed to improve the quality of decoded video in broadcast environments prone to transmission errors and data loss. The method aims to improve the quality of recovered lost frames by optimising the efficiency of the \ac{ec} beforehand at the encoder-side. -The proposed scheme is based on a scalable coding approach where the best \ac{ec} methods to be used at the decoder are optimally determined at the encoder and signalled through \ac{sei} messages. Such optimal \ac{ec} modes are found by simulating transmission losses followed by a lagrangian optimisation of the \textit{signalling rate - \ac{ec} distortion} cost. A generalised saliency-weighted distortion is used and the residue between coded frames and their EC substitutes is encoded using a rate-controlled enhancement layer. When data loss occurs the decoder uses the signalling information to improve the reconstruction quality. +In this chapter an error concealment-aware encoding scheme is proposed to improve the quality of decoded video in broadcast environments prone to transmission errors and data loss. +The method aims to enhance the reconstruction of recovered lost frames by optimising the efficiency of the \ac{ec}. +The proposed scheme is based on a scalable coding approach where the best \ac{ec} methods to be used at the decoder are optimally determined at the encoder and signalled through \ac{sei} messages. Such optimal \ac{ec} modes are found by simulating transmission losses followed by a Lagrangian optimisation of the \textit{signalling rate - \ac{ec} distortion} cost. A generalised saliency-weighted distortion is used and the residue between coded frames and their EC substitutes is encoded using a rate-controlled enhancement layer. When data loss occurs the decoder uses the signalling information to improve the reconstruction quality. This chapter is organized as follows. Section~\ref{Section5:Problem} presents the problem formulation and motivation for the method proposed in this chapter. In Section~\ref{Section5:ECCandidates} does an initial evaluation of different error concealment schemes in order to find the relevant ones to be used in the proposed error concealment-aware coding scheme. -Section~\ref{Section5:OverviewProposedMethod} provides an overview of the architecture of the proposed method followed by a detailed description in Section~\ref{Section5:ProposedMethod}. -Section~\ref{Section5:ECSignallingPerformance} presents a discussion of the performance of the proposed \ac{ec} optimisation and signalling, and Section~\ref{Section5:OverallPerformance} presents an overall performance of the proposed method. -Finally, Section~\ref{Section5:Summary} concludes this chapter. +Section~\ref{Section5:Architecture} provides an overview of the architecture of the proposed method followed by a detailed description in Section~\ref{Section5:ProposedMethod}. +Sections~\ref{Section5:ECSignallingPerformance}~and~\ref{Section5:OverallPerformance} present the performance results and discussion of the proposed \ac{ec} optimisation and signalling. +Finally, Section~\ref{Section5:Summary} concludes the chapter. \clearpage %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -33,41 +36,48 @@ Finally, Section~\ref{Section5:Summary} concludes this chapter. \label{Section5:Problem} \glsreset{mve} -\glsunset{mb} +\glsreset{mc} +%\glsunset{mb} -In Chapter~\ref{Chapter:ErrorResilienceReview} different \ac{ec} algorithms that can effectively reconstruct lost data were described. However, different algorithms might not present the same accuracy for every lost frame or slice. Moreover, selection at the \ac{mb}/\ac{pu} level is not straightforward. -In the past, hybrid \ac{ec} schemes were proposed to improve the reconstruction quality by either combining spatial and temporal \ac{ec} algorithms~\cite{HybridConcTB2008} or by optimising the \ac{mve} at the pixel and block levels~\cite{HybridConcealmentTB2008,2DConcHybridMotion2010}. -Regarding the \ac{hevc}, existing algorithms also aim to improve the reconstruction accuracy of the motion-based recovery by using the information of correctly received neighbours, e.g., block partitions~\cite{HEVCMCBlockAug2013} and prediction residue~\cite{HEVCMCResidueDec2013}. Although such methods are able to reduce the reconstruction artefacts and improve the overall quality, decoder-based decisions are still not fully accurate, as they do not have enough reference information to ease the decision process. Therefore, higher performance can be achieved by performing encoding-side estimation and decision based on the error-free information. +In Chapter~\ref{Chapter:ErrorResilienceReview} different \ac{ec} algorithms that can effectively reconstruct lost video were described, though not all of them exhibit the same accuracy for every lost frame or slice. +Moreover, it was also shown that combining different \ac{ec} approaches at the \ac{mb}/\ac{pu} level to improve the reconstruction accuracy is not straightforward. +In the past, hybrid \ac{ec} schemes were proposed to improve the recovery quality of lost frames by either combining spatial and temporal \ac{ec} algorithms~\cite{HybridConcTB2008} or by optimising the \ac{mve} at the pixel and block levels~\cite{HybridConcealmentTB2008,2DConcHybridMotion2010}. +Regarding the \ac{hevc}, existing algorithms also aim to improve the reconstruction accuracy of motion-based recovery approaches by using the information of correctly received neighbours, e.g., block partitions~\cite{HEVCMCBlockAug2013} and prediction residue~\cite{HEVCMCResidueDec2013}. Although such methods are able to reduce the reconstruction artefacts and improve the overall video quality, decoder-based decisions are still not fully accurate, as they only have limited information to be used as reference in the decision process. +Therefore, higher reconstructed quality can be achieved by estimating the best \ac{ec} techniques at the encoding-side based on the original video signal. %Such approach not only improves quality but also reduces the complexity of the video decoders. %In this chapter a robust coding driven by \ac{ec} methods is developed with the objective of enhancing the quality of reconstructed video at decoders receiving erroneous streams from the transmission network. The proposed approach combines error resilience tools at the encoder-side with \ac{ec} at the decoder-side. -Different approaches were proposed in the past to address the problem of \ac{ec} at the encoder-side \cite{ConcealmentDrivenSliceOrdering2008,ConcealmentAwareME2006,ErrorResilienceModeHEVC2016}. Moreover, extra information was used to aid the \ac{ec} operation at the decoder-side, either by sending redundant data \cite{ResilienceRECAP_ICASSP2009,SaliencyErrorConcealment_TM2013} or by embedding data into the bitstream \cite{ConcealmentIntraDataHidng2009}. Although the above methods facilitate the \ac{ec} performance either through slice reordering or based on \acs{mv}, they always rely on the assumption that a fixed \ac{ec} method is used at the decoder. +Different approaches were proposed in the past to address the problem of \ac{ec}-aware video coding~\cite{ConcealmentDrivenSliceOrdering2008,ConcealmentAwareME2006,ErrorResilienceModeHEVC2016}. +Extra information was also used to aid the \ac{ec} operation at the decoder-side, either by sending redundant information \cite{ResilienceRECAP_ICASSP2009,SaliencyErrorConcealment_TM2013} or by embedding additional coded data into the bitstream \cite{ConcealmentIntraDataHidng2009}. +Although these methods facilitate the \ac{ec} performance either through slice reordering or based on \acs{mv}, they always rely on the assumption that some fixed \ac{ec} method is used at the decoder. % The work presented in \cite{ConcealmentSignalling2008,SignallingRobustScalable2015} proposes testing several methods with transmission of extra symbols to indicate the best method to be applied by the decoder. Although such approaches achieves reasonably good performance, it relies on the spatial neighbouring information or multi-layer video, which may not be available in all cases. -To advance one step further, in this chapter an concealment-aware encoding scheme is proposed to achieve higher flexibility in terms of the \ac{ec} approaches, leading to higher recovered video quality. -This is accomplished by selecting at the encoder, among several \ac{ec} algorithms with different characteristics, the one that yields the lowest reconstruction distortion at the decoder and also minimises the \ac{rd} cost considering the necessary signalling rate. Thus, the proposed approach outperforms similar previous methods \cite{ConcealmentAwareME2006,ErrorResilienceModeHEVC2016} by selecting the best error concealment method without affecting the encoder optimisation. -To further improve the quality of recovered frames, an enhancement layer is encoded with the mismatch residue between the transmitted and recovered frames. This is a dynamic optimisation process, which is more efficient than traditional approaches based on redundant pictures \cite{RedudantPicsJan2009,VPxResiliencePCS2016}, which transmit the whole frame as redundancy. -In contrast with previous work presented in Chapter~\ref{Chapter:ErrorResilienceTechniques} that addressed the error resilience problem by optimising the coding dependencies, the approach followed in this work considers both \ac{ec} performance and the rate required for signalling the best \ac{ec} mode to the decoder. +To advance one step further, this chapter presents n investigation on concealment-aware video coding and proposes a new scheme to achieve high decoding flexibility in terms of the \ac{ec} approaches, leading to better video quality. +This is accomplished by selecting, at the encoder, among several \ac{ec} algorithms with different characteristics, the one that yields the lowest reconstruction distortion at the decoder and also minimises the \ac{rd} cost considering the necessary signalling rate. Thus, the proposed approach differs from similar previous methods \cite{ConcealmentAwareME2006,ErrorResilienceModeHEVC2016} by selecting the best error concealment method without affecting the encoder optimisation. +To further improve the quality of recovered frames, an enhancement layer is encoded with the mismatch residue between the transmitted and recovered frames. This is a dynamic optimisation process, which is more efficient than conventional approaches based on redundant pictures \cite{RedudantPicsJan2009,VPxResiliencePCS2016}, which transmit the whole frame as redundancy. +In contrast with the previous work presented in Chapter~\ref{Chapter:ErrorResilienceTechniques}, which addressed the error resilience problem by optimising the coding dependencies, the approach followed in this chapter considers both \ac{ec} performance and the coded rate required for signalling the best \ac{ec} mode to the decoder. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\section{Traditional error concealment algorithms} +\section{Error concealment algorithms} \label{Section5:ECCandidates} -In order to proposed a novel error concealment-aware technique different algorithms were taken into consideration. This section provides a description and an evaluation of the error concealment candidates used in the proposed method. -Firstly, a detailed description of each approach is provided followed by a performance comparison between each method. -\subsection{Motion-compensation algorithms} +In order to proposed a novel error concealment-aware technique different algorithms were taken into consideration. +This section presents a description and evaluation of the most relevant \ac{ec} algorithms used in the proposed method. +Firstly, a technical description of each method is provided followed by a performance comparison between them. + +\subsection{EC algorithms based on motion-compensation} %\subsection{Motion-compensated reconstruction} -The most common algorithms to recover missing frames takes advantage of the correctly received \acs{mv} from the neighbouring frames. +The most common algorithms to recover missing frames take advantage of the correctly received \acs{mv} from the neighbouring frames. This has been widely used in the past for \ac{h264} standard \cite{H264RefConc2006,BlockBasedMVE2002,BlockBasedMVEOverllaped2011} and also in the \ac{hevc} standard~\cite{HEVCMCBlockAug2013,HEVCMCResidueDec2013,DownhillHEVCCon_ICCE2017}. -Let's consider the example of Figure~\ref{FIG5:MotionVectorExtrapolation}, where a missing frame ($f_0$) at time instance $t_0$ is reconstructed based the information present in the neighbouring frames, $f_{-1}$ and $f_{-2}$. -One of the common approaches, i.e., motion-copy based algorithms reconstructs the missing frame through motion compensation directly using the co-located vectors in the closest neighbour frame ($f_{-1}$). The \acs{mv} for the motion compensation ($v_0$) are obtained by copying the \acs{mv} $v_{-1}$ from the same position in $f_{-1}$, as shown by the example (1) in the figure. +Let's consider the example of Figure~\ref{FIG5:MotionVectorExtrapolation}, where a missing frame ($f_0$) at time instant $t_0$ is reconstructed based on the information present in the temporally adjacent frames, $f_{-1}$ and $f_{-2}$. +One of the common approaches is \ac{mc} where the missing frame is reconstructed through motion compensation directly using the co-located vectors in the closest neighbour frame ($f_{-1}$). The \acs{mv} used for the motion compensation ($v_0$) are obtained by copying the \acs{mv} $v_{-1}$ from the same spatial position in $f_{-1}$, as illustrated in the figure by the \ac{mv} $v^{1}_{0}$ which is obtained by copying $v^{1}_{-1}$. -An alternative approach reconstructs the missing frame by using motion compensation through the correctly received \acs{mv}, of frame $f_{-1}$ at time instant $t_{-1}$, according to the following procedure: +An alternative approach is based on the reconstruction of the missing frame by using motion compensation through a extrapolation of the received \acs{mv}, of frame $f_{-1}$ at time instant $t_{-1}$, according to the following procedure: \begin{enumerate} \item the \acs{mv} ($v_{-1}$) of the neighbouring frame ($f_{-1}$) are extrapolated to compute the set of \acs{mv} ($v_{0}$) for the missing frame as follows: @@ -75,7 +85,7 @@ the \acs{mv} ($v_{-1}$) of the neighbouring frame ($f_{-1}$) are extrapolated to % v^\zeta_0 = \frac{ t_0 - t_{C} }{ t_{-1} - t_{R} } \times v^\zeta_{-1},\ \zeta=s,r v_0 = \frac{ t_0 - t_{-1} }{ t_{-1} - t_{R} } \times v_{-1}, \end{equation} -where $t_{R}$ is the time instant of the reference frame pointed by the original \acs{mv} $v_{-1}$. +where $t_{R}$ is the time instant of the reference frame pointed by the original \acs{mv} $v_{-1}$, which in the example of Figure~\ref{FIG5:MotionVectorExtrapolation} corresponds to $t_{-2}$. \item the \ac{mv} associated to the block $b_{-1}(x,y)$ at spatial position $(x,y)$ in $f_{-1}$ may be used to recover the block $b^{'}_{{0}}(x^{'},y^{'})$ in $f_0$, at position $(x^{'},y^{'})$ obtained from $v_0$ components as follows: \begin{align} @@ -83,7 +93,9 @@ x^{'} &= x - v_{0x}, \\ y^{'} &= y - v_{0y}. \end{align} \end{enumerate} -This is illustrated in Figure~\ref{FIG5:MotionVectorExtrapolation} by the example (2). This method is referred to as \glsfirst{mve}. +This method is illustrated in Figure~\ref{FIG5:MotionVectorExtrapolation} by the \ac{mv} $v^{2}_{0}$ which results from the extrapolation of the \ac{mv} $v^{2}_{-1}$, and is referred to as \glsfirst{mve}. +One should note that these methods may not result in a complete motion field for the entire frame, resulting in an incomplete reconstruction. For those regions the average of the three top-left neighbouring \acs{mv} are used to complete the motion field. +%\todo{Grid-aligment! Incomplete!} \begin{figure}[tb] \centering @@ -96,46 +108,45 @@ Finally, as a base for comparison the default \ac{ec} in the reference \ac{hevc} \subsection{Algorithms based on motion estimation} -In this sub-section different method that can be used to estimate a motion-field are described, which can be used to reconstruct the missing motion information. -The obtained motion-field is used in combination with the \ac{mve} algorithm to recover the lost frame. +In this sub-section a different method to estimate a motion-field is described. This can be used to reconstruct the missing motion information of a lost frame. +Then, the recovered motion-field is used in combination with the \ac{mve} method to recover the lost frame. -Typically, optical flow estimation algorithms are used to estimate a 2-D \ac{mv} field, $v=(v_x, v_y)$, for each pixel, such as the following energy function is minimised: +Typically, optical flow estimation algorithms are used to estimate a 2-D motion field, comprising one \ac{mv} $v=(v_x, v_y)$, for each pixel, which minimises the following energy function: \begin{equation} -E(x,y) = \sum_{x} \sum_{y} \bigg| f_{0}\big(x,y\big) - f_{1}\big( x + v_x(x,y), y+v_y(x,y) \big) \bigg|^2. +E(x,y) = \sum_{x} \sum_{y} \bigg| f_{0}\big(x,y\big) - f_{1}\big( x + v_x(x,y), y+v_y(x,y) \big) \bigg|. \end{equation} -Besides using a full search algorithm where a block-based approach is used to search for the best \acs{mv} based on the sum of absolute errors, in this work three different approaches were tested to evaluate their effectiveness to be used in \ac{ec} algorithms (described below), namely: +Besides a full search algorithm where a block-based approach is used to search for the best \acs{mv} that minimises the sum of absolute errors, three other approaches were tested to evaluate their effectiveness in \ac{ec} algorithms. These three methods are the following: \begin{enumerate} - \item a method based on the total variation based on the minimization of the \mbox{L-1} norm~\cite{DualFlow2016}; - \item a method which approximates the neighbouring information of a given pixel by quadratic polynomials on both frames~\cite{PolyFlow2003}; - \item a method based on a combination of descriptor matching and large displacement estimation~\cite{DeepFlow2013}. + \item a method based on the total variation using the minimization of the \mbox{L-1} norm~\cite{DualFlow2016}; + \item a method which approximates the neighbourhood of each pixel by quadratic polynomials on both frames~\cite{PolyFlow2003}; + \item a method based on a combination of descriptor matching and large displacement estimation~\cite{DeepFlow2013}. \HERE \end{enumerate} \subsubsection{Motion estimation based on total variation and regularisation (\textit{DualFlow})} -In this method the motion field is estimated based on the minimization of the total variation obtained using the L-1 norm \cite{DualFlow2007,DualFlow2016}, taking into account the Horn and Schunck smoothness condition~\cite{HornSchunckOpticalFlow1981}. This condition forces the estimated field to be regular across neighbouring pixels by penalising deviations in a quadratic way. +In this method the motion field is estimated based on the minimization of the total variation obtained using the L-1 norm \cite{DualFlow2007,DualFlow2016}, taking into account the Horn and Schunck smoothness condition~\cite{HornSchunckOpticalFlow1981}. This condition forces the estimated field to be regular across neighbouring pixels by penalising deviations in a quadratic sense. Using this method, for each pixel $\mathbf{p}$, the motion field $v(\mathbf{p})$ is obtained by minimising the following energy function, \begin{equation}\label{EQ5:DualFlow} E(v) = \sum_{\mathbf{p} \in \Omega} \Big( \big| f_{-1}(\mathbf{p}) - f_{-2}( \mathbf{p} + v(\mathbf{p}))\big| + \big|\nabla v(\mathbf{p})\big| \Big)%d\mathbf{p},\ \mathbf{p} \in \Omega \end{equation} -where $f_{-1}$ and $f_{-2}$ are the two previous frames at time instances $t_{-1}$ and $t_{-2}$, as represented in Figure~\ref{FIG5:MotionVectorExtrapolation}. In the equation, $\Omega$ defines the sets of pixels of those frames. -$|\nabla v(\mathbf{p})|$ is a weighting factor that penalises high variations of the optical flow which effectively disallow discontinuities. This is based on the assumption that neighbouring regions belong to the same object, thus such regions should have similar motion. +where $f_{-1}$ and $f_{-2}$ are the two previous frames at time instants $t_{-1}$ and $t_{-2}$, as represented in Figure~\ref{FIG5:MotionVectorExtrapolation}. In Equation~\ref{EQ5:DualFlow}, $\Omega$ defines the sets of pixels of those frames, and $|\nabla v(\mathbf{p})|$ is a weighting factor that penalises high variations of the optical flow which effectively disallow discontinuities. This is based on the assumption that neighbouring regions belong to the same object, thus such regions should have similar motion. \subsubsection{Motion estimation based on polynomial expansion (\textit{PolyFlow})} -In this method the motion field is estimated based on a polynomial expansion a neighbourhood of a pixel~\cite{PolyFlow2003}. +In this method the motion field is estimated based on a polynomial expansion of each pixel neighbourhood~\cite{PolyFlow2003}. In this method only quadratic polynomials are considered, expressed as: \begin{equation}\label{EQ5:PolyFlow} -f(\mathbf{p}) \sim \mathbf{p}A_1\mathbf{p}^T+\mathbf{b}_1\mathbf{p}^T+C_1 +f(\mathbf{p}) \sim \mathbf{p}\mathbf{A}_1\mathbf{p}^T+\mathbf{b}_1\mathbf{p}^T+C_1 \end{equation} -where $\mathbf{p} = [x\ y]$, $\mathbf{A_1}$ is a symmetric matrix, $\mathbf{b_1}$ is a row vector and $C_1$ is a scalar. The coefficients are estimated from a weighted least squares fit to the signal values in the neighbourhood. +where $\mathbf{p} = [x\ y]$, $\mathbf{A_1}$ is a symmetric matrix, $\mathbf{b_1}$ is a row vector and $C_1$ is a scalar. The coefficients are estimated using weighted least squares fit to the signal values in the neighbourhood. Since the result of polynomial expansion is that each neighbourhood is approximated by a polynomial, we start by analysing what happens if a polynomial undergoes an ideal translation. Let's consider the signal $f_1(\mathbf{p})$ represented by Equation (\ref{EQ5:PolyFlow}) and a signal $f_2$ which corresponds to a displacement of $f_1$ given by $\mathbf{d}=[v_x \ v_y]$. This can be expressed by the following: \begin{align} f_2(\mathbf{p}) &= f_1(\mathbf{p}-\mathbf{d}) \nonumber \\ -&= (\mathbf{p}-\mathbf{d})A_1(\mathbf{p}-\mathbf{d})^T+\mathbf{b}_1(\mathbf{p}+\mathbf{d})^T+C_1 \\ +&= (\mathbf{p}-\mathbf{d})\mathbf{A}_1(\mathbf{p}-\mathbf{d})^T+\mathbf{b}_1(\mathbf{p}-\mathbf{d})^T+C_1 \\ &= \mathbf{p}\mathbf{A}_1\mathbf{p}^T - 2\mathbf{d}\mathbf{A}_1\mathbf{p}^T+ \mathbf{d}\mathbf{A}_1\mathbf{d}^T+ \mathbf{b}_1\mathbf{p}^T-\mathbf{b}_1\mathbf{d}^T + C_1 \nonumber \\ &= \mathbf{p}\mathbf{A}_1\mathbf{p}^T + (\mathbf{b}_1-2\mathbf{d}\mathbf{A}_1)\mathbf{p}^T + \mathbf{d}\mathbf{A}_1\mathbf{d}^T - \mathbf{b}_1\mathbf{d}^T + C_1\\ &= \mathbf{p}A_2\mathbf{p}^T+\mathbf{b}_2\mathbf{p}^T+C_2, \nonumber @@ -150,6 +161,8 @@ This allows to find the displacement $\mathbf{d}$ for any signal dimensions by s \subsubsection{Large displacement optical flow with deep matching (\textit{DeepFlow})} +\todo{Completar o Deep flow} + In this method the motion field is estimated based on the large displacement optical flow presented in \cite{LargeDisplacementOptFlow_TPAMI2011}, which extends the total variation methods, e.g. the one expressed by Equation (\ref{EQ5:DualFlow}), by including descriptor matching. The correspondences between descriptions are used to apply a coarse-to-fine wrapping, thus avoiding local minimum values. The matching algorithm is built upon a multi-stage architecture with six layers with different patches sizes, interleaving convolutions and max-pooling \cite{DeepFlow2013}. The combination of both approaches increases the matching accuracy. @@ -175,10 +188,13 @@ The smoothness term is a robust penalisation of the gradient of the flow as used \subsection{Experimental evaluation} -In this section a performance evaluation of the different \ac{ec} schemes previously described is carried out. This evaluation aims at finding the suitable methods to be used for efficiently reconstruct the missing frames in case of data loss. In this evaluation the \ac{fc} method is used as reference for comparison. Then, two methods based on the correctly received \acs{mv} are tested, i.e., \ac{mc} and \ac{mve}. Moreover, the four different approaches based on a estimated motion field are compared: (i) full search based on the minimisation of the sum of absolute errors (\FULLSEARCH), (ii) optical flow based a duality between total variation and smoothness term (\DUALFLOW), (iii) optical flow based on polynomial approximations (\POLYFLOW) and (iv) optical flow based on deep matching (\DEEPFLOW). +In this section a performance evaluation of the different \ac{ec} schemes previously described is carried out. This evaluation aims at finding the suitable methods to be used for efficiently reconstruct the missing frames in case of data loss. +The two \ac{ec} methods based on the correctly received \acs{mv} are tested, i.e., \ac{mc} and \ac{mve}, using the \ac{fc} method is used as reference for comparison. +Moreover, the four different approaches to estimate the motion field for the missing frame are compared: (i) full search based on the minimisation of the sum of absolute errors (\FULLSEARCH), (ii) optical flow based on the duality between total variation and smoothness term (\DUALFLOW), (iii) optical flow based on polynomial approximations (\POLYFLOW) and (iv) optical flow based on deep matching (\DEEPFLOW). -Table~\ref{TAB5:ECPerformaceComparison} shows the average quality results obtained using the \ac{psnr} metric for different methods. The absolute value is shown for the \FC method, and differential values are shown for the remaining methods. -Results reveal that the methods \MC and \MVE are able to overcome \FC which does not use any \acs{mv}, achieving average quality gains of up to 1.41 dB and 1.46 dB, respectively. Moreover, by extrapolating the \acs{mv} from the correctly receiving frame to the lost time instance (\MVE) is able to outperform straightforward motion-copy (\MC). +Table~\ref{TAB5:ECPerformaceComparison} shows the average quality (\ac{psnr}) obtained for the different methods under investigation. +The absolute value of \ac{psnr} is shown for the \FC method, while the differential values are shown for the remaining methods. +The results shown in the table reveal that methods \MC and \MVE are both able to overcome \FC, achieving average quality gains of up to 1.41 dB and 1.46 dB, respectively. Moreover, by extrapolating the \acs{mv} from the correctly receiving frame to the lost one (\MVE), it is also possible to outperform motion-copy (\MC). \begin{table}[tb] \centering @@ -188,12 +204,15 @@ Results reveal that the methods \MC and \MVE are able to overcome \FC which does \input{\ConcealmentAwareDir/Results/PerformanceConcealmentMethods.tex} \end{table} +\HERE + Results for \ac{ec} methods based on motion estimation (see columns 5 to 8 of Table~\ref{TAB5:ECPerformaceComparison}) reveal an overall superior performance when comparing with methods which rely of correctly received \acs{mv}. Results for \FULLSEARCH method reveals that only minimising the sum of absolute errors between two consecutive frames might not always be directly related with objects' motions. Thus, this results in an inaccurate motion estimation to be used for the reconstruction of lost frames. Finally, results from Table~\ref{TAB5:ECPerformaceComparison} reveal that the \DEEPFLOW algorithm leads to the most accurate motion field to be used for \ac{ec}, in comparison with the other methods. It is able to achieve an average quality gain of 1.79 dB, and up to 3.39 dB for Book Arrival sequence, where \POLYFLOW only achieves 3.05 dB. -\subsection{Conclusions} +\subsection*{Remarks} -Summarising, the analysis of the previous results indicates that all methods cope with lost frames and manage to recover the missing data with acceptable quality. Nevertheless, it is noticeable that the recovering lost frames with an estimated motion field leads to higher quality, especially using the \DEEPFLOW method. However, as the other methods tested are also able to achieve acceptable video quality they shall not be neglected, as they provide alternative approaches for lost frame recovery with lower computational complexity (i.e., they not require motion estimation). +Summarising, the analysis of the previous results indicates that all methods are able to cope with lost frames and manage to recover the missing data with acceptable quality. +Nevertheless, it is noticeable that lost frames recovered with an estimated motion field result in higher average quality, especially when using the \DEEPFLOW method. However, as the other methods tested are also able to achieve acceptable video quality, they shall not be neglected, as they provide alternative approaches for lost frame recovery with lower computational complexity (i.e., they not require motion estimation). Thus, in order to achieve higher reconstruction quality different approaches shall be considered, aiming at balancing the overall complexity with the recovery quality. %\HERE @@ -201,17 +220,16 @@ Thus, in order to achieve higher reconstruction quality different approaches sha %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\section{Proposed architecture} -\label{Section5:OverviewProposedMethod} +\section{Architecture for the EC-aware video encoding} +\label{Section5:Architecture} -\glsreset{ec} +%\glsreset{ec} -This section presents an overview of the proposed method architecture describing the functional blocks presented providing an introduction for the subsequent detailed descriptions. - -Figure~\ref{FIG5:BlockDiagram} illustrates the architecture of the proposed encoding scheme. -After the video encoder, it includes data loss simulation at the \ac{nal} for every slice. Subsequently, error concealment is applied to find the best \ac{ec} methods that should be used to reconstruct the lost slices in case of transmission errors. -Saliency-weighted \ac{ec} mode decision is used to select the optimal \ac{ec} mode from the best trade-off between \ac{ec} distortion, using the original input video ($f$) as reference, and the number of bits required for signalling. -Then the optimal mode is sent to the decoder along with the residue formed between the error-free decoded frame and the error concealed one, using the scalable extension of the \ac{hevc} standard. +This section presents the general architecture proposed for \ac{ec}-aware video encoding and describes its main the functional blocks, also providing an introduction for the subsequent detailed descriptions. +Figure~\ref{FIG5:BlockDiagram} illustrates the architecture of the proposed \ac{ec}-aware encoding scheme. +After the video encoder, it includes data loss simulation at the \ac{nal} for every slice. Subsequently, \ac{ec} is applied to the simulated losses in order to find the best methods that should be used to reconstruct the lost slices in case of transmission errors. +Saliency-weighted \ac{ec} mode decision is used to select the optimal \ac{ec} mode from the best trade-off between \ac{ec} distortion, using the original input video~($f$) as reference, and the number of bits required for signalling. +Then, the optimal \ac{ec} mode is sent to the decoder along with the residue formed between the error-free decoded frame and its error concealed counterpart. The scalable extension of the \ac{hevc} standard is used to encode this residue information. \begin{figure}[tb] \centering @@ -223,12 +241,12 @@ Then the optimal mode is sent to the decoder along with the residue formed betwe \end{figure} -For the \ac{ec} mode decision optimization the missing slices are reconstructed, as they would be in the decoder, but using different \ac{ec} methods based on motion estimation and extrapolation techniques, to selected the best one at the encoder-side. +For the \ac{ec} mode decision optimization the missing slices are reconstructed, as they would be in the decoder, i.e., recovered at the encoder by simulating the decoder \ac{ec}, but using different \ac{ec} methods based on motion estimation and extrapolation techniques, to selected the best one at the encoder-side. In this process, each \ac{ctu} is partitioned to form a quadtree structure of \ac{ecu} that are used to test the different \ac{ec} candidate modes. Then, the best trade-off between \ac{ecu} partition and signalling is found through \ac{rd} optimisation. -In order to achieve an efficient use of the signalling overhead a saliency-weighted procedure is embedded in the \ac{rd} optimisation based on a spatio-temporal saliency, where more signalling bits are allocated to regions with higher saliency values. +In order to achieve an efficient use of the signalling overhead, a saliency-weighted procedure is embedded in the \ac{rd} optimisation based on a spatio-temporal saliency, where more signalling bits are allocated to regions with higher saliency values. Finally, the signalling information is transmitted to the decoder by using the \ac{sei} \acs{nalu} type~\cite{HEVCHighLevelReferenceDez2012}. -Since in general, the optimal \ac{ec} mode is still not be able to correctly reconstruct all the missing regions, extra residual information is multiplexed in the stream as a scalable \ac{el}. +Since in general, the optimal \ac{ec} mode is still not able to correctly reconstruct all the missing regions, extra residual information is multiplexed in the stream as a scalable \ac{el}. This residual information contains the difference between the compressed ($\hat f_t$) and the \ac{ec} frames ($\tilde f$). The transmitted information is used to assist the \ac{ec} operation at the decoder to recover erroneous frames with high accuracy, by using the best \ac{ec} mode and also residual information to reduce error propagation. @@ -239,18 +257,19 @@ The remaining of this section describes in detail the building blocks of the pro %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%\section{Saliency-weighted error concealment mode decision} -\section{Error concealment mode decision} +\section{Saliency-weighted \ac{ec} mode decision} +%\section{Error concealment mode decision} \label{Section5:ProposedMethod} -In this section the saliency-weighted \ac{ec} mode decision algorithm is described. Firstly, the list of \ac{ec} candidates from the previously described methods are enumerated. Then the \ac{rd} optimisation is proposed based on a saliency detection to decrease the overall bitrate without compromising the reconstruction quality. Moreover, the signalling coding strategy is also described. Finally, the introducing of \ac{ec} residue as auxiliary information to improve the reconstruction quality is described. +In this section the proposed saliency-weighted \ac{ec} mode decision algorithm is described. Firstly, the list of \ac{ec} candidates is provided based on the previously described methods. +Then, the \ac{rd} optimisation is proposed using a saliency region in each frame to decrease the overall bitrate without compromising the reconstruction quality. +The signalling coding strategy is also described. +Finally, the introducing of \ac{ec} residue as auxiliary information to improve the reconstruction quality is described. -\subsection{Error concealment candidates} +\subsection{\ac{ec} candidates} - - -A set of four \ac{ec} candidate modes (M1...M4) is used in the optimisation process that selects one of them as best for each particular \ac{ecu}. Each \ac{ec} mode corresponds to a different \ac{ec} method, as shown in Figure~\ref{FIG5:concealmentcandidates}, namely: +A set of four \ac{ec} candidate modes (M1...M4) is used in the optimisation process that selects one of them as the best for each particular \ac{ecu}. Each \ac{ec} mode corresponds to a different \ac{ec} method, as shown in Figure~\ref{FIG5:concealmentcandidates}, namely: \begin{itemize} \item M1, M2: These two methods are based on extrapolation of an estimated motion field; \item M3: This method is based on \ac{mv} extrapolation of previously received \acs{mv}; @@ -266,16 +285,17 @@ A set of four \ac{ec} candidate modes (M1...M4) is used in the optimisation pro \FIGSVSPACE \end{figure} -The first two methods M1, M2 rely on an estimated motion field, using the method \DEEPFLOW \cite{DeepFlow2013}, to recover the lost frame. Then, it is extrapolated at the pixel level, i.e., one \ac{mv} is assigned to each pixel, and \ac{pu} level, i.e., one \ac{mv} is assigned to the entire block, at the missing frame instant. This results in two different reconstructions of the missing frame, where the former may increase the accuracy of the pixel extrapolation, but the latter has the advantage of keeping the block structure and generate a smoother reconstruction. +The first two methods M1, M2 rely on an estimated motion field, using the method \DEEPFLOW \cite{DeepFlow2013}, to recover the lost frame. Then, such motion field is extrapolated at the pixel level, i.e., one \ac{mv} is assigned to each pixel, and \ac{pu} level, i.e., one \ac{mv} is assigned to the entire unit. +This results in two different reconstructions of the missing frame, where the former may increase the accuracy of the pixel extrapolation, but the latter has the advantage of keeping the block structure and generate a smoother reconstruction. -The method M3 is based on a traditional \glsfirst{mve}, and finally M4 reconstructs the missing frames using a \glsfirst{mc} algorithm. +The method M3 is based on a convention \ac{mve}, and finally M4 reconstructs the missing frames using the \ac{mc} algorithm. -\subsection{Selection optimization} +\subsection{Optimal \ac{ec} mode selection} -As described above, in the sub-section~\ref{Section5:ECCandidates}, the proposed method includes four \ac{ec} candidates, which result in different recovered frames for each lost frame (simulated). +As previously described in the sub-section~\ref{Section5:ECCandidates}, the proposed method allows to choose between four \ac{ec} candidates, which result in different recovered frames for each lost frame (simulated). The lost frame is partitioned into quadtree structures, each one forming a set of variable size \acs{ecu}. -Then, finding the optimal \ac{ec} method for each \ac{ecu} is formulated as a rate-distortion cost minimisation. To solve the aforementioned problem, a Lagrangian optimisation is used~\cite{LagragianMultiplier1982} to achieve the best trade-off between reconstruction quality and signalling overhead. +Then, the process of finding the optimal \ac{ec} method for each \ac{ecu} is formulated as a rate-distortion cost minimisation. To solve such problem, a Lagrangian optimisation is used~\cite{LagragianMultiplier1982} to achieve the best trade-off between reconstruction quality and signalling overhead. Furthermore, image saliency driven by visual attention modelling is used in the optimisation process to weight the cost of the overhead information of each \ac{ecu}, according to the visual relevance of its content. % \HIGHLIGHT{% @@ -296,19 +316,25 @@ Furthermore, image saliency driven by visual attention modelling is used in the % \label{eq:Lagrangian} % \end{align} -The optimal \ac{ec} mode $m_{l,i}^*$, for the $i_{th}$ \ac{ecu} which size is defined by the quadtree partitioning level $l$, is obtained by minimising the following lagrangian cost: -\begin{equation}\label{eq:optdepth} +The optimal \ac{ec} mode $m_{l,i}^*$, for the $i_{th}$ \ac{ecu} of size defined by the quadtree partitioning level $l$, is obtained by minimising the following lagrangian cost: +\begin{equation}\label{EQ6:OptimalDepth} \small \newcommand{\argmin}{\operatornamewithlimits{arg\ min}} % (m^*,l^*) = \argmin_{m,l} \left\{ D(m,l) + \left(1+\frac{t_0-t_I}{T_{I}}\right) \lambda R(m,l) \right\}, % l^* = \argmin_{l=1..3} \left\{ D(l) + \lambda \left(1-\frac{t_I-t_c}{T_{\ac{ird}}}\right) R(l) \right\}, -m_{l,i}^* = \argmin\limits_{\footnotesize \substack{l\in\left\{0...3\right\} \\ i\in\left\{1...4^l\right\}}}\left\{ D(m_{l,i}) + \left(1+\frac{t_0-t_I}{T_{I}}\right) \lambda R(m_{l,i}) \right\}, +m_{l,i}^* = \argmin\limits_{\footnotesize \substack{l\in\left\{0...3\right\} \\ i\in\left\{1...4^l\right\}}}\left\{ D(m_{l,i}) + \left(1+\frac{t_C-t_I}{T_{I}}\right) \lambda R(m_{l,i}) \right\}, \end{equation} where $R(m_{l,i})$ represents the number of overhead bits required to encode the \ac{ec} mode, and $D(m_{l,i})$ is sum of absolute error of the \ac{ecu}. -In equation \ref{eq:optdepth}, besides the parameter $\lambda$ used to trade-off the cost of distortion and overhead rate in the minimisation process, there is also a weighting factor to control the cost of $R(m_{l,i})$ according to the duration of the error propagation. This is given by the ratio between the temporal distance from the current frame ($t_0$) to the next refresh frame ($t_I$) and the period of refresh frames ($T_{I}$). This weight varies between $\left[0;1 \right]$, achieving the lowest value for intra-coded frames and the maximum for the frame located immediately before an intra-coded one. -Moreover, a dynamic value for $\lambda$ is used to adjust the amount of overhead across the frame, according to the visual relevance of each image region given by a saliency map. +In equation \ref{EQ6:OptimalDepth}, besides the parameter $\lambda$ used to trade-off distortion and overhead rate in the minimisation process, there is also a weighting factor to control the cost of $R(m_{l,i})$ according to the duration of the error propagation. As in the method proposed in the Chapter~\ref{Chapter:ErrorResilienceTechniques} this method uses the ratio between the temporal distance from the current frame ($t_C$) to the next refresh frame ($t_I$) and the period of refresh frames ($T_{I}$). +%This weight varies between $\left[0;1 \right]$, achieving the lowest value for intra-coded frames and the maximum for the frame located immediately before an intra-coded one. +This behaviour follows the fact that when a lost frame is closer to the next intra-coded frame, error propagation is lower. Thus, it is not worthwhile to spend too many overhead bits to reduce it. This is accomplished by increasing the cost of such bits according to the temporal distance between the missing frame and the next I frame. +Moreover, a dynamic value for $\lambda$ is also used to adjust the amount of overhead within the frame, according to the visual relevance of each image region given by a saliency map. + +The algorithm described above results in \acs{ecu} with variable size, which is defined by the depth of the quad-tree structure, i.e., level $l$. The partition size is jointly optimised along with the best \ac{ec} candidate method using Equation~\ref{EQ6:OptimalDepth}. Alternatively, a fixed \ac{ecu} size can also be used by defining a fixed depth for all quad-tree structures, i.e., constant $l$ for the entire frame. This can be achieved by using Equation~\ref{EQ6:OptimalDepth} to solely optimise the \ac{ec} methods for each \ac{ecu}. +Although this reduces the overall complexity of the proposed method, it leads to a non-optimal solution which may required additional overhead. In Section~\ref{Section5:ECSignallingPerformance} a comparison between a dynamic and fixed approach is provided. + \subsection{Saliency detection} @@ -325,8 +351,8 @@ On the one hand, higher values of saliency (i.e. regions with higher visual rele % \lambda = \lambda_{0} + \frac{\Delta S}{S_{\ac{ctu}} - \mean S}, % \lambda = \lambda_{0} e^{\frac{\mean S - S_{\ac{ctu}}}{\Delta S}}, \Delta S > 0, \lambda=\begin{cases} -\lambda_{0} e^{\frac{\mean S - s}{\Delta S}}, & \text{if $\Delta S > 0$}.\\ -\lambda_{0}, & \text{no saliency used}. +\lambda_{0} e^{\frac{\mean S - s}{\Delta S}}, & \text{if $\Delta S > 0$},\\ +\lambda_{0}, & if \Delta S=0 \text{ (no saliency used)}, \end{cases} \end{equation} where $s$ is the saliency value of the current \ac{ctu}, $\mean S$ is the average saliency value and $\Delta S$ is the difference between the maximum and minimum. @@ -363,22 +389,22 @@ One should note that the method is not dependent on the existence of a saliency Figure~\ref{FIG5:signalling} illustrates the structure of the signalling information for each \ac{ctu} comprising a quadtree of \acs{ecu}. -In this structure, one bit is used for the $split$ flag and two bits for the $ecm$ mode associated to each \ac{ecu}. +In this structure, one bit is used for the $split$ flag and two bits for the \ac{ec} mode associated to each \ac{ecu} ($ecm \in \{M1, M2, M3, M4\}$). The $split$ flag indicates whether the \ac{ecu} is further split into the next~level~($split=1$) or \ac{ecu} partitioning stops at the current level ($split=0$) and the corresponding $ecm$ modes are inserted. Since no split is necessary at the highest level, then $split=1$ followed by the four $ecm$ modes for each \ac{ecu}. -This structure allows dynamic selection of different levels within each frame. +This structure allows dynamic selection of different values for the level of the quad-tree partitioning ($l$) within each frame. Adaptive arithmetic coding~\cite{ArithmeticCodingJun1994} is used to efficiently encode both the $split$ and $ecm$ symbols and further reduce the overhead. Note that the \ac{sei} information uses much less bitrate than the video signal, thus it is very unlikely to be hit during transmission. -\subsection{Enhancement-layer concealment residue} +\subsection{Enhancement-layer: \ac{ec} residue coding} -As described earlier, in order to allow further improvement of recovered slices, besides the best \ac{ec} mode, the proposed encoding scheme also encodes a controlled amount of \ac{ec} residue using the scalable extension of \ac{hevc} (see Figure~\ref{FIG5:BlockDiagram}). +As described earlier, in order to allow further improvement of recovered slices, besides the best \ac{ec} mode, the proposed encoding scheme also allows encoding a controlled amount of \ac{ec} residue using the scalable extension of \ac{hevc} (see Figure~\ref{FIG5:BlockDiagram}). The enhancement layer encodes the difference between the coded frame ($\hat f$) and its counterpart recovered by using the optimal \ac{ec} mode ($\tilde f$). Note that, the higher the accuracy of the best \ac{ec} mode, the lower the amount of bits spent in the enhancement layer. At the decoder-side, the reconstruction of a missing frame ($\tilde f$) follows three steps: \begin{enumerate} - \item the missing frame is reconstructed using the optimal signalled \ac{ec} methods for each \ac{ecu} ($\tilde f'$); + \item the missing frame is reconstructed using the optimal \ac{ec} methods for each \ac{ecu} $\tilde f'$ received from the encoder; \item the enhancement layer residue is decoded ($e^c$); \item the recovered frame is obtained as follows: \begin{equation} @@ -386,10 +412,10 @@ At the decoder-side, the reconstruction of a missing frame ($\tilde f$) follows \end{equation} \end{enumerate} -The additional residue received from the enhancement layer is particularly useful to improve the quality of those \acs{ctu} where the \ac{ec} strategies are not able to produce very accurate reconstruction. +The additional residue decoded from the enhancement layer is particularly useful to improve the quality of those \acs{ctu} where the \ac{ec} strategies are not able to produce very accurate reconstruction. Moreover, since the enhancement layer only carries residual information the overhead is affordable. -Note that the generated bitstream is compliant with the standard, as the error concealment signalling is encapsulated in \ac{sei} \ac{nalu} while the error concealment residue is carried in \acs{nalu} related with the scalable extension. Then, an adapted decoder, capable of decoding both the \ac{sei} \acs{nalu} and the scalable \acs{nalu} takes full benefit from the proposed method. Nevertheless, any standard \ac{hevc} decoder is also able to reconstruct the video, because it simply ignores the extra \acs{nalu}. +Note that the generated bitstream is compliant with the standard, as the \ac{ec} signalling is encapsulated in \ac{sei} \ac{nalu}, while the error concealment residue is carried in \acs{nalu} related with the scalable extension. Then, an adapted decoder, capable of decoding both the \ac{sei} \acs{nalu} and the scalable \acs{nalu} is also able to take full benefit from the proposed method. Nevertheless, any standard \ac{hevc} decoder is also able to reconstruct the video, because it simply ignores the extra \acs{nalu}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -398,16 +424,19 @@ Note that the generated bitstream is compliant with the standard, as the error c \section{Error concealment signalling evaluation} \label{Section5:ECSignallingPerformance} -This section presents the performance of the proposed \ac{ec}-aware optimisation scheme using the \ac{hevc} standard as the basis of the encoding framework. -Two different evaluations are presented. In the former, only robust \ac{ec} of intra-frames is evaluated, while in the latter the proposed \ac{ec}-aware optimisation is applied to all frames corresponding to a generic robust video transmission using optimal \ac{ec}. +This section presents the performance of the proposed \ac{ec}-aware optimisation scheme using the \ac{hevc} reference encoder as the basis of the encoding framework. +Two different types of performance evaluation are presented. +In the first one, only robust \ac{ec} of intra-frames is evaluated, while in the second one the proposed \ac{ec}-aware optimisation is applied to all frames corresponding to a generic robust video transmission using optimal \ac{ec}. \subsection{Intra-frame EC optimisation} \label{SEC4:IntraPerformance} -As mention before, in this section the evaluation of the proposed method is performed only for intra frames to study their impact on error propagation. As intra coding units result in higher amount of bits, they also need a higher number of packets, making them more prone to transmission errors and losses. -Results indicate that for an intra-period of 16 frames, approximately 42\% of the packets corresponds to intra coded slices. -%Also, by limiting the proposed method to intra frames, the amount of required overhead is reduced. +In this section the performance of the proposed method is evaluated only for intra frames to demonstrate its efficiency in reducing error propagation. +Since intra \acs{cu} result in higher amount of bits, they also require a higher number of packets, making them more prone to transmission errors and losses. +Results indicate that for an intra-period of 16 frames, approximately 42\% of the packets correspond to intra coded slices, which is a non-negligible probability of one of then being hit by transmission errors. +Moreover, by limiting the proposed method to intra frames, the amount of required overhead is reduced. + % \begin{table}[tb] % \newcommand{\somespaces}{} @@ -435,117 +464,113 @@ Results indicate that for an intra-period of 16 frames, approximately 42\% of th % These results show that all error concealment candidates are used, which indicates that all of them are found relevant by the proposed method to recover the missing frames. % The candidate methods based on the estimated motion field (i.e., M1, M2) are selected more often because they provide more accurate clues about the scene's motion than the others (i.e., M3, M4). -\subsubsection{Overhead analysis} - -\begin{table}[tb] -\newcommand{\somespaces}{} -\centering -\caption{Percentage of overhead (\%)} -\label{tab:redundancy} -\TABLESVSPACE -\footnotesize -% \resizebox{\columnwidth}{!}{ -\begin{tabular}{p{3.2cm}|x{2cm}|x{2cm}|x{3.5cm}} -\toprule -\multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} -& $32\times32$ & $16\times16$ & \PROP \\ -& \textbf{Level: 1} & \textbf{Level: 2} & \textbf{Level: dynamic} \\ -\hline -% Basketball Drill & 0.08 \ 0.11 & 0.27 \ 0.40 & 0.08 \ 0.60 \\ -Basketball Drill & 0.08 & 0.27 & 0.08 \\ -\hline -% Race Horses & 0.16 & 0.65 & 0.08 \\ -% \hline -Kendo & 0.27 & 1.05 & 0.19 \\ -\hline -Park Scene & 0.42 & 1.72 & 0.05 \\ -\hline -People On Street & 0.06 & 0.25 & 0.11\\ -\hline -Traffic & 0.50 & 1.81 & 0.06 \\ -%\hline -%Jockey & 0.38 & 1.53 & 0.24 \\ -\bottomrule -\end{tabular} -% } -\end{table} - -Table \ref{tab:redundancy} shows the overhead introduced by the proposed method and also for fixed size partitions of $32\times32$ and $16\times16$, i.e., $Depth=1$ and $Depth=2$. While the maximum overhead of these fixed partitions is \HIGHLIGHT{$1.81\%$} (Traffic) the proposed dynamic partitioning method is able to reduce the overhead to a maximum of \HIGHLIGHT{$0.19\%$} (Kendo). -% This is considered acceptable, given the gains obtained under lossy transmission. - -\subsubsection{Quality evaluation} +%\subsubsection{Quality evaluation} \begin{figure}[!t] -\centering -\input{./042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameLowDelay.tex} -\caption{Error propagation using Low-Delay for the Basketball Drill sequence.} -\label{FIG5:singlelosslowdelay} -\vspace{2mm} -\input{./042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameRandomAccess.tex} -\caption{Error propagation using Random-Access for the Park Scene sequence.} -\label{FIG5:singlelossramdon} -\FIGSVSPACE + \centering + \input{./042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameLowDelay.tex} + \caption{Error propagation using \ac{ld} for the Basketball Drill sequence.} + \label{FIG5:singlelosslowdelay} + \vspace{2mm} + \input{./042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameRandomAccess.tex} + \caption{Error propagation using \ac{ra} for the Park Scene sequence.} + \label{FIG5:singlelossramdon} + \FIGSVSPACE \end{figure} -Since a relevant performance indicator is the error propagation over temporally predicted frames, this was evaluated using a single frame loss. -Figures \ref{FIG5:singlelosslowdelay} and \ref{FIG5:singlelossramdon} show the \ac{psnr} results when frame $\#16$ is missing, for Low-Delay and Random-Access encoding configurations, respectively. these Figures show the results for the proposed method (\PROP), the motion-copy method (\MC), and also for a fixed partition size ($16\times16$). -These results indicate that the proposed method (\PROP) outperforms the reference one and it is able to achieve quality gains up to \HIGHLIGHT{2 dB} for Low-Delay configuration (see Figure~\ref{FIG5:singlelosslowdelay}), decreasing the error propagation and increasing the quality of the affected frames up to \HIGHLIGHT{3 dB}. -The use of Random-Access configuration leads to higher quality degradation than LD, due the higher temporal distance of the reference frame used for concealment (frame $\#8$ is used to conceal lost frame~$\#16$). Note that when Frame \# 16 is lost, all frames within the range \#9 to \#31 (i.e., next I-frame) are affected by error propagation. + +Since a relevant performance indicator is the error propagation over temporally predicted frames, this was evaluated using a single intra-frame loss. +Figures \ref{FIG5:singlelosslowdelay} and \ref{FIG5:singlelossramdon} show the \ac{psnr} results when frame Intra-frame $\#16$ is missing, for \ac{ld} and \ac{ra} encoding configurations, respectively. These Figures show the results for the proposed method (\PROP), the motion-copy method (\MC), and also an alternative proposed method using fixed \ac{ecu} size of $16\times16$ pixels. +These results indicate that the proposed method (\PROP) outperforms the reference one and it is able to achieve quality gains up to \HIGHLIGHT{2 dB} for \ac{ld} configuration (see Figure~\ref{FIG5:singlelosslowdelay}), decreasing the error propagation and increasing the quality of the frames affected from the loss of the intra frame, by up to \HIGHLIGHT{3 dB}. +The use of \ac{ra} configuration leads to higher quality degradation than LD, due the higher temporal distance of the reference frame used for \ac{ec} (Frame $\#8$ is used to recover the lost Frame~$\#16$). Note that when Intra-frame \#16 is lost, all frames within the range \#9 to \#31 (i.e., next I-frame) are affected by error propagation. Nevertheless, the proposed method still achieves similar quality gains (approximately 2 dB). -Moreover, for both configurations, even when the overhead is reduced, by moving from fixed partition to dynamic partition size, the proposed method presents only a small reduction in the reconstruction quality. This indicates that using dynamic partitioning allows to achieve a better trade-off between the amount of overhead and the error concealment performance. +Further tests were performed to evaluate the effectiveness of the proposed method under different loss rates. In these experiments the packets corresponding to intra frames were randomly discarded to simulate loss events, at different \acs{plr} rates with an average burst length of 5 packets. +For each test condition 25 trials were performed and the average quality of the frames affected by errors (the lost frame and dependent ones) is presented in Table~\ref{tab:randomloss}. The table also shows the \ac{psnr} variance value in parentheses and the difference to the \ac{psnr} obtained with a reference \ac{hevc} decoder. +% The absolute \ac{psnr} value is shown for the reference case (\MC), while the \ac{psnr} difference is presented for the proposed method. +The results confirm the superiority of the proposed method, outperforming the reference \ac{hevc} for both low and high \ac{plr}. Using the \ac{ld} coding configuration, the proposed method is able to achieve quality gains of \HIGHLIGHT{0.88 dB} for 3\% of \ac{plr} and \HIGHLIGHT{1.50 dB} for 10\% of \ac{plr}. In case of RA, higher quality gains are achieved, leading to an average improvement of \HIGHLIGHT{3.44 dB} for 10\% of \ac{plr}. +It is also worthwhile to notice that the \ac{psnr} variance obtained with the \PROP method is lower, which indicates lower quality variations in the video segments affected by errors. +%The quality gains achieved by the proposed method are similar to those achieved by previous works~\cite{ConcealmentIntraDataHidng2009,ConcealmentSignalling2008} for the same \acs{plr}. However, these require a higher amount of overhead (approximately $3\%$), thus the proposed method achieves a better trade-off between quality and overhead. - -\begin{table}[!t] -\newcommand{\somespaces}{} -\centering -% \caption{Average quality (\ac{psnr}) of the frames affected by errors and its variance under random packet loss} -\caption{Average \ac{psnr} and variance of affected frames} -\label{tab:randomloss} -\TABLESVSPACE -\small -\footnotesize -\newcommand{\colwidth}{1.6cm} -\begin{tabular}{p{2.8cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} -\toprule -\multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & \multicolumn{3}{c|}{\textbf{PLR=3\%}} & \multicolumn{2}{c}{\textbf{PLR=10\%}}\\ -& \MC & \PROP & $\Delta$PSNR& \MC & \PROP & $\Delta$PSNR \\ -\midrule -\multicolumn{7}{c}{Low-delay configuration} \\ [2pt] -\hline -Basketball Drill & 33.15 (0.42) & 34.26 (0.24) & \optbold{+1.11} & 32.97 (0.47) & 35.90 (0.27) & \optbold{+2.93}\\ \hline -% Race Horses & 30.78 (1.32) & 31.55 (0.94) & \optbold{+0.77} & 30.26 (1.55) & 31.12 (1.20) & \optbold{+0.86}\\ \hline -Kendo & 34.33 (1.28) & 36.57 (1.11) & \optbold{+2.24} & 33.84 (1.26) & 37.49 (1.05) & \optbold{+3.65}\\ \hline -Park Scene & 35.22 (0.12) & 35.33 (0.08) & \optbold{+0.11} & 34.75 (0.11) & 34.96 (0.09) & \optbold{+0.21}\\ \hline -People On Street & 34.36 (0.32) & 34.98 (0.22) & \optbold{+0.62} & 32.17 (0.65) & 33.45 (0.50) & \optbold{+1.28}\\ \hline -Traffic & 36.77 (0.48) & 36.88 (0.57) & \optbold{+0.11} & 36.04 (0.30) & 36.31 (0.27) & \optbold{+0.27}\\ \hline -Jockey & 36.00 (1.14) & 37.20 (1.05) & \optbold{+1.20} & 35.44 (1.21) & 36.74 (1.18) & \optbold{+1.30}\\ \hline -\textbf{Average} & \textbf{34.37 (0.73)} & \textbf{35.25 (0.60)} & \textbf{+0.88} & \textbf{33.64 (0.79)} & \textbf{35.14 (0.65)} & \textbf{+1.50} \\ -% \midrule -% \multicolumn{7}{c}{Random-access configuration} \\ [2pt] -% \hline -% Basketball Drill & 30.22 (8.46) & 30.81 (7.66) & \optbold{+0.59} & 29.19 (7.60) & 29.84 (6.88) & \optbold{+0.65} \\ \hline -% % Race Horses & 27.35 (8.02) & 29.92 (5.89) & \optbold{+2.57} & 25.75 (7.76) & 28.61 (6.14) & \optbold{+2.86} \\ \hline -% Kendo & 27.45 (23.9) & 32.38 (18.6) & \optbold{+4.93} & 25.22 (28.3) & 36.98 (22.4) & \optbold{+11.8} \\ \hline -% Park Scene & 33.91 (1.40) & 34.46 (1.07) & \optbold{+0.55} & 31.38 (1.85) & 32.50 (1.48) & \optbold{+1.12} \\ \hline -% People On Street & 29.30 (5.12) & 31.44 (3.17) & \optbold{+2.14} & 24.89 (5.37) & 27.69 (4.09) & \optbold{+2.80} \\ \hline -% Traffic & 35.64 (1.34) & 38.12 (1.24) & \optbold{+2.48} & 32.85 (1.88) & 35.60 (1.40) & \optbold{+2.75} \\ \hline -% Jockey & 32.35 (12.7) & 34.06 (8.60) & \optbold{+1.71} & 29.96 (12.5) & 32.11 (8.90) & \optbold{+2.15} \\ \hline -% \textbf{Average} & \textbf{30.89 (5.64)} & \textbf{33.03 (2.14)} & \textbf{+2.14} & \textbf{28.46 (5.68)} & \textbf{31.90 (4.44)} & \textbf{+3.44}\\ -\bottomrule -\end{tabular} -% } +\begin{table}[tbp] + \newcommand{\somespaces}{} + \centering + \caption{Average quality (\ac{psnr}) of the frames affected by errors and its variance under random packet loss.} + %\caption{Average \ac{psnr} and variance of affected frames} + \label{tab:randomloss} + \TABLESVSPACE + \small + \footnotesize + \newcommand{\colwidth}{1.6cm} + \begin{tabular}{p{2.8cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} + \toprule + \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & \multicolumn{3}{c|}{\textbf{PLR=3\%}} & \multicolumn{2}{c}{\textbf{PLR=10\%}}\\ + & \MC & \PROP & $\Delta$PSNR& \MC & \PROP & $\Delta$PSNR \\ + \midrule + \multicolumn{7}{c}{Low-delay configuration} \\ [2pt] + \hline + Basketball Drill & 33.15 (0.42) & 34.26 (0.24) & \optbold{+1.11} & 32.97 (0.47) & 35.90 (0.27) & \optbold{+2.93}\\ \hline + Race Horses & 30.78 (1.32) & 31.55 (0.94) & \optbold{+0.77} & 30.26 (1.55) & 31.12 (1.20) & \optbold{+0.86}\\ \hline + Kendo & 34.33 (1.28) & 36.57 (1.11) & \optbold{+2.24} & 33.84 (1.26) & 37.49 (1.05) & \optbold{+3.65}\\ \hline + Park Scene & 35.22 (0.12) & 35.33 (0.08) & \optbold{+0.11} & 34.75 (0.11) & 34.96 (0.09) & \optbold{+0.21}\\ \hline + People On Street & 34.36 (0.32) & 34.98 (0.22) & \optbold{+0.62} & 32.17 (0.65) & 33.45 (0.50) & \optbold{+1.28}\\ \hline + Traffic & 36.77 (0.48) & 36.88 (0.57) & \optbold{+0.11} & 36.04 (0.30) & 36.31 (0.27) & \optbold{+0.27}\\ \hline + Jockey & 36.00 (1.14) & 37.20 (1.05) & \optbold{+1.20} & 35.44 (1.21) & 36.74 (1.18) & \optbold{+1.30}\\ \hline + \textbf{Average} & \textbf{34.37 (0.73)} & \textbf{35.25 (0.60)} & \textbf{+0.88} & \textbf{33.64 (0.79)} & \textbf{35.14 (0.65)} & \textbf{+1.50} \\ + \midrule + \multicolumn{7}{c}{Random-access configuration} \\ [2pt] + \hline + Basketball Drill & 30.22 (8.46) & 30.81 (7.66) & \optbold{+0.59} & 29.19 (7.60) & 29.84 (6.88) & \optbold{+0.65} \\ \hline + Race Horses & 27.35 (8.02) & 29.92 (5.89) & \optbold{+2.57} & 25.75 (7.76) & 28.61 (6.14) & \optbold{+2.86} \\ \hline + Kendo & 27.45 (23.9) & 32.38 (18.6) & \optbold{+4.93} & 25.22 (28.3) & 36.98 (22.4) & \optbold{+11.8} \\ \hline + Park Scene & 33.91 (1.40) & 34.46 (1.07) & \optbold{+0.55} & 31.38 (1.85) & 32.50 (1.48) & \optbold{+1.12} \\ \hline + People On Street & 29.30 (5.12) & 31.44 (3.17) & \optbold{+2.14} & 24.89 (5.37) & 27.69 (4.09) & \optbold{+2.80} \\ \hline + Traffic & 35.64 (1.34) & 38.12 (1.24) & \optbold{+2.48} & 32.85 (1.88) & 35.60 (1.40) & \optbold{+2.75} \\ \hline + Jockey & 32.35 (12.7) & 34.06 (8.60) & \optbold{+1.71} & 29.96 (12.5) & 32.11 (8.90) & \optbold{+2.15} \\ \hline + \textbf{Average} & \textbf{30.89 (5.64)} & \textbf{33.03 (2.14)} & \textbf{+2.14} & \textbf{28.46 (5.68)} & \textbf{31.90 (4.44)} & \textbf{+3.44}\\ + \bottomrule + \end{tabular} + % } \end{table} -Further tests were performed to evaluate the effectiveness of the proposed method under more different loss rates. In the experiments the packets corresponding to intra frames were randomly discarded to simulate loss events, at different \acs{plr} rates with an average burst length of 5 packets. -For each test condition 25 trials were performed and the average quality of the frames affected by errors (the lost frame and dependent ones) is presented in Table~\ref{tab:randomloss}. The table also shows the \ac{psnr} variance value in parentheses and the difference to the \ac{psnr} obtained with a reference \ac{hevc} decoder -% The absolute \ac{psnr} value is shown for the reference case (\MC), while the \ac{psnr} difference is presented for the proposed method. -The results confirm the superiority of the proposed method, outperforming the reference \ac{hevc} for both low and high \ac{plr}. Using Low-Delay coding configuration, the proposed method is able to achieve quality gains of \HIGHLIGHT{0.88 dB} for 3\% of \ac{plr} and \HIGHLIGHT{1.50 dB} for 10\% of \ac{plr}. In case of RA, higher quality gains are achieved, leading to an average improvement of \HIGHLIGHT{3.44 dB} for 10\% of \ac{plr}. -It is also worthwhile to notice that the \ac{psnr} variance obtained with the \PROP method is lower, which indicates lower quality variations in the video segments affected by errors. +%\subsubsection{Overhead analysis} -%The quality gains achieved by the proposed method are similar to those achieved by previous works~\cite{ConcealmentIntraDataHidng2009,ConcealmentSignalling2008} for the same \acs{plr}. However, these require a higher amount of overhead (approximately $3\%$), thus the proposed method achieves a better trade-off between quality and overhead. +Subsequently, the percentage of overhead bitrate introduced by the proposed method is compared against fixed \ac{ecu} sizes of $32\times32$ and $16\times16$, i.e., $Level=1$ and $Level=2$. +These results are presented in Table \ref{TAB6:RedundancyIntra} and were obtained for a total bitrate as shown in Table~\ref{TableTestSequences}. +It is noticeable from the results that reducing the \ac{ecu} size (see results for $32\times32$ and $16\times16$) increases the overhead ratios as more symbols are transmitted (one for each \ac{ecu}). +While the maximum overhead of these fixed partitions is \HIGHLIGHT{$1.81\%$} (Traffic) the proposed dynamic partitioning method is able to reduce the overhead to a maximum of \HIGHLIGHT{$0.19\%$} (Kendo). +Moreover, comparing the results of Table \ref{TAB6:RedundancyIntra} with the video quality results obtained in case of packet losses reveals that, in the proposed method, only a small reduction in the reconstruction quality even with a significantly lower overhead. +This indicates that using dynamic partitioning allows to achieve a better trade-off between the amount of overhead and the \c{ec} performance. In summary, the proposed technique is able to enhance the performance of the standard decoding of intra coded frames by finding the best error concealment method at the encoder and sending such information with a small amount of signalling overhead. +\begin{table}[tbp] + \newcommand{\somespaces}{} + \centering + \caption{Percentage of overhead (\%)} + \label{TAB6:RedundancyIntra} + \TABLESVSPACE + \footnotesize + % \resizebox{\columnwidth}{!}{ + \begin{tabular}{p{3.2cm}|x{2cm}|x{2cm}|x{3.5cm}} + \toprule + \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} + & $32\times32$ & $16\times16$ & \PROP \\ + & \textbf{Level: 1} & \textbf{Level: 2} & \textbf{Level: dynamic} \\ + \hline + Basketball Drill & 0.08 & 0.27 & 0.08 \\ \hline + Race Horses & 0.16 & 0.65 & 0.08 \\ \hline + Kendo & 0.27 & 1.05 & 0.19 \\ \hline + Park Scene & 0.42 & 1.72 & 0.05 \\ \hline + People On Street & 0.06 & 0.25 & 0.11\\ \hline + Traffic & 0.50 & 1.81 & 0.06 \\ %\hline + %Jockey & 0.38 & 1.53 & 0.24 \\ + \bottomrule + \end{tabular} + % } +\end{table} + + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % SECTION @@ -554,7 +579,9 @@ In summary, the proposed technique is able to enhance the performance of the sta % \subsection{Overall performance evaluation} \label{SEC4:OverallPerformance} -In this section the proposed \ac{ec}-aware optimisation scheme is evaluated for a generalised robust video transmission of intra and inter coded frames. This section is organised in the following subsections: +In this section the proposed \ac{ec}-aware optimisation scheme is evaluated for a generalised robust video transmission of both intra and inter coded frames. +The experimental results presented in this section were obtained by applying the proposed technique to all video frames. +This section is organised in the following subsections: \begin{itemize} \item evaluation of the usage distribution of each \ac{ec} mode; \item performance evaluation of the \ac{ec} optimal mode decision (without using the enhancement layer); @@ -569,7 +596,8 @@ In this section the proposed \ac{ec}-aware optimisation scheme is evaluated for The \ac{ec} mode usage distribution is an indicator of the usefulness of each \ac{ec} mode. Table \ref{tab:useratio} shows the average $Level$ selected for the \ac{ecu} quadtree and the usage percentage of each \ac{ec} mode. -It is noticeable from the results that the proposed method leads to low average $Level$ values (i.e., less than 1), which implicitly results in less signalling information. This also justifies the low overhead shown in Table~\ref{tab:redundancy_signalling}. +It is noticeable from these results that the proposed method leads to low average $Level$ values (lower than 1), resulting in larger \acs{ecu}, which requires a lower amount of \ac{ecu} to be transmitted than if higher values of $Level$ were chosen. This implicitly results in low signalling information. +This also justifies the low overhead shown in Table~\ref{tab:redundancy_signalling}. \begin{table}[b] \newcommand{\somespaces}{} @@ -596,30 +624,31 @@ Tennis & 1.17 & 25.27 & 30.03 & 23.22 & 21.48 \\ %\hline \end{tabular} \end{table} -The results in Table \ref{tab:useratio} show that all \ac{ec} modes are effectively used, which indicates that all of them are found relevant by to recover the missing data. -The \ac{ec} candidate methods based on the estimated motion field (i.e., M1, M2) are selected more often because they provide more accurate representation of the scene's motion than the others (i.e., M3, M4). +The results in Table \ref{tab:useratio} also show that all \ac{ec} modes are effectively used, which indicates that all of them are found relevant to recover the missing data. +The \ac{ec} candidate methods based on motion field estimation (i.e., M1, M2) are selected more often because they provide more accurate representation of the scene's motion than the others (i.e., M3, M4). \subsubsection{Evaluation of optimal \ac{ec} mode decision} \label{sec:ECModeSelectionPerformance} -\input{\ConcealmentAwareDir/Results/TableSignallingPerformance.tex} - -The evaluation of the proposed saliency-weighted \ac{ec} mode decision (\DYNLAMBDA) was carried out under loss events and compared with the reference methods. The video quality was evaluated by using two quality metrics: \ac{psnr} and a \ac{fsim}~\cite{FSIM_TIP2011} in order to validate the performance of proposed method in a consolidated manner. -Realistic conditions were used for transmission, by allowing independent losses for any of type of \acs{nal}, hence there is no guarantee that the decoder conditions are always equal to those at the encoder, which means that a signalling mismatch may occur. +The evaluation of the proposed saliency-weighted \ac{ec} mode decision (\DYNLAMBDA) was carried out under random loss events and compared with the reference methods. The video quality was evaluated by using two quality metrics: \ac{psnr} and a \ac{fsim}~\cite{FSIM_TIP2011} in order to validate the performance of the proposed method in a consolidated manner. +Realistic conditions were used for transmission, by allowing independent losses for any of type of \acs{nal}, hence there is no guarantee that the decoder conditions are always equal to those at the encoder, which means that sub-optimal \ac{ec} may be used at the decoder. The results obtained without saliency weighting are also shown for comparison (\FIXEDLAMBDA). In this case a constant $\lambda_0=10$ is used. -Tables~\ref{TAB4:SignallingPSNR}~and~\ref{TAB4:SignallingFSIM} show the average quality under different \acs{plr} for the Low-delay configuration. +Tables~\ref{TAB4:SignallingPSNR}~and~\ref{TAB4:SignallingFSIM} show the average quality under different \acs{plr} for the \ac{ld} configuration. Table~\ref{TAB4:SignallingPSNR} shows the \ac{psnr} and weighted \ac{psnr} (\ac{wpsnr}) between parentheses. The \ac{mse} weights used in the \ac{wpsnr} are defined by the saliency values~\cite{SaliencyQualityMetric2011}. Table~\ref{TAB4:SignallingFSIM} shows the results obtained for the \ac{fsim} quality metric. Note that, the absolute value is shown for the reference case (\REF), while the differential values are shown for the remaining methods. -From the \ac{psnr} results shown in Table~\ref{TAB4:SignallingPSNR}, one may conclude that the proposed method (\DYNLAMBDA) consistently outperforms the reference ones for both low and high \acs{plr}. Average \ac{psnr} gains up to 2.95~dB for PLR=3\% and 2.58~dB for PLR=10\% are achieved. It is also worthwhile to notice that higher performance is obtained for sequences with high motion, such as Basketball Drill, which leads to \ac{psnr} gains up 4.82~dB Moreover, the results of Table~\ref{TAB4:SignallingFSIM} confirm the quality improvement achieved by the proposed method, leading to an increase of \ac{fsim} up to 4.32, which indicates that the proposed method is able to recover important image features with higher accuracy. +\input{\ConcealmentAwareDir/Results/TableSignallingPerformance.tex} + +From the \ac{psnr} results shown in Table~\ref{TAB4:SignallingPSNR}, one may conclude that the proposed method (\DYNLAMBDA) consistently outperforms the reference ones for both low and high \acs{plr}. Average \ac{psnr} gains up to 2.95~dB for PLR=3\% and 2.58~dB for PLR=10\% are achieved. It is also worthwhile to notice that higher performance is obtained for sequences with high motion, such as Basketball Drill, which leads to \ac{psnr} gains up 4.82~dB. +Moreover, the results of Table~\ref{TAB4:SignallingFSIM} confirm the quality improvement achieved by the proposed method, leading to an increase of \ac{fsim} up to 4.32, which indicates that the proposed method is able to recover important image features with higher accuracy than the others. Table~\ref{tab:redundancy_signalling} shows the percentage of overhead in the total rate with and without saliency-weighted optimisation, i.e., \FIXEDLAMBDA and \DYNLAMBDA, respectively. As shown in the Table, the saliency-weighted optimisation reduces the amount of overhead by dynamically assigning unequal number of bits to image regions of different visual importance. -These results show that using an adaptive value of $\lambda$ based on the saliency values leads to an average overhead reduction of 28\% in comparison with the case where no saliency is used, i.e., constant $\lambda$. +These results show that using an adaptive value of $\lambda$, based on the saliency values, leads to an average overhead reduction of 28\% in comparison with the case where no saliency is used, i.e., constant $\lambda$. It is worth to note that using the saliency-weighted optimisation to reduce the overhead does not compromise the video quality, as can be seen in the results shown in Tables~\ref{TAB4:SignallingPSNR}~and~\ref{TAB4:SignallingFSIM}. The \ac{psnr} values for \FIXEDLAMBDA and \DYNLAMBDA are very close, but in overhead savings are significant for most sequences. The achieved \ac{wpsnr} and \ac{fsim} results further demonstrate the effectiveness of the saliency-based optimisation, because the lower overhead does not affect the most relevant image regions, i.e., those with higher saliency values. Thus, the proposed method using a adaptive $\lambda$ (i.e., using saliency) is able to not only achieve higher \ac{ec} quality than the reference methods, but also higher quality gains than using a constant $\lambda$ (i.e., without saliency). @@ -635,11 +664,7 @@ The figure shows the results obtained by decoding the stream using the proposed \input{\ConcealmentAwareDir/Results/ECMQualitySingleloss.tex} Figure \ref{FIG5:singlelossIntra} shows the \ac{psnr} for the same conditions but with a loss event on the intra-coded Frame \#16, in sequences Kimono and Tennis. These results reveal that the proposed method is able to improve the reconstruction of inter-coded frames and also outperforms the existing techniques whenever a intra-coded frame is missing. -%\HIGHLIGHT{Although in this frame loss event the recovery quality is lower for every method tested, the proposed approach is able to achieve lower error propagation and achieve quality gains up to 2 dB (see results for Tennis sequence) - ESTA FRASE NAO SE ENTENDE} <- Apenas com estes resultados esta frase nao faz sentido - - - -These results indicate that the proposed method outperforms both reference methods since the time instant when the loss occurs. The lost frame itself is recovered with much higher quality than using the reference \ac{ec} methods. This is due to the use of the optimal \ac{ec} mode to recover each \ac{ecu}, following the \ac{sei} information received from the encoder. +These results also indicate that the proposed method outperforms both reference methods since the time instant when the loss occurs. The lost frame itself is recovered with much higher quality than using the reference \ac{ec} methods. This is due to the use of the optimal \ac{ec} mode to recover each \ac{ecu}, following the \ac{sei} information received from the encoder. In this case, the proposed method is able to achieve quality gains up to \HIGHLIGHT{5~dB}, decreasing the error propagation and increasing the \ac{psnr} of affected frames by up to \HIGHLIGHT{6~dB}. The visual impact of using the proposed \ac{ec}-aware encoding can be evaluated by observing the recovered frame (i.e., frame \#6) of Basketball Drill as an example. Figure~\ref{FIG5:RecoveredExample} shows that higher visual quality and less reconstruction artefacts are obtained in comparison with the \MC method. It is worthwhile to note that the proposed method is able to reconstruct the object edges with higher accuracy than the reference \MC. @@ -666,8 +691,8 @@ The visual impact of using the proposed \ac{ec}-aware encoding can be evaluated %\subsection{Quality evaluation} As in the previous simulations, the packets were randomly discarded to simulate loss events, at different \acs{plr}. -The \ac{psnr} obtained with the proposed method is compared with the default method used by reference \ac{hevc} , i.e., frame-copy (\REF), motion-copy (\MC)~\cite{HEVCMCResidueDec2013}, and the method proposed in \cite{ROPERefGenJan2010}, which implements a robust scheme to minimise error propagation. -The following results were obtained using the same constant bitrate for each test condition. +The quality obtained with the proposed method is compared with the default method used by reference \ac{hevc} , i.e., frame-copy (\REF), motion-copy (\MC)~\cite{HEVCMCResidueDec2013}, and the method proposed in \cite{ROPERefGenJan2010}, which implements a robust scheme to minimise error propagation. +In this experiment both layers, i.e., video stream and enhancement layer with the EC residue, may be affected by errors. \subsection{Quality evaluation} @@ -675,8 +700,8 @@ The following results were obtained using the same constant bitrate for each tes Tables~\ref{tab:concealmentResidueRandomLoss_PLR3}~and~\ref{tab:concealmentResidueRandomLoss_PLR10} present the average \ac{psnr} of decoded video for different \acs{plr} and different percentages of total overhead (i.e., signalling rate plus enhancement layer rate). The absolute \ac{psnr} value is shown for the reference case, while the \ac{psnr} difference is presented for the remaining methods. These results show that transmitting the \ac{ec} residue in the enhancement layer further improves the reconstruction quality obtained when only the optimal \ac{ec} method is used (see previous section). -By only using 10\% of the total bit rate, the proposed \ac{ec}-aware coding scheme is able to outperform the reference \ac{hevc}, the \MC and the method proposed in \cite{ROPERefGenJan2010} for both Low-delay and Random-access configurations. -Using 20\% of the total bitrate the quality gains increase up to \HIGHLIGHT{6.90~dB} for the Basketball Drill sequence when comparing with the reference \ac{hevc}, achieving an average gain of \HIGHLIGHT{3.79~dB} for the Low-delay configuration (\HIGHLIGHT{1.33}~dB for the Random-access). +By only using 10\% of the total bit rate, the proposed \ac{ec}-aware coding scheme is able to outperform the reference \ac{hevc}, the \MC and the method proposed in \cite{ROPERefGenJan2010} for both \ac{ld} and \ac{ra} configurations. +Using 20\% of the total bitrate the quality gains increase up to \HIGHLIGHT{6.90~dB} for the Basketball Drill sequence when comparing with the reference \ac{hevc}, achieving an average gain of \HIGHLIGHT{3.79~dB} for the \ac{ld} configuration (\HIGHLIGHT{1.33}~dB for the \ac{ra}). When comparing the results of Tables~\ref{tab:concealmentResidueRandomLoss_PLR3}~and~\ref{tab:concealmentResidueRandomLoss_PLR10} with the spatial and temporal information of each sequence (SI/TI in Table~\ref{TableTestSequences}) it is noticeable that higher quality gains are achieved for sequences with higher motion information, i.e., Basketball~Drill and Tennis sequences. Moreover, for sequences with higher spatial information (e.g., BQSquare and People on Street) considerable quality gains are also achieved. @@ -722,14 +747,14 @@ R = \abs{\frac{\Delta PSNR(\ac{plr}=10\%)}{\Delta PSNR(Error\text{-}free)}}. The results shown in Table~\ref{tab:overall_gain_ratio}, demonstrate that the effectiveness of the \ac{ec}-aware coding scheme is higher than all other methods under comparison. In fact, not only the absolute \ac{psnr} difference (i.e., \ac{psnr} gain) is higher for the same \ac{plr}, but also the efficiency of the overhead is also higher. This means that the rate spent to encode the signalling information and the enhancement residue carry more useful information than the other methods. -For instance, in comparison with \textit{Ref \cite{DualFrameME2015}} the proposed method leads to higher quality degradation in the error-free case. However, for PLR=10\% it clearly outperforms \textit{Ref \cite{DualFrameME2015}} with a much higher value of $R$, indicating a far superior efficiency. +For instance, in comparison with the method in \cite{DualFrameME2015} the proposed method leads to higher quality degradation in the error-free case. However, for PLR=10\% it clearly outperforms the method in \cite{DualFrameME2015} with a much higher value of $R$, indicating a far superior efficiency. Results in the table~\ref{tab:overall_gain_ratio} also show that the small loss of coding efficiency in error-free transmission is clearly surpassed by the quality gains under error-prone transmission conditions. The methods in \cite{ROPERefGenJan2010,DualFrameME2015} are based on modified predictions, while the method in \cite{VPxResiliencePCS2016} uses redundant data without any further optimisation. Both of these approaches require low additional computation complexity. However, as observed in the results shown in Table ~\ref{tab:overall_gain_ratio} the lower additional complexity also corresponds to lower quality gains in case of transmission errors. -The proposed method uses \ac{ec} techniques with different levels of computational complexity for each \ac{ecu}. Since an adaptive approach is used, the average complexity tends to be lower in comparison with other methods that rely on a fixed approach. Thus, less complex video decoding is obtained when using the proposed method when compared with other straightforward methods that always used the same \ac{ec} techniques for all \acs{ctu}. +The proposed method uses \ac{ec} techniques with different levels of computational complexity for each \ac{ecu}. Since an adaptive approach is used, the average complexity tends to be lower in comparison with other methods that rely on a fixed approach. Thus, less complex video decoding is obtained when using the proposed method when compared with other straightforward methods that always used the same \ac{ec} technique for all \acs{ctu}. \begin{table}[t] @@ -761,8 +786,8 @@ The proposed method uses \ac{ec} techniques with different levels of computati \section{Summary} \label{Section5:Summary} -In this chapter an error concealment-aware encoder is proposed by incorporating loss simulation at the encoder-side to enhance the error concealment efficiency at the decoder. +In this chapter an \ac{ec}-aware encoding scheme is proposed by incorporating loss simulation at the encoder-side to enhance the EC efficiency at the decoder. % The proposed method relies on loss simulation at the encoder-side to optimise the error concealment proposed. Then, the optimal error concealment is signalled to the decoder. -The novelty of this method is to include optimal error concealment mode decision at the encoder, to be used by the decoder in case of data loss, and also to allow encoding of the error concealment residue using the enhancement layer of the \ac{hevc} standard. +The novelty of this method is to include optimal \ac{ec} mode decision at the encoder, to be used by the decoder in case of packet loss, and also to allow encoding of the \ac{ec} residue using the enhancement layer of the \ac{hevc} standard. The overall results demonstrate that the proposed method is able to efficiently reconstruct lost frames and reduce the error propagation compared with other reference methods. -A consistent quality improvement is achieved at the cost of low overhead, which allows to use this method in broadcasting video services and applications where packet loss probability should not be ignored. +A consistent quality improvement achieved at the cost of low overhead, allows the use of this method in broadcasting video services and applications where packet loss probability is not negligible. diff --git a/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz b/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz index 5769bb3..dbba8c8 100644 --- a/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz +++ b/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz @@ -45,7 +45,7 @@ %% NALU Loss and EC optimisation -\node [propBlock, above=2.2 of Encoder.west, anchor=south west, text width=3.8cm, minimum height=2.4cm] (ConcOpt) {Saliency-weighted EC Mode Decision}; +\node [propBlock, above=2.2 of Encoder.west, anchor=south west, text width=3.8cm, minimum height=2.8cm] (ConcOpt) {Saliency-weighted EC Mode Decision}; \node [propBlock, below=.8 of ConcOpt.south east, anchor=north east, text width=1.7cm, minimum height=.9cm] (LossSim) {NALU Loss Simulation}; \path[arrow] (EncoderNAL -| LossSim.south) -- (LossSim) node[point] at (EncoderNAL -| LossSim) {};; \path[arrow] (Decoder) -- (ConcOpt.south -| Decoder) node[pos=.6,coordinate] (p2) {} node[point] at (p2) {}; @@ -75,19 +75,19 @@ \path[arrow] (ConcOpt.east |- Signalling) -- (Signalling) node[above, pos=.45, yshift=-1] {\scriptsize M1...M4}; %% Input -\node[text width=.5cm, text centered] at (-1.7,4.5) (Input) {Input video}; +\node[text width=.5cm, text centered] at (-1.7,4.9) (Input) {Input video}; \path[arrow] (Input) |- (Encoder) node[left,pos=.2] {$f$}; \path[arrow] (Input |- ConcOpt) -- (ConcOpt) node[point] at (Input |- ConcOpt) {}; %% TS Mux and output \path (Encoder.south) -- (Encoder |- Signalling.north) node[midway] (middleRef) {}; % Find middle ref -\node [refBlock, right=8.5 of middleRef, text width=.7cm, minimum height=5.5cm] (NAL) {TS Mux}; +\node [refBlock, right=8.5 of middleRef, text width=.7cm, minimum height=5.8cm] (NAL) {Mux}; \node [right of=NAL, node distance=1.5cm, coordinate] (Out) {}; \path[arrow] (EncoderNAL) -- (NAL.west |- EncoderNAL) node[above,pos=1, xshift=-.4cm] {VCL$_0$}; \path[arrow] (ScalableNAL) -- (NAL.west |- Scalable) node[above,pos=1, xshift=-.4cm] {VCL$_1$}; \path[arrow] (SignallingNAL) -- (NAL.west |- Signalling) node[above,pos=1, xshift=-.4cm] {SEI}; -\path[arrow] (NAL) -- (Out) node[above,pos=1, xshift=-.2cm, text width=1.3cm] {Transport Stream}; +\path[arrow] (NAL) -- (Out) node[above,pos=1, xshift=-.2cm, text width=1.3cm] {Video Stream}; \end{tikzpicture} diff --git a/042_ConcealmentAwareEncoding/Figures/MotionVectorExtrapolation.tikz b/042_ConcealmentAwareEncoding/Figures/MotionVectorExtrapolation.tikz index 44274db..00438d1 100755 --- a/042_ConcealmentAwareEncoding/Figures/MotionVectorExtrapolation.tikz +++ b/042_ConcealmentAwareEncoding/Figures/MotionVectorExtrapolation.tikz @@ -31,9 +31,10 @@ \tikzstyle{refblock}=[rectangle,draw=darkgray, thick] \tikzstyle{projectedblock}=[rectangle,draw=black, thick,dashed] \tikzstyle{refmv_linestyle}=[-, thick] +\tikzstyle{frame_label}=[anchor=center,scale=0.8] \tikzstyle{refmv}=[draw=black, refmv_linestyle,decoration={markings,mark=at position 1 with {\arrow[ultra thick,scale=1]{latex}}},postaction={decorate}] -\tikzstyle{extrapolated_mv}=[refmv_linestyle,dashed, draw=blue, decoration={markings,mark=at position 1 with {\arrow[ultra thick,scale=1]{latex}}},postaction={decorate}] +\tikzstyle{extrapolated_mv}=[refmv_linestyle,dashed, draw, decoration={markings,mark=at position 1 with {\arrow[ultra thick,scale=1]{latex}}},postaction={decorate}] \tikzstyle{arrow}=[draw, ->,>=stealth',thick,,decoration={markings,mark=at position 1 with {\arrow[ultra thick,scale=0.5]{stealth}}},postaction={decorate}] @@ -66,42 +67,22 @@ style={yslant=\TikZyslant,xscale=\TikZxscale}] \foreach \y in {1,3} \draw[black, dotted] (\x,\y) +(0,0) rectangle ++(1,1); - \coordinate (TB1) at (1+0.7,4+0.8); - \draw[projectedblock] (TB1) rectangle ++(1,-1); - - \coordinate (TB2) at (3,4); - \draw[projectedblock] (TB2) rectangle ++(1,-1); - + \coordinate (TB1) at (1,4); + %\coordinate (TB2) at (3,4); \coordinate (TB3) at (3+0.2/2,2-0.8/2); - \draw[projectedblock] (TB3) rectangle ++(1,-1); + \coordinate (TB4) at (2+0.9,2+0.6); - \coordinate (TB4) at (3,2); + \draw[projectedblock] (TB1) rectangle ++(1,-1); \draw[projectedblock] (TB4) rectangle ++(1,-1); + +% \coordinate (TB4) at (3,2); +% \draw[projectedblock] (TB4) rectangle ++(1,-1); + \end{scope} \end{pgfonlayer} -% % Frame t+1 -% \begin{pgfonlayer}{L0} -% \begin{scope}[ yshift=4*\yframeshift, xshift=4*\xframeshift, -% style={yslant=\TikZyslant,xscale=\TikZxscale}] -% -% -% \coordinate (TP1B1) at (1,4); -% \coordinate (TP1B2) at (3,4); -% \coordinate (TP1B3) at (1,2); -% \coordinate (TP1B4) at (3,2); -% -% %Draw a frame -% \DrawFrame{white}{4} -% -% % \draw[black,very thick] (TP1B2) +(0,0) rectangle ++(1,-1); -% \draw[black,very thick] (TP1B4) +(0,0) rectangle ++(1,-1); -% -% \end{scope} -% \end{pgfonlayer} - % Frame t-1 \begin{pgfonlayer}{L1} \begin{scope}[ yshift=2*\TikZyfshift, xshift=2*\xframeshift, @@ -110,32 +91,25 @@ style={yslant=\TikZyslant,xscale=\TikZxscale}] \coordinate (T0B1) at (1,4); \coordinate (T0B2) at (3,4); \coordinate (T0B3) at (1,2); - \coordinate (T0B4) at (3,2); + \coordinate (T0B4) at (2,2); - \coordinate (TM1AuxB3) at (1,2); - - \coordinate (Bottom_V1) at (2.5,0); - - \draw[extrapolated_mv] (TB1) to (T0B1); + \coordinate (T0B1_aux) at (1-0.7,4-0.8); % Draw Motion vectors \coordinate (TM1B0) at (3+0.2,2-0.8); \coordinate (TM1B0C) at (3+0.2/2,2-0.8/2); -% \draw[refmv] (TP1B4) to (TM1B0); - \draw[refblock] (TM1B0) rectangle ++(1,-1); - \draw[extrapolated_mv] (TB3) to (TM1B0); - \draw[refblock] (TM1B0C) rectangle ++(1,-1); - \draw[extrapolated_mv] (TB4) to (TM1B0C); + \path[extrapolated_mv] (TB1) -- (T0B1_aux) node[above,pos=.2] {\footnotesize $v^{1}_{0}$};; + \path[extrapolated_mv] (TB4) -- (T0B4) node[above,pos=.4] {\footnotesize $v^{2}_{0}$};; - \coordinate (TM1B1) at (3-0.6,4-0.5); - \draw[extrapolated_mv] (TB2) to (TM1B1); - \DrawFrame{white}{4} \draw[black,very thick] (T0B1) +(0,0) rectangle ++(1,-1); - \draw[black,very thick] (T0B2) +(0,0) rectangle ++(1,-1); + \draw[projectedblock] (T0B1_aux) +(0,0) rectangle ++(1,-1); + \draw[black,very thick] (T0B4) +(0,0) rectangle ++(1,-1); + + \end{scope} \end{pgfonlayer} @@ -148,10 +122,11 @@ style={yslant=\TikZyslant,xscale=\TikZxscale}] %Define Points \coordinate (TM2B1) at (1-0.7,4-0.8); \coordinate (TM2B2) at (3-0.6,4-0.5); + \coordinate (TM2B4) at (2-0.9,2-0.6); % Draw Motion vectors - \draw[refmv] (T0B1) to (TM2B1); - \draw[refmv] (T0B2) to (TM2B2); + \path[refmv] (T0B1) -- (TM2B1) node[above,pos=.2] {\footnotesize $v^{1}_{-1}$}; + \path[refmv] (T0B4) -- (TM2B4) node[above,pos=.3] {\footnotesize $v^{2}_{-1}$}; \DrawFrame{white}{4} @@ -159,19 +134,19 @@ style={yslant=\TikZyslant,xscale=\TikZxscale}] % Draw block \draw[refblock] (TM2B1) rectangle ++(1,-1); - \draw[refblock] (TM2B2) rectangle ++(1,-1); + \draw[refblock] (TM2B4) rectangle ++(1,-1); \end{scope} \end{pgfonlayer} %MVs name -\node[scale=0.5] at (3*\xframeshift-1.0cm,4.9) {(2)}; -\node[scale=0.5] at (3*\xframeshift-1.0cm,3) {(1)}; +% \node[scale=0.5] at (3*\xframeshift-1.0cm,4.9) {(2)}; +% \node[scale=0.5] at (3*\xframeshift-1.0cm,3) {(1)}; % \node[scale=0.5] at (3*\xframeshift-2.0cm,2) {(2)}; % \node[scale=0.5] at (3*\xframeshift-1.0cm,0.3) {(4)}; - \tikzstyle{frame_label}=[anchor=center,scale=0.8] + % \node[frame_label,scale=0.95] (LostFrameLabel) at (4*\xframeshift-0.9cm,-1.8) {Lost frame }; % \path[draw,-latex,thick] (LostFrameLabel) edge[out=90, in=0] (3*\xframeshift+5cm*\TikZxscale,0cm); @@ -193,15 +168,5 @@ style={yslant=\TikZyslant,xscale=\TikZxscale}] % \node[frame_label] at (4*\xframeshift,\yframeshift*4) { $t_{1}$ }; \end{scope} -% \begin{scope}[yshift=-3.2cm,xshift=-0.5cm] -% \begin{scope} -% \draw[refmv] (0,0) to (1.5,0); -% \fill (2cm,0) node[frame_label,scale=0.95,right]{Original Motion Vector}; -% \end{scope} -% \begin{scope}[xshift=14cm] -% \draw[extrapolated_mv] (0,0) to (1.5,0); -% \fill (2cm,0) node[frame_label,scale=0.95,right]{Extrapolated Motion Vector}; -% \end{scope} -% \end{scope} \end{tikzpicture} diff --git a/042_ConcealmentAwareEncoding/Results/ECMQualitySingleloss.tex b/042_ConcealmentAwareEncoding/Results/ECMQualitySingleloss.tex index a20cbd4..4c2a903 100644 --- a/042_ConcealmentAwareEncoding/Results/ECMQualitySingleloss.tex +++ b/042_ConcealmentAwareEncoding/Results/ECMQualitySingleloss.tex @@ -19,6 +19,7 @@ set xtics 5,2,15 set ytics 4 right set key width 6 +set yrange[20:45] set title '\TitleFont Basketball Drill' plot '-' using 1:2 title '\KeyFont \REF' w lp ls 1, '-' using 1:2 title '\KeyFont \MC' w lp ls 2, '-' using 1:2 title '\KeyFont \DEEPFLOW' w lp ls 3, '-' using 1:2 title '\KeyFont \PROP' w lp ls 4 0 42.08 44.10 44.86 @@ -106,6 +107,7 @@ e 19 40.22 42.85 43.05 e +unset yrange set title '\TitleFont Book Arrival' plot '-' using 1:2 title '\KeyFont \REF' w lp ls 1, '-' using 1:2 title '\KeyFont \MC' w lp ls 2, '-' using 1:2 title '\KeyFont \DEEPFLOW' w lp ls 3, '-' using 1:2 title '\KeyFont \PROP' w lp ls 4 0 41.35 41.07 43.41 diff --git a/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameLowDelay.tex b/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameLowDelay.tex index e72c73d..45db658 100644 --- a/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameLowDelay.tex +++ b/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameLowDelay.tex @@ -7,7 +7,7 @@ set xrange[6:6+27] set xtics 6,2,6+27 set key bottom left width 0 #set title '\TitleFont Basketball Drill' -plot '-' using 1:2 title '\KeyFont \MC' w lp ls 1, '-' using 1:2 title '\KeyFont \emph{16x16}' w lp ls 2, '-' using 1:2 title '\KeyFont \PROP' w lp ls 3 +plot '-' using 1:2 title '\KeyFont \MC' w lp ls 1, '-' using 1:2 title '\KeyFont \emph{$16\times16$}' w lp ls 2, '-' using 1:2 title '\KeyFont \PROP' w lp ls 3 2 40.24 43.19 44.01 3 40.24 42.96 43.94 4 41.26 43.62 44.49 diff --git a/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameRandomAccess.tex b/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameRandomAccess.tex index 7a26a23..1688e77 100644 --- a/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameRandomAccess.tex +++ b/042_ConcealmentAwareEncoding/Results/IntraEC_QualityVsFrameRandomAccess.tex @@ -7,7 +7,7 @@ set xrange[6:6+27] set xtics 6,2,6+27 set key bottom left width 0 #set title '\TitleFont Park Scene' -plot '-' using 1:2 title '\KeyFont \MC' w lp ls 1, '-' using 1:2 title '\KeyFont \emph{16x16}' w lp ls 2, '-' using 1:2 title '\KeyFont \PROP' w lp ls 3 +plot '-' using 1:2 title '\KeyFont \MC' w lp ls 1, '-' using 1:2 title '\KeyFont \emph{$16\times16$}' w lp ls 2, '-' using 1:2 title '\KeyFont \PROP' w lp ls 3 2 39.63 41.81 43.47 3 39.22 41.77 43.39 4 39.84 41.86 43.47 diff --git a/042_ConcealmentAwareEncoding/Results/TableSignallingPerformance.tex b/042_ConcealmentAwareEncoding/Results/TableSignallingPerformance.tex index 73d2e6b..b7733c7 100644 --- a/042_ConcealmentAwareEncoding/Results/TableSignallingPerformance.tex +++ b/042_ConcealmentAwareEncoding/Results/TableSignallingPerformance.tex @@ -91,7 +91,7 @@ \begin{sidewaystable} \newcommand{\somespaces}{} \centering -\caption{Average quality (\ac{psnr} and \ac{wpsnr}) of the affected frames for Low-Delay configuration.} +\caption{Average quality (PSNR and WPSNR) of the affected frames for Low-Delay configuration.} \label{TAB4:SignallingPSNR} \TABLESVSPACE \small diff --git a/References.tex b/References.tex index 3ea7a0d..bb7fa66 100644 --- a/References.tex +++ b/References.tex @@ -1,2 +1,2 @@ \bibliographystyle{IEEEtran} -\bibliography{../../References/ResilienceROPE,../../References/HEVCStreamErrors,../../References/ResilienceLeaky,../../References/VideoTransmissionStandards,../../References/ResilienceUEP,../../References/Concealment,../../References/ConcealmentDriven,../../References/ConcealmentHEVC,../../References/H264,../../References/HEVCErrorResilience,../../References/Personal,../../References/Resilience,../../References/ResilienceLocalisation,../../References/ResilienceRPS,../../References/ResilienceRedundant,../../References/VideoStreaming,../../References/HEVC,../../References/Misc,../../References/OpticalFlow,../../References/ResilienceDataPartitioning,../../References/ResilienceDistortion,../../References/ResilienceIntraRefresh} +\bibliography{../../References//ResilienceDataPartitioning,../../References//ResilienceROPE,../../References//NeuralNetworks,../../References//Resilience,../../References//OpticalFlow,../../References//ResilienceRedundant,../../References//VideoStreaming,../../References//HEVC,../../References//HEVCStreamErrors,../../References//H264,../../References//ResilienceDistortion,../../References//ResilienceLocalisation,../../References//ResilienceIntraRefresh,../../References//Concealment,../../References//VideoTransmissionStandards,../../References//Personal,../../References//ResilienceRPS,../../References//Misc,../../References//HEVCErrorResilience,../../References//ResilienceLeaky,../../References//ConcealmentDriven,../../References//ResilienceUEP,../../References//ConcealmentHEVC} diff --git a/TikzThesisTemplate.pgs b/TikzThesisTemplate.pgs index 3d4be51..a322900 100644 --- a/TikzThesisTemplate.pgs +++ b/TikzThesisTemplate.pgs @@ -1,11 +1,12 @@ \documentclass[phdthesis,en]{estgleiria} \usepackage{carreira_phd_thesis} +\usepackage{lipsum} \usepackage{showframe} \usepackage{balance} -\usepackage[active,tightpage]{preview} -\PreviewEnvironment{tikzpicture} +% \usepackage[active,tightpage]{preview} +% \PreviewEnvironment{tikzpicture} \begin{document} @@ -16,6 +17,6 @@ \end{figure} \balance -The increasing diversity of multimedia applications and services, and the emergence of Ultra-HD formats (\emph{e.g.}, \mbox{4k} or \mbox{8k} resolution) are continuously reinforcing the need for high efficiency video coding efficiency, particularly in +\lipsum[1-2] \end{document} diff --git a/carreira_phd_thesis.sty b/carreira_phd_thesis.sty index 801e282..11fd1ae 100644 --- a/carreira_phd_thesis.sty +++ b/carreira_phd_thesis.sty @@ -51,6 +51,10 @@ \newcommand{\KeyFont}[0]{\scriptsize} \newcommand{\PLRLABEL}{\footnotesize Packet Loss Ratio (\%)} \newcommand{\PSNRLABEL}{\footnotesize PSNR (dB)} +\newcommand{\TABLEPLRLABEL}{Packet Loss Ratio} + +%\usepackage[first=0, last=100]{lcg} +%\newcommand{\random}{\rand\arabic{rand}} %% TikZ \usepackage{tikz} @@ -71,13 +75,13 @@ \usetikzlibrary{hobby} \newcommand{\yslant}{-0.26} \newcommand{\yfshift}{0.05cm} -\definecolor{TikZColorFill}{RGB}{200,200,200} -\definecolor{TikZColorFillDark}{RGB}{95,95,95} +\definecolor{TikZColorFill}{RGB}{220,220,220} +\definecolor{TikZColorFillDark}{RGB}{120,120,120} \definecolor{TikZColorFillProp}{RGB}{159, 161, 249} -\definecolor{colorBackground}{RGB}{194,215,249} -\definecolor{colorBackgroundSecondary}{RGB}{158,195,255} -\definecolor{colorErrors}{rgb}{.16,.40,.40} -\definecolor{colorErrorProp}{rgb}{.209,.705,.740} +%\definecolor{colorBackground}{RGB}{194,215,249} +%\definecolor{colorBackgroundSecondary}{RGB}{158,195,255} +\definecolor{TikZColorBlock0}{RGB}{0,95,120} +\definecolor{TikZColorBlock1}{RGB}{0,160,200} \def\TikZyslant{-0.2} \def\TikZxscale{0.75} \def\TikZyfshift{0.05cm} @@ -96,7 +100,7 @@ % *** FLOAT PACKAGES *** % Coloca sempre as figuras dentro das secções a que pertencem \usepackage{float} -\usepackage[section]{placeins} +%\usepackage[section]{placeins} % *** MATH PACKAGES *** @@ -129,7 +133,8 @@ \newcommand{\FIGSIZE}{0.7\columnwidth} \newcommand{\FIGSIZEDOUBLE}{0.45\columnwidth} \newcommand{\FIGSIZETABLE}{0.5\columnwidth} -\newcommand{\TABLESVSPACE}{\vspace{2mm}} +\newcommand{\TABLESVSPACE}{\vspace{-1mm}} +%\newcommand{\TABLESVSPACE}{} \newcommand{\FIGSVSPACE}{} \newcommand{\optbold}[1]{\textbf{#1}} diff --git a/carreira_phd_thesis.tex b/carreira_phd_thesis.tex index c4e8e75..4e6dca6 100644 --- a/carreira_phd_thesis.tex +++ b/carreira_phd_thesis.tex @@ -7,6 +7,8 @@ \usepackage{carreira_phd_thesis} +\usepackage[colorinlistoftodos]{todonotes} + % correct bad hyphenation here \hyphenation{op-tical net-works semi-conduc-tor} @@ -16,13 +18,14 @@ % \newcommand{\HIGHLIGHT}[1]{\textcolor{red}{#1}} \newcommand{\HIGHLIGHT}[1]{\textcolor{red}{#1}} \newcommand{\HERE}{\textcolor{red}{CHECK HERE!}} +\newcommand{\VERSION}[1]{\textcolor{red}{r#1}} %\includeonly{010_Introduction/Introduction} % \includeonly{021_VideoCoding/VideoCodingReview} %\includeonly{022_ErrorResilience/RobustTransmissionReview} %\includeonly{031_ErrorResilienceEvaluation/ErrorResilienceEvaluation} %\includeonly{041_ImprovedErrorResilience/ImprovedErrorResilience} -\includeonly{042_ConcealmentAwareEncoding/ConcealmentAwareEncoding} +%\includeonly{042_ConcealmentAwareEncoding/ConcealmentAwareEncoding} %\includeonly{043_EndToEndRobustSystem/EndToEndRobustSystem} \begin{document} @@ -41,6 +44,8 @@ \titlefigs{\includegraphics[width=10cm]{lborolodonlogo.png}} % make the title area +\listoftodos + \maketitle \frontmatter @@ -53,9 +58,9 @@ % Certificate of Originality %\includepdf[pages=-, pagecommand=, templatesize={5in}{10in}]{000_Preamble/LU/origin.pdf} -\input{000_Preamble/dedic} -\input{000_Preamble/thanks} -\input{000_Preamble/abstract} +%\input{000_Preamble/dedic} +%\input{000_Preamble/thanks} +%\input{000_Preamble/abstract} %\dominitoc \glsunsetall diff --git a/carreira_phd_thesis_abr.sty b/carreira_phd_thesis_abr.sty index 36467ae..45cd7ad 100644 --- a/carreira_phd_thesis_abr.sty +++ b/carreira_phd_thesis_abr.sty @@ -23,8 +23,6 @@ \customnewacronym{iso}{ISO}{International Organization for Standardization} \glsunset{iso} \customnewacronym{iec}{IEC}{International Electrotechnical Commission} \glsunset{iec} - - \customnewacronym{gop}{GOP}{Group of Pictures} \customnewacronym{qp}{QP}{Quantized Parameter} \customnewacronym{rd}{R-D}{\mbox{Rate-Distortion}} @@ -46,6 +44,9 @@ \customnewacronym{dbf}{DBF}{Deblocking Filter} +\customnewacronym{ld}{LD}{Low-Delay} +\customnewacronym{ra}{RA}{Random-Access} + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% H.264 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -102,7 +103,7 @@ \customnewacronym[longplural={Regions of Interest}]{roi}{ROI}{Region of Interest} \customnewacronym{rope}{ROPE}{Recursive Optimal Per-Pixel Estimation} \customnewacronym{rps}{RPS}{Reference Picture Selection} -\customnewacronym{uep}{UEP}{Unequal Error protection} +\customnewacronym{uep}{UEP}{Unequal Error Protection} \customnewacronym{mdc}{MDC}{Multiple Description Coding} \customnewacronym{fc}{FC}{Frame-Copy} @@ -125,7 +126,8 @@ \customnewacronym{ssim}{SSIM}{Structural Similarity Index Metric} \customnewacronym{fsim}{FSIM}{Feature Similarity Index} - +\customnewacronym{nn}{NN}{Neural Network} +\customnewacronym{cnn}{CNN}{Convolutional Neural Network} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Experiemental Results @@ -134,8 +136,8 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Loss types -\def\LOSTALL{$IDR_{Loss}$} -\def\KEEPINTRA{$IDR_{NoLoss}$} +\def\LOSTALL{$IDR_{Loss}$\xspace} +\def\KEEPINTRA{$IDR_{NoLoss}$\xspace} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Reference methods @@ -185,8 +187,10 @@ \def\CHKB{\mbox{\textit{Chkb}}\xspace} \def\EXP{\mbox{\textit{Exp}}\xspace} +\def\FIXED{\mbox{\textit{Dyn}}\xspace} +%\def\DYN{\mbox{\textit{Prop-MV}}\xspace} +\def\DYN{\mbox{\textit{Dyn}}\xspace} \def\NOREDUND{\mbox{Without MVs}\xspace} -\def\DYN{\mbox{\textit{Prop-MV}}\xspace} \newcommand\REDUND[1]{\mbox{#1\%\ of MVs}\xspace} \newcommand\OPTREDUND[1]{\mbox{\DYN ($\alpha$=#1)}\xspace} @@ -197,6 +201,8 @@ \def\EXPELTEN{\textit{\mbox{Exp+EL10\%}}\xspace} \def\EXPELTWEN{\textit{\mbox{Exp+EL20\%}}\xspace} +\def\BEST{\textit{\mbox{Target}}\xspace} + %% Long Labels \def\FIXEDLAMBDALONG{\FIXEDLAMBDA \mbox{\textit{(Constant $\lambda$)}}\xspace} \def\DYNLAMBDALONG{\DYNLAMBDA \mbox{\textit{(Adaptive $\lambda$)}}\xspace}