diff --git a/021_VideoCoding/VideoCodingReview.tex b/021_VideoCoding/VideoCodingReview.tex index 920d435..cd7cbcd 100644 --- a/021_VideoCoding/VideoCodingReview.tex +++ b/021_VideoCoding/VideoCodingReview.tex @@ -94,7 +94,7 @@ Subsequently, the prediction residue is transformed by linear spatial transform \centering \includegraphics[width=\columnwidth]{021_VideoCoding/Figures/HEVCBlockDiagram} %\input{021_VideoCoding/Figures/HEVCBlockDiagram.tikz} - \caption{Block diagram of an HEVC standard encoder.} + \caption{Picture partitioning and main functional blocks of an HEVC encoder.} \label{fig:HEVCBlockDiagram} \end{figure} @@ -166,7 +166,7 @@ Moreover, introducing random-access points into the bitstream increases its erro \begin{table}[tb] \centering - \caption{Picture categories in HEVC standard.}\label{Table2:PictureTypes} + \caption{Brief description of picture types in HEVC standard.}\label{Table2:PictureTypes} \TABLESVSPACE\small \begin{tabular}{m{1.3cm}|m{5.2cm}|m{6.3cm}} \toprule @@ -242,7 +242,7 @@ Due to the introduction of more flexible random-access points using \ac{cra} pic \begin{figure}[tb] \centering \input{021_VideoCoding/Figures/PictureTypes_Leading_Trailing.tikz} - \caption{Leading and trailing pictures associated with the random-access point $I_{1}$ (letters corresponds to the coding type and the numbers to the coding order).} + \caption{Leading and trailing pictures associated with the random-access point $I_{1}$ (the letters corresponds to the coding type and the numerical indexes to the coding order).} \label{Figure2:LeadingTralingPics} \end{figure} @@ -265,7 +265,7 @@ Although these frames allow higher flexibility in terms of layer switching, they \centering % \includegraphics[width=0.75\columnwidth]{021_VideoCoding/Figures/HEVC/Pictures_TSA} \input{021_VideoCoding/Figures/PictureTypes_TSA.tikz} - \caption{Temporal layers access pictures example with three layers ($T_0$ to $T_2$).} + \caption{Temporal layers access pictures example with three layers, $T_0$ to $T_2$ (the letters corresponds to the coding type and the numerical indexes to the coding order).} \label{Figure2:TSAExample} \end{figure} @@ -540,9 +540,9 @@ The splitting into four \acs{pu} is only allowed when the \ac{cu} size is equal Figure~\ref{Figure2:InterPuPartitions} illustrates the possible \ac{cu} partitioning into \acs{pu}. %One should note that the partition type applied to the \ac{cu} is also applied to divide the \ac{cb} into \acs{pb}. Figure~\ref{Figure2:InterPuPartitions}~(a) shows the symmetric partitions. -The $N \times N$ corresponds to the case where the \ac{cu} is not further divided. -The \ac{hevc} standard defines the horizontal and vertical division into two \acs{pu}, corresponding to the $\frac{N}{2}\times N$ and $N \times \frac{N}{2}$ cases, respectively. -Finally, the $\frac{N}{2} \times \frac{N}{2}$ case results in four \acs{pu}. +The $2N \times 2N$ corresponds to the case where the \ac{cu} is not further divided. +The \ac{hevc} standard defines the horizontal and vertical division into two \acs{pu}, corresponding to the $N\times 2N$ and $2N \times N$ cases, respectively. +Finally, the $N \times N$ case results in four \acs{pu}. The partitions shown in Figure~\ref{Figure2:InterPuPartitions}~(b) correspond to the asymmetric partitioning, which is one of the main improvements introduced in \ac{hevc}, as it allows higher flexibility in motion estimation, leading to higher compression efficiency~\cite{HEVCBlock2012}. These partitions types are only allowed for \acs{cu} with $16\times16$ pixels or larger. diff --git a/022_ErrorResilience/RobustTransmissionReview.tex b/022_ErrorResilience/RobustTransmissionReview.tex index 77c096c..d207a1f 100644 --- a/022_ErrorResilience/RobustTransmissionReview.tex +++ b/022_ErrorResilience/RobustTransmissionReview.tex @@ -96,7 +96,7 @@ Normally, such optimisation is equivalent to the minimisation of the Lagrangian J = D + \lambda R, \end{equation} where $D$ is the distortion of all pixels, $R$ the bitrate and $\lambda$ is the Lagrangian multiplier~\cite{LagragianMultiplier1982}. -Such optimisation only takes into account the coding distortion ($D$), which is only suitable for error-free environments. +Such optimisation only takes into account the coding distortion ($D$), which is only suitable for environments without errors and data loss. However, some alternative approaches use error resilient \ac{rd} optimisation, by taking into consideration possible network losses~\cite{DistortionTrackingH264_ICIP2004}. Other advanced approaches can take advantage of a \ac{rope} of the end-to-end distortion at the encoder-side~\cite{ROPEJun2000}. This may be formulated as follows. @@ -198,7 +198,7 @@ In such approach, the reconstructed reference frames are scaled down by applying \end{equation} where $\hat f_t^i$ is a reference frame, $\alpha$ is the leaky factor used to control the amount of information from the reconstructed frame present in the new frame $\check f_t^i$, and $C$ is a constant value. Leaky prediction has been used in the past for error control, especially in scalable video coding \cite{LeakyPredScalable2002}, where $C$ in~(\ref{EQ2:LeakyPredConstant}) is replaced by a lower layer error-free reconstructed frame. In case of single layer coding, the use of a constant value has significant impact on the coding efficiency leading to higher bitrate, as shown in \cite{LeakyPredConstant1973}. The $\alpha$ parameter allows to control the effect of the constant value, and thus the coding efficiency. -By decreasing $\alpha$, the coding efficiency increases, but the exponential decay in error propagation becomes less significant. For the determination of the $\alpha$ parameter, one can use network information such as packet loss probability $p$ leading to $\alpha=(1-p)$. However, as can be seen in previous studies empirical approach using exhaustive can also be used to optimise the $\alpha$ value for each sequence~\cite{ROPERefGenJan2010}. +By decreasing $\alpha$, the coding efficiency increases, but the exponential decay in error propagation becomes less significant. For the determination of the $\alpha$ parameter, one can use network information such as packet loss probability $p$ leading to $\alpha=(1-p)$. However, as can be seen in previous studies, empirical approaches using exhaustive can also be used to optimise the $\alpha$ value for each sequence~\cite{ROPERefGenJan2010}. % MHMCP_TCSVT2006 @@ -210,8 +210,8 @@ Alternatively, the two previously encoded frames can be used to generate an inte \end{equation} Therefore, it is possible to decrease the error propagation using $\check f_t^i$ as reference frame for prediction, if only one of the original encoded frames is lost (i.e., $\hat f_{t}^i$ or $\hat f_{t-1}^i$). This is due to the fact that $\check f_t^i$ contains information from both $\hat f_{t}^i$ and $\hat f_{t-1}^i$. In~(\ref{EQ2:LeakyPredDual}) $\hat f_{t}^i$ and $\hat f_{t-1}^i$ always correspond to the encoded frames and the generated frames ($\check f_t^i $) are not used to interpolate subsequent frames. This corresponds to a finite impulse response filter. -Although the use of two reference frames for leaky prediction is an effective way to reduce the error propagation with a small penalty in coding efficiency, the use of interpolated frame obtained using (\ref{EQ2:LeakyPredDual}) still leads to a severe error propagation in case of burst frame loss. -In this case both frames would be affect by packet loss, which would reduce the effectiveness of this method. +Although the use of two reference frames for leaky prediction is an effective way to reduce the error propagation with a small penalty in coding efficiency, the use of an interpolated frame in (\ref{EQ2:LeakyPredDual}) still leads to a severe error propagation in case of burst frame loss. +In this case both frames would be affect by packet loss, which would reduce the effectiveness of the method. \begin{figure}[tb] @@ -253,7 +253,7 @@ In the decoder-side, an \ac{ec} method was later used to take advantage of such A network feedback mechanism was proposed in \cite{ErrorResilienceRPSFeb2009} to reduce error propagation at end-user decoders, by dynamically selecting the reference frame only among those correctly received. Figure~\ref{fig2:ReferencePictureSelection} shows an example of an error resilience transcoding scheme, where the original temporal predictions (represented by the solid arrows) are transcoded to use only correctly received frames as reference, i.e., prediction represented by the dashed arrow. This is done based on network feedback. -\HIGHLIGHT{As this technique requires transcoding operations, the complexity increase with the \ac{gop} lengths, as reported in~\cite{ErrorResilienceRPSFeb2009}. Typically, the \ac{gop} length is defined by taking into consideration the video application (i.e., synchronisation, random-access or maximum refresh period allowed) and not by the robust coding scheme.} +\HIGHLIGHT{As this technique requires transcoding operations, the complexity increase with the \ac{gop} length, as reported in~\cite{ErrorResilienceRPSFeb2009}. Typically, the \ac{gop} length is defined by taking into consideration the video application (i.e., synchronisation, random-access or maximum refresh period allowed) and not by the robust coding scheme.} A more advanced scheme may be considered, where only the regions affected by mismatch predictions are processed in the compressed domain, and subsequently transcoded to limit the dependencies from lost frames. To achieve low complexity, \ac{mv} concatenation can be used, in order to obtain new inter-predictions for error-free frames \cite{MVConcatenationAug2011}. @@ -291,7 +291,7 @@ Therefore, there is an unequal contribution of different syntax elements to the Data partition techniques have been developed to provide a mechanism to cope with the unequal importance of coded information by grouping different parts of a compressed bitstream, according to their importance to the decoded quality. Thus, some of these groups can be protected more efficiently than others using \ac{uep} schemes. These are normally associated with \ac{fec} algorithms, which are typically applied at the channel level which is not aware of the video content, leading to a sub-optimal error protection~\cite{UEPMotion_TM2006}. -Using \ac{uep}, a source-channel optimisation is applied, whereby multi-levels of \ac{fec} are used to achieve different levels of error protection, overcoming this problem. +Using \ac{uep}, a source-channel optimisation is applied, whereby multi-levels of \ac{fec} are used to achieve different levels of error protection according to the importance of the coded information. This is presented and discussed with more detail in Sub-section~\ref{SEC22:UEP} In single channel transmission, the most important data or syntax elements (e.g., headers) can be protected with stronger channel codes than the remaining ones. @@ -327,7 +327,7 @@ In the \ac{fmo} type 0 each slice group is defined by a maximum number of \acs{m In the \ac{fmo} type 2, slice groups are defined by rectangular shapes bounded by the top-left and bottom-right corners. Although the slice groups are allowed to overlap each other, each \ac{mb} may only belong to one slice group. Similar to \ac{fmo} type 0, in the types 3, 4 and 5 in Figure~\ref{fig2:FMOTypes}, slice groups are defined by a maximum number of \acs{mb}. -However, for these types the configuration is allowed to change for every frame, contrary of \ac{fmo} type 0. +However, contrary of \ac{fmo} type 0, the configuration is allowed to change every frame for these types In order to keep the signalling overhead affordable, only two slice groups are allowed in these types. The \ac{fmo} type 6 is the most generic case, where the slice groups are explicitly defined using high level packets (i.e., signalling \ac{nal} units), allowing more flexibility at the cost of higher signalling overhead. @@ -342,7 +342,7 @@ The \ac{fmo} type 6 is the most generic case, where the slice groups are explici In the past, \ac{fmo} has been widely used to increase the error robustness of video transmissions in different communication scenarios \cite{FMOErrorResilience2007,FMOResilience2010,FMOResilience2011,FMOROI2012,MotionEnergyDataPart2014}. In~\cite{FMOErrorResilience2007} \ac{fmo} is used to increase the \ac{ec} performance and thus reduce the error propagation in prioritised video transmission. The method divides the \acs{mb} into two slice groups based on an estimation of the \ac{ec} distortion using a two-pass approach. -Then, the slice group which has the most impact on the \ac{ec} distortion is transmitted using a reliable channel, and it is assumed to be received error-free. +Then, the slice group which has the most impact on the \ac{ec} distortion is transmitted using a reliable channel, and it is assumed to be received without errors. The experimental results shows that the proposed approach is able to outperform the dispersed mode (\ac{fmo} type 1)~\cite{FMOErrorResilience2007}. Slice classification can also be based on the weighted sum of the motion vector information and the rate-distortion cost~\cite{FMOResilience2010}. As in \cite{FMOErrorResilience2007}, the \acs{mb} are ranked based on their importance and distributed in the two groups with the same number of blocks. @@ -393,7 +393,7 @@ In \cite{UEPSaliency_TCSVT2010} a method based on a combination of two parameter The scalable video coding framework provides implicit support for \ac{uep} through the layered structures, defined with different levels of quality~\cite{UEPScalable_TCE2008,UEPScalable_TCSVT2010,UEPLayerVideo_TM2014, UEPScalable_ICSPICES2015}. In~\cite{UEPScalable_TCSVT2010}, an algorithm to recursively estimate the total expected distortion in the decoder at the picture-level for each layer is proposed. This method takes into consideration \ac{el} truncation, i.e., a specific layer being discarded, and \ac{ec} to measure an accurate estimation of the end-to-end distortion. The estimation is carried out for every layer in the video stream, in order to apply an optimisation of different levels of error protection across several layers. -\HIGHLIGHT{Also based on scalable coding, the work in~\cite{UEPLayerVideo_TM2014} proposes a technique for finding the optimal \ac{fec} coding rates that leads to correctly decode the \ac{el} when errors affect the base layer.} +Also based on scalable coding, the work in~\cite{UEPLayerVideo_TM2014} proposes a technique for finding the optimal \ac{fec} coding rates that leads to correctly decode the \ac{el} when errors affect the base layer. This prevents discarding correctly received \acs{el} frames due to missing dependencies, increasing the overall video quality. In the context of \ac{hevc}, there are also methods to achieve \ac{uep}. @@ -483,7 +483,7 @@ This section reviews several \ac{ec} methods based on spatial and temporal redun A typical approach to recover missing regions in an image is to estimate the lost regions based on linear interpolation of neighbouring pixels \cite{ErrorConcSpatial1995,ConcTCE2001}. -Figure~\ref{FIG3:pixel_interpolation} shows the estimation process of a lost pixel ($lp$), within the lost region ($R_L$), using the four closest pixels ($p_i$) as reference, and the distance between the lost pixel and the reference ones ($d_i$). The pixel value ($lp$) is recovered using a weighted average of $n$ neighbouring pixels, as follows: +Figure~\ref{FIG3:pixel_interpolation} shows the estimation process of a lost pixel ($lp$), within the lost region ($R_L$), using the four closest pixels ($p_i$) as reference, and the distance between the lost pixel and the reference ones ($d_i$). The pixel value ($lp$) is recovered using a weighted average of $n$ neighbouring pixels, defined as follows: \begin{equation} \label{eq:pixel_interpolation} lp = \left. \sum_{i=1}^n{ \frac{p_i}{d_i} } \middle/ \sum_{i=1}^n{ \frac{1}{d_i} } \right. , @@ -499,7 +499,7 @@ improve the estimation accuracy of the lost region, the edges and image transiti \begin{figure}[tb] \centering \input{./022_ErrorResilience/Figures/LinearInterpolation} -\caption{Lost pixel interpolation from neighbouring error-free pixels. } +\caption{Lost pixel interpolation from neighbouring pixels correctly decoded.} \label{FIG3:pixel_interpolation} \end{figure} @@ -581,7 +581,7 @@ An extension to the previous approach was considered in \cite{PelBasedMVE_TCE200 In \cite{2DConcHybridMotion2010} a \ac{hmve} algorithm was proposed, to improve the accuracy of the \acs{mv} of the \ac{pmve} method, by using the extrapolated vectors at the pixel and block level. Moreover, the \acs{mv} that were wrongly extrapolated are discarded in order to obtain a more accurate MV. In this \ac{ec} method, firstly a pixel-based motion extrapolation is performed and the pixels are organised into three groups. Then, through block-based \ac{mve} two new \acs{mv} are determined: the first one corresponds to the \ac{mv} of the most overlapped block with the current one, and second one is obtained by the weighted average value of the \acs{mv} corresponding to different overlapped blocks. -To discard the wrongly extrapolated \acs{mv} and find the more accurate one from the \ac{mv} set described above, further refined is applied based on the distance between each pair of \acs{mv}. +To discard the wrongly extrapolated \acs{mv} and finding the more accurate ones from the MV set described above, further refinement is applied based on the distance between each pair of \acs{mv}. This algorithm is able to outperform the \ac{pmve} for different packet loss probabilities, as shown by the results in \cite{2DConcHybridMotion2010}. The method was used to recover both intra and inter frames in video sequences compressed with the \ac{h264}, achieving \ac{psnr} gains above $1.09$ dB in the erroneous frames. diff --git a/031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNR.tex b/031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNR.tex index 463271e..c1a6e5a 100644 --- a/031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNR.tex +++ b/031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNR.tex @@ -6,7 +6,7 @@ \toprule \multirow{2}{*}{\textbf{Sequence}} & \multirow{2}{*}{\vbox{\centering \textbf{TMVP configuration}}} & -\multirow{2}{*}{\vbox{\centering \textbf{Error free}}} & +\multirow{2}{*}{\vbox{\centering \textbf{\NOLOSS}}} & \multicolumn{2}{c}{\textbf{\TABLEPLRLABEL}}\\ & & & \textbf{1\%} & \textbf{5\%} \\ \midrule diff --git a/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex b/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex index c6ac75e..4c9ada8 100644 --- a/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex +++ b/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex @@ -44,7 +44,7 @@ There are some studies presented in the past regarding the error robustness of t A subjective evaluation study is presented in~\cite{SubjectiveEvalSep2013,QoEHEVCMay2014} revealing that packet loss ratios higher than $3\%$ have significant effect on the perceived video quality. The vulnerability of \ac{mv} prediction is investigated and its impact on the error robustness is evaluated in \cite{DynamicTMVP2011}. Moreover, in the same study, a method to introduce \ac{mv} refresh points is proposed, achieving average \ac{psnr} gains of approximately 3~dB. -Summarising, the decrease in error robustness is mostly due to the strong data dependencies imposed by the highly complex prediction modes of \ac{hevc}. Since these are selected according to a rate distortion optimisation criterion that assumes error-free transmission, the decoder faces different problems whenever errors or data loss occur in the transmission network \cite{ROPE2007}. +Summarising, the decrease in error robustness is mostly due to the strong data dependencies imposed by the highly complex prediction modes of \ac{hevc}. Since these are selected according to a rate distortion optimisation criterion that assumes \ac{plf} transmission, the decoder faces different problems whenever errors or data loss occur in the transmission network \cite{ROPE2007}. The following sub-sections present an error robustness evaluation of \ac{hevc} focused on the impact of the novel coding techniques introduced in the standard, extending the generic studies presented in previous works. This study provides relevant insights for research directions and motivation to investigate solutions for the error robustness problem in \ac{hevc}. The following case-studies are covered in this experimental evaluation: @@ -105,8 +105,9 @@ Moreover, the ratio of inter-coded \acs{cu} of each coded sequence using the def To simulate the lossy transmission environment, each frame is divided into several slices and each slice encoded and packetized into one \ac{nal} unit, which are then transmitted as the payload of independent packets. In these simulations a whole packet is lost whenever an error occurs, originating a lost slice. \HIGHLIGHT{Random packet loss was simulated using a two-state Markov model, derived from the classical Gilbert-Elliot \cite{GilbertModel_1960, ElliotModel_1963}, which can be used to describe error patterns in real time video applications~\cite{GilbertElliotModelPlr_2008}. In this study a simplified version of the Gilbert-Elliot model was used by only defining the loss probability without control over burst loss events. -The application of this model results in loss patterns with approximately the same average burst loss as the ones provided in \cite{JCTVCLossSim_2012}. -For each test condition, 50 runs were performed and the average \ac{psnr} was measured. Four different \acs{plr} are used resulting in the characteristics presented in Table~\ref{TAB3:LossPatterns}.} +The application of this model resulted in loss patterns with approximately the same average burst loss as the ones provided in \cite{JCTVCLossSim_2012}. +For each test condition, 50 runs are tested and the average quality is measured. +Four different \acs{plr} with the characteristics presented in Table~\ref{TAB3:LossPatterns} are used.} The remaining configurations have been kept at their default values, as given in the common test conditions~\cite{HEVCCommonTests2012}. @@ -124,14 +125,14 @@ In the first set of experiments the impact of the maximum \ac{cu} size in the de \end{table} -Table~\ref{Table3:Quality_BlockSize} shows the average \ac{psnr} (dB) obtained for the three test conditions and for three different packet loss ratios. Results for the error-free scenario shows a consistent quality increase as the maximum \ac{cu} size increases, +Table~\ref{Table3:Quality_BlockSize} shows the average \ac{psnr} (dB) obtained for the three test conditions and for three different packet loss ratios. Results for the \ac{plf} scenario shows a consistent quality increase as the maximum \ac{cu} size increases, % up to a maximum of 1.41~dB for the Tennis sequence when comparing the maximum \ac{cu} size of $64\times64$ with $16\times16$. This confirms the principle used by the \ac{hevc} standard, whereby larger coding units lead to higher coding efficiency. For the Tennis sequence, which has the highest \ac{ti} parameter (see Table~\ref{TableTestSequences}), a maximum of 1.41~dB is achieved. Moreover, it is also noticeable that sequences with higher amount of inter-coded blocks take less advantage of the larger \acs{cu}. -This is noticeable in the results of the BQSquare sequence (i.e., highest ratio of inter-coded blocks and lowest quality gain in the error-free case). -Overall, comparing the results for the error-free and data loss cases, it is noticeable that similar quality gains are obtained, mostly due to the superior coding efficiency of larger \acs{cu}. +This is noticeable in the results of the BQSquare sequence (i.e., highest ratio of inter-coded blocks and lowest quality gain in the \ac{plf} case). +Overall, comparing the results for the \ac{plf} and data loss cases, it is noticeable that similar quality gains are obtained, mostly due to the superior coding efficiency of larger \acs{cu}. In most cases the quality gains slightly decrease for increasing \acs{plr}. Therefore, one may conclude that the error robustness of \ac{hevc} streams is not affected by limiting the maximum \ac{cu} size. @@ -153,7 +154,7 @@ Therefore, one may conclude that the error robustness of \ac{hevc} streams is no Figure~\ref{Figure3:BlockPartitionBask} illustrates an example of the frame partitions for a region of the Basketball Drill sequence for two cases: (i) maximum \ac{cu} size of $64\times64$ and (ii) maximum \ac{cu} size of $16\times16$. As shown, in the moving regions, e.g., moving player, the partitions obtained for both cases have a similar pattern, mainly composed of \acs{cu} with small size. Larger \acs{cu} are only used in the floor, which has a lower spatial complexity and suffers small changes between frames, therefore, easier to compress with $64\times64$ \acs{cu}. Moreover, those regions can be more accurately recovered, by using neighbouring information correctly decoded (spatial and temporal), thus not incurring in severe error propagation. -Therefore, the overall error robustness is not affected by reducing the maximum \ac{cu} size. One may conclude that the use of larger coding units is able to improve the overall \ac{hevc} efficiency in error-free and error prone communications. +Therefore, the overall error robustness is not affected by reducing the maximum \ac{cu} size. One may conclude that the use of larger coding units is able to improve the overall \ac{hevc} efficiency communications networks with and without packet loss. @@ -176,7 +177,7 @@ Figure~\ref{Figure3:QualityvsIntra} shows the average decoded video quality (\ac Results in the figure indicate that as the distance between intra frames increases, the decoded video quality significantly decreases. The quality decreasing is especially noticeable for intra periods higher than 32 frames, indicating that temporal distances greater than approximately one second are not suitable for video streaming under error-prone conditions. When a high temporal distance between intra frames is used, the quality decreasing is more noticeable in the People On Street and BQSquare sequences, which have higher spatial complexity (see values for \ac{si}), leading to a reduction of 13 dB and 14 dB, respectively. -As spatial complexity increases, the prediction residue energy also increases, which leads to more erroneous information in case of data loss, reducing the video quality under errors. \HIGHLIGHT{Finally, based on the result of the confidence interval one may conclude that for different error patterns the average \ac{psnr} of all frames does not significantly changes (maximum interval of 1 dB approximately).} +As spatial complexity increases, the prediction residue energy also increases, which leads to more erroneous information in case of data loss, reducing the video quality under errors. \HIGHLIGHT{Finally, based on the confidence interval one may conclude that for different error patterns the average \ac{psnr} of all frames does not significantly changes (maximum interval of 1 dB approximately).} One should also note from the results of Figure~\ref{Figure3:QualityvsIntra} that, using a small distance between intra frames is not always more efficient. Since intra frames require higher amount of bits, the overall coding efficiency is reduced, resulting in lower overall quality in case of data loss, when comparing with other cases. @@ -210,7 +211,7 @@ Although the use of smaller slices increases the overhead required for slice hea This is due to the lack of refresh points within a frame, resulting in errors that affect larger regions of the frame, which are propagated to the subsequent inter predicted frames. The results for of \textit{SS} (slice partitioning disabled) reveals that this approach also leads to lower error robustness when comparing with those cases where the frames are divided into multiple slices. -\HIGHLIGHT{Moreover, in this case the 95\% confidence interval increases, due to the fact that each packet corresponds to whole frames, which vary in size. Therefore, depending on the affected frame, the amount of lost data will vary leading to a different impact of the recovered quality.} +\HIGHLIGHT{Moreover, in this case the 95\% confidence interval increases, due to the fact that each packet corresponds to a whole frame, which vary in size. Therefore, the amount of lost data varies depending on the affected frame, leading to a different impact of the recovered quality.} Although higher coding efficiency is achieved in this mode, the lower error robustness leads to an overall poor performance under frame or slice loss conditions. It is worthwhile to notice that the results for Kendo sequence do not follow the main trend. This can be explained by the low bitrate required to compress the video signal, which is the lowest when compared with the other test sequences (see Table~\ref{TableTestSequences}). @@ -294,7 +295,7 @@ This is the effect of breaking the temporal dependencies between \acs{mv}, which In order to further evaluate the impact of the temporal \ac{mv} dependencies on the error propagation, the \ac{hevc} streams were subject to random packet losses. Table~\ref{Table3:AvgPSNRWithoutTMVP} presents the average quality results, measured using the \ac{psnr}. -As expected, the results indicate that under error-free conditions higher quality is achieved by enabling the \ac{tmvp} candidate, since higher compression efficiency is obtained because more \ac{mv} predictors are used. +As expected, the results indicate that in the \ac{plf} case higher quality is achieved by enabling the \ac{tmvp} candidate, since higher compression efficiency is obtained because more \ac{mv} predictors are used. This is true for both test conditions, \emph{i.e.}, \ac{ld} and \ac{ra}. However, in the presence of errors, the video quality significantly decreases due to low error robustness when temporal \ac{mv} candidates are used. Alternatively, average quality up to 5.17 dB of \ac{psnr} is achieved when the temporal \ac{mv} predictors are disabled in the \ac{ld} configuration. @@ -362,17 +363,20 @@ Therefore, coding efficiency is not significantly penalised. \subsection{Performance evaluation} \label{sec:ProposedMethodEvaluation} -\begin{figure}[tb] - \centering - \input{031_ErrorResilienceEvaluation/DynamicTmvp/PSNRvsFrameSingle.tex} - \caption{Quality obtained with the proposed scheme, \TIDR and reference HEVC when the frame \#5 is missing for Kendo and BQSquare sequences.} - \label{Figure3:ProposedMethodEvaluationSingle} -\end{figure} + The performance of the proposed scheme (\PROP) was evaluated against two other approaches: reference \ac{hevc} with TMVP enabled and the \TIDR method proposed in \cite{DynamicTMVP2011}. % To evaluate the proposed method, the same experimental setup as presented in Section~\ref{Section3:ErrorResilienceEvaluation} was used. +\begin{figure}[tb] + \centering + \input{031_ErrorResilienceEvaluation/DynamicTmvp/PSNRvsFrameSingle.tex} + \caption{Quality obtained with the proposed scheme, \TIDR and reference HEVC when the frame \#5 is missing for Kendo and BQSquare sequences.} + \label{Figure3:ProposedMethodEvaluationSingle} +\end{figure} + + \subsubsection{Error propagation under single loss events} Firstly, the performance of the proposed method was measured under a single loss event (i.e., only one frame is affected by errors). @@ -383,15 +387,17 @@ The results also show the benefits of using the proposed method over the \TIDR m While the \TIDR method is only able to recover from the frame loss after the refreshing frame, \emph{i.e.}, frame \#8, especially for the BQSquare sequence, the proposed method is able to recover right after the frame loss. + \subsubsection{Quality evaluation under random errors} \begin{table}[tbp] - \centering - \caption{Average standard deviation of the PSNR (dB) for the proposed scheme, \TIDR and reference HEVC.} \label{Table3:AverageStdPSNR} - \TABLESVSPACE - \input{./031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNRSTD.tex} + \centering + \caption{Average standard deviation of the PSNR (dB) for the proposed scheme, \TIDR and reference HEVC.} \label{Table3:AverageStdPSNR} + \TABLESVSPACE + \input{./031_ErrorResilienceEvaluation/DynamicTmvp/TableDynamicTMVPPSNRSTD.tex} \end{table} + Secondly, further simulations were performed to evaluate the effectiveness of the proposed error resiliency method under realistic packet loss patterns. %As in previous experiments, each coded frame is transmitted in one packet. Table~\ref{Table3:AverageStdPSNR} illustrates the average standard deviation of the PSNR, for three of the test sequences. @@ -400,6 +406,7 @@ However, reducing the temporal dependencies of the \acs{mv} using the proposed m The proposed method is able to reduce the quality variations up to 0.94 dB for Kendo sequence (3$\%$ of packet loss) compared to the~\TIDR~method. Therefore, since the proposed method achieves the lowest \ac{psnr} standard deviation, one may conclude that it is able to reduce the video quality variations leading to a better visual quality. + \begin{table}[tbp] \centering \caption{BD-PSNR (dB) differences for the proposed method and \TIDR at different frame loss conditions.} \label{Table3:AverageResultsProposed} @@ -408,9 +415,8 @@ Therefore, since the proposed method achieves the lowest \ac{psnr} standard dev Table~\ref{Table3:AverageResultsProposed} shows the Bjontegaard's average \ac{psnr} increase \cite{BJResults2001}. The results are obtained taking the standard \ac{hevc} encoder as reference. -% The results are obtained taking as reference the standard \ac{hevc} encoder, in order to evaluate the quality difference when temporal \ac{mv} candidates are disabled in both error free and under error-prone conditions. The results presented in Table~\ref{Table3:AverageResultsProposed} show that the proposed method presents practically the same rate-distortion performance when compared to the \TIDR method. -In general, the average quality, in an error-free scenario, only decreases 0.028 dB in comparison to the \TIDR method and 0.034 dB in comparison to the reference encoder. +In general, the average quality, in the \ac{plf} case, only decreases 0.028 dB in comparison to the \TIDR method and 0.034 dB in comparison to the reference encoder. However, the proposed method widely outperforms the reference \TIDR technique in terms of error resiliency performance, achieving in average higher overall quality under error-prone conditions. The effectiveness of the proposed method increases at higher packet loss rates (\emph{e.g.},~$5\%$), achieving an average gain of 0.64 dB ($1.34 - 0.70$), and a maximum gain of 1 dB ($1.44-0.43$) for the Book Arrival sequence , when compared to the \TIDR technique. Higher gains are achieved when compared with the reference \ac{hevc} encoder, as shown in Table~\ref{Table3:AverageResultsProposed}. diff --git a/031_ErrorResilienceEvaluation/QualityEvaluation/TableCUSizes.tex b/031_ErrorResilienceEvaluation/QualityEvaluation/TableCUSizes.tex index cce3cef..40d5eeb 100644 --- a/031_ErrorResilienceEvaluation/QualityEvaluation/TableCUSizes.tex +++ b/031_ErrorResilienceEvaluation/QualityEvaluation/TableCUSizes.tex @@ -7,7 +7,7 @@ \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & % \multirow{2}{*}{\vbox{\centering \textbf{Condition}}} & \multirow{2}{*}{\vbox{\centering \textbf{Maximum\ \ CU size}}} & -\multirow{2}{*}{\vbox{\centering \textbf{Error free}}} & +\multirow{2}{*}{\vbox{\centering \textbf{\NOLOSS}}} & \multicolumn{3}{c}{\textbf{\TABLEPLRLABEL}}\\ & & & \textbf{1\%} & \textbf{5\%} &\textbf{10\%}\\ \hline diff --git a/031_ErrorResilienceEvaluation/QualityEvaluation/TableFilters.tex b/031_ErrorResilienceEvaluation/QualityEvaluation/TableFilters.tex index 948a447..3ada114 100644 --- a/031_ErrorResilienceEvaluation/QualityEvaluation/TableFilters.tex +++ b/031_ErrorResilienceEvaluation/QualityEvaluation/TableFilters.tex @@ -5,7 +5,7 @@ \toprule \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & \multirow{2}{*}{\vbox{\centering \textbf{\ac{sao}}}} & -\multirow{2}{*}{\vbox{\centering \textbf{Error Free}}} & +\multirow{2}{*}{\vbox{\centering \textbf{\NOLOSS}}} & \multicolumn{4}{c}{\textbf{Packet loss rate}}\\ & & & \textbf{1\%} & \textbf{3\%} & \textbf{5\%} &\textbf{10\%}\\ \hline diff --git a/031_ErrorResilienceEvaluation/QualityEvaluation/TableMVPs.tex b/031_ErrorResilienceEvaluation/QualityEvaluation/TableMVPs.tex index b3ca551..53cfadb 100644 --- a/031_ErrorResilienceEvaluation/QualityEvaluation/TableMVPs.tex +++ b/031_ErrorResilienceEvaluation/QualityEvaluation/TableMVPs.tex @@ -5,7 +5,7 @@ % \toprule % \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & % \multirow{2}{*}{\vbox{\centering \textbf{Condition}}} & -% \multirow{2}{*}{\vbox{\centering \textbf{Error Free}}} & +% \multirow{2}{*}{\vbox{\centering \textbf{\NOLOSS}}} & % \multicolumn{4}{c}{\textbf{Packet loss rate}}\\ % & & & \textbf{1\%} & \textbf{3\%} & \textbf{5\%} &\textbf{10\%}\\ % \hline @@ -78,7 +78,7 @@ %\newcommand{\colwidth}{1.5cm} %\begin{tabular}{p{2.8cm}|x{2.3cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} %\toprule -%\multirow{2}{*}{\centering\textbf{Sequence}} & \multirow{2}{*}{\vbox{\centering\textbf{TMVP}}} & \multirow{2}{*}{\vbox{\centering\textbf{Error Free}}} & \multicolumn{3}{c}{\textbf{\TABLEPLRLABEL}} \\ +%\multirow{2}{*}{\centering\textbf{Sequence}} & \multirow{2}{*}{\vbox{\centering\textbf{TMVP}}} & \multirow{2}{*}{\vbox{\centering\textbf{\NOLOSS}}} & \multicolumn{3}{c}{\textbf{\TABLEPLRLABEL}} \\ %& & & \textbf{1\%} & \textbf{5\%} & \textbf{10\%} \\ %\hline %\multirow{\nlines}{*}{\vbox{Basketbal Drill}} @@ -127,7 +127,7 @@ \newcommand{\colwidth}{1.5cm} \begin{tabular}{p{2.8cm}|x{2.3cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} \toprule - \multirow{2}{*}{\centering\textbf{Sequence}} & \multirow{2}{*}{\vbox{\centering\textbf{TMVP}}} & \multirow{2}{*}{\vbox{\centering\textbf{Error Free}}} & \multicolumn{3}{c}{\textbf{\TABLEPLRLABEL}} \\ + \multirow{2}{*}{\centering\textbf{Sequence}} & \multirow{2}{*}{\vbox{\centering\textbf{TMVP}}} & \multirow{2}{*}{\vbox{\centering\textbf{\NOLOSS}}} & \multicolumn{3}{c}{\textbf{\TABLEPLRLABEL}} \\ & & & \textbf{1\%} & \textbf{5\%} & \textbf{10\%} \\ \hline \multicolumn{6}{c}{Low-delay configuration (LD)} \\ [2pt] diff --git a/031_ErrorResilienceEvaluation/TablesSequences.tex b/031_ErrorResilienceEvaluation/TablesSequences.tex index dc7e36f..111532f 100644 --- a/031_ErrorResilienceEvaluation/TablesSequences.tex +++ b/031_ErrorResilienceEvaluation/TablesSequences.tex @@ -42,7 +42,7 @@ {Kristen and Sara \newline $1280\times720@60Hz$} & 25.6 / 6.30 & 93.1 & 15.00 & Two persons talking to each other with moderate motion \\ \hline {Park Scene \newline $1920\times1080@24Hz$} & 31.3 / 11.6 & 92.0 & 10.00 & Moderate motion with cyclists passing across the scene \\ \hline {Party Scene \newline $832\times480@50Hz$} & 52.8 / 11.4 & 89.8 & 4.50 & Moderate motion with three children playing \\ \hline -{People on Street \newline $2560\times1600@24Hz$} & 40.0 / 25.4 & 87.4 & 12.00 & With point capture of people moving; high motion and texture complexity \\ \hline +{People on Street \newline $2560\times1600@24Hz$} & 40.0 / 25.4 & 87.4 & 12.00 & Elevated capture point of people moving; high motion and texture complexity \\ \hline {Race Horses \newline $832\times480@30Hz$} & 43.7 / 24.4 & 79.7 & 8.00 & High motion with several horse riders \\ \hline {Tennis \newline $1920\times1080@30Hz$} & 20.3 / 45.3 & 76.2 & 5.00 & High motion with one moving person in the scene \\ \hline {Traffic \newline $2560\times1600@30Hz$} & 28.4 / 22.3 & 93.0 & 6.00 & Elevated camera capturing highway with several cars with moderate speed \\ diff --git a/041_ImprovedErrorResilience/ImprovedErrorResilience.tex b/041_ImprovedErrorResilience/ImprovedErrorResilience.tex index 16acffe..e0a6550 100644 --- a/041_ImprovedErrorResilience/ImprovedErrorResilience.tex +++ b/041_ImprovedErrorResilience/ImprovedErrorResilience.tex @@ -25,7 +25,7 @@ This chapter is organized as follows. Section~\ref{Section4:Problem} presents th \label{Section4:Problem} As previously mentioned, when \ac{hevc} bitstreams are subject to transmission losses, the decoded video signal incurs in significant degradation of both objective and subjective quality. -This is mostly due to the strong decoding dependencies imposed by the highly complex prediction modes, which are selected according to \ac{rd} optimisation criteria, which always assume error-free transmission~\cite{ROPE2007}. +This is mostly due to the strong decoding dependencies imposed by the highly complex prediction modes, which are selected according to \ac{rd} optimisation criteria, which always assume transmission without errors~\cite{ROPE2007}. Therefore, high compression efficiency is achieved, but resilience to errors and data loss is penalised. \begin{figure}[tb] @@ -143,7 +143,7 @@ Note that an exponential cost increase is used in (\ref{eq:Lagrangian}) in order \begin{figure}[t] \centering \includegraphics[width=0.8\columnwidth]{./041_ImprovedErrorResilience/Figures/ExponentialWeight} -\caption{Exponential weight as a function of the global and local deviations} +\caption{Exponential weight as a function of the global and local deviations.} \label{fig:weightfct} \end{figure} @@ -251,7 +251,7 @@ In this section the performance of the proposed two-stage architecture is evalua As references for comparison, three other methods are used: (i) a fixed checkerboard approach (\CHKB), (ii) the reference picture generation method based on leaky predictions proposed in \cite{ROPERefGenJan2010}, identified as ``\AVG'', and (iii) long-term reference frames (\LONG). The \LONG method only uses the key frames as reference, i.e., in the \ac{ra} configuration only one out of eight frames is used as reference for prediction, allowing a more robust transmission, as the reference frames are less likely to be hit by errors. The experimental setup used in this section is the same as the one previously described in Section~\ref{Section3:ErrorResilienceEvaluation}. -Two test cases were considered for simulation: (i) in \emph{\LOSTALL} all frames may be equally affected by packet loss, including \ac{idr} frames; (ii) in \emph{\KEEPINTRA} \ac{idr} frames are assumed to be prioritised in the network and delivered error-free, or using some combined approach as recently proposed in \cite{ErrorResilienceHeterogenousNetworks2016}. +Two test cases were considered for simulation: (i) in \emph{\LOSTALL} all frames may be equally affected by packet loss, including \ac{idr} frames; (ii) in \emph{\KEEPINTRA} \ac{idr} frames are assumed to be prioritised in the network and delivered without errors packet loss, or using some combined approach as recently proposed in \cite{ErrorResilienceHeterogenousNetworks2016}. At the decoder-side, \ac{ec} is based on motion compensation from previously decoded frames. The motion field of a missing frame is obtained by applying \ac{mv} extrapolation from the two closest neighbours. Afterwards, the final \acs{mv} are selected based on the residual energy of their original prediction~\cite{HEVCMCResidueDec2013}. The evaluation of the encoding-stage is carried out separately from the streaming-stage. @@ -329,7 +329,7 @@ The proposed method is also able to outperform the \ac{hevc} reference software In comparison to the reference \ac{hevc}, quality improvements up to $1.42$~dB at PLR=$10\%$ are obtained for Kendo sequence.%, while the average gain across all test sequences is 1~dB. When comparing the results for both test cases, it is noticeable that the proposed method is able to outperform the reference methods for both cases, regardless whether \ac{idr} frame loss is allowed or not (\LOSTALL and \KEEPINTRA, respectively). -As the \ac{plr} increases, the gains of the proposed method decrease in comparison with the case where the \ac{idr} frames are delivered error-free (i.e., case \KEEPINTRA), due to the higher error propagation of the missing \ac{idr} frame. Moreover, the quality gains are lower in the \LOSTALL case when comparing with \KEEPINTRA for the same \ac{plr}, for both the \ac{ld} and \ac{ra}. +As the \ac{plr} increases, the gains of the proposed method decrease in comparison with the case where the \ac{idr} frames are delivered without errors (i.e., case \KEEPINTRA), due to the higher error propagation of the missing \ac{idr} frame. Moreover, the quality gains are lower in the \LOSTALL case when comparing with \KEEPINTRA for the same \ac{plr}, for both the \ac{ld} and \ac{ra}. Taking into account the above results and the spatio-temporal information shown in Table~\ref{TableTestSequences}, one can observe that the performance of the proposed method is lower for sequences with high spatial details and low temporal activity (e.g., BQSquare and Park Scene), which indicates higher effectiveness for video content with higher motion activity, as can be seen by the results of Race Horses and Tennis sequences. @@ -348,25 +348,25 @@ Taking into account the above results and the spatio-temporal information shown The influence of $\gamma$ parameter used in (\ref{eq:KFactor}) on the video quality was also evaluated, in order to check how the cost constrains affect the performance of the proposed method. Figure~\ref{fig:QualityVsGamma} shows the average quality gains of the proposed method in comparison to the reference \ac{hevc} (\REF) across all test sequences, for different values of $\gamma$. -For the error-free case (PLR=0\%), the results in this figure show that increasing $\gamma$ leads to a decrease in the coding efficiency, especially in the \ac{ld} configuration. +For the \ac{plf} case, the results in this figure show that increasing $\gamma$ leads to a decrease in the coding efficiency, especially in the \ac{ld} configuration. The reason is that higher values of $\gamma$ lead to higher values of $W(r_{\phi})$, therefore the reference picture selection cost is severely constrained by the exponential weights. Since $T_{ID}$ is constant for all frames in the \ac{ld}, higher weights are used for all frames ($T_{ID}$ is used to reduce the weights in case of hierarchical coding) and the encoder is not able to select the best reference frame in the \ac{rd} sense. In the case where packet loss occurs, higher values of $\gamma$ lead to improved error resilience. Since higher weights are used in the \ac{rd} cost, the encoder is constrained to distribute the use of the reference frames of different \acs{pu}, reducing the error propagation. For very high values (e.g., $\gamma=1000$) the encoder is forced to uniformly select the reference frames, resulting in a fixed checkerboard pattern. \HIGHLIGHT{Overall, the $\gamma$ parameter is able to control the reference frame selection constrains, -thus achieving better trade-off between improved error resilience and loss of coding efficiency. The value of this parameter should be selected accordingly with the conditions regarding particular applications, e.g., loss ratios or network bandwidth.} +thus achieving better trade-off between improved error resilience and loss of coding efficiency. The value of this parameter should be selected accordingly with the requirements and condition of the streaming application, e.g., network bandwidth or packet loss ratio.} -\subsubsection{BD-PSNR under error-free conditions} +\subsubsection{BD-PSNR under packet loss free conditions} This sub-section evaluates the \ac{rd} penalty incurred by the proposed method in comparison with \CHKB, as well as the other reference methods. -Table~\ref{tab:bd_error_free} shows the Bjontegaard's Delta PSNR (BD-PSNR) values compared to the reference \ac{hevc} encoder for different coding configurations in error-free decoding. Four \acs{qp} were used from the common test conditions: 22, 27, 32, 37. +Table~\ref{tab:bd_error_free} shows the Bjontegaard's Delta PSNR (BD-PSNR) values compared to the reference \ac{hevc} encoder for different coding configurations in error-free video decoding. Four \acs{qp} were used from the common test conditions: 22, 27, 32, 37. \begin{table}[tb] \centering \newcommand{\resultswidth}{1.5cm} - \caption{BD-PSNR (dB) in error-free conditions of the proposed reference frame selection algorithm (\PROP) in comparison with existing methods.} + \caption{BD-PSNR (dB) in comparison with the reference HEVC for the proposed reference frame selection algorithm (\PROP) in comparison with existing methods, for the case without errors.} \TABLESVSPACE \footnotesize \label{tab:bd_error_free} @@ -388,7 +388,7 @@ The results in Table~\ref{tab:bd_error_free} indicate a small loss of coding ef The fixed checkerboard approach leads to an average quality reduction of $1.22$~dB for the \ac{ld} ($0.32$~dB for \ac{ra}), while the proposed method only loses $0.82$~dB and $0.24$~dB in the \ac{ld} and \ac{ra} configurations, respectively. These results clearly show that dynamically selecting the reference frames is more efficient than using a fixed pattern, confirming the previous results for error-prone conditions. Moreover, in comparison to \LONG and the method in~\cite{ROPERefGenJan2010}, the proposed method is still more efficient, because it consistently achieves better results. -Comparing, the results for both error-free and error-prone conditions, one may conclude that the proposed method not only achieves higher quality gains, but also reduces the impact on coding efficiency. +Comparing, the results for both packet loss free and error-prone conditions, one may conclude that the proposed method not only achieves higher quality gains, but also reduces the impact on coding efficiency. Overall, the loss in the quality compared to the reference \ac{hevc} is considered acceptable, given the increase in robustness obtained in lossy transmission. %As expected, this is due to the sub-optimal \ac{rd} optimisation of the proposed method, because the cost function also includes the cost associated to the choice of the reference frame, in addition to the rate and distortion. diff --git a/041_ImprovedErrorResilience/Results/PSNRversusGama.tex b/041_ImprovedErrorResilience/Results/PSNRversusGama.tex index 6a7a840..3b63d01 100644 --- a/041_ImprovedErrorResilience/Results/PSNRversusGama.tex +++ b/041_ImprovedErrorResilience/Results/PSNRversusGama.tex @@ -26,7 +26,7 @@ set grid mxtics set yrange[-1:2.5] set title '\TitleFont Low-Delay Configuration' -plot '-' using 1:2 title '\KeyFont Error-free' w lp ls 1, '-' using 1:2 title '\KeyFont 1.0\% PLR' w lp ls 2, '-' using 1:2 title '\KeyFont 3.0\% PLR' w lp ls 3, '-' using 1:2 title '\KeyFont 5.0\% PLR' w lp ls 4, '-' using 1:2 title '\KeyFont 10.0\% PLR' w lp ls 5 +plot '-' using 1:2 title '\KeyFont \NOLOSS' w lp ls 1, '-' using 1:2 title '\KeyFont 1.0\% PLR' w lp ls 2, '-' using 1:2 title '\KeyFont 3.0\% PLR' w lp ls 3, '-' using 1:2 title '\KeyFont 5.0\% PLR' w lp ls 4, '-' using 1:2 title '\KeyFont 10.0\% PLR' w lp ls 5 .001 -.034 .01 -.034 .1 -.230 @@ -95,7 +95,7 @@ e set yrange[-0.3:1.5] set title '\TitleFont Random-Access Configuration' -plot '-' using 1:2 title '\KeyFont Error-free' w lp ls 1, '-' using 1:2 title '\KeyFont 1.0\% PLR' w lp ls 2, '-' using 1:2 title '\KeyFont 3.0\% PLR' w lp ls 3, '-' using 1:2 title '\KeyFont 5.0\% PLR' w lp ls 4, '-' using 1:2 title '\KeyFont 10.0\% PLR' w lp ls 5 +plot '-' using 1:2 title '\KeyFont \NOLOSS' w lp ls 1, '-' using 1:2 title '\KeyFont 1.0\% PLR' w lp ls 2, '-' using 1:2 title '\KeyFont 3.0\% PLR' w lp ls 3, '-' using 1:2 title '\KeyFont 5.0\% PLR' w lp ls 4, '-' using 1:2 title '\KeyFont 10.0\% PLR' w lp ls 5 .001 -.012 .01 -.012 .1 -.058 diff --git a/041_ImprovedErrorResilience/Results/RefPicSelectionPSNRDifferentialvsGamaAverageThesis.plt b/041_ImprovedErrorResilience/Results/RefPicSelectionPSNRDifferentialvsGamaAverageThesis.plt index c1c9c69..f192c47 100644 --- a/041_ImprovedErrorResilience/Results/RefPicSelectionPSNRDifferentialvsGamaAverageThesis.plt +++ b/041_ImprovedErrorResilience/Results/RefPicSelectionPSNRDifferentialvsGamaAverageThesis.plt @@ -27,7 +27,7 @@ set xtics 0.01, 10, 1000 set logscale x 10 set mxtics 10 set grid mxtics -plot '-' using 1:2 title 'Error-free' w lp ls 1, '-' using 1:2 title '1.0% PLR' w lp ls 2, '-' using 1:2 title '3.0% PLR' w lp ls 3, '-' using 1:2 title '5.0% PLR' w lp ls 4, '-' using 1:2 title '10.0% PLR' w lp ls 5 +plot '-' using 1:2 title '\NOLOSS' w lp ls 1, '-' using 1:2 title '1.0% PLR' w lp ls 2, '-' using 1:2 title '3.0% PLR' w lp ls 3, '-' using 1:2 title '5.0% PLR' w lp ls 4, '-' using 1:2 title '10.0% PLR' w lp ls 5 .001 -.034 .01 -.034 .1 -.230 @@ -100,7 +100,7 @@ set xtics 0.01, 10, 1000 set logscale x 10 set mxtics 10 set grid mxtics -plot '-' using 1:2 title 'Error-free' w lp ls 1, '-' using 1:2 title '1.0% PLR' w lp ls 2, '-' using 1:2 title '3.0% PLR' w lp ls 3, '-' using 1:2 title '5.0% PLR' w lp ls 4, '-' using 1:2 title '10.0% PLR' w lp ls 5 +plot '-' using 1:2 title '\NOLOSS' w lp ls 1, '-' using 1:2 title '1.0% PLR' w lp ls 2, '-' using 1:2 title '3.0% PLR' w lp ls 3, '-' using 1:2 title '5.0% PLR' w lp ls 4, '-' using 1:2 title '10.0% PLR' w lp ls 5 .001 -.012 .01 -.012 .1 -.058 diff --git a/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex b/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex index c2622be..e2a1a7b 100644 --- a/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex +++ b/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex @@ -107,14 +107,14 @@ E(x,y) = \sum_{x} \sum_{y} \bigg| f_{0}\big(x,y\big) - f_{-1}\big( x + v_x(x,y), Besides a full search algorithm, where a block-based approach is used to search for the best \acs{mv} that minimise the sum of absolute errors, three other approaches were tested to evaluate their effectiveness in \ac{ec} algorithms. These three methods are as follows: \begin{enumerate} - \item a method based on the total variation of the pixel values using the minimization of the \mbox{L-1} norm~\cite{DualFlow2016}; + \item a method based on the duality between total pixel difference and regularisation term~\cite{DualFlow2016}; \item a method which approximates the neighbourhood of each pixel by quadratic polynomials on both frames~\cite{PolyFlow2003}; - \item a method based on a combination of feature matching and total variation~\cite{DeepFlow2013}. %\todo{check here!} + \item a method based on a combination of feature matching and total pixel difference~\cite{DeepFlow2013}. %\todo{check here!} \end{enumerate} -\subsubsection{Motion estimation based on total variation and regularisation (\textit{DualFlow})} +\subsubsection{Motion estimation based on a regularisation term (\textit{DualFlow})} -In this method the motion field is estimated based on the minimization of the total variation of pixel values measured using the L-1 norm \cite{DualFlow2007,DualFlow2016}, taking into account the Horn and Schunck smoothness condition~\cite{HornSchunckOpticalFlow1981}. +In this method the motion field is estimated based on the minimization of the total difference of pixel values measured using the L-1 norm \cite{DualFlow2007,DualFlow2016}, taking into account a regularisation term obtained from the Horn and Schunck smoothness condition~\cite{HornSchunckOpticalFlow1981}. This condition forces the estimated field to be regular across neighbouring pixels by penalising deviations in a quadratic sense. Using this method, for each pixel $\mathbf{p}$, the motion field $v(\mathbf{p})$ is obtained by minimising the following energy function, \begin{equation}\label{EQ5:DualFlow} @@ -133,7 +133,7 @@ In this method only quadratic polynomials are considered, expressed as: \begin{equation}\label{EQ5:PolyFlow} f(\mathbf{p}) \sim \mathbf{p}\mathbf{A}_1\mathbf{p}^T+\mathbf{b}_1\mathbf{p}^T+C_1 \end{equation} -where $\mathbf{p} = [x\ y]$, $\mathbf{A_1}$ is a symmetric matrix, $\mathbf{b_1}$ is a row vector and $C_1$ is a scalar. The coefficients are estimated using weighted least squares fit to the signal values in the neighbourhood. The use of quadratic polynomials was chosen primarily due to simplicity and to keep the computational complexity as low as possible. However, as stated in \cite{farneback2002polynomial} the extension to higher degree is straightforward. +where $\mathbf{p} = [x\ y]$, $\mathbf{A_1}$ is a symmetric matrix, $\mathbf{b_1}$ is a row vector and $C_1$ is a scalar. The coefficients are estimated using weighted least squares fit to the signal values in the neighbourhood. The use of quadratic polynomials was chosen primarily due to simplicity and to keep the computational complexity as low as possible, accepting a small penalty in the estimation accuracy. However, as stated in \cite{farneback2002polynomial} the extension to higher degree is straightforward. Since the result of a polynomial expansion is such that each neighbourhood is approximated by a polynomial, it is necessary to analyse what happens if a polynomial undergoes an ideal translation. Let's consider the signal $f_1(\mathbf{p})$ represented by~(\ref{EQ5:PolyFlow}) and a signal $f_2$ which corresponds to a displacement of $f_1$, given by $\mathbf{d}=[v_x \ v_y]$. This can be expressed by the following: @@ -155,7 +155,7 @@ This allows to find the displacement $\mathbf{d}$ for any signal dimensions by s \subsubsection{Large displacement optical flow with deep matching (\textit{DeepFlow})} -In this method the motion field is estimated based on extension of the total variation methods using descriptor (feature) matching. +In this method the motion field is estimated based on extension of the total pixel difference methods using descriptor (feature) matching. This allows the optical flow estimation to accurately find large displacements, because it is not fully bound by any constrains, such as the Horn \& Such presented in~(\ref{EQ5:DualFlow})\cite{LargeDisplacementOptFlow_TPAMI2011}. Moreover, the correspondences between descriptions are used to apply a coarse-to-fine wrapping, which avoids local minimum values. The combination of both approaches increases the matching accuracy. @@ -187,7 +187,7 @@ The smoothness term is a robust penalisation of the gradient of the flow, simila This section presents a simulation study of the different \ac{ec} schemes previously described. This evaluation aims at finding the suitable methods to be used for efficient reconstruction of the missing frames in case of data loss. The two \ac{ec} methods based on the correctly received \acs{mv} are tested, i.e., \ac{mc} and \ac{mve}, while the \ac{fc} method is used as reference for comparison. -Moreover, the four different approaches to estimating the motion field for the missing frame are compared: (i) full search based on the minimisation of the sum of absolute errors (\FULLSEARCH), (ii) optical flow based on the duality between total variation and smoothness term (\DUALFLOW), (iii) optical flow based on polynomial approximations (\POLYFLOW) and (iv) optical flow based on deep matching (\DEEPFLOW). +Moreover, the four different approaches to estimating the motion field for the missing frame are compared: (i) full search based on the minimisation of the sum of absolute errors (\FULLSEARCH), (ii) optical flow based on the duality between total difference and regularisation term (\DUALFLOW), (iii) optical flow based on polynomial approximations (\POLYFLOW) and (iv) optical flow based on deep matching (\DEEPFLOW). Table~\ref{TAB5:ECPerformaceComparison} shows the average quality (\ac{psnr}) obtained for the different methods under investigation. The absolute value of \ac{psnr} is shown for the \FC method, while the differential values are shown for the remaining methods. @@ -208,7 +208,7 @@ According to the results of Table~\ref{TAB5:ECPerformaceComparison} (columns 5 t Results for \FULLSEARCH method reveal that only minimising the sum of absolute errors between two consecutive frames might not always be directly related with objects' motions, which results in inaccurate motion field to be used in the reconstruction of lost frames. Finally, results from Table~\ref{TAB5:ECPerformaceComparison} reveal that the \DEEPFLOW algorithm leads to the most accurate motion field, in comparison with the other methods. It is able to achieve an average quality gain of 1.79 dB, and up to 3.39 dB for Book Arrival sequence, where \POLYFLOW only achieves 3.05 dB. -Since \POLYFLOW combines different techniques also used by other algorithms in study, e.g., total variation also used by \DUALFLOW, it is expected to achieve accurate estimations and consequently higher \ac{ec} reconstruction quality. +Since \POLYFLOW combines different techniques also used by other algorithms in study, e.g., total difference of pixel values also used by \DUALFLOW, it is expected to achieve accurate estimations and consequently higher \ac{ec} reconstruction quality. Moreover, using feature matching in combination with smoothing conditions, the \DEEPFLOW algorithm is able to correctly estimate large displacements, which results in improved performance for video signals with higher motion, e.g., Tennis sequence. \subsection*{Remarks} @@ -382,7 +382,7 @@ Adaptive arithmetic coding~\cite{ArithmeticCodingJun1994} is used to efficiently \end{figure} -\subsection{Enhancement-layer: \ac{ec} residue coding} +\subsection{Enhancement-layer: EC residue coding} As described earlier in this section, in order to allow further improvement of recovered slices, besides the best \ac{ec} mode, the proposed encoding scheme also allows encoding a controlled amount of \ac{ec} residue using the scalable extension of \ac{hevc} (see Figure~\ref{FIG5:BlockDiagram}). The enhancement layer module encodes the difference between the coded frame ($\hat f$) and its counterpart recovered by using the optimal \ac{ec} mode ($\tilde f$). @@ -424,7 +424,8 @@ In this section the performance of the proposed method is evaluated only for int Since intra \acs{cu} result in higher amount of bits, they also require a higher number of packets, making them more prone to errors and data loss. Results indicate that for an intra-period of 16 frames, approximately 42\% of the packets correspond to intra coded slices, which is a non-negligible probability of one of them being hit by transmission errors which would lead to packet loss. Moreover, by limiting the proposed method to intra frames, the amount of required overhead is reduced. -In this case a constant $\lambda_0=10$ was used, without taking into consideration saliency estimation. This value results from an exhaustive search targetting acceptable quality gains with a low overhead (i.e., average overhead under 5\%). +In this case a constant $\lambda_0=10$ was used, without taking into consideration saliency estimation. +This value results from an exhaustive search targetting acceptable quality gains with a low overhead (i.e., average overhead under 5\%). Since a relevant performance indicator is the error propagation over temporally predicted frames, this was evaluated using a single intra-frame loss. Figures~\ref{FIG5:singlelosslowdelay} and~\ref{FIG5:singlelossramdon} show the \ac{psnr} results when frame Intra-frame $\#16$ is missing, for \ac{ld} and \ac{ra} encoding configurations, respectively. @@ -696,7 +697,7 @@ Since the proposed \ac{ec}-aware scheme is able to achieve higher quality withou An advantage of using a feedback channel is the low extra complexity required to implement robust coding because the encoder receives specific information about transmission errors. This allows straightforward spatio-temporal localization of the lost data without computational effort. -Table~\ref{tab:overall_gain_ratio} shows the average quality differences (\textbf{$\mathbf{\Delta}$ PSNR})between the reference method (\REF) and the proposed one plus three other previous works. Two different test conditions are shown in the table: error-free and PLR=10\%. Moreover, an efficiency measure $R$ is defined, as the ratio between the quality gains obtained under packet loss and the quality decrease in the error-free case, i.e., +Table~\ref{tab:overall_gain_ratio} shows the average quality differences (\textbf{$\mathbf{\Delta}$ PSNR})between the reference method (\REF) and the proposed one plus three other previous works. Two different test conditions are shown in the table: \ac{plf} and PLR=10\%. Moreover, an efficiency measure $R$ is defined, as the ratio between the quality gains obtained under packet loss and the quality decrease in the \ac{plf} case, i.e., \begin{equation} R = \frac{\Delta PSNR(\ac{plr}=10\%)}{-\Delta PSNR(Error\text{-}free)}. \end{equation} @@ -704,8 +705,8 @@ R = \frac{\Delta PSNR(\ac{plr}=10\%)}{-\Delta PSNR(Error\text{-}free)}. The results shown in Table~\ref{tab:overall_gain_ratio}, demonstrate that the effectiveness of the \ac{ec}-aware coding scheme is higher than all other methods under comparison. In fact, not only the absolute \ac{psnr} difference (i.e., \ac{psnr} gain) is higher for the same \ac{plr}, but also the efficiency of the overhead is also higher. This means that the overhead bits spent to encode the signalling information and the enhancement residue carry more useful information than the other methods. -For instance, in comparison with the method in \cite{DualFrameME2015} the proposed method leads to higher quality degradation in the error-free case. However, for PLR=10\% it clearly outperforms the method in \cite{DualFrameME2015}, achieving an higher $R$, indicating a superior efficiency. -Results in Table~\ref{tab:overall_gain_ratio} also show that the small loss of coding efficiency in error-free transmission is clearly surpassed by the quality gains under error-prone transmission conditions. +For instance, in comparison with the method in \cite{DualFrameME2015} the proposed method leads to higher quality degradation in those cases without errors. However, for PLR=10\% it clearly outperforms the method in \cite{DualFrameME2015}, achieving an higher $R$, indicating a superior efficiency. +Results in Table~\ref{tab:overall_gain_ratio} also show that the small loss of coding efficiency in packet loss free transmission is clearly surpassed by the quality gains under error-prone transmission conditions. The methods in \cite{ROPERefGenJan2010,DualFrameME2015} are based on modified predictions, while the method in \cite{VPxResiliencePCS2016} uses redundant data without any further optimisation. Both of these approaches require low additional computational complexity. However, as observed in the results shown in Table~\ref{tab:overall_gain_ratio}, the lower additional complexity also corresponds to lower quality gains in case of packet loss. @@ -718,14 +719,14 @@ The proposed method uses \ac{ec} techniques with different levels of computati \begin{table}[t] \newcommand{\somespaces}{} \centering -\caption{Comparison of the proposed EC-aware robust coding architecture against existing in error-free and packet loss environments.}\label{tab:overall_gain_ratio} +\caption{Comparison of the proposed EC-aware robust coding architecture against existing techniques in environments with and without packet loss.}\label{tab:overall_gain_ratio} \footnotesize % \resizebox{\columnwidth}{!}{ \begin{tabular}{p{2.4cm}x{2.4cm}x{2.4cm}x{2cm}} % \begin{tabular}{l|c|c|c} \toprule \multirow{2}{*}{\vbox{\textbf{Method}}} & \multicolumn{2}{c}{\textbf{$\mathbf{\Delta}$ PSNR (dB)}} & \multirow{2}{*}{\centering \vbox{$\mathbf{R}$ }} \\ -& {Error-free} & {PLR=10\%} & \\ +& {\NOLOSS} & {PLR=10\%} & \\ % \hline \midrule \AVG & -0.88 & 1.64 & 1.9 \\ \midrule %\hline diff --git a/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex b/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex index 0398777..b17a435 100644 --- a/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex +++ b/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex @@ -11,7 +11,7 @@ % |x{\colwidth}x{\colwidth}x{\colwidth}x{\colwidth}x{\colwidth}} % \toprule % \multirow{3}{*}{\vbox{\centering \textbf{Sequence}}} -% % & \multicolumn{4}{c||}{\textbf{Error-free}} +% % & \multicolumn{4}{c||}{\textbf{\NOLOSS}} % & \multicolumn{5}{c|}{\textbf{PLR=3\%}} & \multicolumn{5}{c}{\textbf{PLR=10\%}}\\ % & \multirow{2}{*}{\vbox{\centering \REF}} & \multirow{2}{*}{\vbox{\centering \MC}} % & \multirow{2}{*}{\vbox{\centering \AVG}} & \multicolumn{2}{c|}{\PROP} diff --git a/043_DynamicErrorRobustness/DynamicErrorRobustness.tex b/043_DynamicErrorRobustness/DynamicErrorRobustness.tex index a32d5d0..b06c859 100644 --- a/043_DynamicErrorRobustness/DynamicErrorRobustness.tex +++ b/043_DynamicErrorRobustness/DynamicErrorRobustness.tex @@ -14,7 +14,7 @@ The effectiveness of different error resilience techniques is rather variable de Therefore, it is not straightforward to select a single technique capable of guaranteeing the highest efficiency for every possible scenario and also for the whole range of networking conditions. This chapter addresses this problem by proposing an adaptive error robustness scheme that optimises the efficiency of the techniques previously described in Chapters~\ref{Chapter:ErrorResilienceTechniques}~and~\ref{Chapter:ConcealmentAwareTechniques}. At the encoder-side, a pre-trained neural network predicts the video stream that is able to achieve the highest decoded quality for particular loss cases, among various video streams encoded with different levels of compression efficiency and error robustness. -The stream selection is based on different input parameters that in\textsl{}clude information from the video signal, the coded stream and the transmission network. +The stream selection is based on different input parameters that include information from the video signal, the coded stream and the transmission network. Using this approach the overall performance of the proposed adaptive method significantly increases, achieving a better trade-off between compression efficiency and error robustness. This chapter is organised as follows. @@ -70,19 +70,18 @@ The remaining configuration was kept as described in Section~\ref{Section3:Error \subsubsection{Preliminary simulation results and discussion} +\begin{figure}[!t] + \centering + \input{043_DynamicErrorRobustness/Results/Quality_PLR_Methods.tex} + \vspace{-4mm} + \caption{Average decoded video quality for the bitstreams encoded with the main bitrate (bps).} + \label{Figure6:AverageQualityMain} +\end{figure} Figures~\ref{Figure6:AverageQualityMain}~and~\ref{Figure6:AverageQualityReduction} show the average decoded video quality (\ac{psnr}) obtained for different \acs{plr}. Figure~\ref{Figure6:AverageQualityMain} shows the results obtained with the test sequence encoded using a constant target bitrate listed in Table~\ref{Table6:TestMaterialCharacteristics}, referred to as the main bitrate. Results on Figure~\ref{Figure6:AverageQualityReduction} were obtained by reducing the main bitrate by 40\%. This is used to evaluate the impact of the network bandwidth on the decoded video quality. The results shown in the figures correspond to the same test sequences, in order to allow an accurate and fair comparison. -\begin{figure}[!t] - \centering - \input{043_DynamicErrorRobustness/Results/Quality_PLR_Methods.tex} - \vspace{-4mm} - \caption{Average decoded video quality for the bitstreams encoded with the main bitrate (bps).} - \label{Figure6:AverageQualityMain} -\end{figure} - \begin{figure}[!t] \centering @@ -123,7 +122,7 @@ Figure~\ref{Figure6:BlockDiagram} illustrates the proposed architecture, whose m Firstly, the input signals are coded using different levels of error protection. Then, an optimisation algorithm is used to decide which bitstream should be transmitted at any given instant, based on different parameters. The selected stream is expected to obtain the highest quality at the decoder-side for the given conditions. -A reference video coder is used to compress the video signal using highly efficient coding techniques, obtaining the best \ac{rd} performance for error-free environments. +A reference video coder is used to compress the video signal using highly efficient coding techniques, obtaining the best \ac{rd} performance for packet loss free environments. This is used as the reference bitstream. Subsequently, a transcoding operation is applied to this reference bitstream aided by the input video signal to efficiently select the reference frames used for inter prediction. The reference frame selection mechanism proposed in Chapter~\ref{Chapter:ErrorResilienceTechniques} is used for this purpose. @@ -136,8 +135,10 @@ The proposed approach results in four different bitstreams with different charac (iii) \SMEXPELTEN: robust video coding using selective reference frames and optimal \ac{ec} method signalling plus 10\% of bitrate allocated for the scalable enhancement layer \ac{ec} residue; (iv) \SMEXPELTWEN: same as the previous bitstream but with 20\% bitrate allocated for the scalable~\ac{el}. -The video stream with the optimal error robustness is selected using an \ac{nn}, which receives different parameters extracted from the video signal, the compressed bitstream and information from the network conditions. -From the input video, relevant information regarding the spatial and temporal complexity is computed using existing metrics. Moreover, further information is also extracted from the coded bitstream using a stream parser. +The video stream with the optimal error robustness is selected using an \ac{nn}, which receives different parameters extracted from the input video, the compressed bitstream and information from the network conditions. +The relevant video characteristics that are useful for this algorithm are known measures of spatial and temporal complexity~\cite{ITUTP9102008}, defined in Section~\ref{Section6:InputParameters}. +These are computed by the functional block defined as "Video analyser" in Figure~\ref{Figure6:BlockDiagram}. +Moreover, further information is also extracted from the coded bitstream using a stream parser. Finally, the packet loss ratio of the transmission channel is also used. These parameters are fed into the optimisation block to select the best bitstream for transmission. The optimisation process is described in detail in the following section. @@ -177,6 +178,7 @@ Then, the following \acs{plr} were used to cover a wide range of loss ratios gi This results in a total of \NumCases test conditions. \subsection{Input parameters} +\label{Section6:InputParameters} A set of different parameters is used to predict the optimal stream that achieves the highest quality in specific network conditions. Based on these inputs the proposed scheme should be able to select the appropriate robust coding stream that should be transmitted. @@ -233,7 +235,7 @@ Transmission networks with higher loss ratios require a higher bit allocation fo These parameters were compared against the optimal stream, selected from \SMEC to \SMEXPELTWEN. This aims at finding the correlation between the input parameters and the output variable, i.e., target stream. In this study the streams are characterised by a number between 1 and 4. -Table~\ref{Table6:Correlation} shows the absolute Person Correlation coefficient obtained across all sequences, bitrates and \acs{plr}. +Table~\ref{Table6:Correlation} shows the absolute Person Correlation coefficient \cite{PersonCorrelation_2015} obtained across all sequences, bitrates and \acs{plr}. The table shows the results for the parameters previously described, as well as, a new parameter obtained from the average magnitude of the optical flow measured using the method in~\cite{DeepFlow2013} (\textit{OPT\_FLOW}). Results in Table~\ref{Table6:Correlation} show that the selected parameters are all related with the optimal streams, with the ratio of inter-coded blocks having the highest correlation. Moreover, one can also notice that the parameters related with the motion activity, i.e., \ac{ti} and \textit{AVG\_MV}, are also correlated with the output variable. Since both \textit{AVG\_MV} and \textit{OPT\_FLOW} represent similar information (motion insensitive), only \textit{AVG\_MV} was used in the proposed method, due to its higher Person Correlation coefficient. @@ -244,7 +246,8 @@ Since both \textit{AVG\_MV} and \textit{OPT\_FLOW} represent similar informatio A \ac{nn} is a computational model based on the structure and functions of biological neural networks. Mainly, a \ac{nn} consists of three different layers: the input layer which receives data from an external environment, the hidden layer which processes the inputs and an output layer~\cite{NeuralNetworks1994}. In the past \acs{nn} have been widely used in video processing applications such as, quality evaluation~\cite{VideoQualityNN_TNN2006}, video analysis~\cite{VideoAnalysisNN_2018} and video coding~\cite{RateDistortionNN_VCIP2017,HEVCComplexityNN_TIP2018}. -In the proposed method a feed-forward network is used, which connects every neuron in a given layer to the neurons of the subsequent layer. +Neural networks are used in the proposed method since they are universal approximators, which can fit arbitrary complex data that could not be approximated using analytical models~\cite{NeuralNetworks_2016}. +In this work a feed-forward network is used, which connects every neuron in a given layer to the neurons of the subsequent layer. \begin{figure}[tb] \centering @@ -261,10 +264,10 @@ Subsequently, the network transforms the input data through a series of neurons The output layer comprises four neurons corresponding to each available video stream from \SMEC to \SMEXPELTWEN (different error resilience techniques). Finally, the output with the highest value is selected and the corresponding stream is used for transmission, since it is the one expected to achieve the highest video quality for a particular scenario. -In the proposed approach a shallow \ac{nn} using three layer, with 32 nodes in the hidden layer is used for simplicity and to avoid over-fitting, due to the limited test cases~\cite{NeuralNetworkDesign_2014}. +In the proposed approach a shallow \ac{nn} is used for simplicity and to avoid over-fitting, due to the limited test cases~\cite{NeuralNetworkDesign_2014}. As shown in Figure~\ref{FIG6:NeuralNetwork}, the \ac{nn} using in this method has three layer: input layer with 6 nodes, an hidden layer with 32 nodes and an output layer with 4 nodes. By avoiding over-fitting, it is guaranteed that the proposed method would continue to work in future test conditions, not used for the network training. -The number of nodes in the hidden layer was determined by comparing the performance obtained for different number of neurons in this layer using the cross-entropy performance metric~\cite{CrossEntropy_2005}. -However, different network topologies with different number of neurons in each layer could also be used to predict between different error resilience techniques, as in the proposed method. +The number of nodes in the hidden layer was determined by comparing the performance obtained for different cases using the cross-entropy performance metric~\cite{CrossEntropy_2005}. +%However, different network topologies with different number of neurons in each layer could also be used to predict between different error resilience techniques, as in the proposed method. \subsubsection{Training and validation} @@ -379,7 +382,7 @@ It is only used to establish the optimal case for comparison purposes. The results in the figure allow the comparison between three relevant cases: proposed stream selection method, the optimal case and the reference method. As shown in Figure~\ref{Figure6:ProposedQuality}, the proposed method clearly outperforms the reference \ac{hevc} for all \acs{plr}. -Moreover, in the error-free case (\ac{plr}=0), it leads to imperceptible quality reduction, since the proposed approach is able to select the optimal stream, thus disabling any robust coding technique when the transmission network does not have packet losses. +Moreover, in the packet loss free case (\ac{plr}=0), it leads to imperceptible quality reduction, since the proposed approach is able to select the optimal stream, thus disabling any robust coding technique when the transmission network does not have packet losses. In case of data loss, the results show that the proposed method is able to match the quality obtained by the \BEST case in most cases, revealing that the \ac{nn} is able to efficiently select the optimal stream for transmission. \begin{table}[tb] @@ -436,29 +439,30 @@ These results also confirm that the proposed method is able to select the optima The proposed method achieves a quality gain up to 4.46 dB in comparison with the reference \ac{hevc}, outperforming the quality achieved with the stream \SMEXPELTWEN (i.e., 4.19 dB). This confirms the advantage of the proposed method to adaptively select among different streams with different levels of error protections. +\begin{table}[tb] + \centering + \caption{Average quality gain ($\Delta$PSNR in dB) in comparison with the reference \ac{hevc} for video sequences not used in the NN training.} + \label{Table6:AverageGainBasketBallDrive} + \TABLESVSPACE + \newcommand{\colwidth}{1.7cm} + \begin{tabular}{p{3cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} + \toprule + \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & \multicolumn{5}{c}{\textbf{Robust coded stream}} \\ + & \SMEC & \SMEXP & \SMEXPELTEN & \SMEXPELTWEN & \PROP \\ \hline + Basketball Drive & +2.10 & +2.68 & +3.36 & +3.45 & +3.55 \\ \hline + Kendo & +1.92 & +3.24 & +3.94 & +4.19 & +4.46 \\ \hline + Race Horses & +0.85 & +1.02 & +0.88 & +0.75 & +1.06 \\ + \bottomrule + \end{tabular} +\end{table} + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Summary} \label{Section6:Summary} -\begin{table}[tb] - \centering - \caption{Average quality gain ($\Delta$PSNR in dB) in comparison with the reference \ac{hevc} for video sequences not used in the NN training.} - \label{Table6:AverageGainBasketBallDrive} - \TABLESVSPACE - \newcommand{\colwidth}{1.7cm} - \begin{tabular}{p{3cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} - \toprule - \multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & \multicolumn{5}{c}{\textbf{Robust coded stream}} \\ - & \SMEC & \SMEXP & \SMEXPELTEN & \SMEXPELTWEN & \PROP \\ \hline - Basketball Drive & +2.10 & +2.68 & +3.36 & +3.45 & +3.55 \\ \hline - Kendo & +1.92 & +3.24 & +3.94 & +4.19 & +4.46 \\ \hline - Race Horses & +0.85 & +1.02 & +0.88 & +0.75 & +1.06 \\ - \bottomrule - \end{tabular} -\end{table} - In this chapter an adaptive error robustness method is proposed, based on reference frame selection and \ac{ec}-aware optimisation. By adaptively selecting a different error robustness technique at the encoder-side using a pre-trained \ac{nn}, higher quality gains are achieved when comparing against the individual reference techniques. %Moreover, the proposed method is able to outperform those cases where a single error resilience technique is used. The overall results demonstrate that the proposed method is able to increase the reconstruction video quality in case of errors. diff --git a/043_DynamicErrorRobustness/Figures/JointlyRobustnessDiagram.tikz b/043_DynamicErrorRobustness/Figures/JointlyRobustnessDiagram.tikz index ca81da9..8a5dd9f 100644 --- a/043_DynamicErrorRobustness/Figures/JointlyRobustnessDiagram.tikz +++ b/043_DynamicErrorRobustness/Figures/JointlyRobustnessDiagram.tikz @@ -85,7 +85,7 @@ \path[MainArrow] (Scalable0) -- (Mux.south |- Scalable0) node[OutputLabels] {\SMEXPELTWEN}; \path[MainArrow] (Mux) -- (Out) node[above, midway, text width=2cm, text centered] {Optimal \\ Robust Steam}; -\node [text centered, text width=1.5cm, minimum height=1cm, below=.3 of Mux.east, xshift=1.9cm] (Channel) {Channel Conditions}; +\node [text centered, text width=1.5cm, minimum height=1cm, below=.3 of Mux.east, xshift=1.9cm] (Channel) {Network Conditions}; \path[SignallArrow] (StreamParser) -| (Mux.-6) node (b1) [pos=.3, anchor=center] {\footnotesize /} node[right=-.2 of b1, yshift=-6, inner sep=0] {\footnotesize 3}; diff --git a/060_Appendix/Papers.tex b/060_Appendix/Papers.tex index 7eb0658..e7ac8d8 100644 --- a/060_Appendix/Papers.tex +++ b/060_Appendix/Papers.tex @@ -17,10 +17,11 @@ J.~Carreira, P.~Assuncao, S.~Faria, E.~Ekmekcioglu, and A.~Kondoz, ``Error \line \begin{description} \item[\namedlabel{Conf:DynamicTMVP}{C1}] J.~Carreira, V.~D. Silva, E.~Ekmekcioglu, A.~Kondoz, P.~Assuncao, and S.~Faria, ``Dynamic motion vector refreshing for enhanced error resilience in HEVC,'' in \emph{22nd European Signal Processing Conference (EUSIPCO)}, Sep. 2014, pp. 281--285. \newline ISBN:978-0-9928-6261-9 +\pagebreak \item[\namedlabel{Conf:MotionVectorRedundancies}{C2}] J.~Carreira, E.~Ekmekcioglu, A.~Kondoz, P.~Assuncao, S.~Faria, and V.~D. Silva, ``Selective motion vector redundancies for improved error resilience in HEVC,'' in \emph{IEEE International Conference on Image Processing (ICIP)}, Oct. 2014, pp. 2457--2461. \newline DOI:10.1109/ICIP.2014.7025497 \item[\namedlabel{Conf:ErrorResilience}{C3}] -J.~Carreira, S.~Faria, P.~Assuncao, E.~Ekmekcioglu, and A.~Kondoz, ``Error resilience analysis of motion vector prediction in HEVC,'' in \emph{Conference on Telecommunications (Conftele)}, Oct. 2015, pp. 1--4. +J.~Carreira, S.~Faria, P.~Assuncao, E.~Ekmekcioglu, and A.~Kondoz, ``Error resilience analysis of motion vector prediction in HEVC,'' in \emph{Conference on Telecommunications (Conftele)}, Oct. 2015, pp. 1--4. \newline URL: \url{https://www.it.pt/Publications/DownloadPaperConference/20422} \item[\namedlabel{Conf:Chkb}{C4}] J.~Carreira, P.~Assuncao, S.~Faria, E.~Ekmekcioglu, A.~Kondoz, and H.Lim, ``Reference picture selection using checkerboard pattern for resilient video coding,'' in \emph{IEEE International Conference on Visual Communications and Image Processing (VCIP)}, Dec. 2015, pp. 1--5. \newline DOI:10.1109/VCIP.2015.7457852 \item[\namedlabel{Conf:ConcAware}{C5}] diff --git a/carreira_phd_thesis_abr.sty b/carreira_phd_thesis_abr.sty index 872e96b..00269e0 100644 --- a/carreira_phd_thesis_abr.sty +++ b/carreira_phd_thesis_abr.sty @@ -126,7 +126,6 @@ \customnewacronym{si}{SI}{Spatial Information} \customnewacronym{ti}{TI}{Temporal Information} -\customnewacronym{plr}{PLR}{Packet Loss Ratio} \customnewacronym{mse}{MSE}{Mean Square Error} \customnewacronym{psnr}{PSNR}{Peak Signal-to-Noise Ratio} \customnewacronym{wpsnr}{WPSNR}{Weighted Peak Signal-to-Noise Ratio} @@ -146,7 +145,10 @@ \def\LOSTALL{$IDR_{Loss}$\xspace} \def\KEEPINTRA{$IDR_{NoLoss}$\xspace} -\def\NOLOSS{No Loss} +\def\NOLOSS{PLF} + +\customnewacronym{plr}{PLR}{Packet Loss Ratio} +\customnewacronym{plf}{PLF}{Packet Loss Free} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Reference methods