diff --git a/010_Introduction/HEVCTransmissionStack.pdf b/010_Introduction/HEVCTransmissionStack.pdf deleted file mode 100644 index d26c152..0000000 Binary files a/010_Introduction/HEVCTransmissionStack.pdf and /dev/null differ diff --git a/010_Introduction/HEVCTransmissionStack.tikz b/010_Introduction/HEVCTransmissionStack.tikz deleted file mode 100644 index 8466a4c..0000000 --- a/010_Introduction/HEVCTransmissionStack.tikz +++ /dev/null @@ -1,66 +0,0 @@ -\usetikzlibrary{positioning,calc} -\usetikzlibrary{arrows,matrix} -\usetikzlibrary{fit} -\usetikzlibrary{intersections} - -\def\scale{1} - - -\begin{tikzpicture}[ - font=\footnotesize, - auto, - node distance=1cm, - scale=\scale, - every node/.style={scale=\scale}, - template/.style={text centered}, - block/.style={template, rectangle, draw, text width=1.8cm}, - arrow/.style={template, draw, -latex}, - point/.style={template, draw, fill=black, circle,inner sep=0, minimum size=3} -] - -\tikzstyle{mainBlock} = [block] -\tikzstyle{secondBlock} = [block, minimum height=2cm] - - - -\node[secondBlock, anchor=north west] -(MPEG2) {MPEG-2 Transport Stream}; - -\node[secondBlock, text width=1.2cm, right=0 of MPEG2] -(RTP) {RTP}; - -\node[secondBlock, minimum height=1cm, right=0 of RTP.north east, anchor=north west] -(DASH) {MPEG DASH}; - -\node[secondBlock, minimum height=1cm, right=0 of DASH] -(Prog) {Progressive Download}; - -\node[secondBlock, inner sep=0, fit={(DASH) (Prog)}, minimum height=1cm, below=0 of DASH.south west, anchor=north west] -(ISOMediaFormat) {ISO Base Media \\ File Format}; - -\node[secondBlock, right=0 of Prog.north east, anchor=north west] -(MPEG2Systems) {MPEG-2 Systems (TS/PS)}; - -\node[mainBlock, inner sep=0, fit={(MPEG2) (MPEG2Systems)}, above=0 of MPEG2.north west, anchor=south west, minimum height=1cm, -label=center:HEVC / MPEG-H Part 2 / ITU-T Rec H.265] -(HEVC) {}; - -% \node [draw](chapter1) {\textit{Chapter 1}}; -% \node [draw,right=of chapter1] (paper1) {\textit{Paper 1}}; -% -% \node [draw, below=of chapter1](chapter2) {\textit{Chapter 2}}; -% \node [draw,right=of chapter2] (paper2) {\textit{Paper 2}}; -% -% \node [draw,below=of paper2] (paper3) {\textit{Paper 3}}; -% \node [draw,below=of paper3] (paper4) {\textit{Paper 4}}; -% -% \path let -% \p1=(paper3.north), \p2=(paper4.south), \n1 = {\y1-\y2-\pgflinewidth} -% in -% node [draw, below=of chapter2,minimum height=\n1](chapter3) {\textit{Chapter 3}}; - - -\end{tikzpicture} - - - diff --git a/010_Introduction/HEVCTransmissionStackOverview.pdf b/010_Introduction/HEVCTransmissionStackOverview.pdf deleted file mode 100644 index 6d53bc4..0000000 Binary files a/010_Introduction/HEVCTransmissionStackOverview.pdf and /dev/null differ diff --git a/010_Introduction/RobustVideoSystem.tikz b/010_Introduction/RobustVideoSystem.tikz new file mode 100644 index 0000000..8573c1f --- /dev/null +++ b/010_Introduction/RobustVideoSystem.tikz @@ -0,0 +1,122 @@ +\usetikzlibrary{matrix} +\usetikzlibrary{decorations.text} +\usetikzlibrary{arrows.meta} +\usetikzlibrary{shapes.arrows} + + +\makeatletter +\pgfdeclareradialshading[tikz@ball]{cloud}{\pgfpoint{-0.275cm}{0.4cm}}{% + color(0cm)=(tikz@ball!75!white); + color(0.1cm)=(tikz@ball!85!white); + color(0.2cm)=(tikz@ball!95!white); + color(0.7cm)=(tikz@ball!89!black); + color(1cm)=(tikz@ball!75!black) +} +\tikzoption{cloud color}{\pgfutil@colorlet{tikz@ball}{#1}% + \def\tikz@shading{cloud}\tikz@addmode{\tikz@mode@shadetrue}} +\makeatother + +\tikzset{my cloud/.style={ + cloud, draw, aspect=2, + cloud color={TikZColorFill} + } +} + +% for double arrows a la chef +% adapt line thickness and line width, if needed +\tikzstyle{vecArrow} = [thick, decoration={markings,mark=at position + 1 with {\arrow[semithick]{open triangle 60}}}, + double distance=1cm, shorten >= 5.5pt, + preaction = {decorate}, + postaction = {draw,line width=2cm, white,shorten >= 4.5pt}] +\tikzstyle{innerWhite} = [semithick, white,line width=1.4pt, shorten >= 4.5pt] + + +\begin{tikzpicture}[ + template/.style={centered, text centered, inner sep=1pt, text width=2.3cm}, + label/.style={centered, text centered, inner sep=1pt}, + block/.style={template, draw, minimum size=1cm, thick}, + arrow/.style={template, draw,-latex,thick}, + point/.style={template, draw, fill=black, circle,inner sep=1pt}, + node distance=1.2cm, + font=\footnotesize, + node distance=1.2cm] + +\tikzstyle{coders}=[block, ellipse, font=\small, inner sep=0] + +\definecolor{colorAuxiliary}{RGB}{245,238,197} +\definecolor{colorVideo}{RGB}{241,231,163} + +\tikzset{ + arrowMatrix/.style={ + matrix of nodes, + row sep=-\pgflinewidth, + column sep=-\pgflinewidth, + nodes={rectangle,text width=1.5cm,align=center}, + text depth=1.25ex, + text height=.7ex, + nodes in empty cells + } +} + +\matrix (arrow) [arrowMatrix] { + |[fill=TikZColorFillProp]| & |[fill=TikZColorFillProp]| & |[fill=TikZColorFillProp]| & |[fill=TikZColorFillProp] (last1)| \\ + |[fill=TikZColorFill]| & |[fill=TikZColorFill]| & |[fill=TikZColorFill]| & |[fill=TikZColorFill] (last2)| \\ + |[fill=TikZColorFill]| & |[fill=TikZColorFill]| & |[fill=TikZColorFill]| & |[fill=TikZColorFill] (last3)| \\ +}; + +% The labels +\node[text centered, xshift=.7cm] at (arrow-1-1) {\footnotesize SEI}; +\node[text centered, xshift=.7cm, text width=2.2cm] at (arrow-2-1.south) {\footnotesize VCL}; + + +\fill[white] (last1.north) -- (last2.east) -- (last1.north east) -- cycle; +\fill[white] (last3.south) -- (last3.south east) -- (last2.east) -- cycle; + +\coordinate (div0) at (-.8,-1.3); +\coordinate (div1) at ($(div0)+(-20:.2cm)+(-20:.2cm)+(60:3.1cm)$); + +\fill[fill=white] ($(div0)+(-20:-.2cm)$) -- ++(60:1.8cm) -- ++(-30:.4cm) -- ($(div1)+(-20:-.2cm)$) -- ++(-30:.4cm) -- ++(60:-1.85cm) -- ++(-30:-.4cm) -- ($(div0)+(-20:.2cm)$); + +%\draw ( $(div0)$ ) -- ( $(div0)+(60:1.7cm)$ ); +\path[decoration={markings, mark=at position .5 with \node[transform shape] {\scriptsize Error-prone};}, postaction={decorate}] + ( $(div0)$ ) -- ( $(div0)+(60:1.7cm)$ ); + +%\draw ($(div1)+(60:-1.8cm)$) -- ($(div1)$); +\path[decoration={markings, mark=at position .5 with \node[transform shape] {\scriptsize Transmission};}, postaction={decorate}] + ($(div1)+(60:-1.8cm)$) -- ($(div1)$); + +% Draw the arrow tip +\shade[top color=TikZColorFillProp, bottom color=TikZColorFill, + middle color=TikZColorFill, draw=white, ultra thick] + (last1.north west) -- (last2.center) -- (last3.south west) -- + (last3.south) -- (last2.east) -- (last1.north) -- cycle; + + +\path[decoration={markings, mark=at position .5 with \node[transform shape] {\scriptsize Video};}, postaction={decorate}] + ( $ (last1.north west)!0.5!(last1.north) $ ) -- ( $ (last2.center)!0.5!(last2.east) $ ); +\path[decoration={markings, mark=at position .5 with \node[transform shape] {\scriptsize Stream};}, postaction={decorate}] + ( $ (last2.center)!0.5!(last2.east) $ ) -- ( $ (last3.south west)!0.6!(last3.south) $ ); + + +%% Encoding +\node[fill=TikZColorFillProp, coders, minimum height=3.1cm, text width=2.6cm, left=-.5 of arrow] (SideInformation) {}; +\node[fill=TikZColorFillProp, coders, minimum height=2.1cm, text width=2.2cm, yshift=-7] (RobustEncoding) at (SideInformation) {}; +\node[fill=TikZColorFill, coders, minimum height=1.1cm, text width=1.5cm, yshift=-13] (Encoding) at (SideInformation) {Encoding}; + +\path [postaction={decorate,decoration={text along path, text align=center,text={|\small|Robust encoding}}}] +($(RobustEncoding.west)+(.3,0)$) to [bend left=50] ($(RobustEncoding.east)+(-.3,0)$); + +\path [postaction={decorate,decoration={text along path, text align=center,text={|\small|Side-information}}}] +($(SideInformation.west)+(.3,.3)$) to [bend left=50] ($(SideInformation.east)+(-.3,.3)$); + +%% Decoding +\node[fill=TikZColorFillProp, coders, minimum height=2.1cm, text width=2.2cm, right=-.2 of arrow] (RobustDecoding) {}; +\node[fill=TikZColorFill, coders, minimum height=1.1cm, text width=1.5cm, yshift=-7] (Decoding) at (RobustDecoding) {Decoding}; + +\path[postaction={decorate,decoration={text along path,text align=center,text={|\small|Robust Decoding}}}] +($(RobustDecoding.west)+(.3,0)$) to [bend left=50] ($(RobustDecoding.east)+(-.3,0)$); + + +\end{tikzpicture} + diff --git a/021_VideoCoding/VideoCodingReview.tex b/021_VideoCoding/VideoCodingReview.tex index 0b13a50..edc9640 100755 --- a/021_VideoCoding/VideoCodingReview.tex +++ b/021_VideoCoding/VideoCodingReview.tex @@ -21,6 +21,8 @@ In order to increase the coding flexibility and higher video resolutions, the \a In this chapter the new data structures coding techniques specifically introduced to increase the coding efficiency are described. Particular emphasis is given to those which may have a significant impact in the error robustness characteristics of \ac{hevc} streams. +\glsunset{hevc} + This chapter is organized as follows. Section~\ref{Section2:GeneraVideoSystem} introduces the functional blocks of a generic hybrid video coding system, and Section~\ref{Section2:Overview} provides an overview of the \ac{hevc} standard. Then, Section~\ref{Section2:HighLevel} describes the high-level syntax elements which are used to carry relevant information in coded streams. @@ -100,7 +102,7 @@ The encoder uses both intra- and inter-picture prediction to exploit spatial and \centering \includegraphics[width=\columnwidth]{021_VideoCoding/Figures/HEVCBlockDiagram} %\input{\FigsPath/HEVCBlockDiagram.tikz} -\caption{\HIGHLIGHT{Block diagram of an \ac{hevc} standard encoder.}} +\caption{Block diagram of an HEVC standard encoder.} \label{fig:HEVCBlockDiagram} \end{figure} @@ -146,7 +148,7 @@ Similar to \ac{h264}~\cite{H264AVCStandard2003}, the \ac{hevc} streams are based % \bitheader{0-15} \\ \colorbitbox{TikZColorFill}{1}{F} & \colorbitbox{gray!60}{6}{NAL Type} & \colorbitbox{TikZColorFill}{6}{Reserved} & \colorbitbox{gray!60}{3}{TIDP} \end{bytefield} - \caption{Two bytes \ac{nal} unit header in \ac{hevc}.} + \caption{Two bytes \ac{nal} unit header in HEVC.} \label{Figure2:NalHeader} \end{figure} @@ -170,7 +172,7 @@ The new set of parameters \ac{vps} contains information related to the different \begin{table}[tb] \centering - \caption{Picture categories in \ac{hevc} standard.}\label{Table2:PictureTypes} + \caption{Picture categories in HEVC standard.}\label{Table2:PictureTypes} \TABLESVSPACE\small \begin{tabular}{m{1.3cm}|m{5.2cm}|m{6.3cm}} \toprule @@ -420,7 +422,7 @@ In order to overcome this issue and to allow all possible modes, \ac{hevc} uses \input{021_VideoCoding/Figures/IntraPredictionsDirectional.tikz} \\ \end{tabular} } -\caption{Directional intra prediction modes in \ac{hevc} (left) and an example of the intra prediction mode 32 for a $4\times4$ block.} +\caption{Directional intra prediction modes in HEVC (left) and an example of the intra prediction mode 32 for a $4\times4$ block.} \label{Figure2:IntraModesHEVC} \end{figure} @@ -515,7 +517,8 @@ This process discards a given frame whenever the \ac{dpb} has the number of defi \begin{table}[tb] \centering - \caption{Reference Picture Set example.}\label{Table2:ReferencePictureListExample} + \caption{Reference Picture Set example.} + \label{Table2:ReferencePictureListExample} \small \TABLESVSPACE \begin{tabular}{x{1.5cm}|ccc} @@ -555,7 +558,7 @@ Using the \ac{rpset} concept the decoder is always able to detect loss events an \input{\FigsPath/PUSymetric.tikz} & (a)\\ \input{\FigsPath/PUAsymetric.tikz} & (b)\\ \end{tabular} - \caption{Partition types of an inter coded \ac{cu} into \acs{pu}~\cite{HEVCBlock2012}.} + \caption{Partition types of an inter coded CU into PUs~\cite{HEVCBlock2012}.} \label{Figure2:InterPuPartitions} \end{figure} @@ -578,7 +581,7 @@ These partitions types are only allowed for \acs{cu} with $16\times16$ pixels or \centering \input{\FigsPath/MergeModeHEVC.tikz} %\input{\FigsPath/HEVC/MergeModeHEVCTwoFrames.tikz} -\caption{Illustration of the \ac{mv} prediction candidates in the AMVP.} +\caption{Illustration of the MV prediction candidates in the AMVP.} \label{Figure2:MVCandidates} \end{figure} diff --git a/022_ErrorResilience/Figures/LocalizationTechniques.tikz b/022_ErrorResilience/Figures/LocalizationTechniques.tikz new file mode 100644 index 0000000..cde475c --- /dev/null +++ b/022_ErrorResilience/Figures/LocalizationTechniques.tikz @@ -0,0 +1,209 @@ + +\newcommand{\DrawFrame}[1] +{ + \draw[fill=#1,fill opacity=1, thick] (0,0) rectangle (9+1,8); % Opacity +% \draw[thick] (0,0) rectangle (#2+1,#2+1); +% \foreach \x in {1,...,#2} +% \draw (\x,0) -- ++(0,#2+1); +% \foreach \y in {1,...,#2} +% \draw (0,\y) -- ++(#2+1,0); +} + + +\begin{tikzpicture}[use Hobby shortcut,closed=true] + +\def\TikzFrameScale{.35} + +\tikzstyle{labels} =[text centered, font=\footnotesize] +\tikzstyle{arrow} =[draw,-latex, thick] +\tikzstyle{errorBlock}=[draw=colorErrors, fill=colorErrors] +\tikzstyle{errorPropagation}=[draw=colorErrorProp, fill=colorErrorProp] +\tikzstyle{refresh}=[fill=colorBackgroundSecondary,draw=colorBackgroundSecondary] +\def\xfshift{5.5cm} +\def\yfshift{.2cm} + +\def\errorPosX{3} +\def\errorPosY{5} + +\begin{scope} + \node[labels, text width=2.1cm] (ErrorPropLabel) at (5,11*\TikzFrameScale) {Error \\ propagation}; + \node[labels, text width=2.1cm] (Error) at (4.3,2*\TikzFrameScale) {Lost data}; + \begin{scope}[scale=\TikzFrameScale,xshift=-1cm] + + \draw[fill=colorBackground,fill opacity=1] (0,-1) rectangle ++(10,9); + + \draw[errorBlock,draw] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + + \foreach \x in {\errorPosX,...,8} + \draw[errorPropagation] (\x+1,\errorPosY) rectangle ++(1 cm,1 cm); + + \foreach \y in {-1,0,1,2,3,4} + \foreach \x in {0,1,2,3,4,5,6,7,8,9} + \draw[errorPropagation] (\x,\y) rectangle ++(1 cm,1 cm); + + % \draw (0,\errorPosY) -- ++(\errorPosX + 1,0) -- ++(0,1) -- ++(8-\errorPosX,0); + + \path[arrow] (ErrorPropLabel) edge[bend right] (8,5); + \path[arrow] (Error) -- (\errorPosX +.5, \errorPosY +.5); + + \draw[thick] (0,-1) rectangle ++(10,9); + + \node[labels,text width=3.5cm] at (4.5,-3) {\textbf{Error propagation \\ within a frame}}; + + \end{scope} + + \begin{scope}[yshift=.5cm, xshift=5.5cm] + \begin{scope}[scale=\TikzFrameScale, yshift=3*\yfshift, xshift=3*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t+3}$ }; + + \draw[errorPropagation] (\errorPosX ,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +2,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX ,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +2,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +0,\errorPosY -2) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY -2) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +0,\errorPosY -3) rectangle ++(1cm,1cm); + \coordinate (endPoint) at (9,0); + \end{scope} + \begin{scope}[scale=\TikzFrameScale, yshift=2*\yfshift, xshift=2*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t+2}$ }; + + \draw[errorPropagation] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +2,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX+1,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX+1,\errorPosY -2) rectangle ++(1cm,1cm); + \end{scope} + \begin{scope}[scale=\TikzFrameScale, yshift=1*\yfshift, xshift=1*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t+1}$ }; + + \draw[errorPropagation] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY) rectangle ++(1cm,1cm); + \coordinate (tempErrorPropPoint) at (\errorPosX +.5,\errorPosY +.5); + + \end{scope} + \begin{scope}[scale=\TikzFrameScale, yshift=0*\yfshift, xshift=0*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t}$ }; + \draw[errorBlock] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + \coordinate (startPoint) at (0,0); + \path[arrow] (Error) -- (\errorPosX +.5, \errorPosY +.5); + \end{scope} + + \path[arrow] (ErrorPropLabel) edge[bend left] (tempErrorPropPoint); + \path (startPoint) -- (endPoint) node[labels, pos=.5, yshift=-1.5cm] {\textbf{Error propagation over time}}; + \end{scope} +\end{scope} + + + +\begin{scope}[yshift=-5cm] + + + \node[labels, text width=1.2cm] (Refresh) at (4.3,2*\TikzFrameScale) {Refresh blocks}; + \begin{scope}[scale=\TikzFrameScale,xshift=-1cm] + \draw[fill=colorBackground,fill opacity=1] (0,-1) rectangle ++(10,9); + \foreach \y in {0,2,...,6} + \foreach \x in {0,1,2,3,4,5,6,7,8,9} + \draw[refresh] (\x,\y) rectangle ++(1 cm,1 cm); + + \draw[errorBlock,draw] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + + \foreach \x in {\errorPosX,...,8} + \draw[errorPropagation] (\x+1,\errorPosY) rectangle ++(1 cm,1 cm); + + \draw[thick] (0,-1) rectangle ++(10,9); + + % \path[arrow] (ErrorPropLabel.0) -- (7,5.5); + % \path[arrow] (Error) -- (\errorPosX +.5, \errorPosY +.5); + + \node[scale=.6] at (0.8, 0.5) {G1}; + \node[scale=.6] at (0.8, -0.5) {G2}; + \node[labels,text width=3.5cm] at (4.5,-3) {\textbf{Spatial localisation \\ example}}; + \end{scope} + \begin{scope}[xshift=5.5cm] + \begin{scope}[scale=\TikzFrameScale, yshift=3*\yfshift, xshift=3*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t+3}$ }; + + \draw[errorPropagation] (\errorPosX ,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +2,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX ,\errorPosY -1) rectangle ++(1cm,1cm); +% \draw[errorPropagation] (\errorPosX +1,\errorPosY -1) rectangle ++(1cm,1cm); +% \draw[errorPropagation] (\errorPosX +2,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +0,\errorPosY -2) rectangle ++(1cm,1cm); +% \draw[errorPropagation] (\errorPosX +1,\errorPosY -2) rectangle ++(1cm,1cm); +% \draw[errorPropagation] (\errorPosX +0,\errorPosY -3) rectangle ++(1cm,1cm); + + \draw[refresh] (8,4) rectangle ++(1cm,1cm); + \draw[refresh] (4,4) rectangle ++(1cm,1cm); + \draw[refresh] (7,6) rectangle ++(1cm,1cm); + \draw[refresh] (7,1) rectangle ++(1cm,1cm); + \draw[refresh] (3,3) rectangle ++(1cm,1cm); + + \coordinate (endPoint) at (9,0); + \end{scope} + \begin{scope}[scale=\TikzFrameScale, yshift=2*\yfshift, xshift=2*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t+2}$ }; + + \draw[errorPropagation] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +2,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX,\errorPosY -1) rectangle ++(1cm,1cm); +% \draw[errorPropagation] (\errorPosX+1,\errorPosY -1) rectangle ++(1cm,1cm); +% \draw[errorPropagation] (\errorPosX+1,\errorPosY -2) rectangle ++(1cm,1cm); + + \draw[refresh] (8,4) rectangle ++(1cm,1cm); + \draw[refresh] (4,4) rectangle ++(1cm,1cm); + \draw[refresh] (7,6) rectangle ++(1cm,1cm); + \draw[refresh] (7,1) rectangle ++(1cm,1cm); + \draw[refresh] (3,2) rectangle ++(1cm,1cm); + \end{scope} + \begin{scope}[scale=\TikzFrameScale, yshift=1*\yfshift, xshift=1*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t+1}$ }; + + \draw[errorPropagation] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX,\errorPosY -1) rectangle ++(1cm,1cm); + \draw[errorPropagation] (\errorPosX +1,\errorPosY) rectangle ++(1cm,1cm); + + \draw[refresh] (1,2) rectangle ++(1cm,1cm); + \draw[refresh] (2,4) rectangle ++(1cm,1cm); + \draw[refresh] (5,3) rectangle ++(1cm,1cm); + \draw[refresh] (7,6) rectangle ++(1cm,1cm); + \draw[refresh] (7,1) rectangle ++(1cm,1cm); + + \end{scope} + \begin{scope}[scale=\TikzFrameScale, yshift=0*\yfshift, xshift=0*\xfshift, yslant=\TikZyslant,xscale=\TikZxscale] + \DrawFrame{colorBackground} + \node at (12cm/2,-1.5cm) { $f_{t}$ }; + \draw[errorBlock] (\errorPosX,\errorPosY) rectangle ++(1cm,1cm); + \coordinate (startPoint) at (0,0); + + \draw[refresh] (1,2) rectangle ++(1cm,1cm); + \draw[refresh] (2,4) rectangle ++(1cm,1cm); + \draw[refresh] (5,3) rectangle ++(1cm,1cm); + \draw[refresh] (7,6) rectangle ++(1cm,1cm); + \draw[refresh] (7,1) rectangle ++(1cm,1cm); + + \path[arrow] (Refresh) -- (2.5,4.5); + + \end{scope} + \path (startPoint) -- (endPoint) node[labels, pos=.5, yshift=-1.5cm] {\textbf{Temporal localisation example}}; + \end{scope} +\end{scope} + + + + + +\end{tikzpicture} diff --git a/022_ErrorResilience/RobustTransmissionReview.tex b/022_ErrorResilience/RobustTransmissionReview.tex index 7733d66..24b7e88 100755 --- a/022_ErrorResilience/RobustTransmissionReview.tex +++ b/022_ErrorResilience/RobustTransmissionReview.tex @@ -18,16 +18,16 @@ In general, digital communications through networks are prone to transmission errors or data loss. These errors can be handled in different ways, such as retransmission, error correction, and \ac{ec} \cite{ErrorResilience2000,PixelInterlacingIntraConcealment2011}. -This chapter reviews the main robust video coding techniques that can be applied at the \ac{vcl} to reduce the error propagation and increase the decoded video quality. -Several techniques have been proposed in the past to deal with the problem of robust video transmission, especially for video coding standards with high temporal dependencies, such as \ac{h264} and more recently \ac{hevc}. -These techniques are normally focused on the spatio-temporal dependencies in the coded streams, which are used to obtain high coding gains but lead to low error robustness~\cite{ROPE2007}. -In this chapter the error resilience and concealment schemes are divided in different categories depending on their functionality and algorithms. +This chapter reviews the main robust video coding techniques that can be applied at the \ac{vcl} to reduce the error propagation and increase the decoded video quality. +Several techniques have been proposed in the past to deal with the problem of robust video transmission, especially for video coding standards with high temporal dependencies, such as \ac{h264} and more recently \ac{hevc}. +These techniques are normally focused on the spatio-temporal dependencies in the coded streams, which are used to obtain high coding gains but lead to low error robustness~\cite{ROPE2007}. +In this chapter the error resilience and concealment schemes are divided in different categories depending on their functionality and algorithms. Although each approach is based on a different techniques they all aim at striking the most efficient balance between coding efficiency and decoded video quality under lossy conditions. -This chapter is organised as follows: Section~\ref{Section22:Localisation} describes the localisation techniques, which reduce the temporal and spatial dependencies in coded bitstreams. Section~\ref{Section22:DataPartitionRedundant} provides an overview of methods based on data partition, classification and bitstream redundancy. +This chapter is organised as follows: Section~\ref{Section22:Localisation} describes the localisation techniques, which reduce the temporal and spatial dependencies in coded bitstreams. Section~\ref{Section22:DataPartitionRedundant} provides an overview of methods based on data partition, classification and bitstream redundancy. %These techniques ease the introduction of extra information in order to recover from lost data. -Section~\ref{Section22:Concealment} covers the techniques used at the decoder-side as post-processing to reduce the effect of errors and improve visual quality and smoothness. -Section~\ref{Section22:ConcealmentAware} provides a description of methods that use \ac{ec} techniques to achieve improved error robustness. +Section~\ref{Section22:Concealment} covers the techniques used at the decoder-side as post-processing to reduce the effect of errors and improve visual quality and smoothness. +Section~\ref{Section22:ConcealmentAware} provides a description of methods that use \ac{ec} techniques to achieve improved error robustness. Finally, Section~\ref{Section22:Summary} concludes this chapter by providing a overall comparison between the different approaches. \clearpage @@ -38,8 +38,8 @@ Finally, Section~\ref{Section22:Summary} concludes this chapter by providing a o In general, most of the compression efficiency is achieved by exploiting both the spatial and temporal redundancy of video signals. Since there is high correlation between neighbouring information, predictive coding schemes are very effective to achieve higher compression gains by exploiting such correlations. However, the drawback of high predictive coding schemes is the strong dependencies between compressed symbols with great impact on error propagation. -In order to increase the robustness of video transmission, several coding mechanisms can be employed at different elements of a transmission chain to mitigate the artefacts caused by errors and data loss. -These mechanisms can be classified according to the functional characteristics and the type of problem they address~\cite{ErrorResilienceAug2005}. +In order to increase the robustness of video transmission, several coding mechanisms can be employed at different elements of a transmission chain to mitigate the artefacts caused by errors and data loss. +These mechanisms can be classified according to the functional characteristics and the type of problem they address~\cite{ErrorResilienceAug2005}. In this chapter the robust video coding techniques ere grouped into four categories: \begin{itemize} \item Localization techniques; @@ -76,9 +76,7 @@ Since these techniques break the predictive coding, they have a negative impact \begin{figure}[tb] \centering -%\subfloat{\input{./022_ErrorResilience/Figures/LocalizationTechniquesSpatial.tikz}}\\ -%\subfloat{\input{./022_ErrorResilience/Figures/LocalizationTechniquesTemporal.tikz}} -\input{./022_ErrorResilience/Figures/LocalizationTechniques_v2.tikz} +\input{./022_ErrorResilience/Figures/LocalizationTechniques.tikz} \caption{Examples of spatial and temporal error propagation and how it can be reduced using localisation techniques \cite{ErrorResilienceAug2005}.} \end{figure} @@ -183,7 +181,7 @@ Using $\check f_t^i$ as reference frame for prediction significantly decrease er \multicolumn{3}{c}{\includegraphics[width=12cm]{\TransmissionResiliencefigs/ROILeakyPrediction.png}} \\ $f_{n-1}$ & $f_{n}$ & $f_{n+1}$ \\ \end{tabular} - \caption{Example of error propagation with a \ac{roi} (shaded area indicates error, and the arrows indicate temporal predictions to previous frame) \cite{ROILeakyPred2009}.} + \caption{Example of error propagation with a ROI (shaded area indicates error, and the arrows indicate temporal predictions to previous frame) \cite{ROILeakyPred2009}.} \label{fig2:ErrorPropagationROI} \end{figure} @@ -366,7 +364,7 @@ This leads to a poor estimation of lost pixels, resulting in noticeable artefact \begin{figure}[tb] \centering \input{\ErrorConcealmentReviewFigs/LinearInterpolation} -\caption{ Lost pixel interpolation from neighbouring error-free pixels. } +\caption{Lost pixel interpolation from neighbouring error-free pixels. } \label{FIG3:pixel_interpolation} \end{figure} @@ -466,7 +464,7 @@ As mentioned before, the \ac{mc} method is a simple way of concealment a missing \begin{figure}[t] \centering \includegraphics[width=0.9\columnwidth]{\ErrorConcealmentReviewFigs/MVE} -\caption{\ac{mve} extrapolation with the corresponding extrapolated macroblock, based on the method presented in \cite{BlockBasedMVE2002}.} +\caption{Motion vector extrapolation with the corresponding extrapolated macroblock, based on the method presented in \cite{BlockBasedMVE2002}.} \label{FIG3:MVEExtrapolation} \end{figure} @@ -507,7 +505,7 @@ The \ac{hmve} method aims to find the more accurate \ac{mv} from the \ac{mv} set This algorithm is able to outperform the \ac{pmve} for different packet loss probabilities, as shown by the results in \cite{2DConcHybridMotion2010}. The method is used to conceal both I (INTRA) and P frames in video sequence compressed with the \ac{h264}, achieving more $1.09$ dB (\ac{psnr}) in the erroneous frames. %\HIGHLIGHT{HEVC} -In the case of HEVC, there are also a few works worth to be mentioned. The works presented for the \ac{hevc} standard are mainly based on temporal \ac{ec} approaches. +In the case of \ac{hevc}, there are also a few works worth to be mentioned. The works presented for the \ac{hevc} standard are mainly based on temporal \ac{ec} approaches. In~\cite{HEVCMCBlockAug2013} the block partitions of the neighbouring frames were used to improve the performance motion vector extrapolation. Using this the reconstructed frame keeps the boundaries smooth, leading to higher objective and visual quality. Moreover, in order to select only the reliable motion information, the residue information can be exploited, as shown in~\cite{HEVCMCResidueDec2013}. diff --git a/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz b/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz index b81a363..5769bb3 100644 --- a/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz +++ b/042_ConcealmentAwareEncoding/Figures/BlockDiagram.tikz @@ -76,7 +76,7 @@ %% Input \node[text width=.5cm, text centered] at (-1.7,4.5) (Input) {Input video}; -\path[arrow] (Input) |- (Encoder); +\path[arrow] (Input) |- (Encoder) node[left,pos=.2] {$f$}; \path[arrow] (Input |- ConcOpt) -- (ConcOpt) node[point] at (Input |- ConcOpt) {}; %% TS Mux and output diff --git a/043_EndToEndRobustSystem/EndToEndRobustSystem.tex b/043_EndToEndRobustSystem/EndToEndRobustSystem.tex index 081ae9f..dd7f9b0 100644 --- a/043_EndToEndRobustSystem/EndToEndRobustSystem.tex +++ b/043_EndToEndRobustSystem/EndToEndRobustSystem.tex @@ -29,12 +29,12 @@ The evaluation of these methods revealed that they can efficiently increase the Therefore, it is noticeable that each method might be optimised for different applications, video characteristics and network conditions. To advance one step forward, a jointly robust coding method using different techniques can be used to increase the decoder video quality across difference conditions. -Such approach can be based on both the input contents and the network information, such as bitrate and packet loss conditions to optimise the robust coders to each application scenario. In order to overcome the above problem a jointly evaluation study is necessary to identify the most efficient techniques and to evaluate their performance for different conditions. +Such approach can be based on both the input contents and the network information, such as bitrate and packet loss conditions to optimise the robust coders to each application scenario. In order to overcome the above problem, a joint evaluation study is necessary to identify the most efficient techniques across different conditions. \subsection{Comparison of the proposed methods} -In order to develop a jointly robust coder to cope with transmission errors an evaluation study was carried out, to compare the error robustness performance of the proposed methods for different conditions with each others. +In order to develop a joint robust coder to cope with transmission errors an evaluation study was carried out, to compare the error robustness performance of the proposed methods for different conditions with each others. %This study extends the previous evaluations presented in Chapter~\ref{Chapter:ErrorResilienceTechniques}~and~\ref{Chapter:ConcealmentAwareTechniques}, as they target a wider range of test material and network conditions. The experimental evaluation carried out in this section follows the same condition described in Section~\ref{Section3:ErrorResilienceEvaluation}. %In order to perform a wider evaluation different bitrate and \acs{plr} were used. Table~\ref{Table6:TestVariables} illustrates the selected configurations. As shown in the table a total of 15 test sequences were used encoder using 5 different target bitrates. One should note that the main bitrate varies based on the test sequence and it is shown in third column of Table~\ref{TableTestSequences}. Different network conditions were evaluated using a total of 13 different \acs{plr}, with higher special focus on lower loss ratios, which are more relevant to modern video streaming applications. @@ -67,7 +67,7 @@ Figures~\ref{Figure6:AverageQualityMain}~and~\ref{Figure6:AverageQualityReductio \label{Figure6:AverageQualityReduction} \end{figure} -Results on both figures show that the proposed methods (i.e., \EXP, \EXPELTEN and \EXPELTWEN) are all able to outperform the reference \ac{hevc} for high \acs{plr}. In case of the proposed reference picture selection method (\EXP), quality gains in comparison with the \REF case are achieved for lower loss ratio, i.e., \acs{plr} as low as 2\% for all test conditions shown. +Results on both figures show that the proposed methods (\EXP, \EXPELTEN and \EXPELTWEN) are all able to outperform the reference \ac{hevc} for high \acs{plr}. In case of the proposed reference picture selection method (\EXP), quality gains in comparison with the \REF case are achieved for lower loss ratio, i.e., \acs{plr} as low as 2\% for all test conditions shown. Comparing the results for both target bitrates also shows that the proposed methods are able to achieve higher quality gains for higher bitrates. Therefore, the for a lower bitrate (see results of Figure~\ref{Figure6:AverageQualityReduction}) the proposed methods only outperform the reference method for slightly higher \acs{plr}. This is more noticeable in the \EXPELTWEN case. Summarising, the performance of the proposed methods is not similar across different video content and target bitrate, which impose a challenge on which method should be applied to achieve the highest decoded video quality in case of transmission errors. In order to achieve the highest quality the optimal trade-off between auxiliary information and video bitrate should be obtained by selecting the optimal error robust coding algorithm. diff --git a/carreira_phd_thesis.sty b/carreira_phd_thesis.sty index 1f1b1ac..801e282 100644 --- a/carreira_phd_thesis.sty +++ b/carreira_phd_thesis.sty @@ -5,7 +5,8 @@ \usepackage{subfiles} \usepackage{xspace} \usepackage{pdfpages} -\usepackage[colorlinks=false,hidelinks,pdfpagelabels]{hyperref} +%\usepackage[colorlinks=false,hidelinks,pdfpagelabels]{hyperref} +\usepackage[colorlinks=true,hidelinks,pdfpagelabels]{hyperref} %\usepackage[pdfpagelabels]{hyperref} % *** Minitoc *** @@ -91,7 +92,6 @@ \usepackage{tabularx, colortbl, booktabs, array, multirow} \newcolumntype{x}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}m{#1}} \newcolumntype{y}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}b{#1}} -\usepackage{slashbox} % *** FLOAT PACKAGES *** % Coloca sempre as figuras dentro das secções a que pertencem diff --git a/carreira_phd_thesis.tex b/carreira_phd_thesis.tex index 4118793..c4e8e75 100644 --- a/carreira_phd_thesis.tex +++ b/carreira_phd_thesis.tex @@ -22,7 +22,7 @@ %\includeonly{022_ErrorResilience/RobustTransmissionReview} %\includeonly{031_ErrorResilienceEvaluation/ErrorResilienceEvaluation} %\includeonly{041_ImprovedErrorResilience/ImprovedErrorResilience} -%\includeonly{042_ConcealmentAwareEncoding/ConcealmentAwareEncoding} +\includeonly{042_ConcealmentAwareEncoding/ConcealmentAwareEncoding} %\includeonly{043_EndToEndRobustSystem/EndToEndRobustSystem} \begin{document} @@ -58,11 +58,12 @@ \input{000_Preamble/abstract} %\dominitoc +\glsunsetall \tableofcontents \listoffigures \listoftables \printglossary[title=List of Abbreviations] - +\glsresetall \mainmatter % \pagenumbering{gobble} diff --git a/carreira_phd_thesis_abr.sty b/carreira_phd_thesis_abr.sty index c7ef047..36467ae 100644 --- a/carreira_phd_thesis_abr.sty +++ b/carreira_phd_thesis_abr.sty @@ -23,6 +23,8 @@ \customnewacronym{iso}{ISO}{International Organization for Standardization} \glsunset{iso} \customnewacronym{iec}{IEC}{International Electrotechnical Commission} \glsunset{iec} + + \customnewacronym{gop}{GOP}{Group of Pictures} \customnewacronym{qp}{QP}{Quantized Parameter} \customnewacronym{rd}{R-D}{\mbox{Rate-Distortion}} @@ -200,11 +202,3 @@ \def\DYNLAMBDALONG{\DYNLAMBDA \mbox{\textit{(Adaptive $\lambda$)}}\xspace} - - - - - - - - diff --git a/estgleiria.cls b/estgleiria.cls index ea577d4..576a417 100644 --- a/estgleiria.cls +++ b/estgleiria.cls @@ -263,7 +263,7 @@ T\kern-.1667em\lower.5ex\hbox{E}\kern-.125emX\spacefactor1000} % {\sc\chaptertitlename\ \thechapter}{15pt} % {\Huge \thispagestyle{empty}} % [\vspace{1cm}\titlerule\minitoc\glsresetall] - \titleformat{\chapter}[display]{\normalfont\Huge\bfseries}{\sc \chaptertitlename\ \thechapter}{20pt}{\Huge \thispagestyle{empty}}[\vspace{5mm}\glsresetall] + \titleformat{\chapter}[display]{\normalfont\Huge\bfseries}{\sc \chaptertitlename\ \thechapter}{20pt}{\Huge \thispagestyle{empty}}[\vspace{5mm}] \properpagestyle \pagenumbering{arabic}}