diff --git a/021_VideoCoding/Figures/BlockPartitionsWithTree.tikz b/021_VideoCoding/Figures/BlockPartitionsWithTree.tikz index c716eff..89bf433 100644 --- a/021_VideoCoding/Figures/BlockPartitionsWithTree.tikz +++ b/021_VideoCoding/Figures/BlockPartitionsWithTree.tikz @@ -16,7 +16,7 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds} \begin{tikzpicture}[scale=\scale,on grid, ->,>=stealth'] -\tikzstyle{TextStyle}=[scale=.75,text centered] +\tikzstyle{TextStyle}=[scale=.7,text centered] \begin{scope}[scale=0.4] @@ -29,7 +29,7 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds} \tikzstyle{CU}=[draw=DrawColor, thin] \tikzstyle{PU}=[draw=DrawColor, thin, densely dashed] - \node[TextStyle] at (8,16.5) {$64\times64$ - CTU}; + \node[TextStyle] at (8,16.5) {CTU - $64\times64$}; \draw[CTU] (0,0) rectangle ++(16,16); % 64x64 %% 32x32 @@ -53,7 +53,7 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds} \node[TextStyle] at (12,12) {$32\times32$}; \node[TextStyle] at (2,14) {$16\times16$}; - \node[TextStyle] at (5,13) {$8\times8$}; + \node[TextStyle,scale=.9] at (5,13) {$8\times8$}; \node[TextStyle] (CULabel) at (9.5,6) {CU}; \draw[black, -latex] (CULabel) to (7,7); diff --git a/021_VideoCoding/Figures/FrameList/RaceCTU.png b/021_VideoCoding/Figures/FrameList/RaceCTU.png new file mode 100644 index 0000000..b1ac2b1 Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/RaceCTU.png differ diff --git a/021_VideoCoding/Figures/FrameList/RaceCTUCrop.png b/021_VideoCoding/Figures/FrameList/RaceCTUCrop.png new file mode 100644 index 0000000..ed9945f Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/RaceCTUCrop.png differ diff --git a/021_VideoCoding/Figures/FrameList/RaceCU.png b/021_VideoCoding/Figures/FrameList/RaceCU.png new file mode 100644 index 0000000..2bd865e Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/RaceCU.png differ diff --git a/021_VideoCoding/Figures/FrameList/RaceCUCrop.png b/021_VideoCoding/Figures/FrameList/RaceCUCrop.png new file mode 100644 index 0000000..ca9f466 Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/RaceCUCrop.png differ diff --git a/021_VideoCoding/Figures/FrameList/RaceTU.png b/021_VideoCoding/Figures/FrameList/RaceTU.png new file mode 100644 index 0000000..9a79d0e Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/RaceTU.png differ diff --git a/021_VideoCoding/Figures/FrameList/RaceTUCrop.png b/021_VideoCoding/Figures/FrameList/RaceTUCrop.png new file mode 100644 index 0000000..502d21f Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/RaceTUCrop.png differ diff --git a/021_VideoCoding/Figures/FrameList/Race_0.png b/021_VideoCoding/Figures/FrameList/Race_0.png new file mode 100644 index 0000000..79a60fd Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/Race_0.png differ diff --git a/021_VideoCoding/Figures/FrameList/Race_1.png b/021_VideoCoding/Figures/FrameList/Race_1.png new file mode 100644 index 0000000..2da868b Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/Race_1.png differ diff --git a/021_VideoCoding/Figures/FrameList/Race_2.png b/021_VideoCoding/Figures/FrameList/Race_2.png new file mode 100644 index 0000000..c944164 Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/Race_2.png differ diff --git a/021_VideoCoding/Figures/FrameList/Race_3.png b/021_VideoCoding/Figures/FrameList/Race_3.png new file mode 100644 index 0000000..ee70c70 Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/Race_3.png differ diff --git a/021_VideoCoding/Figures/FrameList/Race_Coded.png b/021_VideoCoding/Figures/FrameList/Race_Coded.png new file mode 100644 index 0000000..19da3c8 Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/Race_Coded.png differ diff --git a/021_VideoCoding/Figures/FrameList/Race_CodedCrop.png b/021_VideoCoding/Figures/FrameList/Race_CodedCrop.png new file mode 100644 index 0000000..132abc8 Binary files /dev/null and b/021_VideoCoding/Figures/FrameList/Race_CodedCrop.png differ diff --git a/021_VideoCoding/Figures/HEVCBlockDiagram.pdf b/021_VideoCoding/Figures/HEVCBlockDiagram.pdf new file mode 100644 index 0000000..c188945 Binary files /dev/null and b/021_VideoCoding/Figures/HEVCBlockDiagram.pdf differ diff --git a/021_VideoCoding/Figures/HEVCBlockDiagram.tikz b/021_VideoCoding/Figures/HEVCBlockDiagram.tikz index a0df5a7..40f23b6 100644 --- a/021_VideoCoding/Figures/HEVCBlockDiagram.tikz +++ b/021_VideoCoding/Figures/HEVCBlockDiagram.tikz @@ -1,91 +1,150 @@ \usetikzlibrary{arrows,matrix} % \usepackage{amssymb, amsmath, mathptmx} -\begin{tikzpicture}[font=\scriptsize,node distance=1cm,scale=1.2] +\begin{tikzpicture}[font=\footnotesize,node distance=1.5cm] \tikzstyle{sumBlock}=[draw,circle, inner sep=0cm, minimum size=0.1cm] -\tikzstyle{processBlock}=[draw, fill=\TikZColorFill, inner sep=0.1cm, minimum size=0.7cm,text centered] -\tikzstyle{proposedBlock}=[processBlock, fill=gray!30] -\tikzstyle{blockArrow}=[draw,-latex] -\tikzstyle{point} = [draw, fill=black, circle,inner sep=1pt] - -\coordinate (sumPoint) at (0,0); -\node[sumBlock] (ResidueDiff) at (sumPoint) {$+$}; -% \begin{scope}[xshift=-2cm, yshift=3.2cm, scale=1.5] -% \def\spacing{.2} -% \node at ((.5+3*\spacing*.5,-.5) (Input) {}; -% \foreach \i in {0,...,3} -% \draw[thick,fill=white] (\spacing*\i,-.15*\i) rectangle ++(1,1); -% % \node[text width=.7cm, text centered] at (1.6,1.1) {Video Stream}; -% % \node[text width=.7cm, text centered] at (1.1,.1) {Input video}; +\tikzstyle{templateBlock} = [inner sep=0.1cm, text centered] +\tikzstyle{processBlock} = [templateBlock, draw, fill=\TikZColorFill, minimum height=0.9cm, text width=2cm] +\tikzstyle{proposedBlock} = [processBlock, fill=gray!30] +\tikzstyle{labels} = [templateBlock, text width=2cm, font=\scriptsize] +\tikzstyle{blockArrow} = [draw,-latex] +\tikzstyle{arrowControl} = [blockArrow, dashed] +\tikzstyle{arrowOutput} = [blockArrow, shorten >=6, thick] +\tikzstyle{point} = [draw, fill=black, circle,inner sep=1pt] + +\tikzstyle{frame} = [anchor=north west, inner sep=0] +\tikzstyle{ctu} = [draw, step=12.8mm] + +\def\framescale{0.23753} + +%% CUs +% \begin{scope}[scale=\framescale, every node/.style={scale=\framescale}] +\node[frame, on grid] at (0,0) (FrameCU) {\includegraphics[width=2cm]{FrameList/RaceCUCrop} }; +% \draw[ctu] (0,0) grid ++(84.2mm,-48.0mm); % \end{scope} +\node[labels, above=0 of FrameCU] {Coding Units quad-tree}; + +%% CTUs +\node[frame, on grid] at (0,-2.5cm) (FrameCTU) {\includegraphics[width=2cm]{FrameList/RaceCTUCrop} }; + +\begin{scope}[yshift=-5cm] +\def\spacing{.1} +\node[frame] at (\spacing*-2 ,\spacing*0 ) (is) {\includegraphics[width=2cm]{FrameList/Race_CodedCrop} }; +\node[frame] at (\spacing*-1 ,\spacing*-1) {\includegraphics[width=2cm]{FrameList/Race_CodedCrop} }; +\node[frame] at (\spacing*0 ,\spacing*-2) {\includegraphics[width=2cm]{FrameList/Race_CodedCrop} }; +\node[frame] at (\spacing*1 ,\spacing*-3) {\includegraphics[width=2cm]{FrameList/Race_CodedCrop} }; +\node[frame] at (\spacing*2 ,\spacing*-4) (ie) {\includegraphics[width=2cm]{FrameList/Race_CodedCrop} }; +\node[fit=(is) (ie), inner sep=1pt] (in) {}; +\end{scope} -\node[left of=ResidueDiff,text width=1cm,node distance=2.5cm] (Input) {\textbf{Input video}}; -\path[blockArrow] (Input) -- (ResidueDiff) node [point, midway] (p0) {}; +\path[blockArrow] (in) -- (FrameCTU) node[labels, text width=1.5cm, midway, left] {Partitioning into CTUs}; +\path[blockArrow] (FrameCTU) -- (FrameCU); -\node[processBlock,right=of ResidueDiff] (Transf) {$T$}; -\node[processBlock,right=of Transf] (Quant) {$Q$}; -\node[processBlock,right=of Quant] (VLC) {CABAC}; +\node[sumBlock, right=1.2 of FrameCU] (ResidueDiff) {$+$}; -\node[right=of VLC, text centered,text width=1.5cm] (Output) {\textbf{HEVC bitstream}}; -\path[blockArrow] (VLC) -- (Output); +%% CTUs +\node[frame, right=2.2cm of FrameCU] (FTu) {\includegraphics[width=2cm]{FrameList/RaceTUCrop} }; + +\node[labels, text width=2.3cm, above=0 of FTu] {Transform Units \\quad-tree}; + +\node[processBlock, text width=.9cm, right=0.5 of FTu] (Transf) {$T$}; +\node[processBlock, text width=.9cm, right=0.5 of Transf] (Quant) {$Q$}; +\node[processBlock, text width=3.3cm, above=1 of Transf, xshift=.5cm] (GCC) {Encoder control}; + +\node[labels,right=1.8 of Quant] (QuantCoef) {Quantized Coefficients}; +\node[labels,right=2.7 of GCC] (ControlData) {Control data}; + +\path[blockArrow] (Quant) -- (QuantCoef) node [point,pos=0.5] (pointQuant) {}; + +\node[processBlock, below=0.5 of pointQuant, text width=.9cm,] (InvQuant) {$Q^{-1}$}; +\node[processBlock, below=0.5 of InvQuant, text width=.9cm, ] (InvTransf) {$T^{-1}$}; + +\node[sumBlock, below=0.6 of InvTransf] (ReconResidue) {$+$}; + +\node[processBlock, below=.9 of FTu, xshift=.9cm] (IntraEstm) {Intra-frame estimation}; +\node[labels, scale=.8, below=0 of IntraEstm.200, align=right, yshift=-3] {34 modes}; + +\node[processBlock, below=.6 of IntraEstm] (Intra) {Intra prediction}; +\node[processBlock, below=.9 of Intra] (Inter) {Inter prediction}; +\node[processBlock, below=.6 of Inter] (MotionSearch) {Motion Estimation}; +% \node[labels, scale=.8, below=0 of MotionSearch, text width=2cm] {Symmetric \& asymmetric partitions}; +\node[labels, scale=.8, below=0 of MotionSearch.200, align=right, yshift=-3, text width=4cm] {AMVP, Merge Mode}; + +\node[processBlock, below=1.2 of ReconResidue, text width=2.3cm] (Filters) {Deblocking \& SAO filters}; +% \node[processBlock, text width=2.3cm] at (Filters |- MotionSearch) (RefBuffer) {Decoded Picture Buffer}; +\node[processBlock, text width=2.3cm] at (Filters |- MotionSearch) (RPSet) {Reference Picture Set}; -\path[blockArrow] (ResidueDiff) -- (Transf); -\path[blockArrow] (Transf) -- (Quant); -\path[blockArrow] (Quant) -- (VLC) node [point,midway] (QuantFrame) {}; +%% Intra / Inter prediction arrows +\node [point,left=.7 of Intra] (IntraPred) {}; +\node [point,left=.7 of Inter] (InterPred) {}; +\path[blockArrow,-] (Intra) -- (IntraPred); +\path[blockArrow,-] (Inter) -- (InterPred); +\node[coordinate,left of=IntraPred, below of=IntraPred, node distance=0.15cm] (w1) {}; +\node[coordinate,left of=InterPred, above of=InterPred, node distance=0.15cm] (w2) {}; +\path (IntraPred) -- (InterPred) node [coordinate, midway] (pointPred) {}; +\path (pointPred) -- (pointPred -| ResidueDiff) node [point] (pointPred) {}; +\draw[-latex,densely dashed] (w1) to [bend left=60] (w2); +\draw[thick,blockArrow] (IntraPred)+(0.1,-0.1) -- (pointPred); +\path[blockArrow] (pointPred) -- node[pos=0.96,right] {-} (ResidueDiff); -\node[processBlock,below=.8 of QuantFrame] (InvQuant) {$Q^{-1}$}; -\node[processBlock,right=0.7 of InvQuant] (InvTransf) {$T^{-1}$}; -\node[sumBlock,below=.5 of InvTransf] (ReconResidue) {$+$}; -\node[processBlock,text width=1.9cm, below=.8 of ReconResidue] (Filters) {Deblocking \& SAO filters}; -\path[blockArrow] (QuantFrame) -- (InvQuant); +\path[blockArrow] (FrameCU) -- (ResidueDiff) node [point, midway] (pointIn) {}; +\path[blockArrow] (ReconResidue) -- (Filters) node[point,pos=.4] (pointRecon) {}; + +% \path[blockArrow] (ResidueDiff) -- (Transf); +\path[blockArrow] (ResidueDiff) -- (FTu); +\path[blockArrow] (FTu) -- (Transf); + +\path[blockArrow] (pointIn) |- (GCC); +\path[blockArrow] (GCC) -- (ControlData); + +\path[blockArrow] (pointQuant) -- (InvQuant); \path[blockArrow] (InvQuant) -- (InvTransf); \path[blockArrow] (InvTransf) -- (ReconResidue); -\path[blockArrow] (ReconResidue) -- (Filters) node[point,midway] (p1) {}; -\node[left=.4 of InvQuant, coordinate,yshift=0.5cm] (x1) {}; +\path[blockArrow] (pointRecon) -| (Intra); +% \path[blockArrow] (RefBuffer) -- (RPSet); +% \path[blockArrow] (RPSet) -- (MotionSearch); +\path[blockArrow] (Filters) -- (RPSet); +\path[blockArrow] (RPSet) -- (MotionSearch); + +\path[blockArrow] (IntraEstm) -- (Intra); +\path[blockArrow] (MotionSearch) -- (Inter); +\path[blockArrow] (pointIn) |- (MotionSearch); + +\path[blockArrow] (pointIn |- IntraEstm) node[point] {} -- (IntraEstm); + +\node[left=.6 of InvQuant, coordinate] (x1) {}; \path[blockArrow,-] (x1 -| ResidueDiff) node[point] () {} -- (x1); \path[blockArrow] (x1) |- (ReconResidue); -\node [processBlock,text width=1.5cm] (Intra) at (2.2,-1.8) {Intra prediction}; -\node [processBlock,text width=1.5cm,below=1.2 of Intra] (Inter) {Inter prediction}; -\path[blockArrow] (p1) -| (Intra); +\node[processBlock,right=2.5 of ReconResidue] (VLC) {Header Formatting \& CABAC}; +\begin{pgfonlayer}{background} -\node [point,left=.8 of Intra] (z1) {}; -\node [point,left=.8 of Inter] (z2) {}; +\path[arrowControl] (GCC -| Quant.90) -- (Quant.90); +\path[arrowControl] (GCC -| IntraEstm.30) -- (IntraEstm.30); +\path[arrowControl] (GCC -| MotionSearch.60) -- (MotionSearch.60); -\path[blockArrow,-] (Intra) -- (z1); -\path[blockArrow,-] (Inter) -- (z2); +\path[arrowOutput] (QuantCoef) -- (VLC.135); +\path[arrowOutput] (ControlData.-30) -- (VLC); +\path[arrowOutput] (MotionSearch) -- (VLC.190) node[labels,above, pos=0.7, yshift=.2cm] {Motion \\ vectors}; +\path[arrowOutput] (IntraEstm) -- (VLC) node[labels,above, pos=0.8, yshift=.2cm] {Intra \\ modes}; +\path[arrowOutput] (Filters) -- (VLC.220) node[labels,below, pos=0.7, yshift=-.2cm] {Filter \\ data}; -\node [coordinate,left of=z1,below of=z1,node distance=0.15cm] (w1) {}; -\node [coordinate,left of=z2, above of=z2, node distance=0.15cm] (w2) {}; - -\path (z1) -- (z2) node [coordinate, midway] (w) {}; -\path (w) -- (w -| sumPoint) node [point] (z) {}; - -\draw[-latex,densely dashed] (w1) to [bend left=60] (w2); -\draw [thick,blockArrow] (z1)+(0.1,-0.1) -- (z); - -\path[blockArrow] (z) -- node[pos=0.96,right] {-} (ResidueDiff); - -\node[processBlock,below=of Filters,text width=2cm] (RefBuffer) {Decoded Picture Buffer}; - -\node[processBlock,text width=1.5cm] at(Inter |- RefBuffer) (MotionSearch) {Motion Estimation}; - -\path[blockArrow] (Filters) -- (RefBuffer) node[point,pos=0.4] (p2) {}; -\path[blockArrow] (RefBuffer) -- (MotionSearch); -\path[blockArrow] (MotionSearch) -- (Inter); -\path[blockArrow] (p0) |- (MotionSearch); +\end{pgfonlayer} -\node[right of=p2, text centered,text width=1.5cm, node distance=2.3cm] (Output) {\textbf{Video output}}; -\path[blockArrow] (p2) -- (Output); + +\node[right=1 of VLC, text centered,text width=1.7cm] (Output) {\textbf{HEVC NALUs}}; +\path[blockArrow] (VLC) -- (Output); + \end{tikzpicture} + diff --git a/021_VideoCoding/Figures/IntraPredictionPlanar.tikz b/021_VideoCoding/Figures/IntraPredictionPlanar.tikz new file mode 100644 index 0000000..996ad0f --- /dev/null +++ b/021_VideoCoding/Figures/IntraPredictionPlanar.tikz @@ -0,0 +1,104 @@ + +\begin{tikzpicture}[auto, on grid, scale=.5, y=-1cm, every node/.style={scale=.5, y=-1cm}] + \def\BlockSize{4} + \tikzstyle{TextStyle}=[text centered, scale=1/.5, font=\footnotesize] + + \tikzstyle{PU} =[draw, minimum height=\BlockSize cm, minimum width=\BlockSize cm] + \tikzstyle{Neighbour} =[draw, minimum height=1 cm, minimum width=\BlockSize cm, fill=gray!60] + + + + \tikzstyle{PixelAnchor} =[xshift=.5cm, yshift=-.5cm] + \tikzstyle{Pixel} =[circle, thin, fill=\TikZColorFill, inner sep=0, minimum size=10pt] + + \tikzstyle{PixelNeighbour} =[Pixel, fill=black!70] + \tikzstyle{Arrow} =[-latex,thick, draw=black] + \tikzstyle{NeighbourSquares} =[fill=\TikZColorFillDark] + + \tikzstyle{EllipseDup} =[PixelAnchor, draw, ellipse, minimum width=5cm, minimum height=.7cm, inner sep=0] + + \draw[Pixel] (0,0) grid ++(4.5,4.5); + \node[NeighbourSquares, anchor=north west, minimum width=1cm, minimum height=1cm] at (-1,-1) {}; + \node[NeighbourSquares, anchor=north west, minimum width=4cm, minimum height=1cm] at (0,-1) {}; + \node[NeighbourSquares, anchor=north west, minimum width=4cm, minimum height=1cm] at (4,-1) {}; + \node[NeighbourSquares, anchor=north west, minimum width=1cm, minimum height=4cm] at (-1,0) {}; + \node[NeighbourSquares, anchor=north west, minimum width=1cm, minimum height=4cm] at (-1,4) {}; + + % Pixels + \foreach \x in {0,...,3} + \foreach \y in {0,...,3} + { + \node at (\x,\y) [PixelAnchor,Pixel]() {}; + } + + \path[PixelAnchor, Arrow, shorten >= 2pt] (-1,1) -- ++( 2, 0 ); + \path[PixelAnchor, Arrow, shorten >= 2pt] (4,1) -- ++( -3, 0 ); + + \path[PixelAnchor, Arrow, shorten >= 2pt] (1,-1) -- ++( 0, 2 ); + \path[PixelAnchor, Arrow, shorten >= 2pt] (1,4) -- ++( 0, -3 ); + + + \node[EllipseDup] at (1,4) {}; + \node[EllipseDup, rotate=90] at (4,1) {}; + + \foreach \y in {-1,...,7} + { + \node at (-1,\y) [PixelAnchor,PixelNeighbour]() {}; + } + \foreach \y in {-1,...,3} + { + \node at (4,\y) [PixelAnchor,PixelNeighbour]() {}; + } + +% \path[PixelAnchor, Arrow, dashed, shorten >= 3pt] (4,-1) -- ++( 0 , \BlockSize ); +% \path[PixelAnchor, Arrow, dashed, shorten >= 3pt] (-1,4) -- ++( \BlockSize, 0 ); + \foreach \x in {0,1,...,7} + { + \node at (\x,-1) [PixelAnchor,PixelNeighbour]() {}; + } + \foreach \x in {0,1,...,3} + { + \node at (\x,4) [PixelAnchor,PixelNeighbour]() {}; + } + + \node[TextStyle] at (1.2,-1.3) {$p_1$}; + \node[TextStyle] at (4.2,-1.3) {$p_2$}; + + \node[TextStyle] at (5.3,1.3) {$p'_2$}; + + \node[TextStyle] at (-1.3,1.3) {$p_3$}; + \node[TextStyle] at (-1.3,4.3) {$p_4$}; + + \node[TextStyle] at (1.2,5.3) {$p'_4$}; + + + + + + \path[Arrow, very thick, -] (-1,4) -- ++(6,0); + \path[Arrow, very thick, -] (4,-1) -- ++(0,6); + \path[Arrow, very thick, -] (-1,0) -- ++(1,0); + \path[Arrow, very thick, -] (0,-1) -- ++(0,1); + \path[Arrow, very thick, -] (0,0) -- ++(4,0); + \path[Arrow, very thick, -] (4,0) -- ++(4,0); + \path[Arrow, very thick, -] (0,0) -- ++(0,4); + \path[Arrow, very thick, -] (0,4) -- ++(0,4); +% \node[TextStyle] at (-1.2,-1.5) {Top-left}; + + \node[TextStyle, anchor=south west, align=left, text width=4cm] at (-3,10) (RefLabel) {Reference pixels}; + \node[TextStyle, anchor=south west, align=left, text width=4cm, below=.5cm of RefLabel] (PredLabel) {Predicted pixels}; +% \node[TextStyle, anchor=south west, align=left, text width=4cm, right=4cm of RefLabel] (PixelDup) {Pixel duplication}; +% \node[TextStyle, anchor=south west, align=left, text width=4cm, below=.5cm of PixelDup] (PredDir) {Prediction reference}; + \node[TextStyle, anchor=south west, align=left, text width=4cm, right=4cm of RefLabel] (PredDir) {Prediction reference}; + + \node[Pixel, , minimum size=12pt, left=.1 cm of PredLabel.west] {}; + \node[PixelNeighbour, minimum size=12pt, left=.1 cm of RefLabel.west] {}; +% \path[Arrow, latex-, dashed] (PixelDup.west) -- ++(-1,0); + \path[Arrow, latex-] (PredDir.west) -- ++(-1,0); + + +% \path[PixelAnchor, Arrow, blue] (2.75,-1) -- ++( -2 , \BlockSize ); + +\end{tikzpicture} + + diff --git a/021_VideoCoding/Figures/IntraPredictions.tikz b/021_VideoCoding/Figures/IntraPredictions.tikz index 221b5e0..51210a2 100644 --- a/021_VideoCoding/Figures/IntraPredictions.tikz +++ b/021_VideoCoding/Figures/IntraPredictions.tikz @@ -2,7 +2,6 @@ decorations.pathreplacing,decorations.pathmorphing,shapes,% matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds} - \pgfdeclarelayer{background} \pgfdeclarelayer{foreground} \pgfdeclarelayer{L0} @@ -11,71 +10,65 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds} \pgfdeclarelayer{L3} \pgfsetlayers{background,L0,L1,L2,L3,main,foreground} -\newcommand{\scale}{.13} +\newcommand{\scale}{.12} \begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ] - \definecolor{LCUFill}{rgb}{0,0.5,1} - \definecolor{DrawColor}{rgb}{0,0,0.5} - \definecolor{LCUFill}{rgb}{1,1,1} - \definecolor{DrawColor}{rgb}{0,0,0} + \tikzstyle{arrowstyle}=[scale=1] + \tikzstyle{IntraArrow}=[draw=black, shorten <=0, postaction={decorate,decoration={markings, mark=at position .3 with {\arrow[arrowstyle]{stealth}}}} ] + \tikzstyle{IntraNumber}=[scale=5.5, align=center] - \tikzstyle{TextStyle}=[scale=2, text centered] + \draw[draw=black, very thick] (-31.5,-31.5) rectangle ++(64,64); - \tikzstyle{LCU}=[draw=DrawColor, fill=LCUFill, very thick] - \tikzstyle{CU}=[draw=DrawColor, thin] - \tikzstyle{PU}=[draw=DrawColor, thin, densely dashed] - - \tikzstyle arrowstyle=[scale=1] - \tikzstyle{IntraArrow} =[draw=black, shorten <=0, postaction={decorate,decoration={markings, mark=at position .3 with {\arrow[arrowstyle]{stealth}}}} ] - - -% \newcommand{\gridsize}{70} -% \draw[help lines] (-\gridsize,-\gridsize) grid ++(\gridsize*2,\gridsize*2); - - \draw[draw=black, very thick] (-31,-31) rectangle (31,31); +% \draw[xshift=.5cm] (-32,33) grid ++(64,1); +% \draw[yshift=-.5cm] (-32,33) grid ++(-1,-64); +% \node[IntraNumber] at (-30,33) {30}; +% \node[IntraNumber] at (-20,33) {20}; +% \node[IntraNumber] at (-10,33) {10}; +% \node[IntraNumber] at (0,33) {0}; +% \node[IntraNumber] at (10,33) {10}; +% \node[IntraNumber] at (20,33) {20}; +% \node[IntraNumber] at (30,33) {30}; \coordinate (Center) at (0,0); -% \foreach \x in {-32,...,31} -% \draw[draw=black] (\x-.5,31.5) rectangle ++(1,1); -% \foreach \y in {-31,...,32} -% \draw[draw=black] (-32-.5,\y-.5) rectangle ++(1,1); - - \newcounter{intra} - \setcounter{intra}{1} - \foreach \y in {-31,-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26} + %% Vertical Modes + \setcounter{maincounter}{1} + \foreach \y in {-31,-26,-21,-17,-13,-9,-5,-2} { - \addtocounter{intra}{1} - \node[scale=4, align=center] at (-32,\y) {\theintra}; - \draw[IntraArrow] (-31,\y) to (Center); + \addtocounter{maincounter}{1} + \draw[IntraArrow] (-32,\y) to (Center); + \node[IntraNumber] at (-34,\y-.7) {\themaincounter}; } - \addtocounter{intra}{1} - \node[scale=4, align=center] at (-32,32) {\theintra}; - \draw[IntraArrow] (-31,31) to (Center); - \foreach \x in {-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31} + \addtocounter{maincounter}{1} + \draw[IntraArrow] (-32,0) to (Center); + \node[IntraNumber] at (-34,0) {\themaincounter}; + \foreach \y in {2,5,9,13,17,21,26} { - \addtocounter{intra}{1} - \node[scale=4, align=center] at (\x,32) {\theintra}; - \draw[IntraArrow] (\x,31) to (Center); + \addtocounter{maincounter}{1} + \draw[IntraArrow] (-32,\y) to (Center); + \node[IntraNumber] at (-34,\y+.7) {\themaincounter}; } -% \foreach \x in {-31,-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31} -% { -% \draw[IntraArrow] (Center) to (\x,-31); -% } -% \foreach \y in {-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31} -% { -% \draw[IntraArrow] (Center) to (31,\y); -% } + %% Horizontal Modes + \foreach \x in {-32,-26,-21,-17,-13,-9,-5,-2} + { + \addtocounter{maincounter}{1} + \draw[IntraArrow] (\x,33) to (Center); + \node[IntraNumber] at (\x-.8,34.7) {\themaincounter}; + } + \addtocounter{maincounter}{1} + \draw[IntraArrow] (0,33) to (Center); + \node[IntraNumber] at (0,34.7) {\themaincounter}; + \foreach \x in {2,5,9,13,17,21,26,32} + { + \addtocounter{maincounter}{1} + \draw[IntraArrow] (\x,33) to (Center); + \node[IntraNumber] at (\x+.8,34.7) {\themaincounter}; + } -% \end{scope} -% \foreach \y in {0,...,31} -% \draw[black] (-1,\y) rectangle ++(1,1); - - -% \node[scale=6, text width=2cm, align=left] at (8,-1) {Planar}; -% \node[scale=6, text width=2cm, align=left] at (8,-4) { DC}; +% \node[scale=1/\scale, text width=2cm, align=left] at (10,-3) {\footnotesize 0: Planar}; +% \node[scale=1/\scale, text width=2cm, align=left] at (10,-6) {\footnotesize 1: DC}; % % H.264 diff --git a/021_VideoCoding/Figures/IntraPredictionsExample.tikz b/021_VideoCoding/Figures/IntraPredictionsDirectional.tikz similarity index 52% rename from 021_VideoCoding/Figures/IntraPredictionsExample.tikz rename to 021_VideoCoding/Figures/IntraPredictionsDirectional.tikz index fe51f76..e35746a 100644 --- a/021_VideoCoding/Figures/IntraPredictionsExample.tikz +++ b/021_VideoCoding/Figures/IntraPredictionsDirectional.tikz @@ -1,19 +1,19 @@  \begin{tikzpicture}[auto, on grid, scale=.5, y=-1cm, every node/.style={scale=.5, y=-1cm}] \def\BlockSize{4} - \tikzstyle{TextStyle}=[text centered, font=\Huge] + \tikzstyle{TextStyle}=[text centered, scale=1/.5, font=\footnotesize] + + \tikzstyle{PU} =[draw, minimum height=\BlockSize cm, minimum width=\BlockSize cm] + \tikzstyle{Neighbour} =[draw, minimum height=1 cm, minimum width=\BlockSize cm, fill=gray!60] - \tikzstyle{PU} =[draw, minimum height=\BlockSize cm, minimum width=\BlockSize cm, font=\Huge] - \tikzstyle{Neighbour} =[draw, minimum height=1 cm, minimum width=\BlockSize cm, font=\Large, fill=gray!60] - \def\TikZColorFill{gray!20} -\def\TikZColorFillDark{gray!80} \tikzstyle{PixelAnchor} =[xshift=.5cm, yshift=-.5cm] \tikzstyle{Pixel} =[circle, thin, fill=lightgray, inner sep=0, minimum size=10pt] - \tikzstyle{PixelNeighbour} =[Pixel, fill=black!80] + + \tikzstyle{PixelNeighbour} =[Pixel, fill=black!70] \tikzstyle{Arrow} =[-latex,thick, draw=black] - \tikzstyle{NeighbourSquares} =[fill=darkgray!80] + \tikzstyle{NeighbourSquares} =[fill=\TikZColorFillDark] \draw[Pixel] (0,0) grid ++(4.5,4.5); @@ -41,36 +41,36 @@ } - \path[PixelAnchor, Arrow, dashed, shorten >= 38 pt] (.5,-1) -- ++( -2 , \BlockSize ); - \path[PixelAnchor, Arrow, shorten >= 25 pt] (1,-1) -- ++( -2 , \BlockSize ); - \path[PixelAnchor, Arrow, dashed, shorten >= 13 pt] (1.5,-1) -- ++( -2 , \BlockSize ); + \path[PixelAnchor, Arrow, dashed] (.5,-1) -- ( 0 , 0 ); + \path[PixelAnchor, Arrow, ] (1,-1) -- ( 0 , 1 ); + \path[PixelAnchor, Arrow, dashed] (1.5,-1) -- ( 0 , 2 ); \foreach \x in {2,...,5} { \path[PixelAnchor,Arrow] (\x,-1) -- ++(-2, \BlockSize); } \foreach \x in {2.5,...,4.5} { - \path[PixelAnchor,Arrow,dashed] (\x,-1) -- ++(-2, \BlockSize); + \path[PixelAnchor,Arrow,dashed] (\x,-1) -- ++(-1.5, \BlockSize - 1); } \path[Arrow, very thick, -] (-1,4) -- ++(6,0); \path[Arrow, very thick, -] (4,-1) -- ++(0,6); \path[Arrow, very thick, -] (-1,0) -- ++(1,0); \path[Arrow, very thick, -] (0,-1) -- ++(0,1); - \path[Arrow, very thick, -] (0,0) -- ++(4,0) node[midway,above, yshift=1cm, scale=1.2] {Top}; - \path[Arrow, very thick, -] (4,0) -- ++(4,0) node[midway,above, yshift=1cm, scale=1.2] {Top-right}; + \path[Arrow, very thick, -] (0,0) -- ++(4,0) node[TextStyle, midway,above, yshift=.4cm] {Top}; + \path[Arrow, very thick, -] (4,0) -- ++(4,0) node[TextStyle, midway,above, yshift=.4cm] {Top-right}; - \path[Arrow, very thick, -] (0,0) -- ++(0,4) node[midway,above, xshift=-1cm, scale=1.2, rotate=90] {Left}; - \path[Arrow, very thick, -] (0,4) -- ++(0,4) node[midway,above, xshift=-1cm, scale=1.2, rotate=90] {Bottom-left}; -% \path[Arrow, very thick, -] (8,0) -- ++(0,-1); -% \path[Arrow, very thick, -] (0,8) -- ++(-1,0); + \node[TextStyle] at (-1.2,-1.5) {Top-left}; - \node[anchor=south west, align=left, text width=4cm] at (6,6) (RefLabel) {Reference pixels}; - \node[anchor=south west, align=left, text width=4cm, below=.5cm of RefLabel] (PredLabel) {Predicted pixels}; - \node[anchor=south west, align=left, text width=4cm, below=.5cm of PredLabel] (PredDir) {Prediction direction}; + \path[Arrow, very thick, -] (0,0) -- ++(0,4) node[TextStyle, midway,above, xshift=-.5cm, rotate=90] {Left}; + \path[Arrow, very thick, -] (0,4) -- ++(0,4) node[TextStyle, midway,above, xshift=-.5cm, rotate=90] {Bottom-left}; - \node[Pixel, left=1.3 cm of PredLabel] {}; - \node[PixelNeighbour, left=1.3 cm of RefLabel] {}; + \node[TextStyle, anchor=south west, align=left, text width=4cm] at (4,7) (RefLabel) {Reference pixels}; + \node[TextStyle, anchor=south west, align=left, text width=4cm, below=.5cm of RefLabel] (PredLabel) {Predicted pixels}; + \node[TextStyle, anchor=south west, align=left, text width=4cm, below=.5cm of PredLabel] (PredDir) {Prediction direction}; + + \node[Pixel, , minimum size=12pt, left=.1 cm of PredLabel.west] {}; + \node[PixelNeighbour, minimum size=12pt, left=.1 cm of RefLabel.west] {}; \path[Arrow, latex-] (PredDir.west) -- ++(-1,0); diff --git a/021_VideoCoding/Figures/PictureTypes_TSA.tikz b/021_VideoCoding/Figures/PictureTypes_TSA.tikz index 84c7eee..7df6edf 100644 --- a/021_VideoCoding/Figures/PictureTypes_TSA.tikz +++ b/021_VideoCoding/Figures/PictureTypes_TSA.tikz @@ -26,5 +26,11 @@ % \node[draw,dashed,inner sep=5pt, fit=(B3) (B2) (B4) (B2A)] (Leading) {}; % \node[label,below=.1 of Leading] {Leading}; \node[label,right=.1 of P6, yshift=-.3cm] {TSA}; -\path[dependency] (P0.south west)++(-1,-.8) -- ++(12cm+3cm,0) node[midway, anchor=center,fill=white] {Display order}; +\path[dependency] (P0.south west)++(-1,-.6) -- ++(12cm+3cm,0) node[midway, anchor=center,fill=white] {Display order}; + +\path[dependency] (P0.south west)++(-1,-.4) -- ++(0,+4.2cm) node[midway, anchor=center,fill=white, rotate=90] {Layer}; +\node[label, left=.1 of P0] (T0) {T$_0$}; +\node[label] at (T0 |- P1) {T$_1$}; +\node[label] at (T0 |- P2) {T$_2$}; + \end{tikzpicture} diff --git a/021_VideoCoding/VideoCodingReview.tex b/021_VideoCoding/VideoCodingReview.tex index c29e25c..502f604 100755 --- a/021_VideoCoding/VideoCodingReview.tex +++ b/021_VideoCoding/VideoCodingReview.tex @@ -89,26 +89,33 @@ Similar to the previous coding standards developed by JCT-VC, in \ac{hevc}~\cite coding syntax and semantics are standardised. The constrains and functionalities are defined through profiles and levels, while the semantic meaning of syntax elements define the decoding process, such that every decoder conforming to the standard generates the same output when given a complaint stream. Such delimitation of scope in the standard allows maximum freedom for developers and manufactures to develop proprietary optimisations to cope with various applications’ requirements (\emph{e.g.}, balancing the compression ratio and the coding complexity). - -\begin{figure}[tb] -\centering -% \includegraphics[width=0.75\columnwidth]{\FigsPath/HEVC/HEVCBlockDiagram} -\input{\FigsPath/HEVCBlockDiagram.tikz} -\caption{\HIGHLIGHT{Generic \ac{hevc} standard architecture.}} -\label{fig:HEVCBlockDiagram} -\end{figure} - \HIGHLIGHT{ Figure \ref{fig:HEVCBlockDiagram} illustrates the block diagram of the hybrid \ac{hevc} encoder. It follows the same overall architecture as its predecessors starting from H.261. The encoder uses both intra- and inter-picture prediction to exploit spatial and temporal redundancies, respectively. Subsequently, the prediction residue transformed by linear spatial transform, which is then quantised, and entropy coded. This allows to reduce the bitstream redundancy to a minimum. -The resulting bitstream is combined with control information, which signals the prediction chosen, motion vectors, filtering operations and bitstream headers. +%The resulting bitstream is combined with control information, which signals the prediction chosen, motion vectors, filtering operations and bitstream headers. } -\HIGHLIGHT{ -As depicted in Figure \ref{fig:HEVCBlockDiagram}, each picture can be compressed with intra coding algorithms or inter prediction using previously encoded and reconstructed pictures kept in the \ac{dpb} as reference. -The encoding process follows a block-based approach as in previous standards with the larger units referred to as \ac{ctu} with a maximum size of $64\times64$. In order to improve the overall quality of the reconstructed pictures and smooth the coding artefacts, \ac{hevc} uses the deblocking and \ac{sao} filters. -The encoder has a built-in decoder to generate the reference pictures. In the decoding loop, the residual information is delivered to the inverse quantisation and transform modules. The finally reconstructed samples are obtained by adding the predicted signal and applying the filtering operations. -} + +\begin{figure}[tb] +\centering + \includegraphics[width=\columnwidth]{021_VideoCoding/Figures/HEVCBlockDiagram} +%\input{\FigsPath/HEVCBlockDiagram.tikz} +\caption{\HIGHLIGHT{Block diagram of an \ac{hevc} standard encoder.}} +\label{fig:HEVCBlockDiagram} +\end{figure} + +\glsreset{ctu} +\glsreset{cu} +\glsreset{pu} +\glsreset{tu} +\glsreset{nalu} +\glsreset{cabac} +\glsreset{rpset} + +There are some novelties introduced in the \ac{hevc} standard. As depicted in Figure \ref{fig:HEVCBlockDiagram}, the encoding process follows a block-based approach, therefore each picture is partitioned into larger blocks, referred to as \acs{ctu}, with a maximum size of $64\times64$. Subsequently, a dynamic quad-tree partitioning is used to divide the picture into \acs{cu}. These units are introduced as the basic processing units for prediction, and for the transform and quantisation operations \acs{tu} are used. +In order to generate the prediction residue the \ac{hevc} may use one of 34 intra-prediction modes or take advantage of symmetric and asymmetric partitions for motion estimation and inter-prediction. +The motion estimation is performed in those pictures listed in the \ac{rpset} which was introduced in the \ac{hevc} to increase the error robustness of reference picture lists. +To improve the overall quality of the reconstructed pictures and smooth the coding artefacts, \ac{hevc} not only uses a deblocking filter but also a \ac{sao} filter. Finally, the control data, quantised coefficients, prediction modes, motion vectors and filter information are compressed using the \ac{cabac} entropy coder and generate a compressed bitstream composed of different \acs{nalu}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% @@ -226,7 +233,7 @@ This enables a more efficient prediction structure, while maintaining an access However, it results in leading pictures which may not be decodable, which are explained in the next sub-section. The introduction of \ac{cra} frames in \ac{hevc} was able to improve the coding efficiency by up to 6\% \cite{HEVCCRAPics2011}. -\HERE +%\HERE The \ac{hevc} supports bitstream splicing, which means taking a particular bitstream starting from a \ac{rap} (both \ac{idr} and \ac{cra} frames) and insert it into another bitstream at a random-access point. In case of bitstream splicing starting from an \ac{idr} frame the leading pictures does not have dependencies to frames prior the \ac{rap}, thus all frames are decodable. @@ -258,21 +265,20 @@ Therefore, when a random-access is performed at the associated \ac{rap} frame, t \subsubsection{Temporal sub-layer switching pictures} -\begin{figure}[tb] -\centering -% \includegraphics[width=0.75\columnwidth]{\FigsPath/HEVC/Pictures_TSA} -\input{\FigsPath/PictureTypes_TSA.tikz} -\caption{Temporal layers access pictures example.} -\label{Figure2:TSAExample} -\end{figure} - -\HERE A \ac{tsa} picture is a trailing picture that marks a temporal layer switching point. When decoding a sub-set of the temporal layers and a \ac{tsa} picture is found from the temporal layer just above the maximum temporal layer currently being decoded, it is possible for the decoder to start decoding any number of additional temporal layers. For example, frame $P_6$ in Figure~\ref{Figure2:TSAExample} is considered a \ac{tsa} picture since it only depends on a frame belonging to temporal layer 0 for prediction, as well as, any subsequent frame predicted from the \ac{tsa} frame $P_6$. In this example, if the decoder was only decoding the temporal layer 0 it can after $P_6$ starts decoding both layer 0 and 1 or decode all the available three layers. Although theses frame allows higher flexibility in terms of layer switching, they severely constrain the prediction of frames following the \ac{tsa} picture. +\begin{figure}[tb] + \centering + % \includegraphics[width=0.75\columnwidth]{\FigsPath/HEVC/Pictures_TSA} + \input{\FigsPath/PictureTypes_TSA.tikz} + \caption{Temporal layers access pictures example.} + \label{Figure2:TSAExample} +\end{figure} + To reduce the constrains in frame prediction due to layer switching, \ac{hevc} introduces the \ac{stsa} picture. These pictures have similar purpose as the \ac{tsa} frames but they only guarantee that frames in the same temporal layer as the \ac{stsa} frame in the decoding order are able to be correctly reconstructed. This is guaranteed by preventing any dependency from frames preceding the \ac{stsa} to be used as reference in its temporal layer. One example of a \ac{stsa} frame is also shown in Figure~\ref{Figure2:TSAExample} in frame $P_2$. @@ -366,10 +372,13 @@ Figure~\ref{Figure2:HEVCCodingStructure} illustrates an example of a \ac{ctu} pa The \ac{cu} is the basic processing unit in \ac{hevc}. It is further partitioned into two types of units, specifically \ac{pu} and \ac{tu}, which are data blocks used for prediction and transform operations, respectively. The decision whether to use intra-frame or inter-frame prediction modes is taken at \ac{cu} level. However, depending on the partitioning of the \ac{cu} into \acs{pu} different intra or inter coding modes can be used for each \ac{pu}. +\HERE + %The main key feature of \ac{hevc} is the new block partition structure, which achieves more flexibility and efficiency. Four different block concepts have been introduced in the standard: \ac{ctu}, \ac{cu}, \ac{pu}, and \ac{tu}, all of which have different roles. Figure \ref{fig:HEVCBlockStructure} illustrates an example of a CTU partitioning in HEVC, and it is compared to the \ac{h264} \ac{mb} structure. CTUs are defined as the high-level partition within each slice and can be as large as 64$\times$64, which can be recursively partitioned into four smaller blocks referred to as CUs. The CU is the basic units for the intra/inter prediction and can be as small as 8$\times$8. Subsequently, the CU are divided into up to four PUs which are used for inter prediction. For intra coding PU is not used. Finally, the CU is divided into a transform unit tree, which may include several TUs, in a quad-tree structure. Transform units are the basic units for the transform operations applied to the coding residue. % %Each CU may be split into two or four Pus resulting in either symmetric or asymmetric partitions. Asymmetric partitioning of the CU into two PUs enables more efficient motion estimation since it allows more flexible partitioning of the picture~\cite{BlockPartHEVC2012}. This scheme is able to improve the coding performance by up to $1\%$ when compared with rectangular symmetric partitioning. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -388,7 +397,7 @@ Intra predicted blocks are obtained using the previously encoded pixels around t When the intra prediction mode is chosen for a \ac{cu}, the size of the \ac{pu} need to be the same as the \ac{cu}, except when the smaller \ac{cu} is selected and further splitting into four \acs{pu} is done. For example, when the smaller \ac{cu} size is $8\times8$, the encoder may decide to divide it into four \acs{pu} of $4\times4$ pixels, each one having its own intra prediction mode. This is signalled using a binary flag. The use of small \acs{pu} is useful for complex regions that require fine-granularity prediction. -In \ac{hevc} three types of intra-frame prediction are defined: angular modes, Planar and DC~\cite{IntraCodingHEVC2012,HEVCIntraperformance2012}. +In \ac{hevc} there are 35 different of intra-frame prediction modes: Planar, DC and 33 angular modes as shown on the left of Figure~\ref{Figure2:IntraModesHEVC}~\cite{IntraCodingHEVC2012,HEVCIntraperformance2012}. %Figure~\ref{Figure2:IntraModesHEVC} illustrates these intra prediction modes. %As shown, the predictions are applied to square blocks only. %The border around the square blocks corresponds to the reference pixels from which the prediction is applied (different colours correspond to different pixel values). @@ -406,12 +415,12 @@ In order to overcome this issue and to allow all possible modes, \ac{hevc} uses \begin{figure}[tb] \centering \resizebox{\textwidth}{!}{ -\begin{tabular}{x{.6\columnwidth}x{.4\columnwidth}} +\begin{tabular}{x{.55\columnwidth}x{.45\columnwidth}} \input{021_VideoCoding/Figures/IntraPredictions.tikz} & -\input{021_VideoCoding/Figures/IntraPredictionsExample.tikz} \\ +\input{021_VideoCoding/Figures/IntraPredictionsDirectional.tikz} \\ \end{tabular} } -\caption{List of directional intra prediction modes in \ac{hevc} (left) and an example of the intra prediction mode 32 for a $4\times4$ block.} +\caption{Directional intra prediction modes in \ac{hevc} (left) and an example of the intra prediction mode 32 for a $4\times4$ block.} \label{Figure2:IntraModesHEVC} \end{figure} @@ -420,59 +429,89 @@ In order to overcome this issue and to allow all possible modes, \ac{hevc} uses Directional intra prediction was first introduced in the \ac{h264} standard~\cite{H264OverviewJul2003}, to estimate regions with a structured texture or directional edges. Directional prediction projects the reconstructed samples in the block neighbourhood (see reference pixels in Figure~\ref{Figure2:IntraModesHEVC}) along a specific direction. The \ac{hevc} increases the number of possible directions from 8 (in \ac{h264}) to 33~\cite{IntraCodingHEVC2012}. The direction of each projection is illustrated in the left-side of Figure~\ref{Figure2:IntraModesHEVC} by the directional modes 2 to 34. %In the figure it is possible to notice that directional modes 2 to 10 only project the pixels from the left neighbouring border in a horizontal direction, modes 10 to 25 use both the left and top neighbouring pixels in a diagonal direction and modes 26 to 34 project the top neighbours in vertical direction. -As shown in the figure, the angular modes are designed to provide a dense coverage near the horizontal and vertical directions. Moreover, the angular directions are more coarse as it gets closer to the diagonals. This reflects the observed statistical prevalence of the angles and the prediction efficiency. - -To improve the efficiency of angular prediction, when the reference samples need to be projected a bilinear interpolation is used from the two closest pixels using a $1/32$ accuracy. The dashed arrows in Figure~\ref{Figure2:IntraModesHEVC} correspond to a projection of interpolated pixels. +As shown in the figure, the angular modes are designed to provide a dense coverage near the horizontal (i.e., mode 10) and vertical (i.e., mode 26) directions. Moreover, the angular directions are more coarse as it gets closer to the diagonals. This reflects the observed statistical prevalence of the angles and the prediction efficiency. +To improve the efficiency of angular prediction, when the reference samples need to be projected a bilinear interpolation is used from the two closest pixels using a $1/32$ accuracy. The dashed arrows on right of Figure~\ref{Figure2:IntraModesHEVC} correspond to a projection of interpolated pixels. \subsubsection{Planar and DC} -In contrast, with the angular mode which cover those regions with strong direction edges, \ac{hevc} also implements the Planar and DC modes targeting smooth regions. These modes are useful for smooth regions as they result in a linear or constant prediction. +\begin{figure}[tb] +\centering +\input{021_VideoCoding/Figures/IntraPredictionPlanar.tikz} +\caption{Example of Planar prediction for a $4\times4$ block.} +\label{Figure2:IntraModePlanar} +\end{figure} -The DC mode uses the average or the neighbouring reference samples to generate a constant prediction for the current \ac{pu}. -The Planar mode predicts the \ac{pu} through linear predictions from vertical and horizontal slopes derived from the neighbouring reference samples. As the Planar mode used linear surface for prediction it prevents the discontinuities at \ac{pu} boundaries. Both of these modes are also illustrated in Figure~\ref{Figure2:IntraModesHEVC}. +The Planar and DC modes are particularly efficient in the prediction of smooth regions. +The DC mode uses the average or the neighbouring reference samples to generate a constant prediction for the current \ac{pu} as a whole, i.e., all pixels are predicted from the same value. + +The Planar mode predicts the \ac{pu} through linear interpolation from the four closest neighbouring reference pixels. Figure~\ref{Figure2:IntraModePlanar} shows the reference pixels (i.e., $p_1$, $p'_2$, $p_3$ and $p'_4$) and arrows pointing to the pixel being predicted. +Since the right and bottom neighbourhood is not available (not coded yet), the same pixel $p_2$ is repeated to form a column of reference pixels, marked with an ellipse in Figure~\ref{Figure2:IntraModePlanar}. A similar approach is also used to create a the bottom row of reference pixels from the pixel $p_4$ . +%As the Planar mode used linear surface for prediction it prevents the discontinuities at \ac{pu} boundaries. \subsubsection{Sample smoothing} -In \ac{hevc} sample smoothing is applied at two distinct occasions two improve the overall performance of the prediction accuracy. Firstly, the reference samples are filtered and secondly the \ac{pu} is filtered after prediction. This improves the previous standard \ac{h264} where only reference samples were filtered. +In \ac{hevc} sample smoothing is applied in two distinct cases to improve the overall performance of the prediction accuracy. +Firstly, the reference samples are filtered and secondly the \ac{pu} is filtered after prediction. +%In comparison with previous standard \ac{h264}, where only reference samples were filtered. -In \ac{hevc} the reference sample smoothing is conditionally applied, based on the \ac{pu} and intra prediction mode. Similar to \ac{h264}, a three tap $\left[1\ 2\ 1\right]/4$ smoothing filter is applied, except for \ac{pu} sizes of $4\times4$. The prediction results from Planar mode is also filtered when the \ac{pu} size is greater than $8\times8$, and it is not used for the DC mode as it would not be useful. +In \ac{hevc}, the reference sample smoothing is conditionally applied, based on the \ac{pu} and intra prediction mode. Similar to \ac{h264}, a three tap smoothing filter (i.e., $\left[1\ 2\ 1\right]/4$) is applied, except for \ac{pu} sizes of $4\times4$. The prediction resulting from Planar mode is also filtered when the \ac{pu} size is greater than $8\times8$, and it is not used for the DC mode. -The second filter in \ac{hevc} is the boundary smoothing. This filter aims at removing the discontinuities along block boundaries due to intra prediction. The application of this filter is more exclusive, and it is only applied to the DC and angular mode 10 and 26 (exactly the horizontal and vertical directions, respectively). A two-tap filter $\left[3\ 1\right]/4$ fed by the original samples and the adjacent reference sample. For the DC mode this filter is applied to both the top row and the left-most column. In case of the angular mode 10 and 26 only the left-most column and the top row are filtered, respectively. Only the mentioned boundaries are filtered, as they are the most likely to introduce discontinuities. The result of the boundary smoothing can be noticed in Figure~\ref{Figure2:IntraModesHEVC} for the aforementioned modes. +The second filter in \ac{hevc} is the boundary smoothing. This filter aims at removing the discontinuities along block boundaries due to intra prediction. +This filter is only applied when the block size is smaller than $32\times32$ to the DC and angular modes 10 and 26 (exactly the horizontal and vertical directions, respectively). +For the DC mode a two-tap filter ($\left[3\ 1\right]/4$), fed by the original predicted value and the adjacent reference sample, is applied to both the top row and the left-most column of predicted pixels. +In the case of angular mode 10 (horizontal direction), only the predicted samples of the first column are changed by adding half of the difference between the adjacent reference sample and the top-left reference sample. +A similar filtering is done for angular mode 26 (vertical direction), but only for the first row +of predicted samples. +Only the mentioned boundaries are filtered, as they are the most likely to introduce discontinuities. %\subsubsection{Rough Mode Decision} %\cite{HEVCROughModeDecision} \subsection{Inter-frame prediction} -The efficiency of video coding algorithms consistently relies on inter-frame prediction techniques used to reduce temporal redundancy. The underlying idea of inter-frame prediction is to estimate the current frame from one or more previously encoded reference frames using block-based motion compensated. The most common technique used for motion estimation in \ac{hevc}, as well as in \ac{h264}, is the boundary matching algorithm. In the \ac{hevc} standard an \ac{amvp} for efficient motion vector signalling, which defines the Merge Mode and differential coding of the motion information. +The efficiency of video coding algorithms consistently rely on inter-frame prediction techniques to reduce temporal redundancy. +The underlying idea of inter-frame prediction is to estimate the current frame from one or more previously encoded frames used as reference with block-based motion compensation. The most common technique used for motion estimation in \ac{hevc} is the block matching algorithm. In the \ac{hevc} standard an \ac{amvp} is used for efficient motion vector coding, which defines the Merge Mode and differential coding of the motion information. \subsubsection{Motion estimation and compensation} -Motion estimation consists in searching for the current block with the highest similarity with the target block to be predicted (\ac{pu} in \ac{hevc}), using block-matching algorithms over the previously encoded frames, i.e. the reference frames. Typically, reference frames correspond to past or future temporal frames. The frame encoding order determines which reference frames are selected and whether future frames are available. Typically, video frame type P normally are predicted from the past as B frame are predicted from frames both in past and future. -Block-matching consist on subtracting the current block from one of equal size in the reference frame in a pre-defined window. The reference block that results in the lowest difference is selected for prediction. -In order to signal the selected block to be used for motion compensation the different between the target and the reference block positions are computed to obtain a two-dimensional vector, referred to as \ac{mv}. This vector is signalled to the decoder, in order to apply the same prediction. +Motion estimation consists in searching for a block with the highest similarity with the current block to be predicted (\ac{pu} in \ac{hevc}), using block-matching algorithms over previously encoded frames, i.e. the reference frames. +Typically, reference frames correspond to past or future temporally adjacent frames. +The frame encoding order determines which references are selected and which future frames are available. +The reference block that results in the lowest error is selected as the best block for prediction. +To identify the best matching block to be used for motion compensation, the different between the target and the reference block positions is encoded as a two-dimensional \glsfirst{mv}. -Similar to \ac{h264} the \ac{hevc} allows for either one or two \ac{mv} to be selected to compensate the target \ac{pu}. This results in uni-predictive and bi-predictive coding, respectively. \ac{hevc} also allows for weighted prediction, which consists in scaling and offset operations the predicted block using determined factors which are explicitly sent to the decoder. -The motion vectors are obtained from frames included in the reference frame list. Each frame is identified by an index, therefore to fully describe inter-frame prediction a combination of two parameters is required: \ac{mv} and reference index. +The \ac{hevc} allows for either one or two \acs{mv} to be used for each \ac{pu}. Video frames of type P are predicted from one single reference (one \ac{mv}) while frame of type B are predicted from two references (two \acs{mv}). +This results in uni-direction or bi-directional coding, respectively. +Furthermore, weighted prediction can also be used, which consists in scaling and offset operations performed on the prediction block to improve efficiency. +The \acs{mv} only refer to frames included in a reference frame list, where each frame is identified by an index. Therefore, to fully describe inter-frame prediction a combination of two parameters is required: \ac{mv} and reference frame index. -In recent standards motion compensation has been performed using a variable block size approach. In \ac{h264} block sizes ranging from $4\times4$ up to $16\times$ pixels were allowed. In \ac{hevc} a significant improvement was introduced by extending the block sizes up to $64\times64$ pixels. As mentioned before it also allows asymmetric rectangular block sizes. -By using larger prediction units, \ac{hevc} is able to save bits in the signalling of the selected \ac{mv}, because less \ac{mv} are required. +Since MPEG-2, motion estimation/compensation has been performed using blocks of variable size. +In \ac{h264} block sizes ranging from $4\times4$ up to $16\times$ pixels are allowed, while \ac{hevc} extends the possible block sizes up to $64\times64$ pixels, and also to asymmetric rectangular block sizes. +By using larger prediction units less \ac{mv} are required and fewer bits are spent in coding motion information. -In order to increase the efficiency of motion estimation using fractional \acs{mv} are used. This has been proven to improve accuracy of motion compensation. To accomplish this, frame interpolation is performed to generated non-integer pixels in the reference frame. Both \ac{h264} and \ac{hevc} standards use \acs{mv} up to quarter-pixel accuracy for luma component. For chroma samples, the \ac{mv} accuracy is determined according to the chroma sampling format, which for 4:2:0 results in eighth-pixel accuracy. In contrast, with the two-stage approach used in \ac{h264}, \ac{hevc} uses a single separable approach for the interpolation process without intermediate rounding operations. This improves the precision and simplifies the architecture of the fractional interpolation. +In order to increase the efficiency of motion estimation fractional \acs{mv} are used, which has been proven to improve motion compensation accuracy. +This is accomplished through interpolation to generate non-integer pixels in the reference frame, resulting in \acs{mv} up to quarter-pixel accuracy for luma samples. +For chroma samples, the \ac{mv} accuracy is determined according to the chroma sampling format, which for 4:2:0 results in eighth-pixel accuracy. +In contrast with the two-stage approach used in \ac{h264}, \ac{hevc} uses a single separable approach for the interpolation process without intermediate rounding operations. This improves the precision and simplifies the architecture of the fractional interpolation. -The block matching-based motion estimation is one of the most computationally complex procedures in the \ac{mv} encoder. Normally, the optimal solution for motion estimation is provided by the full-search algorithm, which tests all the possible candidate blocks in the search area. Although the full-search algorithm achieves the optimal performance, it requires the highest computational complexity. Therefore, sub-optimal algorithms have been proposed over the years to reduce the motion estimation complexity and decrease encoding time\cite{HEVCME2012,HEVCME2017,HEVCME2014}. -For example, the HM reference software implements a fast algorithm based on Test Zone Search ~\cite{HEVCME2012}. -Since motion estimation is a non-normative procedure, encoders may implement any alternative search algorithms that suits its needs (i.e., compression ratios or encoding time). +The block matching-based motion estimation is one of the most computationally intensive procedures in video encoders. +In general, the optimal solution for motion estimation is provided by the full-search algorithm, which tests all the possible candidate blocks in the search area. However, although the full-search algorithm achieves the optimal performance, it requires the highest computational complexity. +Therefore, sub-optimal algorithms have been proposed over the years to reduce the motion estimation complexity and decrease encoding time~\cite{HEVCME2012,HEVCME2017,HEVCME2014}. +For example, the HM reference software implements a fast algorithm based on Test Zone Search~\cite{HEVCME2012}. +Since motion estimation is a non-normative procedure, encoders may implement any alternative motion search algorithms. \subsubsection{Reference picture management} -Another improvement in \ac{hevc} is related with the reference picture management scheme. \ac{hevc} introduces the \ac{rpset} concept~\cite{HEVCHighLevelReferenceDez2012}, which defines how previously decoded pictures are managed in the \ac{dpb}. Decoded frame in the \ac{dpb} are mark grouped in one of the following categories: (i) \emph{short-term reference}, (ii) \emph{long-term reference} and (iii) \emph{unused for reference}. Once a frame is marked as \emph{unused for reference} it is no longer used for motion compensation and it will be discarded from the \ac{dpb} after being displayed/outputted. -In \ac{h264} a sliding windows approach was used to implicitly manage the reference picture list. This process discards a given frame whenever the \ac{dpb} has the number of defined reference frames (which is signalled by the encoder). Moreover, \ac{h264} used an explicitly \ac{mmco} to send commands to the decoder and change the normal reference picture management process. +Another improvement in \ac{hevc} is related with the reference picture management scheme. \ac{hevc} introduces the \ac{rpset} concept~\cite{HEVCHighLevelReferenceDez2012}, which defines how previously decoded pictures are managed in the \ac{dpb}. +Decoded frames in the \ac{dpb} are grouped in one of the following categories: (i) \emph{short-term reference}, (ii) \emph{long-term reference} and (iii) \emph{unused for reference}. +Once a frame is marked as \emph{unused for reference} it is no longer used for motion compensation and it will be discarded from the \ac{dpb} after being sent to the displayed. +In \ac{h264} a sliding window approach is used to implicitly manage the reference picture list. +This process discards a given frame whenever the \ac{dpb} has the number of defined reference frames (which is controlled by the encoder). Moreover, \ac{h264} uses an explicitly \ac{mmco} to send commands to the decoder and change the normal reference picture management process. \begin{table}[tb] \centering @@ -492,14 +531,21 @@ In \ac{h264} a sliding windows approach was used to implicitly manage the refere \end{tabular} \end{table} -In the \ac{hevc} standard instead of a signalling relative changes, the status of the \ac{dpb} is signalled for every slice. -The most fundamental difference with the \ac{rpset} concept compared to \ac{mmco}/sliding window of \ac{h264} is that for each particular slice a complete set of the reference frames that are used by the current or any subsequent frames are be provided. -Thus, a complete set of all frames that must be kept in the \ac{dpb} (in the present and future) are signalled. -This is different from the \ac{h264} scheme where only relative changes to the \ac{dpb} are signalled. With the \ac{rpset} concept, no information from earlier frames in decoding order is needed to maintain the correct status of the \ac{dpb}. Table~\ref{Table2:ReferencePictureListExample} shows the \ac{rpset} for some frames illustrated in Figure~\ref{Figure2:LeadingTralingPics}. As expected for the \ac{cra} frame ($I_2$) nothing is signalled. For the remaining frames, two types of signalled are used: frame number and a flag to mark if it is currently used. This is required because any frame not listed in a given \ac{rpset} will be discarded and subsequent frames cannot use it as reference. +In the \ac{hevc} standard the status of the \ac{dpb} is encoded for every slice, instead of the implicit managing used in the past. +The most fundamental difference in \ac{rpset} concept compared to \ac{mmco}/sliding window of \ac{h264} is that each particular frame has a complete set of the reference frames that are used or will be used (or will be used in subsequent frames). +Thus, a complete set of all frames that must be kept in the \ac{dpb} explicitly provided. +This is different from \ac{h264} where only relative changes to the \ac{dpb} encoded. +With the \ac{rpset} concept, no information from earlier frames in decoding order is needed to maintain the correct status of the \ac{dpb}. +Table~\ref{Table2:ReferencePictureListExample} shows the \ac{rpset} for some frames illustrated in Figure~\ref{Figure2:LeadingTralingPics}. +As expected for the \ac{cra} frame ($I_2$) no information is sent to the decoder because there is not inter-prediction and the \ac{dpb} should be empty. +For the remaining frames, two types of information are encoded: frame number and a binary flag to indicate whether it is currently used. This is required because any frame not listed in a given \ac{rpset} will be discarded by the decoder and then subsequent frames cannot use them as reference. -The \ac{rpset} concept in \ac{hevc} provides a basic level of error robustness to the reference picture management. In \ac{h264} whenever a reference frame is lost the decoder can look at the value of the -\emph{FrameNum} (incremental value associated with each reference frame) to detect the loss event. However, as gaps in \emph{FrameNum} are allowed the decoded might not be able to decide the missing frame was intentional or if it resulted from a loss event in the transmission. Moreover, even if the decoded if able to detect the loss event it might not be able to correctly compute the value the \ac{poc} associated with the concealed frame. Using the \ac{rpset} concept in the \ac{hevc} the decoder is always able to detect loss events and correctly identify the missing frames, preventing it from use wrong frames as reference. +The \ac{rpset} concept provides a basic level of error robustness to the reference picture management. +In \ac{h264} whenever a reference frame is lost the decoder can look at the value of the \emph{FrameNum} (incremental value associated with each reference frame) to detect the loss event. +However, as gaps in \emph{FrameNum} are allowed the decoder might not be able to decide whether the missing frame was intentional or resulted of a transmission loss event. +%Moreover, even if the decoded if able to detect the loss event it might not be able to correctly compute the value the \ac{poc} associated with the concealed frame. +Using the \ac{rpset} concept the decoder is always able to detect loss events and correctly identify the missing frames. \subsubsection{Asymmetric block partitions} @@ -514,15 +560,17 @@ The \ac{rpset} concept in \ac{hevc} provides a basic level of error robustness t \end{figure} -When inter prediction mode is signalled, \ac{hevc} defines a higher range of partitioning schemes, allowing the \ac{cu} to be divided into one, two or four \acs{pu}. -The splitting into four \acs{pu} is only allowed when the \ac{cu} size is equal to the minimum. Otherwise, the splitting should be carried out at the \ac{cu} level which leads to the same partition tree. -Figure~\ref{Figure2:InterPuPartitions} illustrates the possible \ac{cu} partitioning into \acs{pu}. One should note that the partition type applied to the \ac{cu} is also applied to divide the \ac{cb} into \acs{pb}. +When inter prediction mode is selected a higher range of partitioning schemes is allowed, thus dividing the \ac{cu} into one, two or four \acs{pu}. +The splitting into four \acs{pu} is only allowed when the \ac{cu} size is equal to the minimum. +%Otherwise, the splitting should be carried out at the \ac{cu} level which leads to the same partition tree. +Figure~\ref{Figure2:InterPuPartitions} illustrates the possible \ac{cu} partitioning into \acs{pu}. +%One should note that the partition type applied to the \ac{cu} is also applied to divide the \ac{cb} into \acs{pb}. Figure~\ref{Figure2:InterPuPartitions}~(a) shows the symmetric partitions. The $N \times N$ corresponds to the case where the \ac{cu} is not further divided. Then, the standard defines the horizontal and vertical division into two \acs{pu}, corresponding to the $\frac{N}{2}\times N$ and $N \times \frac{N}{2}$ cases, respectively. Finally, the $\frac{N}{2} \times \frac{N}{2}$ case results in four \acs{pu}. The partitions shown in Figure~\ref{Figure2:InterPuPartitions}~(b) correspond to the asymmetric partitioning, which is one of the main improvements introduced in \ac{hevc}, as it allows higher flexibility in motion estimation, leading to higher compression efficiency~\cite{HEVCBlock2012}. -These modes are only allowed for \acs{cu} with $16\times16$ pixels or larger. +These partitions types are only allowed for \acs{cu} with $16\times16$ pixels or larger. \subsubsection{Merge mode} @@ -534,21 +582,27 @@ These modes are only allowed for \acs{cu} with $16\times16$ pixels or larger. \label{Figure2:MVCandidates} \end{figure} -The Merge mode was introduced in \ac{hevc} standard, as a new technique to derive motion information from spatial and temporal neighbouring blocks~\cite{MotionCodingHEVC2013}. This technique extends the concepts of the direct and skip modes used in \ac{h264}~\cite{H264OverviewJul2003}, using a more sophisticated approach with two key differences. -Firstly, the Merge Mode allows for more \ac{mv} candidates. Thus, an index value is included to select one out of several \ac{mv} candidates. Secondly, it explicitly identifies the reference picture index used for the selected candidate, as well as, the reference picture list. This results in an improved flexibility, compared with the predefined values used in \ac{h264}. The \ac{mv} candidates allowed in the Merge Mode\cite{MotionCodingHEVC2013} are illustrated in Figure \ref{Figure2:MVCandidates}. The figure illustrates the spatial candidates' positions ($A_0$, $A_1$, $B_0$, $B_1$, $B_2$). -Moreover, it also uses a temporal motion vector candidate, derived from the co-located position on the temporal adjacent frame at positions $T_0$ and $T_1$. From this \ac{mv} candidate list the unavailable candidates are not accounted and duplicated candidates (i.e. with the same motion information) are removed. -In the Merge Mode, a $C-1$ number of candidates are used, selected from the aforementioned order and one temporal \ac{mv} candidate is selected to fulfil the total number of \ac{mv} ($C$). The value of $C$ is defined at the slice header and a flag controlling the use of temporal \ac{mv} candidates is sent at the \ac{pps}-level. -For B-slices, additional \ac{mv} candidates are generated by choosing two existing candidates according to a predefined -order for reference picture list 0 and list 1. A total number of twelve combination are specified in \ac{hevc} which are listed in ~\cite{HEVCOverviewDez2012}. For P-slices if the number of candidates is less than $C$ zero motion vectors are included to fill the remaining entries. +The Merge mode was introduced in \ac{hevc} as a new technique to derive motion information from spatial and temporal neighbouring blocks~\cite{MotionCodingHEVC2013}. This technique extends the concepts of the Direct and Skip modes used in \ac{h264}~\cite{H264OverviewJul2003}, using a more sophisticated approach with two key differences. +Firstly, the Merge Mode allows for more \ac{mv} candidates than the Skip in \ac{h264}. +Thus, an index is used to identify one out of several \ac{mv} candidates. +Secondly, the reference picture index is explicitly identified. +This results in improved flexibility, compared with the predefined values used in \ac{h264}. +The \ac{mv} candidates allowed in the Merge Mode\cite{MotionCodingHEVC2013} are illustrated in Figure \ref{Figure2:MVCandidates}. +The figure shows the positions of the spatial \ac{mv} candidates ($A_0$, $A_1$, $B_0$, $B_1$, $B_2$). +Moreover, it also uses a temporal \ac{mv} candidate, derived from the co-located position on the temporally adjacent frame at positions $T_0$ or $T_1$. From the \ac{mv} candidate list the unavailable candidates are not considered, and duplicated candidates (i.e. with the same motion information) are removed. +The Merge Mode uses a $C-1$ spatial \ac{mv} candidates, selected from the aforementioned order, and one temporal \ac{mv} candidate to fulfil the total number of \acs{mv} ($C$). The value of $C$ is defined at the slice header and a flag controlling the use of temporal \ac{mv} candidates is encoded at the \ac{pps}-level. +For B-slices, additional \ac{mv} candidates are considered by choosing two existing candidates according to a predefined order for reference picture list 0 and list 1. A total number of twelve combinations are specified in \ac{hevc}, which are listed in~\cite{HEVCOverviewDez2012}. For P-slices, if the number of candidates is less than $C$, then zero \acs{mv} are included to fill the remaining entries. -The Merge Mode allows for the signalling of a \ac{mv} candidate and the residual information. In \ac{hevc}, the Skip mode, i.e., transmitting only the \ac{mv} without residual information, is treated as a special case of the Merge Mode. In this case only a skip flag is enabled, and the corresponding merge index is transmitted. +The Merge Mode defines how to encode the \ac{mv} candidate and the residual information. +In \ac{hevc}, the Skip mode, i.e., transmitting only the \ac{mv} without residual information, is treated as a special case of the Merge Mode. In this case only a skip flag is enabled, and the corresponding merge index is transmitted. \subsubsection{Motion vector prediction} -The \ac{hevc} also improves the performance of the motion vector coding for those cased where the Merge Mode is not used. \ac{hevc} always use a difference encoding approach for efficiently represents the motion information. -It considers the same \ac{mv} candidate of the Merge mode, illustrated in Figure~\ref{Figure2:MVCandidates}, to derive the motion vector predictors from which the difference vector is obtained. -From these \ac{mv} candidates only two are selected based on the following: +The \ac{hevc} also improves the performance of the \ac{mv} coding for those cases where the Merge Mode is not used. +\ac{hevc} always use a differential encoding approach for efficiently representing the motion information. +It considers the same \ac{mv} candidates of the Merge mode, illustrated in Figure~\ref{Figure2:MVCandidates}, to derive the \ac{mv} predictors from which the difference vector is obtained. +From these \ac{mv} candidates only the first two available candidates are selected based on the following order: \begin{enumerate} \item Left candidate: first available from $A_0$ and $A_1$. \item Top candidate: first available from $B_0$, $B_1$ and $B_2$. @@ -556,7 +610,8 @@ From these \ac{mv} candidates only two are selected based on the following: \end{enumerate} Therefore, the use of the temporal \ac{mv} candidate is limited to those cases where the spatial neighbours are not able to provide two valid candidates. -\ac{hevc} only allows a much lower number of candidates to be used in the motion vector prediction process, since it can still send a motion vector difference. Moreover, the encoder will perform motion estimation for each candidate, which increase the encoding complexity for every possibility tested. For those cases where the reference index is not the same in the current \ac{pu} and the neighbouring one, a scaling version of the \ac{mv} is used. +For \ac{mv} prediction only a much lower number of candidates is allowed, since the encoder can still send a \ac{mv} difference. Moreover, the encoder performs motion estimation for each candidate, which increases the encoding complexity for every possibility tested. +%For those cases where the reference index is not the same as the current \ac{pu} and the neighbouring one, a scaling version of the \ac{mv} is used. %In the \ac{h264} standard, along different motion compensated coding techniques~\cite{H264OverviewJul2003}, the so-called P\_SKIP mode was introduced. In this mode, neither the quantised residual information, nor the \ac{mv} is transmitted. This mode is useful for large areas with no change or just constant motion, and it is only enabled for $16\times16$ blocks, \emph{i.e.}, the whole \ac{mb}. The \ac{mv} for the P\_SKIP mode is derived based on the surrounding neighbour MBs\cite{H264MotionVectorPrediction2003}. When no neighbours are available, or the left or top \acs{mv} are equal to zero, a zero \ac{mv} is used. Otherwise, the \ac{mv} is derived from the median value of the following three neighbours: left, above and above right, as shown in Figure \ref{SkipMergeMode}~(a) by $MV_A$, $MV_B$ and $MV_C$, respectively. Whenever the $MV_C$ is not available, the $MV_D$ is used instead. % @@ -570,24 +625,27 @@ Therefore, the use of the temporal \ac{mv} candidate is limited to those cases w \label{Section2:TransformQuant} -\glsreset{tb} +\glsreset{tu} -Similarly, to the previous standards, \ac{hevc} uses a transform followed by a quantisation to compress the residual signal. -The residual block is partitioned into multiple square blocks, referred to as \ac{tb}. +Similarly, to the previous standards, \ac{hevc} uses a transform followed by quantisation to compress the residual signal. +The residual block is partitioned into multiple square blocks, referred to as \acs{tu}. \subsubsection{Core transform} A 2D transform of the residual information is obtained by applying 1D transforms in both the horizontal and vertical directions. The core transform matrices are obtained from the \ac{dct} basis functions to define a integer matrix of $32\times32$ points. Sub-sampled versions of this matrix are used to derive the remaining transform matrix sizes down to $4\times4$. -A special case is applied for the residual coding of a $4\times4$ intra coded block. In this case an alternative integer transform based on the \ac{dst}. +A special case is applied for coding the residual of $4\times4$ intra coded block. +In this case an alternative integer transform based on the \ac{dst} is used. The motivation for \ac{dst} is related to the statistics of the residual signal of these blocks, which tend to present higher residue as the distance to the boundary used for prediction (i.e, top or left boundary) increases. -Experiments have shown that DST provides up to 1\% of bitrate savings for the compression of $4\times4$ intra predicted blocks, without significantly increase the computational complexity \cite{DCTDSTComp2013}. +Experiments have shown that \ac{dst} provides up to 1\% of bitrate savings for the compression of $4\times4$ intra predicted blocks, without significantly increase the computational complexity \cite{DCTDSTComp2013}. \subsubsection{Quantisation} -\ac{hevc} standard follows the same principle introduced in \ac{h264} for the quantisation method. After obtaining the residual transform a quantisation is applied using a uniform-reconstruction quantiser scheme based on the \ac{qp}. The standard supports \ac{qp} values ranging from 0 to 51, and a increase of 6 in the \ac{qp} corresponds to doubling the quantisation step. This procedure is the main responsible to introduce the coding errors as it performs many-to-one mapping, which cannot be reversed. - +\glsreset{qp} +The \ac{hevc} follows the same principle introduced previous standards for quantisation. +After applying the transform operation to prediction residue transform a quantisation is applied to the resulting coefficients using a uniform-reconstruction quantiser scheme based on a \ac{qp}. +The standard supports \ac{qp} values ranging from 0 to 51, and an increase of 6 in the \ac{qp} corresponds to doubling the quantisation step. This procedure is the main responsible for the coding distortion as it performs many-to-one mapping, which cannot be reversed. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -596,10 +654,13 @@ Experiments have shown that DST provides up to 1\% of bitrate savings for the c \section{Entropy coding} \label{Section2:Entropy} -In contrast to \ac{h264} standard, which allows two entropy coding methods, \ac{hevc} only uses the \ac{cabac} \cite{HEVCCabac2003}. \Gls{cabac} is an arithmetic coding method that only uses binary symbols, considering different probability models for each symbol. Entropy coding efficiency is closely linked with the context model selected. Therefore, this was carefully designed in \ac{hevc} standard, extending the functionality presented in \ac{h264} +\glsreset{cabac} + +\ac{hevc} only uses the \ac{cabac}, which is an arithmetic coding method that only uses binary symbols, considering different probability models for each symbol~\cite{H264Cabac2003}. +Entropy coding efficiency is closely linked with the context model selected. Therefore, this was carefully designed in the \ac{hevc} standard, extending the functionality previously defined for \ac{h264}. For example, the \ac{hevc} standard exploits the depth of the partition tree or residual transform tree in order to derive the context models for several syntax elements. -Regarding transform coefficient coding, \ac{cabac} uses a scanning method to firstly organise the coefficients and then transmits the position of the last non-zero transform coefficient. -Moreover, it transmits, a significance map, the sign bits and the levels for the transform coefficients. +Regarding transform coefficient coding, \ac{cabac} uses a scanning method to firstly organise the coefficients and then encodes the position of the last non-zero transform coefficient. +Moreover, a significance map is also encoded along the sign bits and the levels of the transform coefficients. Three coefficient scanning methods are available: the diagonal up-right, the horizontal and the vertical scan. The coefficient scanning is implicitly selected and always performed in $4\times4$ sub-blocks for all transform block sizes. % Details about the context models selection and the compression of each syntax element of \ac{hevc} can be found in [1]. @@ -610,10 +671,15 @@ The coefficient scanning is implicitly selected and always performed in $4\times \section{In-loop filters} \label{Section2:Filters} -The \ac{hevc} standard uses two filtering procedures over the reconstructed pixels before writing them into the frame buffer, namely the Deblocking Filter and the \ac{sao} filter. -The main purpose of deblocking filter is to reduce blocking artefacts caused by the block-based coding tools, operating similarly to the deblocking filter of the \ac{h264} standard. -DBF is applied to the samples adjacent to the \ac{pu} or \ac{tu} boundaries, that are aligned on a $8\times8$ grid. Such grid-based restriction reduces the computational complexity and improves parallel-processing -implementation which is one of the main goals of \ac{hevc}. Three strength levels can be used by filter depending on the coded block characteristics. +\glsreset{dbf} +\glsreset{sao} + +The \ac{hevc} standard uses two filtering procedures over the reconstructed pixels before writing them into the frame buffer, namely the \ac{dbf} and the \ac{sao} filter. +The main purpose of deblocking filter is to reduce blocking artefacts caused by the block-based coding tools.%, operating similarly to the deblocking filter of the \ac{h264} standard. +\ac{dbf} is applied to the samples that are spatially adjacent to the \ac{pu} or \ac{tu} boundaries, considering a $8\times8$ grid. +Such grid-based restriction reduces the computational complexity and improves parallel-processing +implementation which is one of the main goals of \ac{hevc}. +Three strength levels can be used by the \ac{dbf} depending on the coded block characteristics. The \ac{sao} is a non-linear filter, that is adaptively applied to all samples of the image, after the deblocking filter. In its operation, the \ac{sao} filter modifies the samples by adding an offset value, extracted from lookup tables transmitted by the encoder. For each \ac{ctu}, the encoder decides whether to apply the \ac{sao} filter or not, and if used, one of the two available filter types is applied, namely the band offset or the edge offset. In the band offset mode, the added offset value depends on the sample amplitude, while the edge offset uses the gradient to classify and derive the offset value. @@ -626,13 +692,15 @@ by adding an offset value, extracted from lookup tables transmitted by the encod \section{Summary} \label{Section2:Summary} -This chapter described the main features of the \glsfirst{hevc} standard, giving a special focus on the high-level features, coding structures and newly introduced prediction modes. +This chapter was devoted to describe the main features of the \ac{hevc} standard, giving a special focus on the high-level features, coding structures and newly introduced prediction modes. -At the high-level \ac{hevc} introduces new \ac{nal} units types, not only for new signalling features but also for the new pictures types introduced. \ac{hevc} introduces new picture types to handle different types of random-access strategies, which allow more flexibility and higher coding efficiency, when compared to the random-access in previous standard such \ac{h264}. +At the high-level, \ac{hevc} introduces new \ac{nalu} types, and also new pictures types. +These allow the implementation of various random-access strategies, more flexibility and higher coding efficiency then its predecessor \ac{h264}. -Another major improvement comes from the inter-frame prediction. On the one hand new high-level reference picture management was introduced, which increase the error robustness, as it explicitly defines the active reference frame and the \ac{dpb} status. On the other hand it introduces the \ac{amvp} and the Merge mode which efficiently reduce the required bits or motion signalling. +Another major improvement comes from the inter-frame prediction. On the one hand new high-level reference picture management was introduced, which increase the error robustness, as it explicitly defines the active reference frames and the \ac{dpb} status. On the other hand it introduces the \ac{amvp} and the Merge mode which efficiently reduce the required bits for coding the motion information. -In general, \ac{hevc} achieves higher coding performance than its predecessors. Previous studies reported an average bitrate saving of approximately 50\% for the same subjective quality. Such amount of bitrate saving is normally related with increased prediction structures with higher dependencies. Such dependencies are more sensible to data loss, which will increase the error propagation to subsequent coding structures. Therefore, it is emergent to evaluate the impact of the transmission errors in the \ac{hevc} and develop efficient techniques to cope with data loss and reduce the error propagation. +The newly developed coding tools and prediction modes described in this chapter have resulted in approximately 50\% of bitrate savings for the same subjective quality in comparison with the previous standard. +However, such high coding efficiency implies higher data dependencies, which are more sensible to data loss and increase the error propagation to subsequent coding structures. Therefore, evaluating the impact of transmission errors and error propagation in \ac{hevc} is of most importance to develop efficient techniques to recover from data loss and reduce the error propagation. diff --git a/022_ErrorResilience/ErrorResilienceCategories.tex b/022_ErrorResilience/ErrorResilienceCategories.tex index 7340610..8e1a1ad 100644 --- a/022_ErrorResilience/ErrorResilienceCategories.tex +++ b/022_ErrorResilience/ErrorResilienceCategories.tex @@ -22,7 +22,7 @@ \end{tabular}& Unequal error protection, transport priority and multi-channel transmission \\ \midrule - Error concealment by post-processing & + Error concealment at the decoder & \begin{tabular}{@{}l@{}} $\bullet$ Linear interpolation \\ $\bullet$ Boundary matching\\ diff --git a/022_ErrorResilience/RobustTransmissionReview.tex b/022_ErrorResilience/RobustTransmissionReview.tex index 9340186..a146761 100755 --- a/022_ErrorResilience/RobustTransmissionReview.tex +++ b/022_ErrorResilience/RobustTransmissionReview.tex @@ -12,24 +12,39 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Introduction} -In general, digital communications through networks are prone to transmission errors and data losses. -These errors can be handled in different ways, such as retransmission, error correction, and \ac{ec} \cite{ErrorResilience2000,PixelInterlacingIntraConcealment2011}. -This chapter reviews the main robust video coding techniques that can be applied at the video encoder/decoder to reduce the error propagation and increase the decode video quality. Several techniques have been proposed in the past to deal with the problem of robust video transmission, especially for video coding standard with high temporal dependencies, such as \ac{h264} and more recently \ac{hevc}. These techniques normally address the spatio-temporal dependencies in video coders which provide higher coding gains~\cite{ROPE2007} but lead to lower error robustness. In this chapter the error resilience and concealment schemes are divided in different categories depending on they functionality and algorithms used. Although different each approach follows a different techniques they all aim at striking the most efficient balance between coding efficiency and decoded video quality. +\glsreset{ec} +\glsreset{vcl} -This chapter is organised as follows: Section~\ref{Section22:Localisation} describes the localisation techniques, which reduce the temporal and spatial dependencies in coded bitstreams. Section~\ref{Section22:DataPartitionRedundant} provides a overview of methods based on data partition and classification and bitstream redundancy. These techniques ease the introduction of extra information in order to recover from lost data. Section~\ref{Section22:Concealment} covers the techniques used at the decoder-side as post-processing tools to reduce the effect of errors and improve visual quality and smoothness. Section~\ref{Section22:ConcealmentAware} provides a description of methods use the \ac{ec} techniques to provide improved error robustness at the encoder-side. Finally, Section~\ref{Section22:Summary} concludes this chapter by providing a overall comparison between the different approaches. + +In general, digital communications through networks are prone to transmission errors or data loss. +These errors can be handled in different ways, such as retransmission, error correction, and \ac{ec} \cite{ErrorResilience2000,PixelInterlacingIntraConcealment2011}. +This chapter reviews the main robust video coding techniques that can be applied at the \ac{vcl} to reduce the error propagation and increase the decoded video quality. +Several techniques have been proposed in the past to deal with the problem of robust video transmission, especially for video coding standards with high temporal dependencies, such as \ac{h264} and more recently \ac{hevc}. +These techniques are normally focused on the spatio-temporal dependencies in the coded streams, which are used to obtain high coding gains but lead to low error robustness~\cite{ROPE2007}. +In this chapter the error resilience and concealment schemes are divided in different categories depending on their functionality and algorithms. +Although each approach is based on a different techniques they all aim at striking the most efficient balance between coding efficiency and decoded video quality under lossy conditions. + +This chapter is organised as follows: Section~\ref{Section22:Localisation} describes the localisation techniques, which reduce the temporal and spatial dependencies in coded bitstreams. Section~\ref{Section22:DataPartitionRedundant} provides an overview of methods based on data partition, classification and bitstream redundancy. +%These techniques ease the introduction of extra information in order to recover from lost data. +Section~\ref{Section22:Concealment} covers the techniques used at the decoder-side as post-processing to reduce the effect of errors and improve visual quality and smoothness. +Section~\ref{Section22:ConcealmentAware} provides a description of methods that use \ac{ec} techniques to achieve improved error robustness. +Finally, Section~\ref{Section22:Summary} concludes this chapter by providing a overall comparison between the different approaches. \clearpage %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\section{Categories of robust video coding techniques} +\section{Robust video coding techniques} -%This chapter provides a review of the existing techniques used for robust video transmission under error-prone networks. -In order to increase the robustness of video transmission, robust coding mechanisms can be employed in a transmission chain to mitigate the artefacts caused by data loss. These techniques can be divided into several categories depending on the functional characteristics and the main problem they address~\cite{ErrorResilienceAug2005}. In this chapter we divide the robust video coding techniques into four categories: +In general, most of the compression efficiency is achieved by exploiting both the spatial and temporal redundancy of video signals. Since there is high correlation between neighbouring information, predictive coding schemes are very effective to achieve higher compression gains by exploiting such correlations. +However, the drawback of high predictive coding schemes is the strong dependencies between compressed symbols with great impact on error propagation. +In order to increase the robustness of video transmission, several coding mechanisms can be employed at different elements of a transmission chain to mitigate the artefacts caused by errors and data loss. +These mechanisms can be classified according to the functional characteristics and the type of problem they address~\cite{ErrorResilienceAug2005}. +In this chapter the robust video coding techniques ere grouped into four categories: \begin{itemize} \item Localization techniques; \item Data partitioning and redundant coding; - \item Error concealment by post-processing; + \item Error concealment at the decoder; \item Concealment-aware error resilience. \end{itemize} @@ -40,12 +55,13 @@ In order to increase the robustness of video transmission, robust coding mechani \input{022_ErrorResilience/ErrorResilienceCategories.tex} \end{table} -Table~\ref{tab:errorresilience} provides a summary of the techniques covered in this section and the key advantages associated with each class of techniques. -Localisation techniques are able to reduce the spatial and temporal dependencies in the bitstreams~\cite{ErrorResilienceModeHEVC2016,ErrorResilienceModeHEVC2017}. -Data partitioning groups the coded data into different categories depending on their relative importance. -Redundant coding makes use of redundant blocks of data to correct the transmission errors and achieve robust decoding in error prone conditions. -The problem of robust transmission may also be addressed from the decoder side, by using post-processing \ac{ec} methods to ease the effects of data loss. -Finally, concealment-aware techniques might be adopted at the encoder to increase the performance of \ac{ec} algorithms and increase the reconstruction quality of lost data at the decoder~\cite{ConcealmentDrivenSliceOrdering2008,ConcealmentAwareME2006,ConcealmentIntraDataHidng2009,ConcealmentSignalling2008}. +Table~\ref{tab:errorresilience} provides a summary of the techniques covered in this section, some examples and the key advantages associated with each group. +Localisation techniques mainly localise and break the predictive coding loop, reducing the spatial and temporal dependencies, so that if an error occurs it is not likely to affect other parts of the coded video~\cite{ErrorResilienceModeHEVC2016,ErrorResilienceModeHEVC2017}. +Data partitioning rely on grouping the coded data into different categories depending on their relative importance. +This ease the introduction of unequal error protection at the transmission layer. +Redundant coding consists in using additional data blocks to enable correction of transmission errors and to achieve robust decoding in error-prone conditions. +In case of the post-processing \ac{ec} methods, the problem of robust transmission is addressed from the decoder-side. +Finally, concealment-aware techniques are used at the encoder, to either adapt the coding loop or introduce extra information which is used to increase the performance of \ac{ec} algorithms and increase the reconstruction quality of lost data at the decoder \cite{ConcealmentDrivenSliceOrdering2008,ConcealmentAwareME2006,ConcealmentIntraDataHidng2009,ConcealmentSignalling2008}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -54,8 +70,7 @@ Finally, concealment-aware techniques might be adopted at the encoder to increas \section{Localisation techniques} \label{Section22:Localisation} -In general, video coding standard the compression efficiency is achieved by exploiting both the spatial and temporal redundancy of video sources. Since there is high correlation between neighbouring information, predictive coding schemes are very effective and obtain high compression gains exploiting such correlations. -The drawback of applying such techniques is the fact that the compressed symbols becomes very dependent on each other's which highly increase the error propagation. + Localization techniques break the predictive coding loop in order to reduce the likelihood of errors affect large regions of the video stream, both in the spatial and temporal domains. As expected since localization techniques break the predictive coding, it reduces the efficiency of such methods. This leads to lower compression ratios and increasing bitrate. @@ -77,8 +92,8 @@ A common localization technique addresses the error resilience problem of the ra \end{equation} where $D$ is the distortion of all pixels, $R$ the required bitrate, and $\lambda$ is the Lagrangian multiplier. In \cite{ROPEJun2000,ROPE2007,ROPERefGenJan2010} a rate distortion optimisation for error resilience was presented. -In order to take into consideration possible network losses a recursive optimal per-pixel estimation (ROPE) of the end-to-end distortion is applied at the encoder-side. -Let $f_n^i$ denote the original value of pixel $i$ in frame $n$, $\hat f_n^i$ denote its encoder reconstruction and $tilde f_n^i$ the decoder reconstruction after error concealment if needed. +In order to take into consideration possible network losses a \ac{rope} of the end-to-end distortion is applied at the encoder-side. +Let $f_n^i$ denote the original value of pixel $i$ in frame $n$, $\hat f_n^i$ denote its encoder reconstruction and $\tilde f_n^i$ the decoder reconstruction after error concealment if needed. Since data loss may occur in the transmission channel, $tilde f_n^i$ is considered a random variable at the encoder. The expected mean square error of each pixel is given by: \begin{align} @@ -145,7 +160,7 @@ In order to generate such reference frames the following can be applied where $\check f_n^i$ is a reference frame pixel, $\alpha$ is the leak factor, and $C$ is a constant value. The use of leaky predictions has been used in the past for error control especially in scalable video coding \cite{LeakyPredScalable2002}, where $C$ in Equation~(\ref{EQ2:LeakyPredConstant}) is replace by a lower layer error-free reconstruction. In case of single layer coding, the use of a constant value has significant impact on the coding efficiency leading to higher bitrate as shown in \cite{LeakyPredConstant1973}. -In order to overcome the coding efficiency problem weighted prediction can be used with similar advantages. Since \ac{h264} motion compensation can be applied from two individual previously encoded frames and combined together to generate a single prediction for the current frame \cite{WeightedBiPred2005}. In the case of \ac{hevc}, a method was proposed to force the use of dual inter-frame prediction~\ctie{DualFrameME2015} leading to lower error propagations. +In order to overcome the coding efficiency problem weighted prediction can be used with similar advantages. Since \ac{h264} motion compensation can be applied from two individual previously encoded frames and combined together to generate a single prediction for the current frame \cite{WeightedBiPred2005}. In the case of \ac{hevc}, a method was proposed to force the use of dual inter-frame prediction~\cite{DualFrameME2015} leading to lower error propagations. In case of leaky prediction instead of applying motion compensation is applied once, but with an interpolated frame obtained as follow: \begin{equation}\label{EQ2:LeakyPredDual} \check f_n^i = \alpha \hat f_n^i + (1- \alpha) \hat f_{n-1}^i. @@ -276,9 +291,9 @@ For example, the primary picture may be coded with fine quantization, while the One the one hand if the primary picture is received, the redundant information is discarded. One the other hand if errors occur and the primary picture is lost, the redundant one is used to provide a lower level of reconstructed quality and contain the error propagation. -Zhu \emph{et al.} investigated the paradigm of redundant picture coding in \ac{h264} standard~\cite{RedudantPicsJan2009}. -Different approaches were investigated in order to achieve a mixed method based on both reference picture selection to obtain a clean decoding without error propagation and a hierarchical allocation of the redundant pictures to increase the error robustness. In~\cite{VPxResiliencePCS2016} the key frames are encoded and transmitted at lower bitrate to be used in case of error to replace the missing frame. -Redundant pictures achieve good performance under error prone conditions, since they introduce secondary pictures that can be decoded when the primary picture is not available. +Zhu \emph{et al.} investigated the paradigm of redundant picture coding in \ac{h264} standard~\cite{RedudantPicsJan2009}. +Different approaches were investigated in order to achieve a mixed method based on both reference picture selection to obtain a clean decoding without error propagation and a hierarchical allocation of the redundant pictures to increase the error robustness. In~\cite{VPxResiliencePCS2016} the key frames are encoded and transmitted at lower bitrate to be used in case of error to replace the missing frame. +Redundant pictures achieve good performance under error prone conditions, since they introduce secondary pictures that can be decoded when the primary picture is not available. Moreover, redundant pictures might also work as localisation techniques when different reference frames are selected. However, such approaches generate large amount of redundant data and overheads. In~\cite{RedundantMVsJune2008} a newer approach was proposed to reduce the amount of information in the redundant pictures of the \ac{h264} standard, by introducing only redundant \acs{mv}. This was also used in~\cite{VPxResilience2016}, where the motion information is used to recover the missing frame. These approaches increases the overall system performance by presenting an efficient trade-off between the redundant bitrate and the error robustness. diff --git a/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex b/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex index c2fd5aa..fd23e98 100644 --- a/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex +++ b/031_ErrorResilienceEvaluation/ErrorResilienceEvaluation.tex @@ -52,6 +52,7 @@ The following cases studies are covered in this experimental evaluation: \end{enumerate} \subsection{Characterisation of the experiments} +\label{ExperimentalTestSetup} \input{031_ErrorResilienceEvaluation/TablesSequences.tex} @@ -61,7 +62,7 @@ The experimental setup used in this work to evaluate the error robustness of the Table~\ref{TableTestSequences} presents a summary of the main characteristics of the video sequences. Appendix~\ref{Appen:Sequences} shows a list of the first frame of each test sequence. In order to evaluation the spatial and temporal complexity of the video sequences the following metrics \cite{ITUTP9112008} were used: \begin{itemize} - \item {\ac{si})} based on Sobel filter. Each video frame ($f_t$) is firstly filtered, then the standard deviation is computed. Finally, the maximum value over time is chosen to represent the \ac{si} metric. This process can be represented in the following equation: + \item {\ac{si}} based on Sobel filter. Each video frame ($f_t$) is firstly filtered, then the standard deviation is computed. Finally, the maximum value over time is chosen to represent the \ac{si} metric. This process can be represented in the following equation: \begin{equation} SI = max_{time}\Big\{ std_{space} \big( Sobel\left(f_t\right) \big) \Big\}. \end{equation} diff --git a/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex b/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex index 10e9fb2..8700474 100644 --- a/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex +++ b/042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex @@ -4,7 +4,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % CHAPTER %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\chapter{Error Concealment-Aware Coding for Robust Video Transmission} +\chapter{Error Concealment-Aware for Robust Video Decoding} \label{Chapter:ConcealmentAwareTechniques} %\vspace{-6mm} %\clearpage @@ -42,7 +42,7 @@ Chapter~\ref{Chapter:ErrorResilienceReview} described different \ac{ec} algorith In this chapter a robust coding driven by \ac{ec} methods with the objective of enhancing the quality of reconstructed video at decoders receiving erroneous streams from the transmission network. The proposed approach combines error resilience tools at the encoder-side with \ac{ec} at the decoder-side. Different approaches were proposed in the past to that address the problem of \ac{ec} at the encoder-side \cite{ConcealmentDrivenSliceOrdering2008,ConcealmentAwareME2006,ErrorResilienceModeHEVC2016}. Moreover, extra information was used to aid the \ac{ec} operation at the decoder-side, either by sending redundant data \cite{ResilienceRECAP_ICASSP2009,SaliencyErrorConcealment_TM2013} or by embedding data into the bitstream \cite{ConcealmentIntraDataHidng2009}. Although the above methods facilitate the EC performance either through slice reordering or based on \acs{mv}, they always rely on the assumption that a fixed \ac{ec} method is used at the decoder. The work presented in \cite{ConcealmentSignalling2008,SignallingRobustScalable2015} proposes testing several methods with transmission of extra symbols to indicate the best method to be applied by the decoder. Although such approaches achieves reasonably good performance, it relies on the spatial neighbouring information or multi-layer video, which may not be available in all cases. -To advance one step further, in this chapter an EC-aware encoding scheme is proposed to achieve higher flexibility in terms of the error concealment approaches which leads to higher recovered video quality. +To advance one step further, in this chapter an EC-aware encoding scheme is proposed to achieve higher flexibility in terms of the error concealment approaches which leads to higher recovered video quality. This is accomplished by selecting at the encoder, among several \ac{ec} algorithms with different characteristics, the one that yields the lowest reconstruction distortion at the decoder and also minimises the \ac{rd} cost considering the necessary signalling rate. Thus, the proposed approach outperforms similar previous methods \cite{ConcealmentAwareME2006,ErrorResilienceModeHEVC2016} by selecting the best error concealment method without affecting the encoder optimisation. To further improve the quality of recovered frames, an enhancement layer is encoded with the mismatch residue between the transmitted and recovered frames. This is a dynamic optimisation process, which is more efficient than traditional approaches based on redundant pictures \cite{RedudantPicsJan2009,VPxResiliencePCS2016}, which transmit the whole frame as redundancy. In contrast with previous work presented in Chapter~\ref{Chapter:ErrorResilienceTechniques} that addressed the error resilience problem by optimising the coding dependencies, the approach followed in this work considers both \ac{ec} performance and the rate required for signalling the best \ac{ec} mode to the decoder. diff --git a/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex b/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex index 5a1bdb7..43a6233 100644 --- a/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex +++ b/042_ConcealmentAwareEncoding/Results/TableOveralPerformance.tex @@ -76,7 +76,7 @@ BQSquare & 35.64 & +1.47 & +1.18 & +2.01 & +1.77 \\ \hline Four People & 39.16 & +0.47 & -0.65 & +0.40 & -0.06 \\ \hline Kimono & 36.19 & +1.41 & -0.54 & +3.09 & +3.47 \\ \hline Park Scene & 34.54 & +2.06 & +2.01 & +3.22 & +3.15 \\ \hline -People & 29.48 & +1.50 & +1.35 & +2.64 & +3.08 \\ \hline +People On Street & 29.48 & +1.50 & +1.35 & +2.64 & +3.08 \\ \hline Race Horses & 29.08 & +2.34 & +4.52 & +5.35 & +6.02 \\ \hline Tennis & 32.71 & +1.32 & +3.13 & +3.81 & +4.97 \\ \hline \textbf{Average} & \textbf{--} & \textbf{+1.55} & \textbf{+1.88} & \textbf{+3.10} & \textbf{+3.33} \\ @@ -119,7 +119,7 @@ BQSquare & 31.36 & +2.50 & +2.40 & +3.33 & +3.55 \\ \hline Four People & 37.12 & +1.24 & -0.99 & +1.82 & +1.51 \\ \hline Kimono & 32.24 & +0.12 & -1.92 & -0.04 & -0.21 \\ \hline Park Scene & 30.42 & +1.22 & +1.56 & +2.92 & +3.64 \\ \hline -People & 25.05 & +0.40 & +0.74 & +0.97 & +2.09 \\ \hline +People On Street & 25.05 & +0.40 & +0.74 & +0.97 & +2.09 \\ \hline Race Horses & 24.19 & +0.63 & +2.53 & +2.75 & +3.74 \\ \hline Tennis & 27.08 & +1.30 & +3.51 & +4.58 & +6.61 \\ \hline \textbf{Average} & \textbf{--} & \textbf{+1.36} & \textbf{+1.96} & \textbf{+3.13} & \textbf{+3.79} \\ diff --git a/043_EndToEndRobustSystem/EndToEndRobustSystem.tex b/043_EndToEndRobustSystem/EndToEndRobustSystem.tex index 33f23df..081ae9f 100644 --- a/043_EndToEndRobustSystem/EndToEndRobustSystem.tex +++ b/043_EndToEndRobustSystem/EndToEndRobustSystem.tex @@ -1,7 +1,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % CHAPTER %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\chapter{\HIGHLIGHT{Optimal Robust Video Coding for High Efficient Video Coding}} +\chapter{\HIGHLIGHT{Error Robustness Optimisation for Highly Efficient Video Coding}} \label{Chapter:EndToEndRobustSystem} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -50,19 +50,19 @@ In this experimental evaluation the following methods are evaluated, namely: \begin{figure}[!t] \centering \input{043_EndToEndRobustSystem/Results/Quality_PLR_Methods.tex} -\vspace{-2mm} +\vspace{-4mm} \caption{Average decoded video quality using the main target bitrate.} \label{Figure6:AverageQualityMain} \end{figure} -\subsubsection{Experimental results} +\subsubsection{Experimental results and discussion} Figures~\ref{Figure6:AverageQualityMain}~and~\ref{Figure6:AverageQualityReduction} shows the average decoded video quality obtained from 50 different trials, measured with the \ac{psnr} quality metric. The figures shows the quality obtained for different loss conditions. Figure~\ref{Figure6:AverageQualityMain} shows the results obtained with the test sequence encoded using a constant target bitrate shown in the third column of Table~\ref{TableTestSequences}. Results on Figure~\ref{Figure6:AverageQualityReduction} were obtained using a lower amount of coded bits, corresponding to a 40\% reduction of the main target bitrate. This aims at finding the impact of the video and network bitrate on the performance of the proposed methods. \begin{figure}[!t] \centering \input{043_EndToEndRobustSystem/Results/Quality_PLR_Methods_Reduced.tex} -\vspace{-2mm} +\vspace{-4mm} \caption{Average decoded video quality using a 40\% reduction of the main target bitrate} \label{Figure6:AverageQualityReduction} \end{figure} @@ -78,8 +78,10 @@ Summarising, the performance of the proposed methods is not similar across diffe \section{Proposed architecture} \label{Section6:ProposedMethod} -In this section the architecture of the proposed method is described. The goal of the proposed method is to obtain an optimal error robustness method for a wide range of applications, i.e., video content and network conditions. -In order to accomplish that, at the encoder-side, the input signals are coded using different levels of error protections. Subsequently, an optimisation algorithm is used to decide which bitstream should be transmitted at any given instant, depending of different parameters. +This section provides an overview of the proposed method, which aims at obtaining the optimal error robustness method for different streaming conditions, i.e., video content and network conditions. +The main advantage of using the proposed algorithm is that it leads to an optimal trade-off between error robustness tools and coding efficiency. +This scheme can be used in transmission systems where different video bitstreams are available at the encoder-side and are selected based the current conditions, i.e., video signal and network conditions. This architecture is suitable for dynamic streaming, e.g., HTTP streaming based on MPEG-DASH~\cite{HEVCHTTPStreamAug2016} or content delivery over heterogeneous networks using the \ac{mmt}~\cite{ATSCMMT2016}. +In order to accomplish that, at the encoder-side, the input signals are coded using different levels of error protections. Subsequently, an optimisation algorithm is used to decide which bitstream should be transmitted at any given instant, based of different parameters. The selected stream is expected to obtain the highest quality at the decoder-side for the given conditions. \begin{figure}[tb] \centering @@ -91,62 +93,181 @@ In order to accomplish that, at the encoder-side, the input signals are coded us \end{figure} Figure~\ref{FIG6:BlockDiagram} illustrates the architecture of the proposed encoding scheme. -Firstly, a reference video coder is used to compress the bitstream using highly efficiency coding techniques in order to obtain the best \ac{rd} performance for error-free environments. This is used as the reference bitstream in the proposed approach. -Subsequently, a transcoding operation based on the reference bitstream and the input video is used to efficiently select the reference pictures used for inter-frame prediction in terms of error propagation. The reference picture selection is applied as proposed in Chapter~\ref{Chapter:ErrorResilienceTechniques}. -Moreover, a loss simulation and saliency-based \ac{ec} optimisation is also applied on the error robust bitstream. This is used to select the optimal \ac{ec} algorithm which should be used at the decoder side. -Since the optimal \ac{ec} mode might not be able to correctly reconstruct all the missing regions, extra residual information is multiplexed in the stream as a scalable \ac{el} using two different bitrates, i.e., with 10\% and 20\% of the total bitrate. This provides two different levels of error robustness. This \ac{ec} optimisation and the \ac{el} coding is applied as proposed in Chapter~\ref{Chapter:ConcealmentAwareTechniques} +Firstly, a reference video coder is used to compress the bitstream using highly efficient coding techniques in order to obtain the best \ac{rd} performance for error-free environments. +This is used as the reference bitstream in the proposed approach. +Subsequently, a transcoding operation based on the reference bitstream and the input video is used to efficiently select the reference pictures used for inter-frame prediction in terms of error propagation. +The reference picture selection is applied as proposed in Chapter~\ref{Chapter:ErrorResilienceTechniques}. +Moreover, a loss simulation and a saliency-based \ac{ec} optimisation is also applied on the robust bitstream. This is used to select the optimal \ac{ec} algorithm which should be selected at the decoder-side. +Since the optimal \ac{ec} mode might not be able to correctly reconstruct all the missing regions, extra residual information is multiplexed in the stream as a scalable \ac{el} using two different bitrates, i.e., with 10\% and 20\% of the total bitrate. This provides two different levels of error robustness. The \ac{ec} optimisation and \ac{el} coding is performed as proposed in Chapter~\ref{Chapter:ConcealmentAwareTechniques}. -The proposed approach results in four different bitstreams: (i) reference stream without error robustness; (ii) robust video coded using a reference picture selection algorithm which leads to lower error propagation in case of data-loss; (iii) robust video coding and decoding using optimal reference pictures and optimal \ac{ec} method signalling and 10\% of bitrate allocated for the scalable enhancement layer \ac{ec} residue; (iv) same as the previous bitstream but with 20\$ of the bitrate allocated for the scalable \ac{el}. These bitstreams are referred in Figure~\ref{FIG6:BlockDiagram} as \textit{Stream\_1} to \textit{Stream\_4}, respectively. +The proposed approach results in four different bitstreams: (i) reference stream without error robustness; (ii) robust video coding using a reference picture selection algorithm which leads to lower error propagation in case of data-loss; (iii) robust video coding and decoding using selective reference pictures and optimal \ac{ec} method signalling plus 10\% of bitrate allocated for the scalable enhancement layer \ac{ec} residue; (iv) same as the previous bitstream but with 20\% of the bitrate allocated for the scalable \ac{el}. These bitstreams are referred to as "S1" to "S4", respectively, as shown in Figure~\ref{FIG6:BlockDiagram}. -Finally, the optimal robust stream is selected using a selection algorithm based on +Finally, the video stream with the optimal error robustness is selected using an algorithm based on different parameters extracted from the video signal, the compressed bitstream and from the transmission channel. +From the input video signal relevant information regarding the spatial and temporal complexity is computed using existing metrics. Moreover, informations is also extracted from the coded bitstream using a stream parser. +%the predictions used by the video encoder as well as the target bitrate are extracted using a stream parser. +Finally, a parameter regarding the transmission channel is also used to predict the data loss. +These parameters are fed into the error robustness optimisation block to select the suitable bitstream which should be transmitted to obtain the highest video quality. The optimisation process is described in detail in Section~\ref{Section6:OptimisationMethod}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\section{Error robustness optimisation method} +\section{Error robustness selection method} +\label{Section6:OptimisationMethod} + +In this section the optimisation method used to select a robust highly efficiently coded stream is presented. +As mentioned before, the proposed scheme aims at predict the best method which should be used at the encoder-side, that achieves the highest quality. The selection method is based on a shallow neural network which predicts the optimal stream based on different parameters. Moreover, in this section a description of the input parameters obtained from the stream parser and the video analyser is also described. + +\subsection{Input parameters} + +\glsreset{si} +\glsreset{ti} + +A set of different parameters are used to predict the optimal approach that achieves the higher quality in specific network conditions. Based on these inputs the proposed scheme is able to select the appropriate robust coding scheme to be used. These parameters aim at covering three different aspects of the video transmission: two parameters from the input video, three parameters from the video stream and two parameters from the transmission network. + +To characterise the input video signal the following parameters are used: \ac{si} and \ac{ti}~\cite{ITUTP9112008}. +These parameters are obtained as described in Chapter~\ref{Chapter:HEVCErrorResilience}, and aim at evaluating the spatial complexity of the video scenes and the motion complexity, which have impact on the error robustness of the coded streams. +As shown in the results of Figures~\ref{Figure6:AverageQualityMain}~and~\ref{Figure6:AverageQualityReduction}, test material with higher \ac{si} or \ac{ti} suffer from higher quality degradation as the \ac{plr} increases, e.g., Book Arrival and Tennis sequences. + +From the coded video streams complementary information to characterise the streaming content is also extracted. A stream parser is applied to the reference bitstream without error robustness techniques. Therefore, the information extracted regarding the prediction information is related to a stream coded based on an optimal \ac{rd} optimisation which was not affected by robust coding techniques. The following information is extracted: +\begin{itemize} +\item \textit{Inter(\%)}: the relation between ration of the image which is coded based on intra and inter frame predictions. This is expressed by the ration of blocks coded using inter frame predictions. +The number of inter-coded blocks indicates the distribution of spatial and temporal dependencies. +Normally, coded streams with higher number of temporal dependencies are more prone to error propagation, thus higher number of bits should be allocated for error protection \HIGHLIGHT{[XX]}. + +\item \textit{AVG\_MV}: from those regions coded using inter frame predictions the motion information is analysed and the average \ac{mv} size is used to provide information regarding the motion intensity. Motion based \ac{ec} algorithms are less efficient \HIGHLIGHT{[XX]} for video signals with higher motion intensity as \acs{mv} are less predictable using algorithms such as \ac{mc} and \ac{mve}. + +\item \textit{Bitrate}: information regarding the bit allocation is extracted, since it will affect the video quality and the amount of bits that can be allocated for robust coding algorithms and auxiliary information. +\end{itemize} + +Finally, since the amount of data loss affects the decoded video quality the transmission network \ac{plr} is also used as input parameter. Transmission networks with higher loss ratios require a higher bit allocation for robust techniques, therefore the error resilience technique should adapt based on the value of the \ac{plr}. + + +\subsection{Neural network} + +Neural Networks are a computational model based on the structure and functions of biological neural networks. +In terms of computer science, a neural network works artificial human nervous system for receiving, processing, and transmitting information between neurons. Mainly, a neural network consists of three different layers: the input layer which receives data from an external environment, the hidden layer which process the inputs and an output layer~\cite{NeuralNetworks1994}. +In the proposed method a feed-forward network is used which connects every neuron a given layer to the neurons of the subsequent layer. + +\begin{figure}[tb] +\centering +%\includegraphics[width=.9\columnwidth]{\ConcealmentAwareDir/Figures/BlockDiagram} +\input{043_EndToEndRobustSystem/Figures/NeuralNetwork.tikz} +\caption{Architecture of the proposed scheme.} +\label{FIG6:BlockDiagram} +\FIGSVSPACE +\end{figure} + +Figure~\ref{FIG6:BlockDiagram} shows the diagram of the neural network used in the proposed method. \HERE +In the proposed method the input layer receives the six input described above which are related with the output video quality which is the optimisation variable. Subsequently, the neural network transforms the input data through a series of neurons in two hidden layers. \section{Performance evaluation} + + +%\begin{table}[t] +% \centering +% \caption{List of the test conditions tested.} +% \label{Table6:TestVariables} +% \TABLESVSPACE +% \begin{tabular}{m{4cm}|y{4cm}|y{6cm}} +% \toprule +% \textbf{Sequence} & \textbf{Bitrate} & \textbf{\ac{plr}} \\ \midrule +% Basketball Drill & Main (see Table~\ref{TableTestSequences}) & Error-free \\ +% Book Arrival & 10\% reduction & 0.5\% \\ +% Bosphorus & 20\% reduction & 1.0\% \\ +% BQSquare & 30\% reduction & 1.2\% \\ +% Cactus & 40\% reduction & 1.6\% \\ +% Four People & 50\% reduction & 2.0\% \\ +% Jockey & & 2.2\% \\ +% Kendo & & 2.4\% \\ +% Kimono & & 2.6\% \\ +% Kristen and Sara & & 2.8\% \\ +% Park Scene & & 3.0\% \\ +% People on Street & & 5.0\% \\ +% Race Horses & & 10.0\% \\ +% Tennis & & \\ +% Traffic & & \\ +% \bottomrule +% \end{tabular} +%\end{table} + + +\subsection{Selection Accuracy} + + \begin{table}[t] - \centering - \caption{List of the test conditions tested.} - \label{Table6:TestVariables} - \TABLESVSPACE - \begin{tabular}{m{4cm}|y{4cm}|y{6cm}} - \toprule - \textbf{Sequence} & \textbf{Bitrate} & \textbf{\ac{plr}} \\ \midrule - Basketball Drill & Main (see Table~\ref{TableTestSequences}) & Error-free \\ - Book Arrival & 10\% reduction & 0.5\% \\ - Bosphorus & 20\% reduction & 1.0\% \\ - BQSquare & 30\% reduction & 1.2\% \\ - Cactus & 40\% reduction & 1.6\% \\ - Four People & 50\% reduction & 2.0\% \\ - Jockey & & 2.2\% \\ - Kendo & & 2.4\% \\ - Kimono & & 2.6\% \\ - Kristen and Sara & & 2.8\% \\ - Park Scene & & 3.0\% \\ - People on Street & & 5.0\% \\ - Race Horses & & 10.0\% \\ - Tennis & & \\ - Traffic & & \\ - \bottomrule - \end{tabular} +\centering +\caption{Confusion matrix of the training and validation set.} +\label{Table6:ConfusionTraining} +\TABLESVSPACE +\renewcommand{\arraystretch}{1.2} +\begin{tabular}{x{.5cm}x{1cm}|x{1.1cm}|x{1.1cm}|x{1.1cm}|x{1.1cm}|} +& & \multicolumn{4}{c}{Optimal} \\ +\multirow{4}{*}{\rotatebox[origin=c]{90}{\parbox[c]{3.8cm}{\centering Selected}}} +& & S1 & S2 & S3 & S4 \\ \hline + & S1 & 122 & 17 & 3 & 0 \\ +\cline{2-6} & S2 & 13 & 80 & 19 & 2 \\ +\cline{2-6} & S3 & 0 & 12 & 96 & 9 \\ +\cline{2-6} & S4 & 0 & 4 & 11 & 202 \\ +\hline +\multicolumn{2}{x{1.7cm}|}{Accuracy} & 90.3\% & 70.8\% & 74.4\% & 94.8\% \\ +\end{tabular} \end{table} -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% SECTION %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%\section{Error concealment optimisation based on MV differences} +\begin{table}[t] +\centering +\caption{Confusion matrix of the validation test set.} +\label{Table6:ConfusionTest} +\TABLESVSPACE +\renewcommand{\arraystretch}{1.2} +\begin{tabular}{x{.5cm}x{1cm}|x{1.1cm}|x{1.1cm}|x{1.1cm}|x{1.1cm}|} + & & \multicolumn{4}{c}{Optimal} \\ +\multirow{4}{*}{\rotatebox[origin=c]{90}{\parbox[c]{3.8cm}{\centering Selected}}} +& & S1 & S2 & S3 & S4 \\ \hline + & S1 & 56 & 7 & 2 & 0 \\ +\cline{2-6} & S2 & 10 & 28 & 6 & 2 \\ +\cline{2-6} & S3 & 1 & 5 & 31 & 8 \\ +\cline{2-6} & S4 & 0 & 5 & 7 & 86 \\ +\hline + \multicolumn{2}{x{1.7cm}|}{Accuracy} & 83.6\% & 62.2\% & 67.4\% & 89.6\% \\ +%\bottomrule +\end{tabular} +\end{table} - -\subsection{Experimental results} +\subsection{Quality evaluation} +\begin{table}[t] +\centering +\caption{Average quality gains ($\Delta$PSNR) in comparison with the reference \ac{hevc} across different bitrates and loss ratios.} +\label{Table6:AverageGains} +\TABLESVSPACE +\newcommand{\colwidth}{2cm} +\begin{tabular}{p{3cm}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}|x{\colwidth}} +\toprule +\multirow{2}{*}{\vbox{\centering \textbf{Sequence}}} & \multicolumn{5}{c}{\textbf{Robust coded stream}} \\ + & S1 & S2 & S3 & S4 & Optimal (\textit{Proposed}) \\ \hline +Basketball Drill & 1.61 & 2.66 & 2.48 & 2.44 & 2.75 \\ \hline +Basketball Drive & 2.17 & 3.46 & 3.22 & 3.63 & 3.69 \\ \hline +Book Arrival & 0.96 & 1.66 & 1.43 & 1.59 & 1.74 \\ \hline +BQSquare & 0.92 & 0.96 & 0.80 & 0.62 & 1.09 \\ \hline +Cactus & 1.28 & 2.04 & 1.89 & 1.96 & 2.09 \\ \hline +Four People & 0.28 & -0.23 & -0.88 & -0.99 & 0.25 \\ \hline +Kendo & 0.81 & 0.78 & 0.20 & 0.61 & 1.02 \\ \hline +Kimono & 1.65 & 3.04 & 3.26 & 3.27 & 3.41 \\ \hline +Kristen Sara & 0.36 & 0.15 & -0.33 & -0.50 & 0.38 \\ \hline +Park Scene & 1.64 & 2.18 & 1.88 & 2.02 & 2.25 \\ \hline +People on Street & 1.00 & 1.27 & 0.91 & 1.49 & 1.63 \\ \hline +Race Horses & 1.93 & 3.89 & 4.06 & 4.30 & 4.39 \\ \hline +Tennis & 1.15 & 2.24 & 2.17 & 3.12 & 3.18 \\ \hline +\textbf{Average} & \textbf{1.21} & \textbf{1.86} & \textbf{1.62} & \textbf{1.81} & \textbf{2.15} \\ +\bottomrule +\end{tabular} +\end{table} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% SECTION %% diff --git a/043_EndToEndRobustSystem/Figures/JointlyRobustnessDiagram.tikz b/043_EndToEndRobustSystem/Figures/JointlyRobustnessDiagram.tikz index 27d1acd..8a24272 100644 --- a/043_EndToEndRobustSystem/Figures/JointlyRobustnessDiagram.tikz +++ b/043_EndToEndRobustSystem/Figures/JointlyRobustnessDiagram.tikz @@ -1,4 +1,4 @@ - + \usetikzlibrary{arrows,matrix} \usetikzlibrary{fit} \usetikzlibrary{intersections} @@ -86,46 +86,62 @@ % \path[arrow] (ConcOpt.east |- Signalling) -- (Signalling) node[above, pos=.45, yshift=-1] {\scriptsize M1...M4}; % -\node [propBlock, text width=2cm, right=2 of Encoder, yshift=-1.3cm] (Parser) {Stream Parser}; -\path[MainArrow] (Encoder -| Parser) node[point] {} -- (Parser); +\node [propBlock, minimum height=.8cm, text width=2cm, right=2 of Encoder, yshift=-1cm] (StreamParser) {Stream Parser}; +\path[MainArrow] (Encoder -| StreamParser) node[point] {} -- (StreamParser); +\node [propBlock, minimum height=.8cm, text width=2cm, below=.3 of StreamParser] (VideoParser) {Video analyser}; %% Input -\begin{scope}[xshift=-2.5cm, yshift=4cm] - \def\spacing{.2} - \coordinate(Input) at (0.5,0.3); - \foreach \i in {0,...,3} - \draw[thick,fill=\TikZColorFill] (\spacing*\i,-.15*\i) rectangle ++(1,1); -% \node[text width=.7cm, text centered] at (1.6,1.1) {Video Stream}; - \node[text width=.7cm, text centered] at (1.1,.1) {Input video}; -\end{scope} +% \begin{scope}[xshift=-2.5cm, yshift=4cm] +% \def\spacing{.2} +% \coordinate(Input) at (0.5,0.3); +% \foreach \i in {0,...,3} +% \draw[thick,fill=\TikZColorFill] (\spacing*\i,-.15*\i) rectangle ++(1,1); +% % \node[text width=.7cm, text centered] at (1.6,1.1) {Video Stream}; +% \node[text width=.7cm, text centered] at (1.1,.1) {Input video}; +% \end{scope} +% \path[MainArrow, shorten <=.9cm] (Input) |- (Encoder); +% \path[MainArrow] (Input |- EXP) node[point] {} |- (EXP); +% \path[MainArrow, shorten <=1.2cm] (Input.east |- ConcOpt) -- (ConcOpt); - -% \node[text width=.5cm, text centered] at (-2,5.5) (Input) {Input video}; -\path[MainArrow, shorten <=.9cm] (Input) |- (Encoder); +\node[text width=.7cm, text centered] (Input) at (-2,4.3) {Input video}; +\path[MainArrow] (Input |- Encoder) node[point] {} |- (VideoParser); +\path[MainArrow] (Input) |- (Encoder); \path[MainArrow] (Input |- EXP) node[point] {} |- (EXP); -% \path[MainArrow] (Input |- ConcOpt) node[point] {} |- (ConcOpt); -\path[MainArrow, shorten <=1.2cm] (Input.east |- ConcOpt) -- (ConcOpt); +\path[MainArrow] (Input.east |- ConcOpt) -- (ConcOpt); + + % % %% TS Mux and output \path (Encoder.south) -- (Encoder |- ConcOpt.north) node[midway] (middleRef) {}; % Find middle ref -\node [propBlock, right=9.5 of middleRef, text width=5.5cm, rotate=-90, anchor=center, minimum height=1cm] (Mux) {Optimal robust coding selection}; -\node [right of=Mux, node distance=2.5cm, coordinate] (Out) {}; +\node [propBlock, right=9 of middleRef, text width=5.5cm, rotate=-90, anchor=center, minimum height=1.1cm] (Mux) {Error robustness optimisation}; +\node [right of=Mux, node distance=2.8cm, coordinate] (Out) {}; \tikzstyle{OutputLabels} = [pos=1, anchor=south east, text width=2cm, align=right] -\path[MainArrow] (Encoder) -- (Mux.south |- Encoder) node[OutputLabels] {\textit{Stream\_1}}; -\path[MainArrow] (EXP) -- (Mux.south |- EXP) node[OutputLabels] {\textit{Stream\_2}}; -\path[MainArrow] (Scalable1) -- (Mux.south |- Scalable1) node[OutputLabels] {\textit{Stream\_3}}; -\path[MainArrow] (Scalable0) -- (Mux.south |- Scalable0) node[OutputLabels] {\textit{Stream\_4}}; +\path[MainArrow] (Encoder) -- (Mux.south |- Encoder) node[OutputLabels] {{S1}}; +\path[MainArrow] (EXP) -- (Mux.south |- EXP) node[OutputLabels] {{S2}}; +\path[MainArrow] (Scalable1) -- (Mux.south |- Scalable1) node[OutputLabels] {{S3}}; +\path[MainArrow] (Scalable0) -- (Mux.south |- Scalable0) node[OutputLabels] {{S4}}; \path[MainArrow] (Mux) -- (Out) node[above, midway, text width=2cm, text centered] {Optimal \\ Robust Steam}; -\path[SignallArrow] (Parser) -| (Mux.-7); -\path[SignallArrow] (Input |- Encoder) node[point] {} -- ++(0,-2) -| (Mux.0); +\node [text centered, text width=1.5cm, minimum height=1cm, below=.3 of Mux.east, xshift=1.9cm] (Channel) {Channel Conditions}; + +% \path[SignallArrow] (StreamParser.9) -| (Mux.-12); +% \path[SignallArrow] (StreamParser.0) -| (Mux.-8); +% \path[SignallArrow] (StreamParser.-9) -| (Mux.-4); +% \path[SignallArrow] (VideoParser.7) -| (Mux.2); +% \path[SignallArrow] (VideoParser.-7) -| (Mux.6); +% \path[SignallArrow] (Channel) -| (Mux.12); + + +\path[SignallArrow] (StreamParser) -| (Mux.-6) node (b1) [pos=.3, anchor=center] {\footnotesize /} node[right=-.2 of b1, yshift=-6, inner sep=0] {\footnotesize 3}; +\path[SignallArrow] (VideoParser) -| (Mux.0) node (b2) at( b1 |- VideoParser) [anchor=center] {\footnotesize /} node[right=-.2 of b2, yshift=-6, inner sep=0] {\footnotesize 2}; +\path[SignallArrow] (Channel) -| (Mux.6); + +% \node at ($ (ah) - (0.55, 0) $) {\scriptsize /}; -\node [text centered, text width=1.5cm, minimum height=1cm, below=.3 of Mux.east, xshift=1.5cm] (Channel) {Channel Conditions}; -\path[SignallArrow] (Channel) -| (Mux.7); \end{tikzpicture} diff --git a/043_EndToEndRobustSystem/Figures/NeuralNetwork.tikz b/043_EndToEndRobustSystem/Figures/NeuralNetwork.tikz index a42dce8..efa6ca0 100644 --- a/043_EndToEndRobustSystem/Figures/NeuralNetwork.tikz +++ b/043_EndToEndRobustSystem/Figures/NeuralNetwork.tikz @@ -1,6 +1,5 @@ -\def\layersep{3cm} - -\begin{tikzpicture}[shorten >=1pt,->, node distance=\layersep] +\def\layersep{3cm} +\begin{tikzpicture}[shorten >=1pt,->, node distance=\layersep,font=\small] \tikzstyle{neuron}=[circle,fill=black!25,minimum size=17pt,inner sep=0pt] \tikzstyle{input neuron}=[neuron, fill=\TikZColorFillDark]; \tikzstyle{hidden neuron}=[neuron, fill=\TikZColorFill]; @@ -8,31 +7,39 @@ \tikzstyle{annot} = [text width=4em, text centered] % Draw the hidden layer nodes - \foreach \name / \y in {1,...,6} - \path[yshift=0.5cm] - node[hidden neuron] (H-\name) at (\layersep,-\y cm) {}; + \foreach \hidden / \y in {1,2,3,4,6} + \path[yshift=0cm] node[hidden neuron] (H-\hidden) at (\layersep,-\y cm) {}; + \path[yshift=0cm] node[rotate=90] (H-5) at (\layersep,-5 cm) {...}; - \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Input \#1}] (I-1) at (0,-1) {}; - \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Input \#2}] (I-2) at (0,-2) {}; - \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Input \#3}] (I-3) at (0,-3) {}; - \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Input \#4}] (I-4) at (0,-4) {}; - \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Input \#5}] (I-5) at (0,-5) {}; + \node[input neuron, pin={[pin edge={latex-,thick,black}]left:SI}] (I-1) at (0,-1) {}; + \node[input neuron, pin={[pin edge={latex-,thick,black}]left:TI}] (I-2) at (0,-2) {}; + \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Inter(\%)}] (I-3) at (0,-3) {}; + \node[input neuron, pin={[pin edge={latex-,thick,black}]left:AVG\_MV}] (I-4) at (0,-4) {}; + \node[input neuron, pin={[pin edge={latex-,thick,black}]left:Bitrate}] (I-5) at (0,-5) {}; + \node[input neuron, pin={[pin edge={latex-,thick,black}]left:PLR}] (I-6) at (0,-6) {}; % Draw the output layer nodes \foreach \name / \y in {1,...,4} % This is the same as writing \foreach \name / \y in {1/1,2/2,3/3,4/4} - \node[output neuron, pin={[pin edge={-latex,thick,black}]right:M\y}] (O-\name) at (2*\layersep,-0.5-\y) {}; + %\node[output neuron, pin={[pin edge={-latex,thick,black}]right:M\y}] (O-\name) at (2*\layersep,-0.5-\y) {}; + \node[yshift=-1cm, output neuron] (O-\name) at (2*\layersep,-\y) {}; + \node[yshift=-1cm,output neuron] (O-O) at (3*\layersep,-2.5) {}; % Connect every node in the input layer with every node in the hidden layer. - \foreach \source in {1,...,5} - \foreach \dest in {1,...,6} - \path (I-\source) edge (H-\dest); + \foreach \source in {1,...,6} + \foreach \hidden in {1,2,3,4,6} + \path (I-\source) edge (H-\hidden); % Connect every node in the hidden layer with the output layer - \foreach \source in {1,...,6} + \foreach \hidden in {1,2,3,4,6} \foreach \dest in {1,...,4} - \path (H-\source) edge (O-\dest); + \path (H-\hidden) edge (O-\dest); + + \foreach \out in {1,...,4} + \path[draw] (O-\out) -- (O-O) node[pos=.3,above] {\scriptsize S\out}; + + \path[draw] (O-O) -- ++(1.5,0) node[pos=0, above, anchor=south west] {S1...S4}; % Annotate the layers \node[annot,above of=H-1, node distance=.8cm] (hl) {Hidden}; diff --git a/carreira_phd_thesis.sty b/carreira_phd_thesis.sty index c668f00..11af4a6 100644 --- a/carreira_phd_thesis.sty +++ b/carreira_phd_thesis.sty @@ -66,24 +66,26 @@ \usetikzlibrary{graphs} \usetikzlibrary{trees} \usetikzlibrary{intersections} - +\usetikzlibrary{matrix} \newcommand{\yslant}{-0.26} \newcommand{\yfshift}{0.05cm} -\usepackage{balance} - -\def\TikZColorFill{gray!20} +\def\TikZColorFill{gray!30} \def\TikZColorFillDark{gray!80} \def\TikZColorFillProp{blue!15} \def\TikZyslant{-0.2} \def\TikZxscale{0.75} \def\TikZyfshift{0.05cm} \newcounter{maincounter} +% We need layers to draw the block diagram +\pgfdeclarelayer{background} +\pgfdeclarelayer{foreground} +\pgfsetlayers{background,main,foreground} % *** TABLE PACKAGES *** \usepackage{tabularx, colortbl, booktabs, array, multirow} \newcolumntype{x}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}m{#1}} \newcolumntype{y}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}b{#1}} - +\usepackage{slashbox} % *** FLOAT PACKAGES *** % Coloca sempre as figuras dentro das secções a que pertencem @@ -103,6 +105,7 @@ \usepackage{adjustbox} \usepackage{rotating} +\usepackage{balance} % *** ABREVIATIONS *** \usepackage[acronym,toc,nomain,nonumberlist]{glossaries} diff --git a/carreira_phd_thesis.tex b/carreira_phd_thesis.tex index 28ba7fc..0edd4c1 100644 --- a/carreira_phd_thesis.tex +++ b/carreira_phd_thesis.tex @@ -19,7 +19,7 @@ %\includeonly{010_Introduction/Introduction} % \includeonly{021_VideoCoding/VideoCodingReview} -%\includeonly{022_ErrorResilience/RobustTransmissionReview} +\includeonly{022_ErrorResilience/RobustTransmissionReview} %\includeonly{031_ErrorResilienceEvaluation/ErrorResilienceEvaluation} %\includeonly{041_ImprovedErrorResilience/ImprovedErrorResilience} %\includeonly{042_ConcealmentAwareEncoding/ConcealmentAwareEncoding} @@ -47,7 +47,9 @@ % PDF exports of Word Documents available (Exported August 2012) % Thesis Access Form \clearpage -\includepdf[pages=-, pagecommand=]{000_Preamble/LU/access.pdf} +\includepdf[pages=1, pagecommand=]{000_Preamble/LU/Certificate_Originality_Thesis_Access.pdf} +\cleardoublepage +\includepdf[pages=2, pagecommand=]{000_Preamble/LU/Certificate_Originality_Thesis_Access.pdf} % Certificate of Originality %\includepdf[pages=-, pagecommand=, templatesize={5in}{10in}]{000_Preamble/LU/origin.pdf} @@ -76,7 +78,7 @@ \include{043_EndToEndRobustSystem/EndToEndRobustSystem} \include{050_Conclusions/Conclusion} -% + \backmatter \appendix \include{060_Appendix/Sequences} diff --git a/carreira_phd_thesis_abr.sty b/carreira_phd_thesis_abr.sty index 2467171..c9fb27b 100644 --- a/carreira_phd_thesis_abr.sty +++ b/carreira_phd_thesis_abr.sty @@ -42,6 +42,8 @@ \customnewacronym{el}{EL}{Enhancement Layer} +\customnewacronym{dbf}{DBF}{Deblocking Filter} + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% H.264 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -75,7 +77,7 @@ \customnewacronym{cu}{CU}{Coding Unit} \customnewacronym{cb}{CB}{Coding Block} \customnewacronym{pu}{PU}{Prediction Unit} -\customnewacronym{pb}{PU}{Prediction Block} +\customnewacronym{pb}{PB}{Prediction Block} \customnewacronym{tu}{TU}{Transform Unit} \customnewacronym{tb}{TB}{Transform Block} @@ -101,6 +103,8 @@ \customnewacronym{uep}{UEP}{Unequal Error protection} \customnewacronym{mdc}{MDC}{Multiple Description Coding} + +\customnewacronym{rope}{ROPE}{Recursive Optimal Per-pixel Estimation} \customnewacronym{fc}{FC}{Frame-Copy} \customnewacronym{mc}{MC}{Motion-Copy} \customnewacronym{bma}{BMA}{Boundary Match Algorithms}