jfmcarreira/doc_phd_thesis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Review chapter 2 and some acronims in chapter 6

Browse Source
This commit is contained in:
jfmcarreira
2018-07-21 01:47:05 +01:00
parent 6aaae78ce9
commit 5f27864ecf
58 changed files with 372 additions and 1724 deletions
Download Patch File Download Diff File Expand all files Collapse all files
021_VideoCoding/Figures/HEVC/BlockPartitionsWithTree.tikz → 021_VideoCoding/Figures/BlockPartitionsWithTree.tikz
+24 -9
View File
@@ -11,10 +11,13 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{1}
\def\NodesSpacing{-1.6}
\newcommand{\scale}{.8}
\begin{tikzpicture}[scale=\scale,on grid, ->,>=stealth']
\tikzstyle{TextStyle}=[scale=.75,text centered]
\begin{scope}[scale=0.4]
\definecolor{LCUFill}{rgb}{0,0.5,1}
@@ -22,8 +25,6 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=.75,text centered]
\tikzstyle{CTU}=[draw=DrawColor, fill=LCUFill, very thick]
\tikzstyle{CU}=[draw=DrawColor, thin]
\tikzstyle{PU}=[draw=DrawColor, thin, densely dashed]
@@ -73,13 +74,13 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\tikzset{
level 1/.style={level distance = 1.6cm, sibling distance=1.6cm},
level 2/.style={level distance = 1.6cm, sibling distance=.7cm},
level 3/.style={level distance = 1.6cm, sibling distance=.55cm},
level 3/.style={level distance = 1.6cm, sibling distance=.65cm},
treenode/.style = {align=center, inner sep=0pt, text centered, font=\sffamily},
node_split/.style = {treenode, circle,inner sep=1pt, white, font=\sffamily\bfseries, draw=black, fill=black},
node_cu/.style = {treenode, circle, draw, thick, font=\sffamily\bfseries\scriptsize, text width=.95em}
}
\node [node_split,scale=1/\scale] at (11cm,15*.4cm) {}
\begin{scope}[xshift=12cm,yshift=15*.4cm]
\node [node_split,scale=1/\scale] (CUD0) at (0,0) {}
child{ node [node_split] {}
child{ node [node_cu] {1} }
child{ node [node_split] {}
@@ -98,12 +99,26 @@ child{ node [node_split] {}
child{ node [node_cu] {9} }
child{ node [node_cu] {10} }
child{ node [node_cu] {11} }
child{ node [node_cu] {12} }
child{ node [node_cu] (CUD3) {12} }
}
child{ node [node_cu] {13} }
child{ node [node_cu] {14} }
child{ node [node_cu] (CUD2) {14} }
}
child{ node [node_cu] {15} }
child{ node [node_cu] (CUD1) {15} }
;
\node[TextStyle, right=3.5 of CUD0] (D0) {Depth 0};
\node[TextStyle] (D1) at (D0 |- CUD1) {Depth 1};
\node[TextStyle] (D2) at (D0 |- CUD2) {Depth 2};
\node[TextStyle] (D3) at (D0 |- CUD3) {Depth 3};
\foreach \name / \y in {0,...,3}
\node[coordinate, left=1 of D\name] (P-\name) {};
\foreach \name / \y in {0,...,3}
\path[draw, -] (D\name) -- (P-\name);
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/AssymetricBlockPartition.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/AssymetricBlockPartition.tikz
-81
View File
@@ -1,81 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{0.7}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=1.5, text centered]
\tikzstyle{LCU}=[draw=DrawColor, fill=LCUFill,thick]
\tikzstyle{CU}=[draw=DrawColor,thick]
\tikzstyle{PU}=[draw=DrawColor, densely dashed]
% \newcommand{\gridsize}{70}
% \draw[help lines] (-\gridsize,-\gridsize) grid ++(\gridsize*2,\gridsize*2);
\begin{scope}[xshift=0]
\draw[LCU] (0,0) rectangle ++(4,4); % 64x64
\draw[CU] (0,0) rectangle ++(1,4); % 64x64
\draw[CU] (1,0) rectangle ++(3,4); % 64x64
\node[TextStyle] at (2.5,2) {$\frac{3N}{4}\times N$};
\end{scope}
\begin{scope}[xshift=5cm]
\draw[LCU] (0,0) rectangle ++(4,4); % 64x64
\draw[CU] (0,0) rectangle ++(3,4); % 64x64
\draw[CU] (3,0) rectangle ++(1,4); % 64x64
\node[TextStyle] at (1.5,2) {$\frac{3N}{4}\times N$};
\end{scope}
\begin{scope}[xshift=10cm]
\draw[LCU] (0,0) rectangle ++(4,4); % 64x64
\draw[CU] (0,0) rectangle ++(4,3); % 64x64
\draw[CU] (0,3) rectangle ++(4,1); % 64x64
\node[TextStyle] at (2,1.5) {$N\times\frac{3N}{4}$};
\end{scope}
\begin{scope}[xshift=15cm]
\draw[LCU] (0,0) rectangle ++(4,4); % 64x64
\draw[CU] (0,0) rectangle ++(4,1); % 64x64
\draw[CU] (0,1) rectangle ++(4,3); % 64x64
\node[TextStyle] at (2,2.5) {$N\times\frac{3N}{4}$};
\end{scope}
% \draw[CU] (0,0) rectangle ++(16,16); % 64x64
%
% \node[TextStyle] at (4,12) {$32\times32$};
% \draw[CU] (0,0) rectangle ++(8,8); % 32x32
% \draw[CU] (8,8) rectangle ++(8,8); % 32x32
%
% \node[TextStyle] at (10,14) {$16\times16$};
% \draw[CU] (0,0) rectangle ++(4,4); % 16x16
% \draw[CU] (4,4) rectangle ++(4,4); % 16x16
%
% \draw[CU] (8,8) rectangle ++(4,4); % 16x16
% \draw[CU] (12,12) rectangle ++(4,4); % 16x16
%
% \node[TextStyle] at (13,11) {$8\times8$};
% \draw[CU] (12,8) rectangle ++(2,2); % 8x8
% \draw[CU] (14,10) rectangle ++(2,2); % 8x8
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/BitstreamSyntax/.directory
-4
View File
@@ -1,4 +0,0 @@
[Dolphin]
Timestamp=2013,6,23,21,48,12
Version=3
ViewMode=1
021_VideoCoding/Figures/HEVC/BitstreamSyntax/BitstreamSyntax.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/BitstreamSyntax/BitstreamSyntax.tikz
-56
View File
@@ -1,56 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{1}
\begin{tikzpicture}[yscale=0.5, scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=.7, text centered]
\tikzstyle{Square}=[draw=DrawColor,thick]
\tikzstyle{Byte}=[draw=DrawColor,thick, dashed]
\begin{scope}[xshift=0, yshift=-5cm]
\begin{scope}[xshift=0, yshift=0cm]
\node[TextStyle] at (1,.5) {Slice header};
\draw[Square] (0,0) rectangle ++(2,1cm);
\draw[Square] (2,0) rectangle ++(7,1cm);
\foreach \x in {1,...,3}
{
\draw[Square,thin] (2*\x+.1*\x,.1) rectangle ++(2,0.8cm);
\node[TextStyle] at (1+2*\x+.1*\x,.1+0.8/2) {CTU};
}
\node[TextStyle] at (1+2*3+.1*3+1.35,.1+0.8/2) {...};
\coordinate (EdgeL) at (2+.1,.1);
\coordinate (EdgeR) at (4+.1,.1);
\end{scope}
\begin{scope}[xshift=.2cm, yshift=-2cm, scale=1]
\draw[Square] (0,0) rectangle ++(8,1cm);
\foreach \x in {1,7,13}
\draw[draw=black] (\x*.5,0) -- ++(0,.5);
\draw[Square,dashed] (EdgeL) -- (0,1);
\draw[Square,dashed] (EdgeR) -- (8,1);
\end{scope}
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/BitstreamSyntax/CodingUnits.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/BitstreamSyntax/CodingUnits.tikz
-129
View File
@@ -1,129 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{1}
\begin{tikzpicture}[yscale=0.5, scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=.5, text centered]
\tikzstyle{Square}=[draw=DrawColor,thick]
\tikzstyle{Connection}=[Square,dashed]
\tikzstyle{Byte}=[draw=DrawColor,thick, dashed]
\begin{scope}[xshift=0, yshift=-5cm]
\begin{scope}[xshift=0, yshift=0cm]
\node[TextStyle] at (1,.5) {Slice header};
\draw[Square] (0,0) rectangle ++(2,1cm);
\draw[Square] (2,0) rectangle ++(7,1cm);
\node[TextStyle] at (2+.25,0.5) {...};
\foreach \x in {0,...,2}
{
\draw[Square,thin] (2.5+2*\x,0) rectangle ++(2,1cm);
\node[TextStyle] at (2.5+2*\x+1,.5) {CTU$_{n+\x}$};
}
\node[TextStyle] at (2.5+2*3+.25,.1+0.8/2) {...};
\coordinate (EdgeL) at (2+.5,0);
\coordinate (EdgeR) at (2+.5+2,0);
\end{scope}
\begin{scope}[xshift=.25*1cm, yshift=-2.5*1cm, scale=1]
\draw[Connection] (EdgeL) -- (0,1);
\draw[Connection] (EdgeR) -- (8,1);
\draw[Square] (0,0) rectangle ++(8,1cm);
\draw[Square] (0,0) rectangle ++(2,1cm);
% \draw[Square] (1.5,0) rectangle ++(2.5,1cm);
% \node[TextStyle] at (.75,.5) {CTU Position};
\node[TextStyle] at (2/2,.5) {SAO (optional)};
\node[TextStyle] at (6/2+2,.5) {Coding quatree (CT)};
\coordinate (EdgeL) at (2,0);
\coordinate (EdgeR) at (8,0);
\end{scope}
\begin{scope}[xshift=.25*2cm, yshift=-2.5*2cm, scale=1]
\coordinate (BraceT) at (.35,1.2);
\draw[Connection] (EdgeL) -- (0,1);
\draw[Connection] (EdgeR) -- (8,1);
\draw[Square] (0,0) rectangle ++(8,1cm);
\draw[Square] (0,0) rectangle ++(1.5,1cm);
\node[TextStyle,scale=.95] at (1.5/2,.5) {split\_cu\_flag==1};
\draw[Square] (1.5,0) rectangle ++(8/4-1.5/4,1cm);
\draw[Square] (1.5+8/4-1.5/4,0) rectangle ++(8/4-1.5/4,1cm);
\draw[Square] (1.5+2*8/4-2*1.5/4,0) rectangle ++(8/4-1.5/4,1cm);
\node[TextStyle] at (1.5+8/8-1.5/8,.5) {Coding quatree};
\node[TextStyle] at (1.5+8/4-1.5/4+8/8-1.5/8,.5) {Coding quatree};
\node[TextStyle] at (1.5+2*8/4-2*1.5/4+8/8-1.5/8,.5) {Coding quatree};
\node[TextStyle] at (1.5+3*8/4-3*1.5/4+8/8-1.5/8,.5) {Coding quatree};
\end{scope}
\begin{scope}[xshift=.25*2cm, yshift=-2.5*2.7cm, scale=1]
\coordinate (BraceB) at (.35,-.2);
\draw[Square] (0,0) rectangle ++(8,1cm);
\draw[Square] (0,0) rectangle ++(1.5,1cm);
\node[TextStyle,scale=.95] at (1.5/2,.5) {split\_cu\_flag==0};
\node[TextStyle] at (1.5+8/2-1.5/2,.5) {Coding unit};
\coordinate (EdgeL) at (1.5,0);
\coordinate (EdgeR) at (8,0);
\end{scope}
\draw [thick, decoration={brace, mirror, raise=0.5cm }, decorate] (BraceT) -- (BraceB) ;
\end{scope}
\begin{scope}[xshift=.8cm, yshift=-14cm]
\begin{scope}[xshift=.5*0cm, yshift=-2.5*0, scale=1]
\draw[Connection] (EdgeL) -- (0,1);
\draw[Connection] (EdgeR) -- (8,1);
\coordinate (BraceB) at (.35,-.2);
\draw[Square] (0,0) rectangle ++(8,1cm);
\node[TextStyle] at (.25,.5) {...};
\draw[Square] (0.5,0) rectangle ++(1.8,1cm);
\node[TextStyle] at (1.8/2+0.5,.5) {Prediction unit$_{1}$};
\node[TextStyle] at (.5+1.8+.3,.5) {...};
\draw[Square] (0.5+1.8+0.6,0) rectangle ++(1.8,1cm);
\node[TextStyle] at (1.8/2+0.5+1.8+0.6,.5) {Prediction unit$_{n}$};
\node[TextStyle] at (.5+1.8*2+.3*3,.5) {...};
\draw[Square] (0.5+1.8*2+0.6*2,0) rectangle ++(8-0.5-1.8*2-0.6*2,1cm);
\node[TextStyle] at (1.8/2+0.5+1.8+0.6+2.8,.5) {Transform tree};
\end{scope}
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/BitstreamSyntax/NALUnit.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/BitstreamSyntax/NALUnit.tikz
-56
View File
@@ -1,56 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{1}
\begin{tikzpicture}[yscale=0.5, scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=.5, text centered]
\tikzstyle{Square}=[draw=DrawColor,thick]
\tikzstyle{Byte}=[draw=DrawColor,thick, dashed]
\begin{scope}[xshift=0, yshift=0]
\begin{scope}[xshift=0, yshift=0]
\draw[Square] (0,0) rectangle ++(2,1cm);
\draw[Square] (2,0) rectangle ++(7,1cm);
\node[TextStyle] at (2/2,.5) {NAL header};
\node[TextStyle] at (2+7/2,.5) {Slice data};
\coordinate (EdgeL) at (0,0);
\coordinate (EdgeR) at (2,0);
\end{scope}
\begin{scope}[xshift=.2cm, yshift=-2cm, scale=1]
\draw[Square] (0,0) rectangle ++(8,1cm);
\foreach \x in {1,7,13}
\draw[draw=black] (\x*.5,0) -- ++(0,.5);
\draw[Square,dashed] (EdgeL) -- (0,1);
\draw[Square,dashed] (EdgeR) -- (8,1);
\node[TextStyle] at (.25,.5) {F};
\node[TextStyle] at (6*.25+0.5,.5) {NAL type (6)};
\node[TextStyle] at (7*.25+3.25,.5) {Layer ID (6)};
\node[TextStyle] at (3*.25+6.5,.5) {TIDP (3)};
\node[TextStyle] at (8/2,-0.5) {2 bytes header};
\end{scope}
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/BitstreamSyntax/Slice.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/BitstreamSyntax/Slice.tikz
-52
View File
@@ -1,52 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{1}
\begin{tikzpicture}[yscale=0.5, scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=.5, text centered]
\tikzstyle{Square}=[draw=DrawColor,thick]
\tikzstyle{Connection}=[Square,dashed]
\tikzstyle{Byte}=[draw=DrawColor,thick, dashed]
\begin{scope}[xshift=0, yshift=-5cm]
\begin{scope}[xshift=0, yshift=0cm]
\node[TextStyle] at (1,.5) {Slice header};
\draw[Square] (0,0) rectangle ++(2,1cm);
\draw[Square] (2,0) rectangle ++(7,1cm);
\node[TextStyle] at (2+.25,0.5) {...};
\foreach \x in {0,...,2}
{
\draw[Square,thin] (2.5+2*\x,0) rectangle ++(2,1cm);
\node[TextStyle] at (2.5+2*\x+1,.5) {CTU$_{n+\x}$};
}
\node[TextStyle] at (2.5+2*3+.25,.1+0.8/2) {...};
\coordinate (EdgeL) at (2+.5,0);
\coordinate (EdgeR) at (2+.5+2,0);
\end{scope}
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/BlockPartitions.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/BlockPartitions.tikz
-71
View File
@@ -1,71 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{0.4}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=1.8, text centered]
\tikzstyle{CTU}=[draw=DrawColor, fill=LCUFill, very thick]
\tikzstyle{CU}=[draw=DrawColor, thin]
\tikzstyle{PU}=[draw=DrawColor, thin, densely dashed]
\node[TextStyle] at (8,16.5) {$64\times64$ - CTU};
\draw[CTU] (0,0) rectangle ++(16,16); % 64x64
%% 32x32
\draw[CU] (0,0) rectangle ++(8,8); % 32x32
\draw[CU] (8,8) rectangle ++(8,8); % 32x32
%% Q1
\draw[CU] (0,8) rectangle ++(4,4); % 16x16
\draw[CU] (4,12) rectangle ++(4,4); % 16x16
\draw[CU] (4,12) rectangle ++(2,2); % 8x8
\draw[CU] (6,14) rectangle ++(2,2); % 8x8
%% Q3
\draw[CU] (0,0) rectangle ++(4,4); % 16x16
\draw[CU] (4,4) rectangle ++(4,4); % 16x16
\draw[CU] (4,4) rectangle ++(2,2); % 8x8
\draw[CU] (6,6) rectangle ++(2,2); % 8x8
\draw[PU] (4,4) rectangle ++(1,1); % 4x4
\draw[PU] (5,5) rectangle ++(1,1); % 4x4
\node[TextStyle] at (12,12) {$32\times32$};
\node[TextStyle] at (2,14) {$16\times16$};
\node[TextStyle] at (5,13) {$8\times8$};
\node[TextStyle] (CULabel) at (9.5,6) {CU};
\draw[black, -latex] (CULabel) to (7,7);
\node[TextStyle] (PULabel) at (2.5,5.5) {PU};
\draw[black, -latex] (PULabel) to (4.5,4.5);
% %H.264
% \node[TextStyle] (PULabel) at (10.5,4.5) {\textcolor{red}{MB}};
% \draw[black, -latex] (PULabel) to (13,3);
% \draw[CU, draw=red] (12,0) rectangle ++(4,4); % 16x16
% \draw[CU, draw=red] (14,0) rectangle ++(2,2); % 8x8
% \draw[CU, draw=red] (12,2) rectangle ++(2,2); % 8x8
% \draw[CU, draw=red] (15,0) rectangle ++(1,1); % 4x4
% \draw[CU, draw=red] (14,1) rectangle ++(1,1); % 4x4
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/BlockPartitionsTree.tikz
-39
View File
@@ -1,39 +0,0 @@
\tikzset{
treenode/.style = {align=center, inner sep=0pt, text centered, font=\sffamily},
node_split/.style = {treenode, circle,inner sep=1pt, white, font=\sffamily\bfseries, draw=black, fill=black},
node_cu/.style = {treenode, circle, draw, thick, font=\sffamily\bfseries\scriptsize, text width=.95em}
}
\begin{tikzpicture}[->,>=stealth',
level 1/.style={level distance = 1.4cm, sibling distance=1.65cm},
level 2/.style={level distance = 1.4cm, sibling distance=.85cm},
level 3/.style={level distance = 1.4cm, sibling distance=.55cm},
]
\node [node_split] {}
child{ node [node_split] {}
child{ node [node_cu] {1} }
child{ node [node_split] {}
child{ node [node_cu] {2} }
child{ node [node_cu] {3} }
child{ node [node_cu] {4} }
child{ node [node_cu] {5} }
}
child{ node [node_cu] {5} }
child{ node [node_cu] {6} }
}
child{ node [node_cu] {7} }
child{ node [node_split] {}
child{ node [node_cu] {8} }
child{ node [node_split] {}
child{ node [node_cu] {9} }
child{ node [node_cu] {10} }
child{ node [node_cu] {11} }
child{ node [node_cu] {12} }
}
child{ node [node_cu] {13} }
child{ node [node_cu] {14} }
}
child{ node [node_cu] {15} }
;
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/IntraPrediction.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/IntraPredictionsExample.tikz
-52
View File
@@ -1,52 +0,0 @@
\newcommand{\scale}{.4}
\begin{tikzpicture}[auto, on grid, scale=\scale]
\def\BlockSize{8}
\tikzstyle{TextStyle}=[text centered, font=\Huge]
\tikzstyle{PU} =[draw, scale=\scale, minimum height=\BlockSize cm, minimum width=\BlockSize cm, font=\Huge]
\tikzstyle{Neighbour} =[draw, scale=\scale, minimum height=1 cm, minimum width=\BlockSize cm, font=\Large, fill=gray!60]
\tikzstyle{Arrow} =[-latex,thick, draw=black]
\tikzstyle{Pixel} =[circle, rounded corners ,fill=lightgray, thick, scale=0.3]
\tikzstyle{PixelNeighbour} =[Pixel, fill=gray]
\node[PU, anchor=north west] at (0,0) { }; % PU
% \node[Neighbour, anchor=south west] at (0,0) { top }; % Top
% \node[Neighbour, anchor=south west] at (\BlockSize,0) { top-right }; % Top-right
% \node[Neighbour, anchor=south east, rotate=90] at (0,0) { left }; % Left
% \node[Neighbour, anchor=south east, rotate=90] at (0,-\BlockSize) { bottom-left }; % Bottom-left
% \draw[Neighbour, scale=1/\scale] (0,0) rectangle ++(-1cm,1cm);
% Pixels
\foreach \x in {0,1,...,7}
\foreach \y in {-1,...,-8}
{
\node at (\x+0.5,\y+0.5) [Pixel]() {};
}
\foreach \x in {-1}
\foreach \y in {0,...,-9}
{
\node at (\x+0.5,\y+0.5) [PixelNeighbour]() {};
}
\foreach \x in {0,1,...,12}
\foreach \y in {0}
{
\node at (\x+0.5,\y+0.5) [PixelNeighbour]() {};
}
\path[Arrow, shorten >= 48 pt] (1+.5,.5) -- ++( -3 , -\BlockSize );
\path[Arrow, shorten >= 25 pt] (2+.5,.5) -- ++( -3 , -\BlockSize );
\foreach \x in {3,...,10}
{
\path[Arrow] (\x +.5,.5) -- ++(-3, -\BlockSize);
}
\node at (15,-6) [scale=.4] (PredLabel) {Predicted Pixels};
\node[below=.3cm of PredLabel, scale=.4] (RefLabel) {Reference Pixels};
\node[Pixel, left=.8 cm of PredLabel] (Pred) {};
\node[PixelNeighbour, left=.8 cm of RefLabel] (Ref) {};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/MergeModeHEVCTwoFrames.tikz
-36
View File
@@ -1,36 +0,0 @@
\begin{tikzpicture}
\newcommand{\xfshift}{8cm}
\newcommand{\scale}{1}
\tikzstyle{BlockMain} =[draw=black, thick, fill=\TikZColorFillDark]
\tikzstyle{Block} =[draw=black, thick, loosely dashed]
\tikzstyle{Label} =[scale=1/\scale,font=\small]
\newcommand{\MainBlockSize}{2}
\newcommand{\NeighbourBlockSize}{.4}
\begin{scope}[yshift=1*\TikZyfshift, xshift=1*\xfshift, style={yslant=\TikZyslant,xscale=\TikZxscale}]
\draw[very thick] (-\MainBlockSize,-\MainBlockSize) rectangle ++(3*\MainBlockSize, 3*\MainBlockSize);
\draw[BlockMain] (0,0) rectangle ++(\MainBlockSize,\MainBlockSize);
\draw[Block] (0,0) rectangle ++(-2*\NeighbourBlockSize,-2*\NeighbourBlockSize);
\node[Label] at (-\NeighbourBlockSize,-\NeighbourBlockSize) {$A_0$};
\draw[Block] (0,0) rectangle ++(-2*\NeighbourBlockSize,2*\NeighbourBlockSize);
\node[Label] at (-\NeighbourBlockSize,\NeighbourBlockSize) {$A_1$};
\draw[Block] (\MainBlockSize,\MainBlockSize) rectangle ++(2*\NeighbourBlockSize,2*\NeighbourBlockSize);
\node[Label] at (\MainBlockSize + \NeighbourBlockSize,\MainBlockSize + \NeighbourBlockSize) {$B_0$};
\draw[Block] (\MainBlockSize,\MainBlockSize) rectangle ++(-2*\NeighbourBlockSize,2*\NeighbourBlockSize);
\node[Label] at (\MainBlockSize-\NeighbourBlockSize,\MainBlockSize + \NeighbourBlockSize) {$B_1$};
\draw[Block] (0,\MainBlockSize) rectangle ++(-2*\NeighbourBlockSize,2*\NeighbourBlockSize);
\node[Label] at (-\NeighbourBlockSize,\MainBlockSize + \NeighbourBlockSize) {$B_2$};
\end{scope}
\begin{scope}[yshift=0*\TikZyfshift, xshift=0*\xfshift, style={yslant=\TikZyslant,xscale=\TikZxscale}]
\draw[very thick] (-\MainBlockSize,-\MainBlockSize) rectangle ++(3*\MainBlockSize, 3*\MainBlockSize);
\draw[BlockMain] (0,0) rectangle ++(\MainBlockSize,\MainBlockSize);
\draw[Block] (\MainBlockSize,0) rectangle ++(2*\NeighbourBlockSize,-2*\NeighbourBlockSize);
\node[Label] at (\MainBlockSize + \NeighbourBlockSize,-\NeighbourBlockSize) {$T_{0}$};
\draw[Block] (\MainBlockSize/2,\MainBlockSize/2) rectangle ++(2*\NeighbourBlockSize,-2*\NeighbourBlockSize);
\node[Label] at (\MainBlockSize/2 + \NeighbourBlockSize,\MainBlockSize/2 -\NeighbourBlockSize) {$T_{1}$};
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/MotionMergeMode.tikz
-37
View File
@@ -1,37 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\begin{tikzpicture}
\newcommand{\scale}{0.06}
\begin{scope}[xshift=3cm,yshift=3cm,node distance = 2.5cm, auto,every node/.style={minimum size=\scale,scale=\scale},scale=\scale]
\newcommand{\ScaleBlocks}{3}
\tikzstyle{BlockMain} =[draw=black, thick]
\tikzstyle{Block} =[draw=black, thick, loosely dashed]
\tikzstyle{Label} =[scale=1/\scale,font=\small]
\newcommand{\MainBlockSize}{30}
\draw[BlockMain] (0,0) rectangle ++(\MainBlockSize,\MainBlockSize);
\node[Label,text width=\MainBlockSize+25,text centered] at (\MainBlockSize/2,\MainBlockSize/2) {Current Block};
\draw[Block] (0,0) rectangle ++(-10*\ScaleBlocks,-10*\ScaleBlocks);
\node[Label] at (-5*\ScaleBlocks,-5*\ScaleBlocks) {$A_0$};
\draw[Block] (0,0) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (-5*\ScaleBlocks,5*\ScaleBlocks) {$A_1$};
\draw[Block] (\MainBlockSize,\MainBlockSize) rectangle ++(10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (\MainBlockSize+5*\ScaleBlocks,\MainBlockSize+5*\ScaleBlocks) {$B_0$};
\draw[Block] (\MainBlockSize,\MainBlockSize) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (\MainBlockSize-5*\ScaleBlocks,\MainBlockSize+5*\ScaleBlocks) {$B_1$};
\draw[Block] (0,\MainBlockSize) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (-5*\ScaleBlocks,\MainBlockSize+5*\ScaleBlocks) {$B_2$};
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/MotionSkipMode.tikz
-48
View File
@@ -1,48 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\begin{tikzpicture}
% \draw[draw=black] (0,0) rectangle ++(21.58cm,27.94cm);
\newcommand{\scale}{0.06}
\begin{scope}[xshift=3cm,yshift=3cm,node distance = 2.5cm, auto,every node/.style={minimum size=\scale,scale=\scale},scale=\scale]
\newcommand{\ScaleBlocks}{3}
\tikzstyle{BlockMain} =[draw=black, thick]
\tikzstyle{Block} =[draw=black, thick, loosely dashed]
\tikzstyle{Label} =[scale=1/\scale,font=\small]
\newcommand{\MainBlockSize}{30}
\draw[Block,opacity=0] (0,0) rectangle ++(-10*\ScaleBlocks,-5*\ScaleBlocks);
% \draw[Block,opacity=0] (0-10*\ScaleBlocks,\MainBlockSize) rectangle ++(-5*\ScaleBlocks,10*\ScaleBlocks);
% \draw[Block,opacity=0] (\MainBlockSize,\MainBlockSize) rectangle ++(5*\ScaleBlocks,10*\ScaleBlocks);
\draw[BlockMain] (0,0) rectangle ++(\MainBlockSize,\MainBlockSize);
\node[Label,text width=\MainBlockSize+20,text centered] at (\MainBlockSize/2,\MainBlockSize/2) {Current Block};
\draw[Block] (0,0) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (-5*\ScaleBlocks,5*\ScaleBlocks) {$MV_A$};
\draw[Block] (\MainBlockSize,\MainBlockSize) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (\MainBlockSize-5*\ScaleBlocks,\MainBlockSize+5*\ScaleBlocks) {$MV_B$};
\draw[Block] (\MainBlockSize,\MainBlockSize) rectangle ++(10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (\MainBlockSize+5*\ScaleBlocks,\MainBlockSize+5*\ScaleBlocks) {$MV_{C}$};
\draw[Block] (0,\MainBlockSize) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
\node[Label] at (-5*\ScaleBlocks,\MainBlockSize+5*\ScaleBlocks) {$MV_{D}$};
% \draw[Block] (\MainBlockSize,0) rectangle ++(10*\ScaleBlocks,-10*\ScaleBlocks);
% \node[Label] at (\MainBlockSize+5*\ScaleBlocks,-5*\ScaleBlocks) {$T_{BR}$};
% \draw[Block] (32,32) rectangle ++(-10*\ScaleBlocks,10*\ScaleBlocks);
% \node[Label] at (32-5*\ScaleBlocks,32+5*\ScaleBlocks) {$T_{CTR}$};
\end{scope}
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/NALHeader.tikz
-29
View File
@@ -1,29 +0,0 @@
\newcommand\bithead[1]{\bitbox[]{#1}{\scriptsize #1}}
\newcommand{\colorbitbox}[3]{%
\rlap{\bitbox{#2}{\color{#1}\rule{\width}{\height}}}%
\bitbox{#2}{#3}}
% \begin{figure*}
% \centering
\begin{tikzpicture}
% \begin{scope}
% \coordinate (A) at (0cm,0);
% \coordinate (B) at (7.73cm,0);
% \draw[transform canvas={xshift = -3.5cm, yshift = 0.1cm}, <->] (A) -- (B) node[midway,fill=white] {{Frame 1}};
% \end{scope}
% \begin{scope}
% \coordinate (A) at (0cm,0);
% \coordinate (B) at (7.73cm,0);
% \draw[transform canvas={xshift = 0.04cm - 3.5cm + 7.73cm + 0.65cm, yshift = 0.1cm}, <->] (A) -- (B) node[midway,fill=white] {{Frame N}};
% \end{scope}
\end{tikzpicture}
\begin{bytefield}[bitwidth=20pt]{50}
\bitheader{0-31} \\
\colorbitbox{gray!20}{1}{F} & \colorbitbox{gray!60}{6}{NAL Type} & \colorbitbox{gray!20}{6}{Reserved} & \colorbitbox{gray!60}{3}{TIDP}
\end{bytefield}
% \caption{MRP stream frame-based structure.}\label{fig:mrp-bitstream}
% \end{figure*}
021_VideoCoding/Figures/HEVC/NALUnitHeader.tikz
-60
View File
@@ -1,60 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{1}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=.5, text centered]
\tikzstyle{Square}=[draw=DrawColor,thick]
\tikzstyle{Byte}=[draw=DrawColor,thick, dashed]
% \begin{scope}[xshift=0, yshift=0]
% \draw[Square] (0,0) rectangle ++(2,1cm);
% \draw[Square] (2,0) rectangle ++(7,1cm);
% \node[TextStyle] at (2/2,.5) {NAL header};
% \node[TextStyle] at (2+7/2,.5) {Slice data};
% \coordinate (EdgeL) at (0,0);
% \coordinate (EdgeR) at (2,0);
% \end{scope}
\foreach \x in {0,1,...,15}
\node at (\x cm,1cm) {$\x$};
\node at (0 cm,0cm) {F};
% \draw[Square] (0,0) rectangle ++(8,1cm);
%
% \foreach \x in {1,7,13}
% \draw[draw=black] (\x*.5,0) -- ++(0,.5);
%
% % \draw[Square,dashed] (EdgeL) -- (0,1);
% % \draw[Square,dashed] (EdgeR) -- (8,1);
%
% \node[TextStyle] at (.25,.5) {F};
% \node[TextStyle] at (6*.25+0.5,.5) {NAL type (6)};
% \node[TextStyle] at (7*.25+3.25,.5) {Layer ID (6)};
% \node[TextStyle] at (3*.25+6.5,.5) {TIDP (3)};
\node[TextStyle] at (8/2,-0.5) {2 bytes header};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/Slices/SliceStructuring.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/Slices/SliceStructuring.tikz
-12
View File
@@ -1,12 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\newcommand{\width}{6cm}
\newcommand{\vdisp}{6cm}
\begin{tikzpicture}
\node[inner sep=0] at (0,0) {\includegraphics[width=\width]{Slices}};
\node[inner sep=0] at (\width+.5cm,0) {\includegraphics[width=\width]{Tiles}};
\node[inner sep=0] at (2*\width+1cm,0) {\includegraphics[width=\width]{WPP}};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/Slices/Slices.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/Slices/Tiles.pdf
-135
View File
File diff suppressed because one or more lines are too long
021_VideoCoding/Figures/HEVC/Slices/WPP.pdf
-135
View File
File diff suppressed because one or more lines are too long
021_VideoCoding/Figures/HEVC/WPP.pdf
-135
View File
File diff suppressed because one or more lines are too long
021_VideoCoding/Figures/HEVC/hevc_review_figures/.directory
-4
View File
@@ -1,4 +0,0 @@
[Dolphin]
Timestamp=2013,6,23,17,50,12
Version=3
ViewMode=1
021_VideoCoding/Figures/HEVC/hevc_review_figures/.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/hevc_review_figures/BlockPartitions.tikz
-69
View File
@@ -1,69 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{0.3}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=1.8, text centered]
\tikzstyle{LCU}=[draw=DrawColor, fill=LCUFill, very thick]
\tikzstyle{CU}=[draw=DrawColor, thin]
\tikzstyle{PU}=[draw=DrawColor, thin, densely dashed]
% \newcommand{\gridsize}{70}
% \draw[help lines] (-\gridsize,-\gridsize) grid ++(\gridsize*2,\gridsize*2);
\node[TextStyle] at (8,16.5) {$64\times64$ - LCU};
\draw[LCU] (0,0) rectangle ++(16,16); % 64x64
\node[TextStyle] at (4,12) {$32\times32$};
\draw[CU] (0,0) rectangle ++(8,8); % 32x32
\draw[CU] (8,8) rectangle ++(8,8); % 32x32
\node[TextStyle] at (10,14) {$16\times16$};
\draw[CU] (0,0) rectangle ++(4,4); % 16x16
\draw[CU] (4,4) rectangle ++(4,4); % 16x16
\draw[CU] (8,8) rectangle ++(4,4); % 16x16
\draw[CU] (12,12) rectangle ++(4,4); % 16x16
\node[TextStyle] at (13,11) {$8\times8$};
\draw[CU] (12,8) rectangle ++(2,2); % 8x8
\draw[CU] (14,10) rectangle ++(2,2); % 8x8
\node[TextStyle] (CULabel) at (5.5,9) {CU};
\draw[black, -latex] (CULabel) to (7,7);
\draw[CU] (4,4) rectangle ++(2,2); % 8x8
\draw[CU] (6,6) rectangle ++(2,2); % 8x8
\node[TextStyle] (PULabel) at (2.5,5.5) {PU};
\draw[black, -latex] (PULabel) to (4.5,4.5);
\draw[PU] (4,4) rectangle ++(1,1); % 4x4
\draw[PU] (5,5) rectangle ++(1,1); % 4x4
%H.264
\node[TextStyle] (PULabel) at (10.5,4.5) {\textcolor{red}{MB}};
\draw[black, -latex] (PULabel) to (13,3);
\draw[CU, draw=red] (12,0) rectangle ++(4,4); % 16x16
\draw[CU, draw=red] (14,0) rectangle ++(2,2); % 8x8
\draw[CU, draw=red] (12,2) rectangle ++(2,2); % 8x8
\draw[CU, draw=red] (15,0) rectangle ++(1,1); % 4x4
\draw[CU, draw=red] (14,1) rectangle ++(1,1); % 4x4
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/hevc_review_figures/EntropyCodingHEVC.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/hevc_review_figures/EntropyCodingHEVC.tikz
-40
View File
@@ -1,40 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\def\MVE{\mbox{MVE}}
\def\MVBASE{\mbox{BASE\_MC}}
\def\INTERVIEW{\mbox{INTERVIEW\_COPY}}
\def\COMBMC{\mbox{COMB\_MC}}
\def\COMBMCDISP{\mbox{COMB\_MC\_Disp}}
% Define block styles
\tikzstyle{block} = [rectangle, draw, text width=5cm, text centered, rounded corners, minimum height=1.5cm]
\tikzstyle{node_circle} = [circle, draw, text centered]
\tikzstyle{decision} = [diamond, draw, text centered, node distance=3cm]
\tikzstyle{line} = [draw, -latex']
\newcommand{\scale}{0.75}
\begin{tikzpicture}[scale=\scale,node distance = 2.5cm, auto,every node/.style={scale=\scale}]
\node [block] at (0,0) (A) {Decode Split Info\\ \textbf{CABAC}};
\node [block,below of=A] (B) {Decode Mode Info \\ \textbf{CABAC}};
\node [block,below of=B] (C) {Decode Prediction Info \\ \textbf{CABAC}};
\node [block,below of=C, left of=C, xshift=-.5cm, yshift=-.5cm] (D1) {Decode Residual Info \\ \textbf{CABAC}};
\node [block,below of=C, right of=C, xshift=.5cm, yshift=-.5cm] (D2) {CABAC Bypass};
\node [block,below of=D2] (E) {Decode Residual Info \\ \textbf{CAVLC}};
\path [line] (A) -- (B);
\path [line] (B) -- (C);
\path [line] (C) -- node[above, xshift=-.5cm] {HEB} (D1);
\path [line] (C) -- node[above, xshift=.5cm] {HTB} (D2);
\path [line] (D2) -- (E);
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/hevc_review_figures/HEVCBlockDiagram.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/hevc_review_figures/HEVCBlockDiagram.tikz
-92
View File
@@ -1,92 +0,0 @@
\newcommand{\scale}{0.7}
\usetikzlibrary{arrows,matrix}
% \usepackage{amssymb, amsmath, mathptmx}
\begin{tikzpicture}[font=\scriptsize,node distance=1.5cm,scale=0.9]
\tikzstyle{sumBlock}=[draw,circle, inner sep=0cm, minimum size=0.1cm]
\tikzstyle{processBlock}=[draw, inner sep=0.1cm, minimum size=0.7cm,text centered]
\tikzstyle{proposedBlock}=[processBlock, fill=gray!30]
\tikzstyle{blockArrow}=[draw,-latex]
\tikzstyle{point} = [draw, fill=black, circle,inner sep=1pt]
\coordinate (sumPoint) at (0,0);
\node[sumBlock] (ResidueDiff) at (sumPoint) {$+$};
% \begin{scope}[xshift=-2cm, yshift=3.2cm, scale=1.5]
% \def\spacing{.2}
% \node at ((.5+3*\spacing*.5,-.5) (Input) {};
% \foreach \i in {0,...,3}
% \draw[thick,fill=white] (\spacing*\i,-.15*\i) rectangle ++(1,1);
% % \node[text width=.7cm, text centered] at (1.6,1.1) {Video Stream};
% % \node[text width=.7cm, text centered] at (1.1,.1) {Input video};
% \end{scope}
\node[left of=ResidueDiff,text width=1cm,node distance=2.5cm] (Input) {\textbf{Input video}};
\path[blockArrow] (Input) -- (ResidueDiff) node [point, midway] (p0) {};
\node[processBlock,right of=ResidueDiff] (Transf) {$T$};
\node[processBlock,right of=Transf] (Quant) {$Q$};
\node[processBlock,right of=Quant,node distance=2.5cm] (VLC) {CABAC};
\node[right of=VLC, text centered,text width=1.5cm, node distance=2.3cm] (Output) {\textbf{HEVC bitstream}};
\path[blockArrow] (VLC) -- (Output);
\path[blockArrow] (ResidueDiff) -- (Transf);
\path[blockArrow] (Transf) -- (Quant);
\path[blockArrow] (Quant) -- (VLC) node [point,midway] (QuantFrame) {};
\node[processBlock,below of=QuantFrame,node distance=1.2cm] (InvQuant) {$Q^{-1}$};
\node[processBlock,right of=InvQuant] (InvTransf) {$T^{-1}$};
\node[sumBlock,below of=InvTransf, node distance=1cm] (ReconResidue) {$+$};
\node[processBlock,text width=1.9cm, below of=ReconResidue] (Filters) {Deblocking \& SAO filters};
\path[blockArrow] (QuantFrame) -- (InvQuant);
\path[blockArrow] (InvQuant) -- (InvTransf);
\path[blockArrow] (InvTransf) -- (ReconResidue);
\path[blockArrow] (ReconResidue) -- (Filters) node[point,midway] (p1) {};
\node[left of=InvQuant, coordinate, node distance=1cm,yshift=0.5cm] (x1) {};
\path[blockArrow,-] (x1 -| ResidueDiff) node[point] () {} -- (x1);
\path[blockArrow] (x1) |- (ReconResidue);
\node [processBlock,text width=1.5cm] (Intra) at (2,-2) {Intra prediction};
\node [processBlock,text width=1.5cm,below of=Intra, node distance=1.8cm] (Inter) {Inter prediction};
\path[blockArrow] (p1) -| (Intra);
\node [point,left of=Intra,node distance=1.2cm] (z1) {};
\node [point,left of=Inter,node distance=1.2cm] (z2) {};
\path[blockArrow,-] (Intra) -- (z1);
\path[blockArrow,-] (Inter) -- (z2);
\node [coordinate,left of=z1,below of=z1,node distance=0.15cm] (w1) {};
\node [coordinate,left of=z2, above of=z2, node distance=0.15cm] (w2) {};
\path (z1) -- (z2) node [coordinate, midway] (w) {};
\path (w) -- (w -| sumPoint) node [point] (z) {};
\draw[-latex,densely dashed] (w1) to [bend left=60] (w2);
\draw [thick,blockArrow] (z1)+(0.1,-0.1) -- (z);
\path[blockArrow] (z) -- node[pos=0.96,right] {-} (ResidueDiff);
\node[processBlock,below of=Filters,text width=1.5cm, node distance=1.5cm] (RefBuffer) {Reference Picture Buffer};
\node[processBlock,text width=1.5cm] at(Inter |- RefBuffer) (MotionSearch) {Motion Estimation};
\path[blockArrow] (Filters) -- (RefBuffer) node[point,pos=0.4] (p2) {};
\path[blockArrow] (RefBuffer) -- (MotionSearch);
\path[blockArrow] (MotionSearch) -- (Inter);
\path[blockArrow] (p0) |- (MotionSearch);
\node[right of=p2, text centered,text width=1.5cm, node distance=2.3cm] (Output) {\textbf{Video output}};
\path[blockArrow] (p2) -- (Output);
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/hevc_review_figures/IntraPredictions.pdf
BIN
View File
Binary file not shown.
021_VideoCoding/Figures/HEVC/hevc_review_figures/IntraPredictions.tikz
-93
View File
@@ -1,93 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{.12}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
\definecolor{LCUFill}{rgb}{0,0.5,1}
\definecolor{DrawColor}{rgb}{0,0,0.5}
\definecolor{LCUFill}{rgb}{1,1,1}
\definecolor{DrawColor}{rgb}{0,0,0}
\tikzstyle{TextStyle}=[scale=2, text centered]
\tikzstyle{LCU}=[draw=DrawColor, fill=LCUFill, very thick]
\tikzstyle{CU}=[draw=DrawColor, thin]
\tikzstyle{PU}=[draw=DrawColor, thin, densely dashed]
\tikzstyle arrowstyle=[scale=1]
\tikzstyle{IntraArrow} =[draw=black, shorten <=0, postaction={decorate,decoration={markings, mark=at position .3 with {\arrow[arrowstyle]{stealth}}}} ]
% \newcommand{\gridsize}{70}
% \draw[help lines] (-\gridsize,-\gridsize) grid ++(\gridsize*2,\gridsize*2);
\draw[draw=black, thick] (-31,-31) rectangle (31,31);
\coordinate (Center) at (0,0);
\foreach \x in {-32,...,31}
\draw[draw=black] (\x-.5,31.5) rectangle ++(1,1);
\foreach \y in {-31,...,32}
\draw[draw=black] (-32-.5,\y-.5) rectangle ++(1,1);
\foreach \x in {-31,-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31}
{
% \pgfmathsetmacro{\intra}{\x+26}
% \node[scale=2.7, align=center] at (\x*1.1,32.5) {$\intra$};
\draw[IntraArrow] (\x,31) to (Center);
}
\foreach \y in {-31,-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26}
{
\draw[IntraArrow] (-31,\y) to (Center);
}
% \foreach \x in {-31,-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31}
% {
% \draw[IntraArrow] (Center) to (\x,-31);
% }
% \foreach \y in {-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31}
% {
% \draw[IntraArrow] (Center) to (31,\y);
% }
% \end{scope}
% \foreach \y in {0,...,31}
% \draw[black] (-1,\y) rectangle ++(1,1);
% \node[scale=6, text width=2cm, align=left] at (8,-1) {Planar};
% \node[scale=6, text width=2cm, align=left] at (8,-4) { DC};
% % H.264
% \foreach \x in {-31,-15,0,15,32}
% \draw[ red,-latex] (Center) to (\x,-32);
%
% \foreach \y in {-15,0,15}
% \draw[red,-latex] (Center) to (32,\y);
\foreach \x in {-31,-15,0,15,31}
\draw[IntraArrow, red] (\x,31) to (Center);
\foreach \y in {-15,0,15}
\draw[IntraArrow, red] (-31,\y) to (Center);
\node[scale=6, text width=2cm, align=left,red] at (8,-1) {Planar};
\node[scale=6, text width=2cm, align=left,red] at (8,-4) { DC};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/hevc_review_figures/PixelBasedMVE.tex
-45
View File
@@ -1,45 +0,0 @@
\usetikzlibrary{calc,trees,positioning,arrows,chains,shapes.geometric,%
decorations.pathreplacing,decorations.pathmorphing,shapes,%
matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfdeclarelayer{L0}
\pgfdeclarelayer{L1}
\pgfdeclarelayer{L2}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{0.20}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
\tikzstyle{LostMB}=[black, thick]
\tikzstyle{ExtrapolatedMB}=[black, dashed, thick]
\tikzstyle{TextNode}=[scale=4.2, text centered, text width=3.3cm]
\tikzstyle{SymbolNode}=[scale=5.2, text centered]
\newcommand{\gridsize}{30}
%\draw[help lines] (-\gridsize,-\gridsize) grid ++(\gridsize*2,\gridsize*2);
\draw[LostMB] (0,0) rectangle ++(8,8);
\draw[ExtrapolatedMB] (6-16,6) rectangle ++(16cm,16cm);
\draw[ExtrapolatedMB] (4,4) rectangle ++(16cm,16cm);
\draw[ExtrapolatedMB] (6,-14) rectangle ++(16cm,16cm);
\draw[ExtrapolatedMB] (3-16,-14) rectangle ++(16cm,16cm);
\node[TextNode](EMB1) at (5-16+16/2,7+16/2) {Extrapolated MB$_1$};
\node[TextNode](EMB1) at (5+16/2,4+16/2) {Extrapolated MB$_2$};
\node[TextNode](EMB1) at (6+16/2,-14+16/2) {Extrapolated MB$_4$};
\node[TextNode](EMB1) at (3-16+16/2,-14+16/2) {Extrapolated MB$_3$};
\node[SymbolNode] at (2,1) {$\circ$};
\node[SymbolNode] at (7,1) {$\circ$};
\node[SymbolNode] at (7,7) {$\circ$};
\node[SymbolNode] at (5,7) {$\diamond$};
\node[SymbolNode,scale=0.7] at (2,4) {$\bigtriangleup$};
\node[SymbolNode] at (-2,4) {$\star$};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/hevc_review_figures/RPS.jpg
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 132 KiB

021_VideoCoding/Figures/HEVC/hevc_review_figures/build_figures
-45
View File
@@ -1,45 +0,0 @@
#!/bin/bash
currdir=$(pwd)
tikz_files=$( find . -name '*.tikz' );
for file_list in $tikz_files; do
dirname="${file_list%/*}"
file="${file_list##*/}"
cd ${dirname}
cat > build.tex <<_EOF_
\documentclass{article}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows}
\usepackage{verbatim}
\usepackage[active,tightpage]{preview}
\PreviewEnvironment{tikzpicture}
\setlength\PreviewBorder{5pt}
\begin{document}
\pagestyle{empty}
\begin{figure}[p]
\centering
\input{ $file }
\end{figure}
\end{document}
_EOF_
pdflatex build.tex
rm build.aux
rm build.log
rm build.tex
mv build.pdf ${file%%.*}.pdf
cd ${currdir}
done
021_VideoCoding/Figures/HEVC/HEVCBlockDiagram.tikz → 021_VideoCoding/Figures/HEVCBlockDiagram.tikz
View File
021_VideoCoding/Figures/HEVC/IntraPredictions.tikz → 021_VideoCoding/Figures/IntraPredictions.tikz
+5 -5
View File
@@ -11,7 +11,7 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\pgfdeclarelayer{L3}
\pgfsetlayers{background,L0,L1,L2,L3,main,foreground}
\newcommand{\scale}{.1}
\newcommand{\scale}{.13}
\begin{tikzpicture}[scale=\scale,every node/.style={scale=\scale},on grid ]
@@ -33,7 +33,7 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
% \newcommand{\gridsize}{70}
% \draw[help lines] (-\gridsize,-\gridsize) grid ++(\gridsize*2,\gridsize*2);
\draw[draw=black, thick] (-31,-31) rectangle (31,31);
\draw[draw=black, very thick] (-31,-31) rectangle (31,31);
\coordinate (Center) at (0,0);
@@ -47,16 +47,16 @@ matrix,shapes.symbols,plotmarks,decorations.markings,shadows,backgrounds}
\foreach \y in {-31,-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26}
{
\addtocounter{intra}{1}
\node[scale=3.5, align=center] at (-32,\y) {\theintra};
\node[scale=4, align=center] at (-32,\y) {\theintra};
\draw[IntraArrow] (-31,\y) to (Center);
}
\addtocounter{intra}{1}
\node[scale=3.5, align=center] at (-32,32) {\theintra};
\node[scale=4, align=center] at (-32,32) {\theintra};
\draw[IntraArrow] (-31,31) to (Center);
\foreach \x in {-26,-21,-17,-13,-9,-5,-2,0,2,5,9,13,17,21,26,31}
{
\addtocounter{intra}{1}
\node[scale=3.5, align=center] at (\x,32) {\theintra};
\node[scale=4, align=center] at (\x,32) {\theintra};
\draw[IntraArrow] (\x,31) to (Center);
}
021_VideoCoding/Figures/IntraPredictionsExample.tikz
+81
View File
@@ -0,0 +1,81 @@
\begin{tikzpicture}[auto, on grid, scale=.5, y=-1cm, every node/.style={scale=.5, y=-1cm}]
\def\BlockSize{4}
\tikzstyle{TextStyle}=[text centered, font=\Huge]
\tikzstyle{PU} =[draw, minimum height=\BlockSize cm, minimum width=\BlockSize cm, font=\Huge]
\tikzstyle{Neighbour} =[draw, minimum height=1 cm, minimum width=\BlockSize cm, font=\Large, fill=gray!60]
\def\TikZColorFill{gray!20}
\def\TikZColorFillDark{gray!80}
\tikzstyle{PixelAnchor} =[xshift=.5cm, yshift=-.5cm]
\tikzstyle{Pixel} =[circle, thin, fill=lightgray, inner sep=0, minimum size=10pt]
\tikzstyle{PixelNeighbour} =[Pixel, fill=black!80]
\tikzstyle{Arrow} =[-latex,thick, draw=black]
\tikzstyle{NeighbourSquares} =[fill=darkgray!80]
\draw[Pixel] (0,0) grid ++(4.5,4.5);
\node[NeighbourSquares, anchor=north west, minimum width=1cm, minimum height=1cm] at (-1,-1) {};
\node[NeighbourSquares, anchor=north west, minimum width=4cm, minimum height=1cm] at (0,-1) {};
\node[NeighbourSquares, anchor=north west, minimum width=4cm, minimum height=1cm] at (4,-1) {};
\node[NeighbourSquares, anchor=north west, minimum width=1cm, minimum height=4cm] at (-1,0) {};
\node[NeighbourSquares, anchor=north west, minimum width=1cm, minimum height=4cm] at (-1,4) {};
% Pixels
\foreach \x in {0,...,3}
\foreach \y in {0,...,3}
{
\node at (\x,\y) [PixelAnchor,Pixel]() {};
}
\foreach \x in {-1}
\foreach \y in {-1,...,7}
{
\node at (\x,\y) [PixelAnchor,PixelNeighbour]() {};
}
\foreach \x in {0,1,...,7}
\foreach \y in {-1}
{
\node at (\x,\y) [PixelAnchor,PixelNeighbour]() {};
}
\path[PixelAnchor, Arrow, dashed, shorten >= 38 pt] (.5,-1) -- ++( -2 , \BlockSize );
\path[PixelAnchor, Arrow, shorten >= 25 pt] (1,-1) -- ++( -2 , \BlockSize );
\path[PixelAnchor, Arrow, dashed, shorten >= 13 pt] (1.5,-1) -- ++( -2 , \BlockSize );
\foreach \x in {2,...,5}
{
\path[PixelAnchor,Arrow] (\x,-1) -- ++(-2, \BlockSize);
}
\foreach \x in {2.5,...,4.5}
{
\path[PixelAnchor,Arrow,dashed] (\x,-1) -- ++(-2, \BlockSize);
}
\path[Arrow, very thick, -] (-1,4) -- ++(6,0);
\path[Arrow, very thick, -] (4,-1) -- ++(0,6);
\path[Arrow, very thick, -] (-1,0) -- ++(1,0);
\path[Arrow, very thick, -] (0,-1) -- ++(0,1);
\path[Arrow, very thick, -] (0,0) -- ++(4,0) node[midway,above, yshift=1cm, scale=1.2] {Top};
\path[Arrow, very thick, -] (4,0) -- ++(4,0) node[midway,above, yshift=1cm, scale=1.2] {Top-right};
\path[Arrow, very thick, -] (0,0) -- ++(0,4) node[midway,above, xshift=-1cm, scale=1.2, rotate=90] {Left};
\path[Arrow, very thick, -] (0,4) -- ++(0,4) node[midway,above, xshift=-1cm, scale=1.2, rotate=90] {Bottom-left};
% \path[Arrow, very thick, -] (8,0) -- ++(0,-1);
% \path[Arrow, very thick, -] (0,8) -- ++(-1,0);
\node[anchor=south west, align=left, text width=4cm] at (6,6) (RefLabel) {Reference pixels};
\node[anchor=south west, align=left, text width=4cm, below=.5cm of RefLabel] (PredLabel) {Predicted pixels};
\node[anchor=south west, align=left, text width=4cm, below=.5cm of PredLabel] (PredDir) {Prediction direction};
\node[Pixel, left=1.3 cm of PredLabel] {};
\node[PixelNeighbour, left=1.3 cm of RefLabel] {};
\path[Arrow, latex-] (PredDir.west) -- ++(-1,0);
% \path[PixelAnchor, Arrow, blue] (2.75,-1) -- ++( -2 , \BlockSize );
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/MergeModeHEVC.tikz → 021_VideoCoding/Figures/MergeModeHEVC.tikz
View File
021_VideoCoding/Figures/HEVC/PUAsymetric.tikz → 021_VideoCoding/Figures/PUAsymetric.tikz
View File
021_VideoCoding/Figures/HEVC/PUSymetric.tikz → 021_VideoCoding/Figures/PUSymetric.tikz
View File
021_VideoCoding/Figures/HEVC/PictureTypes_Leading_Trailing.tikz → 021_VideoCoding/Figures/PictureTypes_Leading_Trailing.tikz
+2 -1
View File
@@ -1,4 +1,4 @@
\begin{tikzpicture}[auto]
\begin{tikzpicture}[auto]
\def\yshift{1.2cm}
\tikzstyle{frame}=[thick,draw, fill=\TikZColorFill, yslant=0,xscale=1,minimum size=1cm]
\tikzstyle{dependency}=[thick,draw,-latex, shorten <=3 pt, shorten >=3 pt]
@@ -31,4 +31,5 @@
\node[label,below=.1 of Leading] {Leading};
\node[draw,dashed,inner sep=5pt, fit=(B7) (B6) (B8) (P5)] (Trailing) {};
\node[label,below=.1 of Trailing] {Trailing};
\path[dependency] (P0.south west)++(-1,-1.3) -- ++(12cm+3cm,0) node[midway, anchor=center,fill=white] {Display order};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/PictureTypes_TSA.tikz → 021_VideoCoding/Figures/PictureTypes_TSA.tikz
+1
View File
@@ -26,4 +26,5 @@
% \node[draw,dashed,inner sep=5pt, fit=(B3) (B2) (B4) (B2A)] (Leading) {};
% \node[label,below=.1 of Leading] {Leading};
\node[label,right=.1 of P6, yshift=-.3cm] {TSA};
\path[dependency] (P0.south west)++(-1,-.8) -- ++(12cm+3cm,0) node[midway, anchor=center,fill=white] {Display order};
\end{tikzpicture}
021_VideoCoding/Figures/HEVC/Tiles_Slices_Example_1.tikz → 021_VideoCoding/Figures/Tiles_Slices_Example_1.tikz
View File
021_VideoCoding/Figures/HEVC/Tiles_Slices_Example_2.tikz → 021_VideoCoding/Figures/Tiles_Slices_Example_2.tikz
View File
021_VideoCoding/Figures/HEVC/WPP.tikz → 021_VideoCoding/Figures/WPP.tikz
View File
021_VideoCoding/VideoCodingReview.tex
+206 -112
View File
@@ -15,10 +15,18 @@
\glsunset{h264}
\glsreset{hevc}
This chapter describes the H.265~-~\ac{hevc}~\cite{H265StandardFeb2018} standard, focusing on the most relevant features of the encoding algorithm for the work developed in this thesis. The \ac{hevc} standard aims to address essentially all existing applications of \ac{h264}~\cite{H264AVCStandard2003}, extending it in two keys aspects: increased video resolution and increased use of parallel processing architectures. In order to increase the flexibility and follow the increase in the video resolution, the \ac{hevc} standard adopts a new block partition structure, as well as, new prediction modes. In this chapter these new techniques introduced are described, especially those which may have a significant impact in the error robustness characteristics of the \ac{hevc}.
%The standard also improves the intra~\cite{IntraCodingHEVC2012} and inter~\cite{MotionCodingHEVC2013} coding and includes new high-level features~\cite{HEVCHighLevelReferenceDez2012}. This chapter will describe main features and improvements introduced in the standard.
This chapter describes the H.265~-~\ac{hevc}~\cite{H265StandardFeb2018} standard, focusing on the most relevant features of the encoding algorithm in the scope of the work developed in this thesis.
In short, the \ac{hevc} standard essentially aims to support all existing applications of \ac{h264}~\cite{H264AVCStandard2003}, while extending two keys aspects: increased video resolution with higher compression ratio and also more efficient use of parallel processing architectures.
In order to increase the coding flexibility and higher video resolutions, the \ac{hevc} standard adopts a new block partition structure, as well as, new prediction modes.
In this chapter the new data structures coding techniques specifically introduced to increase the coding efficiency are described.
Particular emphasis is given to those which may have a significant impact in the error robustness characteristics of \ac{hevc} streams.
This chapter is organized as follows. Section~\ref{Section2:GeneraVideoSystem} introduced the functional blocks of a generic video coding system, and Section~\ref{Section2:Overview} provides the overview of the \ac{hevc} standard features. Subsequently, Section~\ref{Section2:HighLevel} describes the high-level features which are used to carry relevant information regarding the bitstream, Section~\ref{Section2:CodingStructures} describes the coding structures used to divide the video frames into smaller partitions and Section~\ref{Section2:PredictionModes} describes the main predictions techniques, namely intra and inter coding. Finally, a briefly description of the transform and quantisation, entropy coding and in-loop filters is given in Sections~\ref{Section2:TransformQuant}, \ref{Section2:Entropy} and \ref{Section2:Filters}, respectively. Section~\ref{Section2:Summary} concludes this chapter.
This chapter is organized as follows.
Section~\ref{Section2:GeneraVideoSystem} introduces the functional blocks of a generic hybrid video coding system, and Section~\ref{Section2:Overview} provides an overview of the \ac{hevc} standard.
Then, Section~\ref{Section2:HighLevel} describes the high-level syntax elements which are used to carry relevant information in coded streams.
Section~\ref{Section2:CodingStructures} describes the data structures used to divide the video frames into smaller partitions and Section~\ref{Section2:PredictionModes} describes particular aspects of the main predictions techniques, namely intra and inter coding.
Finally, a brief description of the transform and quantisation, entropy coding and in-loop filters is provided in Sections~\ref{Section2:TransformQuant}, \ref{Section2:Entropy} and \ref{Section2:Filters}, respectively.
Section~\ref{Section2:Summary} concludes this chapter.
\clearpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -27,9 +35,11 @@ This chapter is organized as follows. Section~\ref{Section2:GeneraVideoSystem} i
\section{General hybrid video coding}
\label{Section2:GeneraVideoSystem}
A digital video is a temporal succession of images, also referred to as pictures or frames, which usually presents a significant amount of temporal and spatial correlations. In general, video coding and distribution systems follow a generic set of tools: pre-processing, encoding, storage or transmission, decoding and post-processing.
The pre-processing unit is used adapt the raw video signal, in which if the format of raw videos from video cameras is analogue, it will be firstly digitized into RGB images and then converted into YUV representation~\cite{DataCompression1997}. In the next processing tool, the video data will be compressed by the respective encoding unit by reducing its redundancy. The encoding process is able to compress the raw video in a high compression ratio and maintains a good trade-off between video quality and bitrate through high efficient algorithms.
On the contrary, the output video for displaying can be acquired through decoding and post-processing modules.
A digital video signal is a temporal succession of images, also referred to as pictures or frames, which usually presents a significant amount of temporal and spatial correlations.
In general, video coding and distribution systems follow through a generic chain of processing and delivery stages: pre-processing, encoding, storage or transmission, decoding and post-processing.
The pre-processing stage is used adapt the raw video signal, either analogue or digital, to an appropriate format, such as RGB or YUV~\cite{DataCompression1997}.
Then, the video data is compressed by the respective encoding module, mostly reducing its redundancy and visual irrelevancy. The encoding process is responsible for representing the raw video in a highly compressed format, while maintaining a good trade-off between video quality and bitrate through high efficient algorithms and coding tools.
In the last stages of the chain, the compressed format is decoded video and presented by displaying the signal content after adaptation to the consumer equipment through post-processing.
% \begin{figure}[tb]
% \centering
@@ -40,20 +50,33 @@ On the contrary, the output video for displaying can be acquired through decodin
Since H.261, the hybrid video coding scheme has been the basic structure for all video coding standards and recommendations of ITU-T \ac{vceg} and \ac{iso}/\ac{iec} \ac{mpeg}. While the structure has not been changed, the algorithms represented by the building blocks have been refined and the applicable configuration for the algorithms has become more and more flexible over years. The coding scheme is called hybrid as it combines temporal prediction between pictures of the video sequence with transform coding techniques for the prediction error \cite{VideoCodingFundamentals2015} as shown in Figure~\ref{Figure2:HybridCodec}~\cite{BookLucas2017}.
Since the early days of video compressing, hybrid video coding has been the basic structure for all video coding standards and recommendations of ITU-T \ac{vceg} and \ac{iso}/\ac{iec} \ac{mpeg}, e.g., from H.261 to \ac{hevc}.
Figure~\ref{Figure2:HybridCodec} shows the block diagram of a generic hybrid video coding system~\cite{BookLucas2017}.
While the algorithmic structure has not been changed, the specific techniques represented by the building blocks have been refined and the applicable coding configuration has become more and more flexible over years through the use of an increasing variety of coding tools.
This is the reason why coding scheme is called hybrid, as it combines quiet different prediction modes between pictures with transform and entropy coding~\cite{VideoCodingFundamentals2015}.
\begin{figure}[tb]
\begin{figure}[t]
\centering
\input{\FigsPath/HybridCodingBlockDiagramPrePost.tikz}
\caption{Block diagram of a generic video coding system, including the inner blocks of a generic hybrid encoding and decoding scheme.}
\label{Figure2:HybridCodec}
\end{figure}
As aforementioned, the main purpose of video coding is to compress the video by reducing its redundancy.
In general, there are two kinds of redundancies existing in video signal: spatial and temporal.
On the one hand, the spatial redundancy is exploited through intra frame prediction, of which the mechanism is to find the best matching block for current one from its neighbours. This is due to a high correlation between neighbouring pixels inside a frame. As result, in intra frame prediction a block is reconstructed by the blocks that has already been encoded and stored in the bitstream for output. On the other hand, inter frame prediction is designed to deal with the temporal redundancy existed between consecutive frames such as the frame in the past or in the future. Temporal redundancy is normally referred as the correlation between two adjacent frames since they usually have similar backgrounds or a same object with a relative motion. Thus, the purpose of inter-frame prediction is to search for the best matching block for the current one in the reference frames, ranging from past to future frames. After the best matching candidate for the current block is found by either intra- or inter-frame prediction, only the difference between the original block and the predicted one needs to be encoded.
As seen from Figure~\ref{Figure2:HybridCodec}, these differences (also called residual signal) will be further processed and converted into the coefficients of transform domain (T). In general, redundant information can be eliminated through transforming. Subsequently, the transformed coefficients are quantized (Q) to reduce the range of the signal and save the number of bits required for signalling. Finally, the quantized transform coefficients together with the prediction information are compressed to form an output bitstream for transmission by the entropy encoder, as shown in Figure~\ref{Figure2:HybridCodec}.
On contrary, the decoding part performs the inverse operations on the bitstream to reconstruct the video for playing. Subsequently, the aforementioned blocks are described in the context of \ac{hevc} standard.
%As aforementioned, the main purpose of video coding is to compress the video by reducing its redundancy.
In general, there are two kinds of redundancy in video signal: spatial and temporal.
On the one hand, the spatial redundancy is exploited through intra frame prediction, where the underlying mechanism is to find the best matching block for the current one by searching the neighbouring region.
This is possible due to a high correlation between neighbouring pixels within a frame. As result, in intra frame prediction, a block of pixels can be reconstructed from the blocks previously processed.
On the other hand, inter frame prediction is designed to exploit the temporal redundancy that exists between consecutive frames such as those temporally adjacent, e.g., the past frame or the future frame in regard to the current one
Temporal redundancy is normally referred to as the correlation between two adjacent frames since they usually have quiet similar visual content, such as visual objects and background.
Thus, the purpose of inter-frame prediction is to search for the best matching block for the current one in a set of reference frames.
After finding the best matching candidate for the current block, either by intra- or inter-frame prediction, only the difference between the original block and the predicted one needs to be encoded.
As shown in Figure~\ref{Figure2:HybridCodec}, these differences (also known as residual signal) are further processed and transformed (T) into the coefficients in the frequency domain.
Such a transformation decorrelates the residue signal and concentrates the signal energy in few coefficients, mostly at low frequencies.
Subsequently, the transformed coefficients are quantized (Q) to reduce the representation accuracy of the signal, and consequently reduce the number of bits required for coding.
Finally, the quantized transform coefficients together with the prediction information are further compressed by the entropy encoder, forming an output stream suitable for transmission or storage, as shown in Figure~\ref{Figure2:HybridCodec}.
On the other side of the communication chain, the decoding stage performs the inverse operations on the stream to reconstruct the video signal. In the next section, the aforementioned blocks are described in the context of \ac{hevc} standard.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -62,27 +85,30 @@ On contrary, the decoding part performs the inverse operations on the bitstream
\section{Overview the standard features}
\label{Section2:Overview}
As for all the previous coding standards developed by JCT-VC, in \ac{hevc}~\cite{HEVCOverviewJul2012,HEVCOverviewDez2012,BookHEVC2014} only the bitstream structure and the coding syntax are standardised.
Included are the bitstream constraints, the definition of the semantic meaning of syntax elements and the decoding process, such that every decoder conforming to the standard generates the same output when given a correct bitstream.
Such limitation of scope in the standard allows maximal freedom for developers and enables specific optimisations to cope with various applications’ requirements (\emph{e.g.}, balancing the compression ratio and the coding complexity). Therefore, in order to be considered as an \ac{hevc} encoder, the algorithm needs to ensure that the produced bitstream complies with the standard specifications.
Figure \ref{fig:HEVCBlockDiagram} illustrates the block diagram of the hybrid \ac{hevc} encoder. It follows the same overall architecture as its predecessors starting from H.261.
The encoder uses both intra- and inter-picture prediction to exploit spatial and temporal redundancies, respectively. Subsequently, the prediction residue transformed by linear spatial transform, which is then quantised, and entropy coded. This allows to reduce the bitstream redundancy to a minimum.
The resulting bitstream is combined with control information, which signals the prediction chosen, motion vectors, filtering operations and bitstream headers.
Similar to the previous coding standards developed by JCT-VC, in \ac{hevc}~\cite{HEVCOverviewJul2012,HEVCOverviewDez2012,BookHEVC2014} only the
coding syntax and semantics are standardised.
The constrains and functionalities are defined through profiles and levels, while the semantic meaning of syntax elements define the decoding process, such that every decoder conforming to the standard generates the same output when given a complaint stream.
Such delimitation of scope in the standard allows maximum freedom for developers and manufactures to develop proprietary optimisations to cope with various applications’ requirements (\emph{e.g.}, balancing the compression ratio and the coding complexity).
\begin{figure}[tb]
\centering
% \includegraphics[width=0.75\columnwidth]{\FigsPath/HEVC/HEVCBlockDiagram}
\input{\FigsPath/HEVC/HEVCBlockDiagram.tikz}
\caption{Generic \ac{hevc} standard architecture.}
\input{\FigsPath/HEVCBlockDiagram.tikz}
\caption{\HIGHLIGHT{Generic \ac{hevc} standard architecture.}}
\label{fig:HEVCBlockDiagram}
\end{figure}
\HIGHLIGHT{
Figure \ref{fig:HEVCBlockDiagram} illustrates the block diagram of the hybrid \ac{hevc} encoder. It follows the same overall architecture as its predecessors starting from H.261.
The encoder uses both intra- and inter-picture prediction to exploit spatial and temporal redundancies, respectively. Subsequently, the prediction residue transformed by linear spatial transform, which is then quantised, and entropy coded. This allows to reduce the bitstream redundancy to a minimum.
The resulting bitstream is combined with control information, which signals the prediction chosen, motion vectors, filtering operations and bitstream headers.
}
\HIGHLIGHT{
As depicted in Figure \ref{fig:HEVCBlockDiagram}, each picture can be compressed with intra coding algorithms or inter prediction using previously encoded and reconstructed pictures kept in the \ac{dpb} as reference.
The encoding process follows a block-based approach as in previous standards with the larger units referred to as \ac{ctu} with a maximum size of $64\times64$. In order to improve the overall quality of the reconstructed pictures and smooth the coding artefacts, \ac{hevc} uses the deblocking and \ac{sao} filters.
The encoder has a built-in decoder to generate the reference pictures. In the decoding loop, the residual information is delivered to the inverse quantisation and transform modules. The finally reconstructed samples are obtained by adding the predicted signal and applying the filtering operations.
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% SECTION %%
@@ -90,21 +116,21 @@ The encoder has a built-in decoder to generate the reference pictures. In the de
\section{High-level syntax}
\label{Section2:HighLevel}
This section describes the high-level components of the \ac{hevc} standard that provide signalling information and enable bitstream partitioning.
\ac{hevc} inherits several high-level features from \ac{h264}, e.g., the \ac{nal} units, parameter sets, the use of \ac{poc}, and \ac{sei} messages for auxiliary data.
Some high-level features that exist in \ac{h264} were not included in \ac{hevc}, for example \ac{fmo}, redundant slices, arbitrary slice order, data partitioning, and switching slices.
Several new high-level features are introduced, such as the \ac{vps}, the tiles and wavefront tools for parallel processing, the dependent slices for reduced delay, and a new reference picture management concept.
Moreover, new pictures types were introduced new picture types to increase the random-access flexibility, and temporal sub-layer access pictures.
\glsreset{nal}
This section describes the high-level components of the \ac{hevc} standard that provide support for signalling and stream partitioning which are relevant elements in the context of robust video communications.
\ac{hevc} inherits several high-level features from \ac{h264}, e.g., the \ac{nal}, parameter sets, the use of \ac{poc}, and \ac{sei} messages for auxiliary data.
However, some high-level features defined in \ac{h264} were not included, such as \ac{fmo}, redundant slices, arbitrary slice order, data partitioning, and switching slices.
Several new high-level features are introduced, such as the \ac{vps}, tiles and wavefront tools for parallel processing, dependent slices for reduced delay, and a new reference picture management concept.
Moreover, new pictures types were introduced to increase the random-access flexibility and temporal sub-layer switching.
\subsection{Network Abstraction Layer}
Similar to \ac{h264}~\cite{H264AVCStandard2003}, the \ac{hevc} standard used a bitstream based on \glsfirst{nal} units, which are useful to deal with lossy network environments, as it is useful for bitstream partitioning and synchronization \cite{H264OverIPJul2003}.
The \ac{nal} provides the ability to map the \ac{vcl} data that represents video frames on various transport layers, such as \ac{rtp}, MPEG-2 Systems and MPEG DASH. \Gls{nal} units are classified into two categories: (i) \ac{vcl} and (ii) non-\ac{vcl}, as they carry coded video frames or associated data, respectively.
Similar to \ac{h264}~\cite{H264AVCStandard2003}, the \ac{hevc} streams are based on \glsfirst{nal} for stream partitioning and synchronization, which is useful to deal with lossy network environments~\cite{H264OverIPJul2003}. Using this concept the stream is composed of different \acs{nalu}. The \ac{nal} concept provides the ability to map the \ac{vcl} data that represents coded video slices (or frames) on various transport layers, such as \ac{rtp}, MPEG-2 Systems and MPEG DASH. The~\acs{nalu} are classified into two categories: (i) \ac{vcl} and (ii) non-\ac{vcl}, as they carry coded video or associated data, respectively.
In the \ac{hevc} standard two bytes are allocated to describe the content of each \ac{nal} unit. These bytes are used to identify which type of \ac{nal} is being decoded and are transmitted in a fixed format form without variable length codes. This allow extra error robustness but increases the bit allocation. Figure~\ref{Figure2:NalHeader} show the structure of the \ac{hevc} \ac{nal} unit header. The first bit \textit{F}, referred to as \textit{forbiden\_bit}, is always zero in order to ensure that a start code can be detected across different protocol standards. This is followed by the \ac{nal} unit type which identifies its content, enabling the decoder to access if it is required for a particular decoding purpose, or if it can be discarded. Another syntax element defined by the \ac{hevc} standard in the \ac{nal} unit header is the TIDP which marks the temporal layer to which the unit belongs. This allows for temporal scalability (layers ranging from 0 to 6) by immediately discarding the \ac{nal} unit if it belongs to a layer higher than the desired one. Finally, six bits are reserved for future extensions, preventing the creating of extra \gls{nal} units for scalable and multiview extensions as it was done in \ac{h264}.
\begin{figure}[tb]
\begin{figure}[b]
\centering
\vspace{5mm}
\newcommand\bithead[1]{\bitbox[]{#1}{\scriptsize #1}}
\newcommand{\colorbitbox}[3]{%
\rlap{\bitbox{#2}{\color{#1}\rule{\width}{\height}}}%
@@ -117,14 +143,20 @@ In the \ac{hevc} standard two bytes are allocated to describe the content of eac
\label{Figure2:NalHeader}
\end{figure}
The \ac{hevc} standard defines two bytes to describe the content of each \ac{nalu}. These bytes are used to identify which type of \ac{nalu} is being \HIGHLIGHT{decoded} and they are transmitted using a fixed coding format length, i.e., without variable length codes. This allow extra error robustness but increases the bit overhead. Figure~\ref{Figure2:NalHeader} shows the structure of the \ac{hevc} \ac{nal} unit header. The first bit \textit{F}, referred to as \textit{forbiden\_bit}, is always zero in order to ensure that a start code can be detected across different protocol standards. This is followed by the \ac{nal} type which identifies its content, enabling the decoder to access if it is required for a particular decoding purpose, or if it can be discarded. Another syntax element defined by the \ac{hevc} standard is the \textit{temporal\_id\_plus\_one} (TIDP) which marks the temporal layer to which the \ac{nalu} belongs. This allows implicit temporal scalability (layers ranging from 0 to 6) by immediately discarding the \ac{nalu} if it belongs to a layer higher than the desired one. In the header, six bits are reserved for future extensions, preventing the creation of extra \acs{nalu} for scalable and multiview extensions as done in \ac{h264}.
\subsection{Parameters set}
\glsreset{vps}
The encoder uses a set of signalling packets as a robust method to share data with the decoder, referred to as parameter set units. This information was introduced in previous standards to deal with vulnerabilities in the transport of signalling information between the encoder and decoder, which compromised the streaming applications.
Within this approach the set of parameters are included in the bitstream and are repeated as many times as required by the application. This structure is inherited from the \ac{h264} standard\cite{H264OverIPJul2003} with modifications to cope with new coding tools.
The \ac{hevc} includes three sets of parameters, \ac{sps}, \ac{pps} and the newly introduced \glsfirst{vps}\cite{HEVCHighLevelReferenceDez2012}.
Within this approach such set of parameters is multiplexed in the bitstream and repeated as many times as required by the application and communication environment. In \ac{hevc}, this structure is inherited from \ac{h264}\cite{H264OverIPJul2003} with modifications to cope with new coding tools.
The \ac{hevc} includes three sets of parameters, \ac{sps}, \ac{pps} and the newly introduced \ac{vps}\cite{HEVCHighLevelReferenceDez2012}.
% \ac{sps} include sequence related information, that include the decoder operation point, flags to enable or disable certain tools within a given profile. \Gls{pps} include picture related information, that may change for every picture, e.g., \ac{qp} and tilling information.
The new set of parameters introduced in the \ac{hevc} standard (\ac{vps}) contains information related to the different layers of the video, avoiding duplications like in the \ac{h264} and its extensions. \Gls{vps} aims to provide signalling for every layer, guaranteeing a clean and extendible standard, capable of supporting multiple layers. Moreover, \ac{vps} carries information for session negotiation (\emph{e.g.} profile, level and tier). The other sets are the \ac{sps} and \ac{pps}. \Glspl{sps} are related to the whole sequence (\emph{i.e.}, should affect all coded slices), and contain information related with the decoder operation point, as well as, flags to control optional tools and scalability control. \Glspl{pps} carry information that may change for every picture, such as, initial \ac{qp}, flags for picture related tools and tilling information.
The new set of parameters \ac{vps} contains information related to the different layers of the video signal, avoiding duplications like in the \ac{h264} and its extensions. \Gls{vps} aims to provide signalling for every layer, guaranteeing an extendible standard, capable of supporting multiple layers. Moreover, \ac{vps} carries information for session negotiation (\emph{e.g.} profile, level and tier). The \ac{sps} contains information also related to the whole sequence (\emph{i.e.}, should affect all coded slices), and also related with the decoder operation point, as well as flags to control optional tools and scalability. The \ac{pps} is responsible to carry information that may change for every picture, such as, initial \ac{qp}, flags for picture related tools and tilling information.
\subsection{Picture types}
@@ -170,50 +202,83 @@ The new set of parameters introduced in the \ac{hevc} standard (\ac{vps}) contai
\end{tabular}
\end{table}
The \ac{hevc} standard introduces several pictures types as show in Table~\ref{Table2:PictureTypes}. These pictures are divided in the following categories: (i) random-access points, (ii) leading pictures and (iii) temporal sub-layer access pictures~\cite{HEVCHighLevelReferenceDez2012,BookHEVC2014}.
The \ac{hevc} standard supports several pictures types as show in Table~\ref{Table2:PictureTypes}, which are divided in the following categories: (i) random-access points, (ii) leading pictures and (iii) temporal sub-layer access pictures~\cite{HEVCHighLevelReferenceDez2012,BookHEVC2014}.
The use of different types of pictures allows the \ac{hevc} stream to be more flexible and support a wider range of applications and a robust video communications.
They are important to perform random-access, which is a critical feature for channel switching, seek operations and dynamic streaming services.
Moreover, introducing random-access points into the bitstream increases its error robustness as they provide anchor points for a clean decoding after data loss or synchronisation problems.
%\HIGHLIGHT{adicionar texto mencionar robustez}
\subsubsection{Random-Access Point}
The \ac{rap} pictures are used to provide points in the bitstream where it is possible to start the decoding process. The standard defines that a conforming bitstream must start with a \ac{rap} picture. These frames must belong to temporal sub-layer 0 and be coded using intra coding techniques, i.e., must not using previous decoded frames as reference. One should not that frames may be compressed with solely intra prediction and not be marked as random-access points. It is always possible to decode from the \ac{rap} frame and onwards to output any subsequent pictures in output order even if all pictures that precede the \ac{rap} in decoding order are discarded from the bitstream.
The \ac{rap} pictures are used to provide points in the bitstream with no dependencies, where it is possible to start the decoding process. The standard defines that a conforming bitstream must start with a \ac{rap} picture.
These frames must belong to temporal sub-layer 0 and coded using intra coding techniques, i.e., must not using previous decoded frames as reference.
One should note that there may be frames solely compressed with solely intra prediction but not be marked as random-access points.
It is always possible to start decoding from a \ac{rap} frame onwards, and to output any subsequent pictures in display order even if all pictures that precede the \ac{rap} in decoding order are discarded from the bitstream.
The \ac{idr} frame is an intra coded picture that completely refresh the decoding process and starts a new conforming video stream. This means that neither the \ac{idr} nor any other frame following in the decoding order have dependencies to a prior frame. \ac{idr} frames are allowed to have leading pictures, but they must be decodable, thus creating a true random-access point where all subsequent frames in the decoding order can be decoder and displayed.
The \ac{idr} frame is an intra coded picture that completely refreshes the decoding process and starts a new conforming video stream. This means that neither the \ac{idr} nor any other subsequent frame in the decoding order have prior dependencies. \ac{idr} frames are allowed to have leading pictures, but they must be decodable, thus creating a true random-access point where all subsequent frames in the decoding order can be decoded and displayed.
In order to increase the coding efficiency, the \ac{hevc} standard introduces a new \ac{rap} picture, referred to as \ac{cra} picture. Theses frames do not refresh the decoding process and allows leading pictures to depend on frames that precede the \ac{cra} picture in the decoding order. This enables a more efficient prediction structure, while maintaining an access point to begin the decoding process. The introduction of \ac{cra} frames in \ac{hevc} were able to improve the coding efficiency by up to 6\% \cite{HEVCCRAPics2011}.
Introducing \ac{rap} reduces the coding efficiency since no prior reference frames can be used.
In order to reduce this impact, the \ac{hevc} standard introduces a new \ac{rap} picture, referred to as \ac{cra} picture.
These frames do not refresh the decoding process and allow leading pictures to depend on frames that precede the \ac{cra} picture in the decoding order.
This enables a more efficient prediction structure, while maintaining an access point to begin the decoding process.
However, it results in leading pictures which may not be decodable, which are explained in the next sub-section.
The introduction of \ac{cra} frames in \ac{hevc} was able to improve the coding efficiency by up to 6\% \cite{HEVCCRAPics2011}.
\HERE
The \ac{hevc} supports bitstream splicing, which means taking a particular bitstream starting from a \ac{rap} (both \ac{idr} and \ac{cra} frames) and insert it into another bitstream at a random-access point.
In case of bitstream splicing starting from an \ac{idr} frame the leading pictures does not have dependencies to frames prior the \ac{rap}, thus all frames are decodable.
However, whenever this is performed for a bitstream starting from a \ac{cra} frame, the leading associated frames might not be correctly decoded because some of their reference pictures are not present in the combined bitstream.
Therefore, in order to make splicing operation straightforward the \acs{nalu} containing the \ac{cra} picture are changed to \acs{nalu} of type \ac{bla}. This way the decoder is aware that the leading frames should be discarded, namely the \ac{rasl} pictures.
The \ac{hevc} supports bitstream splicing, which means taking a particular bitstream starting from a \ac{rap} (both \ac{idr} and \ac{cra} frames) and introduce it in another bitstream at a random-access instance. Therefore, whenever this is performed for a bitstream starting from a \ac{cra} frame, the leading frames associated might not be correctly decoded. Therefore, in order to make splicing operation straightforward the \ac{nal} units of the \ac{cra} picture are changed to \ac{nal} units of type \ac{bla}. In case of bitstream splicing starting from an \ac{idr} frame the leading pictures does not have dependencies to frames prior the \ac{rap}, thus converting it to \ac{bla} is not required.
\subsubsection{Leading and trailing pictures}
\begin{figure}[tb]
\centering
\input{\FigsPath/HEVC/PictureTypes_Leading_Trailing.tikz}
\input{\FigsPath/PictureTypes_Leading_Trailing.tikz}
\caption{Leading and trailing pictures associated with a random-access point.}
\label{Figure2:LeadingTralingPics}
\end{figure}
Leading pictures correspond to those frame that follow a particular \ac{rap} frame in the decoding order but precedes it in the output order.
A trailing picture is a frame that follows a particular \ac{rap} frame in both decoding and output order and must not depend on any leading frames nor any trailing frame belonging to a previous \ac{rap}.
Figure~\ref{Figure2:LeadingTralingPics} shows examples of leading and trailing pictures. In the figure, frames are illustrated in the output order and the number associated with the frame type (i.e., I, P and B) represents the decoding order. In this example frame $I_1$ corresponds to a \ac{cra} picture.
Leading pictures correspond to those frames that follow a particular \ac{rap} frame in the decoding order but precede it in the display order.
A trailing picture is a frame that follows a particular \ac{rap} frame in both decoding and display order.
%A trailing picture is a frame that follows a particular \ac{rap} frame in both decoding and display order and must not depend on any leading frames nor any trailing frame associated to a previous \ac{rap}.
Figure~\ref{Figure2:LeadingTralingPics} shows examples of leading and trailing pictures. In the figure, frames are illustrated in the display order and the number associated with the frame type (i.e., I, P and B) represents the decoding order. In this example frame $I_1$ corresponds to a \ac{cra} picture.
Leading and trailing pictures are considered to be associated with the closest previous \ac{rap} picture in decoding order, such as frame $I_1$ in Figure~\ref{Figure2:LeadingTralingPics}.
Also, all leading frame of a \ac{rap} frame must precede in decoding order all trailing frames that are associated with the same \ac{rap}. This means that the following order is imposed by the \ac{hevc} standard: 1) \ac{rap} picture, 2) associated leading pictures and 3) associated trailing pictures.
Due to the introduction of more flexible random-access points using \ac{cra} pictures, it is important to mark the leading and trailing pictures, so the decoder can be aware of which frames can be correctly decodable whenever it starts decoding the bitstream.
There is two types of leading pictures as shown in Table~\ref{Table2:PictureTypes}. On the one hand, the \ac{radl} pictures only depend on the associated \ac{rap} pictures and do not have any prior dependencies to trailing pictures associated with previous \ac{rap} frame. These frames can be correctly decoded. On the other hand, \ac{rasl} picture may have dependencies to a prior trailing picture. Therefore, when a random-access is performed to the associated \ac{rap} frame, these frames cannot be correctly decoded, and should be skipped.
There are two types of leading pictures as shown in Table~\ref{Table2:PictureTypes}.
The \ac{radl} pictures which only depend on the associated \ac{rap} picture and do not have any prior dependencies to trailing pictures associated with previous \ac{rap} frame and can be correctly decoded.
On contrary, the \ac{rasl} pictures may have dependencies to a prior trailing pictures.
Therefore, when a random-access is performed at the associated \ac{rap} frame, these frames cannot be correctly decoded, and should be skipped. The example of ~\ref{Figure2:LeadingTralingPics} the leading pictures represented correspond to \ac{rasl} pictures.
\subsubsection{Temporal sub-layer switching pictures}
\begin{figure}[tb]
\centering
% \includegraphics[width=0.75\columnwidth]{\FigsPath/HEVC/Pictures_TSA}
\input{\FigsPath/HEVC/PictureTypes_TSA.tikz}
\caption{Temporal sub-layer access pictures example.}
\input{\FigsPath/PictureTypes_TSA.tikz}
\caption{Temporal layers access pictures example.}
\label{Figure2:TSAExample}
\end{figure}
A \ac{tsa} picture is a trailing picture that marks a temporal sub-layer switching point. When a decoder is decoding a sub-set of the temporal sub-layers in the bitstream and encounters a \ac{tsa} picture type of the temporal sub-layer just above the maximum temporal sub-layer it is decoding, it is possible for the decoder to switch up to and decode any number of additional temporal sub-layers. For example, picture $P_6$ in Figure~\ref{Figure2:TSAExample} can use the \ac{tsa} frame type since it only depends on frame belonging to temporal sub-layer 0 for prediction, as well as, any subsequent frame predicted from the \ac{tsa} frame $P_6$. In the example of figure if the decoder was only decoding the temporal sub-layer 0 it can after $P_6$ starts decoding both sub-layer 0 and 1 or decode all the available three sub-layers. Although theses frame allows higher flexibility in terms of sub-layer switching, they severely constrain the prediction of trailing the \ac{tsa} frame.
\HERE
To reduce the constrains in frame prediction due to sub-layer switching, \ac{hevc} introduces the \ac{stsa} picture. These pictures have similar purpose as the \ac{tsa} frames but they only guarantee that frames in the same temporal sub-layer as the \ac{stsa} frame in the decoding order are able to be correctly reconstructed. This is guaranteed by preventing any dependency to frame preceding the \ac{stsa} to be used as reference in its temporal sub-layer.
One example of a \ac{stsa} frame is given in also given in Figure~\ref{Figure2:TSAExample} in frame $P_2$. This frame can be used to switch to the sub-layer 1 as it does not have dependencies to any prior frame in the same sub-layer ($P_2$ only depends on $I_0$). However, it cannot be classified as \ac{tsa} picture since $P_3$ has dependencies to prior frame in the decoding order to $P_1$.
Summarising, \ac{stsa} frames can be used to switch to a particular sub-layer as the \ac{tsa} can be used to switch to any sub-layer.
A \ac{tsa} picture is a trailing picture that marks a temporal layer switching point.
When decoding a sub-set of the temporal layers and a \ac{tsa} picture is found from the temporal layer just above the maximum temporal layer currently being decoded, it is possible for the decoder to start decoding any number of additional temporal layers.
For example, frame $P_6$ in Figure~\ref{Figure2:TSAExample} is considered a \ac{tsa} picture since it only depends on a frame belonging to temporal layer 0 for prediction, as well as, any subsequent frame predicted from the \ac{tsa} frame $P_6$.
In this example, if the decoder was only decoding the temporal layer 0 it can after $P_6$ starts decoding both layer 0 and 1 or decode all the available three layers. Although theses frame allows higher flexibility in terms of layer switching, they severely constrain the prediction of frames following the \ac{tsa} picture.
To reduce the constrains in frame prediction due to layer switching, \ac{hevc} introduces the \ac{stsa} picture.
These pictures have similar purpose as the \ac{tsa} frames but they only guarantee that frames in the same temporal layer as the \ac{stsa} frame in the decoding order are able to be correctly reconstructed. This is guaranteed by preventing any dependency from frames preceding the \ac{stsa} to be used as reference in its temporal layer.
One example of a \ac{stsa} frame is also shown in Figure~\ref{Figure2:TSAExample} in frame $P_2$.
This frame can be used to switch to the layer 1 as it does not have dependencies to any prior frame in the same layer ($P_2$ only depends on $I_0$).
However, it cannot be classified as \ac{tsa} picture because $P_3$ has dependencies to prior frame in the decoding order to $P_1$.
Summarising, \ac{stsa} frames can be used to switch to a particular layer as the \ac{tsa} can be used to switch to any layer.
\subsection{Picture partitioning}
@@ -223,22 +288,24 @@ The high-level segmentation of a picture in \ac{hevc} is achieved through four t
\item[Error robustness:] partitioning the picture into smaller self-contained units in
order to gain error robustness by the ability to re-synchronize both the parsing and decoding
processes in case of data losses.
\item[Network adaptation:] adapt to the network constraint of \ac{mtu} size, found for example in \ac{ip} networks.
Such packetisation scheme restricts the maximum number of payload bits within a slice regardless of the size of the coded frame. To keep each slice within this limit and minimise the packetisation overhead, a variable number of coding units is used for each slice.
\item[Parallel processing:] partitioning the coded frame into units which can be processed in parallel. This is achieved by dividing the coding units in such way that they can be encoded and decoded independently of each other’s.
\item[Parallel processing:] partitioning the coded frame into processing data units which can be encoded in parallel. This is achieved by dividing the coding units in such way that they can be encoded and decoded independently of each other’s.
\end{description}
\subsubsection{Slices}
The \ac{hevc} standard preserves the slice structure present in the \ac{h264} standard. A slice may comprise either the entire frame or a section of it, and all the associated data (\emph{i.e.}, entropy symbols, prediction and residue information) can be independently decoded. Each slice is transported in a different \ac{nal} unit. As dependencies across slice boundaries are disabled, each slice can be independently reconstructed, regardless if other slices were lost or incorrectly decoded. Slice partitioning is the only tool that can be used for parallel processing in a virtually identical form as in \ac{h264}. As they are independent from each other, it is straightforward to process multiple slices in parallel, without inter-process communication (except for inter-frame prediction). Although this is a simple approach, it incurs in substantial coding overhead due to the higher number of slice headers, and reduction of causal neighbours for prediction (due to lack of predictions across slice boundaries). Another disadvantage of using slices for parallel processing is that they are also used to meet the \gls{mtu} size constrains. Therefore, it might be impossible to meet both requirements using only one technique.
The \ac{hevc} standard preserves the slice structure defined in the \ac{h264} standard. A slice may comprise either the entire frame or a section of it, and all the associated data (\emph{i.e.}, entropy symbols, prediction and residue information) can be independently decoded. Each slice is transported in a different \ac{nal} unit. Some dependencies across slice boundaries are disabled, each slice can be independently reconstructed, regardless whether previous slices were lost or incorrectly decoded. Slice partitioning is the only tool that can be used for parallel processing in a virtually identical form as in \ac{h264}. As they are independent from each other, it is straightforward to process multiple slices in parallel, without inter-process communication (except for inter-frame prediction). Although this is a simple approach, it incurs in substantial coding overhead due to the higher number of slice headers, and reduction of causal neighbours for prediction (due to lack of predictions across slice boundaries). Another disadvantage of using slices for parallel processing is that they are also used to meet the \gls{mtu} size constrains. Therefore, it might be impossible to meet both requirements using only one technique.
Another partitioning tool in \ac{hevc} standard is the dependent slices. These slices allow the partitioning of the frame at coding unit boundaries without breaking intra-frame predictions. Moreover, as they have a smaller slice header they are more efficient in terms of signalling overhead than regular slices. Dependent slices are normally used to reduce the end-to-end delay by allowing part of the slice to transmitted while the rest of it are still being processed.
Another partitioning tool in \ac{hevc} standard is the dependent slices. These slices allow the partitioning of a frame at coding unit boundaries without breaking intra-frame predictions. Moreover, as they have a smaller slice header they are more efficient in terms of signalling overhead than regular slices. Dependent slices are normally used to reduce the end-to-end delay by allowing part of the slice to transmitted while the rest of its data is still being processed.
\begin{figure}[tb]
\centering
\begin{tabular}{x{.45\columnwidth}x{.45\columnwidth}}
\input{\FigsPath/HEVC/Tiles_Slices_Example_1.tikz} &
\input{\FigsPath/HEVC/Tiles_Slices_Example_2.tikz} \\
\input{\FigsPath/Tiles_Slices_Example_1.tikz} &
\input{\FigsPath/Tiles_Slices_Example_2.tikz} \\
(a) & (b) \\
\end{tabular}
\caption{Examples of the interaction between tiles and slices.}
@@ -248,65 +315,56 @@ Another partitioning tool in \ac{hevc} standard is the dependent slices. These s
\subsubsection{Tiles}
A new data partition has been included in the \ac{hevc} standard, referred to as Tiles\cite{HEVCTilesDec2013}. Tiles enable the partition of a coded frame into a smaller group of \acs{ctu} using horizontal and/or vertical boundaries. Figure~\ref{Figure2:HEVCTiles} shows two examples of slice partitioning using tiles. In both cases the frame is partitioned into three tiles by using two vertical boundaries. By defining tiles using only boundaries it allows for an efficient signalling with a small overhead. These boundaries are defined in the \ac{pps} \ac{nal} units.
A new data partition has been included in the \ac{hevc} standard, referred to as Tiles\cite{HEVCTilesDec2013}. Tiles enable the partition of a coded frame into a smaller group of \acs{ctu} using horizontal and/or vertical boundaries. Figure~\ref{Figure2:HEVCTiles} shows two examples of slice partitioning using tiles. In both cases the frame is partitioned into three tiles by using two vertical boundaries. By only using boundaries to define tiles, signalling is achieved with a small overhead. These boundaries are defined in the \ac{pps} \acs{nalu} .
Although the \ac{hevc} standard enables the use of both slices and tiles at the same time, it impose constrains in order to reduce the implementation complexity. One of two cases must be used: (i) all \acs{ctu} within a tile must belong to the same slice, or (ii) all \acs{ctu} within a slice must belong to the same tile. These cases are illustrated in Figure~\ref{Figure2:HEVCTiles} (a) and (b), respectively.
Although the \ac{hevc} standard enables the use of both slices and tiles at the same time, some constrains are imposed in order to reduce the implementation complexity.
One of two cases must be used: (i) all \acs{ctu} within a tile must belong to the same slice, or (ii) all \acs{ctu} within a slice must belong to the same tile. These cases are illustrated in Figure~\ref{Figure2:HEVCTiles} (a) and (b), respectively.
The use of tiles provide several advantages. Specifically, they improve frame partitioning for parallel processing~\cite{WPPTilesParallelHEVC2017} when compared to slices, by reducing the required overhead. Moreover, they also improve the \ac{mtu} size matching and reduce the line buffer memory (by dividing the frame as in Figure~\ref{Figure2:HEVCTiles}). Finally, as tiles allow flexible partitioning, they enable the definition of \acs{roi} for asymmetric video coding.
\subsubsection{Wavefront Parallel Processing}
As mentioned before, the use of either slices or tiles to process each frame using parallel threads require the use of different entropy coding contexts for each slice/tile, in order to make them independently decodable. In consequence the compression efficiency decreases. In order to overcome this issue, the concept of \ac{wpp} was introduced, which enables coding efficient parallel processing~\cite{HEVCParallelCSVT2016}.
As mentioned before, the use of either slices or tiles to process each frame using parallel threads require the use of different entropy coding contexts for each slice/tile, in order to make them independently decodable. As consequence, the compression efficiency decreases. In order to overcome this issue, the concept of \ac{wpp} was introduced, which enables coding using efficient parallel processing~\cite{HEVCParallelCSVT2016}.
\begin{figure}[tb]
\centering
\input{\FigsPath/HEVC/WPP.tikz}
\input{\FigsPath/WPP.tikz}
\caption{Illustration of the wavefront parallel processing.}
\label{Figure2:WPP}
\end{figure}
In \ac{wpp}, each slice is divided into single rows of \acs{ctu} which can be processed in parallel. Entropy decoding and prediction are enabled across \acs{ctu} of different rows. Therefore, higher compression efficiency can be achieved. In order to re-use the same context in the whole slice each row requires a two \ac{ctu} delay in order to make sure the required dependencies have already been coded/decoded. Figure~\ref{Figure2:WPP} illustrates an example of \ac{wpp} using at least five different threads. Each square corresponds to a \ac{ctu} and the numbers corresponds to the time instance in which a \ac{ctu} may start being processed. Using this staggered start (which appears like a wavefront as represented in the figure), the parallelisation may use as many threads as the number of \acs{ctu} rows available in the video frame. Although, the number of threads does not affect the coding efficiency, the required inter-process communication substantially increases.
In \ac{wpp}, each slice (or tile) is divided into single rows of \acs{ctu} which can be processed in parallel, but entropy coding and prediction are enabled across \acs{ctu} of different rows in order to avoid loss in coding efficiency.
To re-use the same context in the whole slice (or tile) each row requires a delay of two \acs{ctu} in order to ensure that all necessary dependencies have already been encoded/decoded. Figure~\ref{Figure2:WPP} illustrates an example of \ac{wpp} using at least five different threads. Each square corresponds to a \ac{ctu} and the numbers corresponds to the time instance in which a \ac{ctu} may start being processed. Using this staggered start (which appears like a wavefront as represented in the figure), the parallelisation may use as many threads as the number of \acs{ctu} rows available in the video frame. However, although, the number of threads does not affect the coding efficiency, the required inter-process communication substantially increases.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% SECTION %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Coding structures}
\section{Data structures}
\label{Section2:CodingStructures}
\glsreset{ctu}
Despite the fact that the \ac{hevc} standard follows a traditional hybrid video coding architecture, it introduces significant changes in the data structures when compared to the previous standards (e.g., \ac{h264}).
In \ac{h264} each frame is divided into units so-called \acs{mb}, which are composed by a $16\times16$ block of luma samples and two $8\times8$ blocks of chroma samples, assuming video data in YUV 4:2:0 sub-sampling. The \ac{mb} is considered the basic processing unit in the \ac{h264}, which define if the containing pixels are compressed with intra or inter prediction techniques. Each \ac{mb} could also be partitioned into sub-blocks which should follow the same prediction type.
\begin{figure}[tb]
\begin{figure}[!t]
\centering
\input{\FigsPath/HEVC/BlockPartitionsWithTree.tikz}
\input{\FigsPath/BlockPartitionsWithTree.tikz}
\caption{Representation of a CTU ($64\times64$) and its partitioning into CUs and PUs (left); corresponding quad-tree with coding order (right)}
\label{Figure2:HEVCCodingStructure}
\end{figure}
In \ac{hevc} each picture is divided into blocks so-called \acs{ctu}, which are composed by the luma and chroma \acs{ctb} and corresponding signalling~\cite{BlockPartHEVC2012}. The maximum \ac{ctu} size is configured by the encoder and signalled to the decoder, and should be one of the following: $64\times64$, $32\times32$ and $16\times16$. The use of coding units with larger size in \ac{hevc} was adopted to achieve higher coding efficiency especially in high resolution video. Each \ac{ctu} can be recursively partitioned into four smaller units, referred to as \acs{cu}, until it reaches the minimum unit size, which cannot be inferior to $8\times8$ pixels. Similar to the maximum \ac{ctu} size the minimum size is also defined at the encoding time. Figure~\ref{Figure2:HEVCCodingStructure} illustrates an example of a \ac{ctu} partitioning into \acs{cu}. The partitioning illustrated in the figure leads to the quad-tree structure represented on the right side with the corresponding coding order.
Despite the fact that the \ac{hevc} standard follows a traditional hybrid video coding architecture, it introduces significant changes in the data structures when compared to the previous standards (e.g., \ac{h264}).
In \ac{h264} each frame is divided into units so-called \acs{mb}, which are composed by a $16\times16$ block of luma samples and two $8\times8$ blocks of chroma samples, assuming video data in YUV 4:2:0 sub-sampling. The \ac{mb} is considered the basic processing unit in the \ac{h264}, and the respective coding mode define whether the corresponding pixels are coded as intra or intra prediction. Each \ac{mb} could also be partitioned into sub-blocks which should follow the same prediction mode.
\begin{figure}[tb]
\centering
\begin{tabular}{x{.8\columnwidth}x{.1\columnwidth}}
\input{\FigsPath/HEVC/PUSymetric.tikz} & (a)\\
\input{\FigsPath/HEVC/PUAsymetric.tikz} & (b)\\
\end{tabular}
\caption{Partitioning modes of an inter coded \ac{cu} into \acs{pu}~\cite{HEVCBlock2012}.}
\label{Figure2:InterPuPartitions}
\end{figure}
The \ac{cu} is the basic processing unit in \ac{hevc} for prediction. It is further partitioned into two types of units, specifically \ac{pu} and \ac{tu}, which are used for prediction and transform operations, respectively. Each \ac{pu} is composed by \acs{pb}, and each \ac{tu} is composed by \acs{tb}.
The decision whether to use intra-frame or inter-frame prediction modes is applied at the \ac{cu} level. However, depending on the partitioning of the \ac{cu} into \acs{pu} different intra or inter coding modes can be used for each \ac{pu}.
\glsreset{ctu}
\glsreset{cu}
When the intra prediction mode is signalled for a \ac{cu}, the size of the \ac{pb} need to be the same as the \ac{cb}, except when the smaller \ac{cb} is selected. In this case further splitting into four \ac{pb} of smaller size. For example, when the smaller \ac{cb} size is $8\times8$, the encoder may decide to divide it into four \acs{pb} of $4\times4$ pixels, each one having its own intra prediction mode. This is signalled using an one bit flag.
The use of small \acs{pb} is useful for complex regions that required a fine-granularity prediction.
The basic data structures in \ac{hevc} are quite different from \ac{h264}. Each picture is divided into blocks named as \ac{ctu}~\cite{BlockPartHEVC2012}. The maximum \ac{ctu} size is an encoder configuration parameter and signalled to the decoder, and should be one of the following: $64\times64$, $32\times32$ and $16\times16$. The use of data structures with larger size in \ac{hevc} was adopted to achieve higher coding efficiency especially in high resolution video. Each \ac{ctu} can be recursively partitioned into four smaller units, referred to as \acs{cu}, until it reaches the minimum unit size, which cannot be smaller than to $8\times8$ pixels. Similar to the maximum \ac{ctu} size, the minimum size is also defined at the encoding time.
Figure~\ref{Figure2:HEVCCodingStructure} illustrates an example of a \ac{ctu} partitioning into \acs{cu}. The partitioning illustrated in the figure leads to the quad-tree structure represented on the left side with the corresponding coding order shown on the right side.
When inter prediction mode is signalled, \ac{hevc} defines a higher range of partitioning schemes, allowing the \ac{cu} to be divided into one, two or four \acs{pu}. The splitting into four \acs{pu} is only allowed when the \ac{cu} size is equal to the minimum. Otherwise, the splitting should be carried out at the \ac{cu} level which leads to the same partition tree. Figure~\ref{Figure2:InterPuPartitions} illustrates the possible \ac{cu} partitioning into \acs{pu}. One should note that the partition applied to the \ac{cu} is also applied to divide the \ac{cb} into \acs{pb}.
Figure~\ref{Figure2:InterPuPartitions}~(a) shows the symmetric partitions. The $N \times N$ corresponds to the case where the \ac{cu} is not further divided. Then, the standard defines the horizontal and vertical division into two \acs{pu}, corresponding to the $\frac{N}{2}\times N$ and $N \times \frac{N}{2}$ cases, respectively. Finally, the $\frac{N}{2} \times \frac{N}{2}$ case result in four \acs{pu}.
The partitions on Figure~\ref{Figure2:InterPuPartitions}~(b) correspond to the asymmetric partitioning, which is one of the main improvements introduced in \ac{hevc}, as it allow for higher flexibility when performing motion estimation and compensation, leading to higher compression efficiency~\cite{HEVCBlock2012}. These modes are only allowed for \acs{cu} with $16\times16$ pixels or larger.
\glsreset{pu}
\glsreset{tu}
The \ac{cu} is the basic processing unit in \ac{hevc}. It is further partitioned into two types of units, specifically \ac{pu} and \ac{tu}, which are data blocks used for prediction and transform operations, respectively.
The decision whether to use intra-frame or inter-frame prediction modes is taken at \ac{cu} level. However, depending on the partitioning of the \ac{cu} into \acs{pu} different intra or inter coding modes can be used for each \ac{pu}.
%The main key feature of \ac{hevc} is the new block partition structure, which achieves more flexibility and efficiency. Four different block concepts have been introduced in the standard: \ac{ctu}, \ac{cu}, \ac{pu}, and \ac{tu}, all of which have different roles. Figure \ref{fig:HEVCBlockStructure} illustrates an example of a CTU partitioning in HEVC, and it is compared to the \ac{h264} \ac{mb} structure. CTUs are defined as the high-level partition within each slice and can be as large as 64$\times$64, which can be recursively partitioned into four smaller blocks referred to as CUs. The CU is the basic units for the intra/inter prediction and can be as small as 8$\times$8. Subsequently, the CU are divided into up to four PUs which are used for inter prediction. For intra coding PU is not used. Finally, the CU is divided into a transform unit tree, which may include several TUs, in a quad-tree structure. Transform units are the basic units for the transform operations applied to the coding residue.
%
@@ -315,44 +373,59 @@ The partitions on Figure~\ref{Figure2:InterPuPartitions}~(b) correspond to the a
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% SECTION %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Prediction methods}
\section{Prediction modes}
\label{Section2:PredictionModes}
Prediction methods are responsible for an important role in image and video coding standards, due to their ability to reduce the signal redundancy based on the previously encoded pixels.
This section describes the prediction techniques used in the \ac{hevc} standard.
Prediction techniques are responsible for an important role in image and video coding standards, due to their ability to reduce the signal redundancy based on the previously encoded pixels.
These techniques include the directional intra prediction for efficiently reduce spatial redundancy and inter-frame motion compensation for remove the temporal redundancies.
This section describes the prediction techniques used in the \ac{hevc} standard that result from different techniques.
\subsection{Intra-frame}
In previous standards, intra-frame prediction was used to efficiently reduce the spatial redundancy in video frames. Intra predicted blocks are obtained using the previously encoded pixels around the current \ac{pu}. \ac{hevc} adopts three types of intra-frame prediction: directional, Planar and DC~\cite{IntraCodingHEVC2012,HEVCIntraperformance2012}. Figure~\ref{Figure2:IntraModesHEVC} shows the available intra prediction mode available in \ac{hevc}. As shown, the predictions are applied to square blocks only. The example in the figure corresponds to \ac{pu} of $8\times8$ pixels, but the standard allows block sizes ranging from $4\times4$ to $32\times32$
The encoder tests all the available intra prediction modes and signals the best mode to the decoder. The decoder must generate an identical prediction block, based on the reconstructed pixels.
\subsection{Intra-frame prediction}
The reconstructed reference samples used in the prediction process belong to
Intra-frame prediction is used to efficiently reduce the spatial redundancy in video frames.
Intra predicted blocks are obtained using the previously encoded pixels around the current \ac{pu}.
When the intra prediction mode is chosen for a \ac{cu}, the size of the \ac{pu} need to be the same as the \ac{cu}, except when the smaller \ac{cu} is selected and further splitting into four \acs{pu} is done. For example, when the smaller \ac{cu} size is $8\times8$, the encoder may decide to divide it into four \acs{pu} of $4\times4$ pixels, each one having its own intra prediction mode. This is signalled using a binary flag.
The use of small \acs{pu} is useful for complex regions that require fine-granularity prediction.
In \ac{hevc} three types of intra-frame prediction are defined: angular modes, Planar and DC~\cite{IntraCodingHEVC2012,HEVCIntraperformance2012}.
%Figure~\ref{Figure2:IntraModesHEVC} illustrates these intra prediction modes.
%As shown, the predictions are applied to square blocks only.
%The border around the square blocks corresponds to the reference pixels from which the prediction is applied (different colours correspond to different pixel values).
%The example in the figure corresponds to a \ac{pu} size of $8\times8$ pixels, but block sizes ranging from $4\times4$ to $32\times32$ are also allowed.
The encoder chooses among all the available intra prediction modes and signals the best mode to the decoder, which must generate an identical prediction block, based on the previously reconstructed pixels.
The reconstructed reference pixels used in the prediction process belong to
the neighbouring blocks at left-down, left, top-left, top and top-right positions, as
can be noticeable in Figure~\ref{Figure2:IntraModesHEVC}. Given a \ac{pu} with size $N \times N$, intra prediction requires a top neighbouring row of $2N$ pixels, a left column of $2N$ pixels plus the top-left neighbouring pixel.
Due to unit boundaries, e.g., slice or tile boundaries, neighbouring pixels might not be all available to apply the intra prediction. In addition, when constrained intra prediction is enabled, reference samples belonging to inter-predicted \acs{pu} are not used in order to remove error propagation from potentially erroneously reconstructed reference frames.
In order to overcome this issue and being able to apply all possible modes, \ac{hevc} uses reference sample substitution to replace the unavailable reference samples with the closer available ones, so that all intra prediction modes can be used.
can be seen in the example shown on the right in Figure~\ref{Figure2:IntraModesHEVC}.
Given a $N \times N$ \ac{pu}, intra prediction requires the top neighbouring row of $2N$ pixels, the left column of $2N$ pixels and the top-left neighbouring pixel.
Due to data unit boundaries, e.g., slice or tile boundaries, neighbouring pixels might not be all available to use intra prediction.
In addition, when constrained intra prediction is enabled, i.e., reference pixels belonging to inter-predicted \acs{pu} are not used in order to remove error propagation from potentially erroneous reconstructed reference frames.
This may results in an incomplete set of neighbouring reference pixels.
In order to overcome this issue and to allow all possible modes, \ac{hevc} uses reference sample substitution to replace the unavailable reference pixels with the closest available ones, so that all intra prediction modes can be used.
\begin{figure}[tb]
\centering
%\begin{tabular}{x{.5\columnwidth}x{.5\columnwidth}}
%\input{\FigsPath/HEVC/IntraPredictions.tikz} &
%\input{\FigsPath/HEVC/IntraPredictionsExample.tikz} \\
%\end{tabular}
%\caption{List of the available \ac{hevc} angular intra prediction modes (left) and example of the angular mode 29 (right).}
\includegraphics[width=0.9\columnwidth]{\FigsPath/HEVC/IntraPrediction}
\caption{Example of \ac{hevc} intra prediction modes for a $8\times8$ block~\cite{BookHEVC2014}.}
\resizebox{\textwidth}{!}{
\begin{tabular}{x{.6\columnwidth}x{.4\columnwidth}}
\input{021_VideoCoding/Figures/IntraPredictions.tikz} &
\input{021_VideoCoding/Figures/IntraPredictionsExample.tikz} \\
\end{tabular}
}
\caption{List of directional intra prediction modes in \ac{hevc} (left) and an example of the intra prediction mode 32 for a $4\times4$ block.}
\label{Figure2:IntraModesHEVC}
\end{figure}
\subsubsection{Angular modes}
Directional intra prediction was first introduced in the \ac{h264} standard~\cite{H264OverviewJul2003}, to estimate regions with a structured texture or directional edge.
In its process, directional prediction projects the reconstructed samples in the block neighbourhood along a specific direction. The \ac{hevc} improves the number of available direction from 8 in \ac{h264} to 33 possible directions~\cite{IntraCodingHEVC2012}. The result of the projections in each possible direction is illustrated in Figure~\ref{Figure2:IntraModesHEVC}.
Directional intra prediction was first introduced in the \ac{h264} standard~\cite{H264OverviewJul2003}, to estimate regions with a structured texture or directional edges.
Directional prediction projects the reconstructed samples in the block neighbourhood (see reference pixels in Figure~\ref{Figure2:IntraModesHEVC}) along a specific direction. The \ac{hevc} increases the number of possible directions from 8 (in \ac{h264}) to 33~\cite{IntraCodingHEVC2012}. The direction of each projection is illustrated in the left-side of Figure~\ref{Figure2:IntraModesHEVC} by the directional modes 2 to 34.
%In the figure it is possible to notice that directional modes 2 to 10 only project the pixels from the left neighbouring border in a horizontal direction, modes 10 to 25 use both the left and top neighbouring pixels in a diagonal direction and modes 26 to 34 project the top neighbours in vertical direction.
As shown in the figure, the angular modes are designed to provide a dense coverage near the horizontal and vertical directions. Moreover, the angular directions are more coarse as it gets closer to the diagonals. This reflects the observed statistical prevalence of the angles and the prediction efficiency.
To improve the efficiency of angular prediction, when the reference samples need to be projected a bilinear interpolation is used from the two closest pixels using a $1/32$ accuracy. The dashed arrows in Figure~\ref{Figure2:IntraModesHEVC} correspond to a projection of interpolated pixels.
The angular modes are designed to provide a dense coverage near the horizontal and vertical directions. It is not noticeable in the Figure~\ref{Figure2:IntraModesHEVC}, but the angular directions are more coarse as it gets closer to the diagonals. This reflects the observed statistical prevalence of the angles and the prediction efficiency.
In order to improve the efficiency of angular prediction the when the reference samples need to be projected a bilinear interpolation is used from the two closest pixels using a $1/32$ accuracy.
\subsubsection{Planar and DC}
@@ -373,7 +446,7 @@ The second filter in \ac{hevc} is the boundary smoothing. This filter aims at re
%\subsubsection{Rough Mode Decision}
%\cite{HEVCROughModeDecision}
\subsection{Inter-frame}
\subsection{Inter-frame prediction}
The efficiency of video coding algorithms consistently relies on inter-frame prediction techniques used to reduce temporal redundancy. The underlying idea of inter-frame prediction is to estimate the current frame from one or more previously encoded reference frames using block-based motion compensated. The most common technique used for motion estimation in \ac{hevc}, as well as in \ac{h264}, is the boundary matching algorithm. In the \ac{hevc} standard an \ac{amvp} for efficient motion vector signalling, which defines the Merge Mode and differential coding of the motion information.
@@ -425,16 +498,37 @@ Thus, a complete set of all frames that must be kept in the \ac{dpb} (in the pre
This is different from the \ac{h264} scheme where only relative changes to the \ac{dpb} are signalled. With the \ac{rpset} concept, no information from earlier frames in decoding order is needed to maintain the correct status of the \ac{dpb}. Table~\ref{Table2:ReferencePictureListExample} shows the \ac{rpset} for some frames illustrated in Figure~\ref{Figure2:LeadingTralingPics}. As expected for the \ac{cra} frame ($I_2$) nothing is signalled. For the remaining frames, two types of signalled are used: frame number and a flag to mark if it is currently used. This is required because any frame not listed in a given \ac{rpset} will be discarded and subsequent frames cannot use it as reference.
The \ac{rpset} concept in \ac{hevc} provides a basic level of error robustness to the reference picture management. In \ac{h264} whenever a reference frame is lost the decoder can look at the value of the
\emph{FrameNum} (incremental value associated with each reference frame) to detect the loss event. However, as gaps in \emph{FrameNum} are allowed the decoded might not be able to decide the missing frame was intentional or if it resulted from a loss event in the transmission. Moreover, even if the decoded if able to detect the loss event it might not be able to correctly compute the value the \ac{poc} associated with the concealed frame. Using the \ac{rps} concept in the \ac{hevc} the decoder is always able to detect loss events and correctly identify the missing frames, preventing it from use wrong frames as reference.
\emph{FrameNum} (incremental value associated with each reference frame) to detect the loss event. However, as gaps in \emph{FrameNum} are allowed the decoded might not be able to decide the missing frame was intentional or if it resulted from a loss event in the transmission. Moreover, even if the decoded if able to detect the loss event it might not be able to correctly compute the value the \ac{poc} associated with the concealed frame. Using the \ac{rpset} concept in the \ac{hevc} the decoder is always able to detect loss events and correctly identify the missing frames, preventing it from use wrong frames as reference.
\subsubsection{Asymmetric block partitions}
\begin{figure}[tb]
\centering
\begin{tabular}{x{.8\columnwidth}x{.1\columnwidth}}
\input{\FigsPath/PUSymetric.tikz} & (a)\\
\input{\FigsPath/PUAsymetric.tikz} & (b)\\
\end{tabular}
\caption{Partition types of an inter coded \ac{cu} into \acs{pu}~\cite{HEVCBlock2012}.}
\label{Figure2:InterPuPartitions}
\end{figure}
When inter prediction mode is signalled, \ac{hevc} defines a higher range of partitioning schemes, allowing the \ac{cu} to be divided into one, two or four \acs{pu}.
The splitting into four \acs{pu} is only allowed when the \ac{cu} size is equal to the minimum. Otherwise, the splitting should be carried out at the \ac{cu} level which leads to the same partition tree.
Figure~\ref{Figure2:InterPuPartitions} illustrates the possible \ac{cu} partitioning into \acs{pu}. One should note that the partition type applied to the \ac{cu} is also applied to divide the \ac{cb} into \acs{pb}.
Figure~\ref{Figure2:InterPuPartitions}~(a) shows the symmetric partitions.
The $N \times N$ corresponds to the case where the \ac{cu} is not further divided.
Then, the standard defines the horizontal and vertical division into two \acs{pu}, corresponding to the $\frac{N}{2}\times N$ and $N \times \frac{N}{2}$ cases, respectively.
Finally, the $\frac{N}{2} \times \frac{N}{2}$ case results in four \acs{pu}.
The partitions shown in Figure~\ref{Figure2:InterPuPartitions}~(b) correspond to the asymmetric partitioning, which is one of the main improvements introduced in \ac{hevc}, as it allows higher flexibility in motion estimation, leading to higher compression efficiency~\cite{HEVCBlock2012}.
These modes are only allowed for \acs{cu} with $16\times16$ pixels or larger.
\subsubsection{Merge mode}
\begin{figure}[tb]
\centering
\input{\FigsPath/HEVC/MergeModeHEVC.tikz}
\input{\FigsPath/MergeModeHEVC.tikz}
%\input{\FigsPath/HEVC/MergeModeHEVCTwoFrames.tikz}
\caption{Illustration of the \ac{mv} prediction candidates in the AMVP.}
\label{Figure2:MVCandidates}
042_ConcealmentAwareEncoding/ConcealmentAwareEncoding.tex
+2 -2
View File
@@ -216,7 +216,7 @@ For the \ac{ec} mode decision optimization the missing slices are reconstructed,
In this process, each \ac{ctu} is partitioned to form a quadtree structure of \ac{ecu} that are used to test the different \ac{ec} candidate modes.
Then, the best trade-off between \ac{ecu} partition and signalling is found through \ac{rd} optimisation.
In order to achieve an efficient use of the signalling overhead a saliency-weighted procedure is embedded in the \ac{rd} optimisation based on a spatio-temporal saliency, where more signalling bits are allocated to regions with higher saliency values.
Finally, the signalling information is transmitted to the decoder by using the \ac{sei} \ac{nal} unit type~\cite{HEVCHighLevelReferenceDez2012}.
Finally, the signalling information is transmitted to the decoder by using the \ac{sei} \ac{nalu} type~\cite{HEVCHighLevelReferenceDez2012}.
Since in general, the optimal \ac{ec} mode is still not be able to correctly reconstruct all the missing regions, extra residual information is multiplexed in the stream as a scalable \ac{el}.
This residual information contains the difference between the compressed ($\hat f_t$) and the \ac{ec} frames ($\tilde f$).
@@ -378,7 +378,7 @@ At the decoder-side, the reconstruction of a missing frame ($\tilde f$) follows
The additional residue received from the enhancement layer is particularly useful to improve the quality of those \acs{ctu} where the \ac{ec} strategies are not able to produce very accurate reconstruction.
Moreover, since the enhancement layer only carries residual information the overhead is affordable.
Note that the generated bitstream is compliant with the standard, as the error concealment signalling is encapsulated in \ac{sei} \acs{nal} units (NALU) while the error concealment residue is carried in NALUs related with the scalable extension. Then, an adapted decoder, capable of decoding both the \ac{sei} NALUs and the scalable NALUs takes full benefit from the proposed method. Nevertheless, any standard \ac{hevc} decoder is also able to reconstruct the video, because it simply ignores the extra NALUs.
Note that the generated bitstream is compliant with the standard, as the error concealment signalling is encapsulated in \ac{sei} \ac{nalu} while the error concealment residue is carried in \acs{nalu} related with the scalable extension. Then, an adapted decoder, capable of decoding both the \ac{sei} \acs{nalu} and the scalable \acs{nalu} takes full benefit from the proposed method. Nevertheless, any standard \ac{hevc} decoder is also able to reconstruct the video, because it simply ignores the extra \acs{nalu}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
TikzThesisTemplate.pgs
+3
View File
@@ -4,6 +4,9 @@
\usepackage{showframe}
\usepackage{balance}
% \usepackage[active,tightpage]{preview}
% \PreviewEnvironment{tikzpicture}
\begin{document}
\begin{figure}[t]
carreira_phd_thesis.pdf
BIN
View File
Binary file not shown.
carreira_phd_thesis.sty
+9 -1
View File
@@ -54,15 +54,23 @@
%% TikZ
\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{tkz-euclide}
\usetikzlibrary{fit}
\usetikzlibrary{calc}
\usetikzlibrary{snakes}
\usetikzlibrary{arrows}
\usetikzlibrary{shapes}
\usetikzlibrary{positioning}
\usetikzlibrary{decorations}
\usetikzlibrary{decorations.markings}
\usetikzlibrary{graphs}
\usetikzlibrary{trees}
\usetikzlibrary{decorations.markings}
\usetikzlibrary{intersections}
\newcommand{\yslant}{-0.26}
\newcommand{\yfshift}{0.05cm}
\usepackage{balance}
\def\TikZColorFill{gray!20}
\def\TikZColorFillDark{gray!80}
\def\TikZColorFillProp{blue!15}
carreira_phd_thesis.tex
+3 -4
View File
@@ -18,9 +18,9 @@
\newcommand{\HERE}{\textcolor{red}{CHECK HERE!}}
%\includeonly{010_Introduction/Introduction}
%\includeonly{021_VideoCoding/VideoCodindgReview}
% \includeonly{021_VideoCoding/VideoCodingReview}
%\includeonly{022_ErrorResilience/RobustTransmissionReview}
\includeonly{031_ErrorResilienceEvaluation/ErrorResilienceEvaluation}
%\includeonly{031_ErrorResilienceEvaluation/ErrorResilienceEvaluation}
%\includeonly{041_ImprovedErrorResilience/ImprovedErrorResilience}
%\includeonly{042_ConcealmentAwareEncoding/ConcealmentAwareEncoding}
%\includeonly{043_EndToEndRobustSystem/EndToEndRobustSystem}
@@ -76,9 +76,8 @@
\include{043_EndToEndRobustSystem/EndToEndRobustSystem}
\include{050_Conclusions/Conclusion}
%
\backmatter
\appendix
\include{060_Appendix/Sequences}
\include{060_Appendix/Papers}
carreira_phd_thesis_abr.sty
+4 -3
View File
@@ -28,6 +28,7 @@
\customnewacronym{rd}{R-D}{\mbox{Rate-Distortion}}
\customnewacronym{nal}{NAL}{Network Abstraction Layer}
\customnewacronym{nalu}{NALU}{Network Abstraction Layer Unit}
\customnewacronym{vcl}{VCL}{Video Coding Layer}
\customnewacronym{poc}{POC}{Picture Order Count}
\customnewacronym{sei}{SEI}{Supplemental Enhancement Information}
@@ -54,7 +55,7 @@
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\customnewacronym{hevc}{HEVC}{High Efficiency Video Coding}
\customnewacronym{vps}{VPS}{Video paramenter set}
\customnewacronym{vps}{VPS}{Video Paramenter Set}
\customnewacronym{sps}{SPS}{Sequence Paramenter Set}
\customnewacronym{pps}{PPS}{Picture Paramenter Set}
\customnewacronym{rpset}{RPSet}{Reference Picture Set}
@@ -63,8 +64,8 @@
\customnewacronym{bla}{BLA}{Broken Link Access}
\customnewacronym{cra}{CRA}{Clean Random Access}
\customnewacronym{idr}{IDR}{Instantaneous Decoding Refresh}
\customnewacronym{radl}{RADL}{Random access decodable leading picture}
\customnewacronym{rasl}{RASL}{Random access skipped leading picture}
\customnewacronym{radl}{RADL}{Random Access Decodable Leading}
\customnewacronym{rasl}{RASL}{Random Access Skipped Leading}
\customnewacronym{tsa}{TSA}{Temporal Sub-layer Access}
\customnewacronym{stsa}{STSA}{Step-wise Temporal Sub-layer Access}