jfmcarreira/calyp
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add modules to calyp

Browse Source
This commit is contained in:
jfmcarreira committed 2021-05-08 22:56:07 +01:00
1 parent 864c3253f1
commit 33bf02ba0f
9 files changed
+1613
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

modules/CMakeLists.txt
+29
View File
@@ -76,6 +76,35 @@ ADD_MODULE_USE_OPENCV(FrameResampling "FrameResampling" VERSION 3.3.0 MODULES co
ADD_MODULE_USE_OPENCV(ThreeSixtySpatialtoTemporal "ThreeSixtySpatialtoTemporal" VERSION 3.1.0 MODULES core)
ADD_MODULE_USE_OPENCV(ThreeSixtyFaceExtraction "ThreeSixtyFaceExtraction" VERSION 3.1.0 MODULES core)
ADD_MODULE_USE_OPENCV(ThreeSixtyFaceConcatenation "ThreeSixtyFaceConcatenation" VERSION 3.1.0 MODULES core)
ADD_MODULE_USE_OPENCV(
ThreeSixtyDownsampling
"ThreeSixtyDownsampling"
VERSION
3.1.0
MODULES
core
imgproc
photo
)
ADD_MODULE_USE_OPENCV(
ThreeSixtyDynamicDownsampling
"ThreeSixtyDynamicDownsampling"
VERSION
3.1.0
MODULES
core
imgproc
)
ADD_MODULE_USE_OPENCV(
ThreeSixtyNxN
"ThreeSixtyNxN"
VERSION
3.1.0
MODULES
core
imgproc
photo
)
IF(EXISTS ${CALYP_EXTRA_MODULES_DIR})
ADD_EXTRA_MODULES(${CALYP_EXTRA_MODULES_DIR})
modules/SaliencyBasedFiltering.cpp
+166
View File
@@ -0,0 +1,166 @@
/* This file is a part of plaYUVer project
* Copyright (C) 2014-2015 by Luis Lucas (luisfrlucas@gmail.com)
* Joao Carreira (jfmcarreira@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file SaliencyBasedFiltering.cpp
*/
#include "SaliencyBasedFiltering.h"
#include <algorithm>
#include <cmath>
#include <opencv2/opencv.hpp>
#define CLAMP_RANGE( X, MIN, MAX ) ( ( X ) < MIN ? MIN : ( ( X ) > MAX ? MAX : X ) )
SaliencyBasedFiltering::SaliencyBasedFiltering()
{
/* Module Definition */
m_iModuleAPI = MODULE_API_2;
m_iModuleType = FRAME_PROCESSING_MODULE;
m_pchModuleCategory = "Quality";
m_pchModuleName = "SaliencyBasedFiltering";
m_pchModuleTooltip = "Image filter based on a map";
m_uiModuleRequirements = MODULE_REQUIRES_NOTHING;
m_uiNumberOfFrames = MODULE_REQUIRES_TWO_FRAMES;
m_pcProcessedFrame = NULL;
}
Bool SaliencyBasedFiltering::create( std::vector<PlaYUVerFrame*> apcFrameList )
{
_BASIC_MODULE_API_2_CHECK_
for( UInt i = 0; i < apcFrameList.size(); i++ )
{
if( !apcFrameList[i]->haveSameFmt( apcFrameList[0], PlaYUVerFrame::MATCH_RESOLUTION ) )
return false;
}
if( apcFrameList[1]->getNumberChannels() != 1 )
{
return false;
}
m_pcProcessedFrame = new PlaYUVerFrame( apcFrameList[0] );
return true;
}
static cv::Mat* getCvFrame( Pel* pInputPel, UInt uiWidth, UInt uiHeight )
{
cv::Mat* cvInput = new cv::Mat( uiHeight, uiWidth, CV_MAKETYPE( CV_8U, 1 ) );
UChar* pCvPel = cvInput->data;
for( UInt i = 0; i < uiWidth * uiHeight; i++ )
{
*pCvPel++ = *pInputPel++;
}
return cvInput;
}
static cv::Mat* filterframe( cv::Mat* cvFrame, UInt kernelSize )
{
cv::Mat* cvFiltered = new cv::Mat( cvFrame->rows, cvFrame->cols, CV_MAKETYPE( CV_8U, 1 ) );
/// Initialize arguments for the filter
cv::Point anchor = cv::Point( -1, -1 );
Double delta = 0;
Int ddepth = -1;
/// Update kernel size for a normalized box filter
cv::Mat kernel = cv::Mat::ones( kernelSize, kernelSize, CV_32F ) / (Double)( kernelSize * kernelSize );
/// Apply filter
cv::filter2D( *cvFrame, *cvFiltered, ddepth, kernel, anchor, delta, cv::BORDER_DEFAULT );
return cvFiltered;
}
static void filterComponent( PlaYUVerFrame* pInput, PlaYUVerFrame* pOutput, PlaYUVerFrame* Map, UInt uiComp )
{
Pel* pMapPel = Map->getPelBufferYUV()[0][0];
Int iMapStep = 1;
Pel* pInputPel = pInput->getPelBufferYUV()[uiComp][0];
Pel* pOutputPel = pOutput->getPelBufferYUV()[uiComp][0];
Int iHeight = pInput->getHeight();
Int iWidth = pInput->getWidth();
if( uiComp > 0 )
{
iHeight = pInput->getChromaHeight();
iWidth = pInput->getChromaWidth();
iMapStep = 2;
}
cv::Mat* cvInput = getCvFrame( pInputPel, iWidth, iHeight );
UInt numberFilters = 1;
UChar* pCvPel[3];
cv::Mat* cvOutput[3];
for( UInt i = 0; i < numberFilters; i++ )
{
cvOutput[i] = filterframe( cvInput, ( 1 << ( 2 * i + 7 ) ) + 1 );
pCvPel[i] = cvOutput[i]->data;
}
Int iCheckNeighbour = 8;
UInt uiRelevance = 0;
for( Int y = 0; y < iHeight; y++ )
{
for( Int x = 0; x < iWidth; x++ )
{
uiRelevance = 0;
// uiRelevance = *pMapPel;
for( Int j = CLAMP_RANGE( y - iCheckNeighbour, 0, iHeight ); j < CLAMP_RANGE( y + iCheckNeighbour + 1, 0, iHeight ); j++ )
{
for( Int i = CLAMP_RANGE( x - iCheckNeighbour, 0, iWidth ); i < CLAMP_RANGE( x + iCheckNeighbour + 1, 0, iWidth ); i++ )
{
uiRelevance += *( pMapPel + ( ( j - y ) * iWidth ) + ( i - x ) );
}
}
if( uiRelevance > 0 )
{
*pOutputPel = *pInputPel;
}
else
{
*pOutputPel = *pCvPel[0];
//*pOutputPel = 0;
}
for( UInt i = 0; i < numberFilters; i++ )
pCvPel[i]++;
pInputPel++;
pOutputPel++;
pMapPel += iMapStep;
}
pMapPel += ( iMapStep - 1 ) * iWidth;
}
}
PlaYUVerFrame* SaliencyBasedFiltering::process( std::vector<PlaYUVerFrame*> apcFrameList )
{
PlaYUVerFrame* pcInputFrame = apcFrameList[0];
for( UInt c = 0; c < pcInputFrame->getNumberChannels(); c++ )
filterComponent( pcInputFrame, m_pcProcessedFrame, apcFrameList[1], c );
return m_pcProcessedFrame;
}
Void SaliencyBasedFiltering::destroy()
{
}
modules/SaliencyBasedFiltering.h
+48
View File
@@ -0,0 +1,48 @@
/* This file is a part of plaYUVer project
* Copyright (C) 2014-2015 by Luis Lucas (luisfrlucas@gmail.com)
* Joao Carreira (jfmcarreira@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file SaliencyBasedFiltering.h
*/
#ifndef __SALIENCYBASEDFILTERING_H__
#define __SALIENCYBASEDFILTERING_H__
#include <cassert>
#include "lib/PlaYUVerModuleIf.h"
class SaliencyBasedFiltering : public PlaYUVerModuleIf
{
REGISTER_CLASS_FACTORY( SaliencyBasedFiltering )
private:
PlaYUVerFrame* m_pcProcessedFrame;
public:
SaliencyBasedFiltering();
virtual ~SaliencyBasedFiltering()
{
}
Bool create( std::vector<PlaYUVerFrame*> apcFrameList );
PlaYUVerFrame* process( std::vector<PlaYUVerFrame*> apcFrameList );
Void destroy();
};
#endif // __SALIENCYBASEDFILTERING_H__
modules/ThreeSixtyDownsampling.cpp
+764
View File
@@ -0,0 +1,764 @@
/* This file is a part of Calyp project
* Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com)
* Luis Lucas (luisfrlucas@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file ThreeSixtyDownsampling.cpp
* \brief Adaptive Latitude Downsampling
*/
#include "ThreeSixtyDownsampling.h"
#include <opencv2/core/core.hpp>
//#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/photo.hpp>
//using cv::InputArray;
//using cv::OutputArray;
using cv::Rect;
using cv::Scalar;
using cv::Size;
ThreeSixtyDownsampling::ThreeSixtyDownsampling()
{
/* Module Definition */
m_iModuleAPI = CLP_MODULE_API_2;
m_iModuleType = CLP_FRAME_PROCESSING_MODULE;
m_pchModuleCategory = "360Video";
m_pchModuleName = "ThreeSixtyDownsampling";
m_pchModuleLongName = "Adaptive Latitude Downsampling";
m_pchModuleTooltip = "Adaptive Latitude Downsampling";
m_uiNumberOfFrames = 1;
m_uiModuleRequirements = CLP_MODULE_REQUIRES_SKIP_WHILE_PLAY |
CLP_MODULE_REQUIRES_OPTIONS |
CLP_MODULES_HAS_INFO |
CLP_MODULE_USES_KEYS;
m_cModuleOptions.addOptions() /**/
( "Downsampling", m_uiDownsampling, "ERP to NxN (0 for up, 1 for down and 2 for recon) [1]" ) /**/
( "y0", m_uiY0, "Vertical anchor point ratio (0-100) [0]" ) /**/
( "x0", m_uiX0, "Horizontal anchor point ratio (0-100) [25%]" ) /**/
// INTER_NEAREST - a nearest-neighbor interpolation
// INTER_LINEAR - a bilinear interpolation (used by default)
// INTER_AREA - resampling using pixel area relation. It may be a preferred
// method for image decimation, as it gives moire’-free results. But when
// the image is zoomed, it is similar to the INTER_NEAREST method.
// INTER_CUBIC - a bicubic interpolation over 4x4 pixel neighborhood
// INTER_LANCZOS4 - a Lanczos interpolation over 8x8 pixel neighborhood
( "Interpolation", m_iInterpolation, "Interpolation method (1-4) [2]" ) /**/
( "IntegerSlope", m_bForceIntSlope, "Force the downsampling slope to be integer (binary) [0]" ) /**/
( "Packing", m_iRearrange, "Rearrange image into a rectangle (0-3) [0]" ) /**/
( "Inpainting", m_iInpaintMethod, "Use inpainting to fill empty pixels (0-2) [0]" ) /**/
( "UpsampWidth", m_iWidth, "Image width after upsampling [-1]" );
m_uiDownsampling = 1;
m_uiY0 = 0;
m_uiX0 = 0;
m_iInterpolation = 2;
m_iInpaintMethod = 0;
m_bForceIntSlope = 0;
m_iRearrange = 0;
m_iWidth = -1;
m_pcOutputFrame = NULL;
m_pcDownsampled = NULL;
m_dPixelRatio = 0;
//#ifdef DEBUG
// m_uiY0 = 25;
// m_uiX0 = 0;
// m_iInterpolation = 4;
// m_bForceIntSlope = 1;
// m_iRearrange = 1;
//#endif
}
ClpString ThreeSixtyDownsampling::moduleInfo()
{
ClpString info( "" );
info += "Interpolation: ";
switch( m_iInterpolation )
{
case cv::INTER_NEAREST:
info += "Nearest";
break;
case cv::INTER_LINEAR:
info += "Linear";
break;
case cv::INTER_AREA:
info += "Area";
break;
case cv::INTER_CUBIC:
info += "Cubic";
break;
case cv::INTER_LANCZOS4:
info += "Lanczos4";
break;
default:
assert( 0 );
}
info += "\ny0 = " + std::to_string( m_uiY0 ) + "\nx0 = " + std::to_string( m_uiX0 );
if( m_bForceIntSlope == 0 )
info += "\nNo integer slope";
else
info += "\nUisng integer slope";
info += "\nPixel ratio = " + std::to_string( m_dPixelRatio );
return info;
}
bool ThreeSixtyDownsampling::create( std::vector<CalypFrame*> apcFrameList )
{
_BASIC_MODULE_API_2_CHECK_
m_uiDownsampling = m_uiDownsampling > 2 ? 1 : m_uiDownsampling;
if( !createDownsamplingMask( apcFrameList[0] ) )
{
return false;
}
switch( m_iInterpolation )
{
case 2:
m_iInterpolation = cv::INTER_LINEAR;
break;
case 3:
m_iInterpolation = cv::INTER_CUBIC;
break;
case 4:
m_iInterpolation = cv::INTER_LANCZOS4;
break;
default:
m_iInterpolation = cv::INTER_NEAREST;
}
switch( m_iInpaintMethod )
{
case 1:
m_iInterpolation = cv::INPAINT_NS;
break;
case 2:
m_iInpaintMethod = cv::INPAINT_TELEA;
break;
}
return true;
}
#define WIDTH_TRIM_START_POINT 0.0
#define WIDTH_FINAL 1.0
bool ThreeSixtyDownsampling::createDownsamplingMask( CalypFrame* pcInputFrame )
{
if( m_iRearrange == 3 )
{
m_uiX0 = 0;
m_uiY0 = std::ceil( 100.0 / 3.0 );
}
if( m_iRearrange == 4 )
{
m_uiX0 = 0;
m_uiY0 = m_uiY0 > 37 ? 40 : 34;
}
m_iHeight = -1;
if( m_uiDownsampling == 0 )
{
if( m_iWidth == -1 )
{
// Guess the output dimensions when upsampling
switch( m_iRearrange )
{
case 1: // Cannot get this - user input needed
break;
case 2:
m_iWidth = pcInputFrame->getWidth();
break;
}
}
m_iHeight = m_iWidth / 2;
}
else
{
m_iWidth = pcInputFrame->getWidth();
m_iHeight = pcInputFrame->getHeight();
}
if( m_iWidth == -1 || m_iHeight == -1 ) // output dimensions are needed
return false;
//m_pcDownsampled = new CalypFrame( m_iWidth, m_uiHeight, pcInputFrame->getPelFormat(), pcInputFrame->getBitsPel() );
m_pcDownsampled = new CalypFrame( m_iWidth, m_iHeight, CalypFrame::findPixelFormat( "GRAY" ), pcInputFrame->getBitsPel() );
m_isOdd = !( m_iHeight % 2 );
for( unsigned ch = 0; ch < m_pcDownsampled->getNumberChannels(); ch++ )
{
Mat* downsamplingMaskPtr = new Mat( m_pcDownsampled->getHeight( ch ), m_pcDownsampled->getWidth( ch ), CV_8UC1, Scalar( 0 ) );
Mat downsamplingMask = *downsamplingMaskPtr;
Mat_<Point> initialReshapePoints( m_pcDownsampled->getHeight( ch ), m_pcDownsampled->getWidth( ch ), Point( -1, -1 ) );
int imH = m_pcDownsampled->getHeight( ch );
int imN = imH / 2;
// Find lines
double theta = atan2( imN, imH - WIDTH_TRIM_START_POINT - WIDTH_FINAL );
double y0 = std::ceil( float( int( m_uiY0 ) * imH ) / 200.0 );
double x0 = std::round( float( int( m_uiX0 ) * imH ) / 100.0 );
double xl = 0;
double slope2 = 0;
if( m_iRearrange == 3 || m_iRearrange == 4 )
{
y0 = std::ceil( y0 / 64.0 ) * 64;
}
Point p0;
Point p1;
Point p2;
if( m_uiY0 > 0 )
{
double x0min = ( 1 + y0 / tan( theta ) - imH + WIDTH_TRIM_START_POINT ) * double( imN - y0 ) / y0 + y0 / tan( theta );
x0 = std::max( x0, x0min );
xl = double( imH - WIDTH_TRIM_START_POINT ) - y0 / tan( theta ) - y0 * y0 / double( imN - y0 ) * ( 1.0 / tan( theta ) - x0 / y0 );
xl = std::floor( xl );
if( m_iRearrange == 3 )
xl = imH / 2;
else if( m_iRearrange == 4 && m_uiY0 == 34 )
xl = imH / 2;
else if( m_iRearrange == 4 && m_uiY0 == 40 )
xl = imH / 3;
slope2 = xl / ( double( imN ) - y0 );
if( m_bForceIntSlope )
slope2 = std::round( slope2 );
// calculate the final coordinates of point corresponding to y0
// p0 corresponds to a coordinate system with reference ate left-most equator
p0 = Point( int( ( double( y0 + 1 ) - double( imH ) / 2.0 ) * slope2 + double( imH - WIDTH_FINAL ) ), y0 );
// give the y value of the triangle which can be used to fill the gap
if( m_iRearrange == 2 )
{
int y1 = double( -p0.x + WIDTH_FINAL ) / slope2 + imN;
p1 = Point( int( ( double( y1 + 1 ) - double( imH ) / 2.0 ) * slope2 + double( imH - WIDTH_FINAL ) ), y1 );
p2 = Point( -1, ( imH - p1.y ) / 2 );
}
}
else
{
slope2 = 2;
p0 = Point( 0, 0 );
p1 = Point( -1, imN );
}
// Reset mask
downsamplingMask = Scalar( 0 );
for( int line = 0; line < imN; line++ )
{
for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 )
{
// calculate destination sizes
int ypos = ( ( imH + ( m_isOdd ? hemisfer : 0 ) ) >> 1 ) + line * hemisfer;
int lineWidth = 0;
if( m_uiY0 == 0 )
{
lineWidth = ( 2 * imH - 2 ) - 4 * line;
}
else
{
double lp = 0;
if( line < y0 )
lp = line * x0 / y0;
else
lp = ( line + 1 - imN ) * slope2 + double( imH - WIDTH_FINAL );
lp = std::round( lp );
assert( imH >= lp );
lineWidth = 2 * imH - 2 * lp;
}
assert( lineWidth > 0 );
int xpos = imH - lineWidth / 2; // N - WIDTH/2
// create mask and init reshape matrix
Mat downsamplingMaskRow = downsamplingMask.row( ypos );
Mat_<Point> reshapePointsRow = initialReshapePoints.row( ypos );
for( int x = xpos; x < xpos + lineWidth; x++ )
{
*( downsamplingMaskRow.data + x ) = ch == 0 ? 255 : 128;
reshapePointsRow( x ) = Point( x, ypos );
}
}
}
if( m_iRearrange > 0 )
{
// initialReshapePoints -> Original coordinates without re-arrange
// intermediateReshapePoints -> Coordinates with re-arrange <- Set this
Mat_<Point> intermediateReshapePoints;
int imH = int( m_pcDownsampled->getHeight( ch ) );
int imN = imH / 2;
// Strecth into wide rectangle
if( m_iRearrange == 1 )
{
intermediateReshapePoints = Mat_<Point>( initialReshapePoints.rows, initialReshapePoints.cols, Point( -1, -1 ) );
Size maskSize( imH - p0.x, imN - p0.y );
for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 )
{
// copy center
Point r0_pt( 0, imN );
Point r0_size( 2 * imH, hemisfer * p0.y );
initialReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ).copyTo( intermediateReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ) );
// main triangle
Mat currMask;
downsamplingMask( Rect( Point( p0.x, 0 ), maskSize ) ).copyTo( currMask );
if( hemisfer == 1 )
cv::flip( currMask, currMask, 0 ); // Filp vertically
initialReshapePoints( Rect( Point( p0.x, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) )
.copyTo( intermediateReshapePoints( Rect( Point( 1, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ), currMask );
// rotated triangle
Mat tmpTriangle;
initialReshapePoints( Rect( Point( imH, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ).copyTo( tmpTriangle );
cv::flip( tmpTriangle, tmpTriangle, 0 );
cv::flip( currMask, currMask, 1 ); // Filp horizontaly
cv::flip( currMask, currMask, 0 );
tmpTriangle.copyTo( intermediateReshapePoints( Rect( Point( 0, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ), currMask );
tmpTriangle.release();
}
// At this point the two triangle were merged into a rectangle
if( ( ( imN - p0.y ) % p0.y ) == 0 ) // check if we can divide the rectangles into equally heighted slices
{
int numberRectSlices = ( imN - p0.y ) / p0.y;
int rectSlicesWidth = 2 * imN - p0.x + 1;
// create extended reshapePoints
Mat_<Point> extendedReshapePoints( 2 * p0.y, intermediateReshapePoints.cols + rectSlicesWidth * numberRectSlices, Point( -1, -1 ) );
intermediateReshapePoints( Rect( Point( 0, imN - p0.y ), Size( intermediateReshapePoints.cols, 2 * p0.y ) ) )
.copyTo( extendedReshapePoints( Rect( Point( 0, 0 ), Size( intermediateReshapePoints.cols, 2 * p0.y ) ) ) );
for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 )
{
Point initSize( rectSlicesWidth, hemisfer * p0.y );
Point finalPoint = Point( 0, p0.y ) + Point( 2 * imH, 0 );
for( int i = 0; i < numberRectSlices; i++ )
{
Point initPoint = Point( 0, imN + hemisfer * ( p0.y + i * p0.y ) );
intermediateReshapePoints( Rect( initPoint, initPoint + initSize ) ).copyTo( extendedReshapePoints( Rect( finalPoint, finalPoint + initSize ) ) );
finalPoint += Point( rectSlicesWidth, 0 );
}
}
intermediateReshapePoints.release();
extendedReshapePoints.copyTo( intermediateReshapePoints );
extendedReshapePoints.release();
}
}
// Rotate triangles into a rectangle and spread the top small triangles
else if( m_iRearrange == 2 )
{
intermediateReshapePoints = Mat_<Point>( initialReshapePoints.rows, initialReshapePoints.cols, Point( -1, -1 ) );
for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 )
{
if( p0.x < imN )
{
for( int side = -1; side < 2; side += 2 )
{
// center
Point r0_pt( imH, imN );
Point r0_size( side * imH, hemisfer * p0.y );
initialReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ).copyTo( intermediateReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ) );
// copy center unmodified
Point r1_pt = r0_pt + Point( 0, hemisfer * p0.y );
Point r1_size( side * imH, hemisfer * ( p1.y - p0.y ) / 2 );
Rect mainRoi( r1_pt, r1_pt + r1_size );
initialReshapePoints( mainRoi ).copyTo( intermediateReshapePoints( mainRoi ), downsamplingMask( mainRoi ) );
// rotated polygon
Point r2_pt = r1_pt + Point( 0, hemisfer * ( p1.y - p0.y ) / 2 );
Mat_<Point> auxPointMap;
Mat auxMask;
initialReshapePoints( Rect( r2_pt, r2_pt + r1_size ) ).copyTo( auxPointMap );
downsamplingMask( Rect( r2_pt, r2_pt + r1_size ) ).copyTo( auxMask );
cv::flip( auxPointMap, auxPointMap, -1 );
cv::flip( auxMask, auxMask, -1 );
auxPointMap.copyTo( intermediateReshapePoints( mainRoi ), auxMask );
auxPointMap.release();
auxMask.release();
// top triangle
// the top triangle is converted to a rectangle and then reshape into and new rectangle with a larger width
Point r34_size( side * ( imH - p1.x + 1 ), hemisfer * ( imN - p1.y ) / 2 );
Point r3_pt( imH, imN + hemisfer * p1.y );
Point r4_pt = r3_pt + Point( 0, r34_size.y );
Point r3l_pt( r3_pt.x, r2_pt.y );
Mat_<Point> topTriangleReshape;
initialReshapePoints( Rect( r3_pt, r3_pt + r34_size ) ).copyTo( topTriangleReshape );
// rotate the top of the top triangle to form a rectangle
initialReshapePoints( Rect( r4_pt, r4_pt + r34_size ) ).copyTo( auxPointMap );
downsamplingMask( Rect( r4_pt, r4_pt + r34_size ) ).copyTo( auxMask );
cv::flip( auxPointMap, auxPointMap, -1 );
cv::flip( auxMask, auxMask, -1 );
//cv::imwrite( "auxMask.png", auxMask );
auxPointMap.copyTo( topTriangleReshape, auxMask );
long shape_area = std::abs( r34_size.x ) * std::abs( r34_size.y );
int reshapedWidth = imH;
while( 1 )
{
if( !( shape_area % reshapedWidth ) )
break;
reshapedWidth--;
}
int reshaped_height = shape_area / reshapedWidth;
if( shape_area % reshapedWidth )
{
return false;
}
topTriangleReshape = topTriangleReshape.reshape( 1, reshaped_height );
topTriangleReshape.copyTo( intermediateReshapePoints( Rect( r3l_pt, r3l_pt + Point( side * topTriangleReshape.cols, hemisfer * topTriangleReshape.rows ) ) ) );
}
}
else
{
return false;
}
}
}
else if( m_iRearrange == 3 )
{
intermediateReshapePoints = Mat_<Point>( 8 * p0.y, initialReshapePoints.cols, Point( -1, -1 ) );
for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 )
{
Point r_pt;
Point r_size;
Point r_pt_final;
Mat currMask;
// copy half of rectangle 6 and 5
r_pt = Point( p0.x, imN );
r_size = Point( 2 * ( imH - p0.x ), hemisfer * p0.y );
r_pt_final = Point( p0.x, 3 * p0.y );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) );
// copy triangle 2 or 4
r_pt = Point( imH, imN + hemisfer * p0.y );
r_size = Point( -( imH - p0.x ), hemisfer * ( imN - p0.y ) );
r_pt_final = Point( imH, 3 * p0.y + hemisfer * p0.y );
downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask );
currMask.release();
// copy triangle 3 or 1
r_pt = Point( imH, imN - hemisfer * p0.y );
r_size = Point( imH - p0.x, -1 * hemisfer * ( imN - p0.y ) );
r_pt_final = Point( p0.x, hemisfer == -1 ? 0 : 6 * p0.y );
downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask );
// copy rectangle 7 and 8
r_pt = Point( imH - hemisfer * ( imH - p0.x ), imN - p0.y );
r_size = Point( -hemisfer * p0.x, p0.y * 2 );
if( hemisfer == -1 )
r_pt_final = Point( imH, 0 );
else
r_pt_final = Point( 2 * imH - p0.x, 4 * p0.y );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) );
}
}
else if( m_iRearrange == 4 )
{
intermediateReshapePoints = Mat_<Point>( 10 * p0.y, initialReshapePoints.cols, Point( -1, -1 ) );
//unsigned hor_pad = 0;
unsigned ver_pad = 0;
if( m_uiY0 == 40 )
{
if( unsigned( 4 * p0.y + ( imH - p0.y ) ) % 2 )
{
ver_pad = 1;
}
}
else
{
if( unsigned( 6 * p0.y + ( imH - p0.y ) ) % 2 )
{
ver_pad = 1;
}
}
for( int side = -1; side < 2; side += 2 )
{
Point r_pt;
Point r_size;
Point r_pt_final;
Mat currMask;
// copy half of rectangle 6 and 5
r_pt = Point( p0.x, imN - p0.y );
r_size = Point( 2 * ( imH - p0.x ), 2 * p0.y );
r_pt_final = Point( p0.x, imN - p0.y );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) );
// copy triangle 2 or 1
r_pt = Point( imH, imN - p0.y );
r_size = Point( side * ( imH - p0.x ), -( imN - p0.y ) );
r_pt_final = Point( imH, imN - p0.y );
downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask );
currMask.release();
// copy triangle 4 or 3
r_pt = Point( imH, imN + p0.y );
r_size = Point( side * ( imH - p0.x ), imN - p0.y );
r_pt_final = Point( imH - side * ( imH - p0.x ), 0 );
downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask );
// copy rectangle 7 and 8
if( m_uiY0 == 40 )
{
r_pt = Point( imH + side * ( imH - p0.x ), imN - p0.y );
r_size = Point( side * p0.x, p0.y * 2 );
r_pt_final = Point( imH - side * ( imH - p0.x ), ver_pad + imN + ( side == -1 ? p0.y : 3 * p0.y ) );
if( side == -1 )
{
Mat flipRect;
cv::flip( initialReshapePoints( Rect( r_pt, r_pt + r_size ) ), flipRect, -1 );
flipRect.copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) );
}
else
{
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) );
}
}
else
{
r_pt = Point( imH + side * ( imH - p0.x ), imN - p0.y );
r_size = Point( side * p0.x, p0.y * 2 );
r_pt_final = Point( imH - side * p0.x, imN + p0.y + ver_pad );
initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) );
}
}
}
initialReshapePoints.release();
intermediateReshapePoints.copyTo( initialReshapePoints ); // copy back to the initial Mat
}
// Create a mask with valid reshape points in order to create the final rectangular frame
Mat reshapePointsMask( initialReshapePoints.rows, initialReshapePoints.cols, CV_8UC1, Scalar( 0 ) );
m_dPixelRatio = 0;
for( int y = 0; y < initialReshapePoints.rows; y++ )
for( int x = 0; x < initialReshapePoints.cols; x++ )
if( initialReshapePoints.at<Point>( y, x ).x != -1 )
{
reshapePointsMask.at<uchar>( y, x ) = 1;
m_dPixelRatio++;
}
m_dPixelRatio = m_dPixelRatio / double( pcInputFrame->getWidth() * pcInputFrame->getHeight() );
Rect reshapeArea = boundingRect( reshapePointsMask );
Mat_<Point>* finalReshapePointsPtr = new Mat_<Point>( reshapeArea.height, reshapeArea.width );
initialReshapePoints( reshapeArea ).copyTo( *finalReshapePointsPtr );
m_cvDownsamplingMask.push_back( downsamplingMaskPtr ); // Save Mat's for each channel
m_cvReshapePoints.push_back( finalReshapePointsPtr );
}
switch( m_uiDownsampling )
{
case 0:
m_pcOutputFrame = new CalypFrame( m_pcDownsampled->getWidth(), m_pcDownsampled->getHeight(), pcInputFrame->getPelFormat(), pcInputFrame->getBitsPel() );
break;
case 1:
m_pcOutputFrame = new CalypFrame( m_cvReshapePoints[0]->cols, m_cvReshapePoints[0]->rows, pcInputFrame->getPelFormat(), pcInputFrame->getBitsPel() );
break;
case 2:
m_pcOutputFrame = new CalypFrame( pcInputFrame );
m_pcOutputFrame->reset();
break;
}
return true;
}
void ThreeSixtyDownsampling::downsamplingOperation( CalypFrame* pcInputFrame )
{
unsigned numBytes = m_pcDownsampled->getBitsPel() > 8 ? 2 : 1;
m_pcDownsampled->reset();
m_pcOutputFrame->reset();
for( unsigned ch = 0; ch < m_pcDownsampled->getNumberChannels(); ch++ )
{
int imH = int( m_pcDownsampled->getHeight( ch ) );
Mat cvInputFrame;
pcInputFrame->toMat( cvInputFrame, true, false, ch );
for( int y = 0; y < imH; y++ )
{
// calculate destination sizes
int lineWidth = cv::countNonZero( m_cvDownsamplingMask[ch]->row( y ) );
int xpos = imH - lineWidth / 2; // (2*HEIGHT - WIDTH)/2
Mat cvDownsample( 1, lineWidth, numBytes > 2 ? CV_16UC1 : CV_8UC1 );
cv::resize( cvInputFrame.row( y ), cvDownsample, cvDownsample.size(), 0, 0, m_iInterpolation );
// copy to output
ClpPel* outpel = &m_pcDownsampled->getPelBufferYUV()[ch][y][xpos];
unsigned char* cv_data = cvDownsample.data;
for( int i = 0; i < lineWidth; i++ )
{
*outpel = 0;
for( unsigned b = 0; b < numBytes; b++ )
{
*outpel = ClpPel( *outpel + ( *cv_data++ << ( 8 * b ) ) );
}
outpel++;
}
}
Mat_<Point> reshapePoints = *( m_cvReshapePoints[ch] );
ClpPel** downSampPelBuff = m_pcDownsampled->getPelBufferYUV()[ch];
ClpPel* pelOutputPtr = m_pcOutputFrame->getPelBufferYUV()[ch][0];
Mat outputMask( m_pcOutputFrame->getHeight( ch ), m_pcOutputFrame->getWidth( ch ), CV_8UC1, Scalar( 1 ) );
for( unsigned y = 0; y < m_pcOutputFrame->getHeight( ch ); y++ )
{
for( unsigned x = 0; x < m_pcOutputFrame->getWidth( ch ); x++ )
{
Point pt = reshapePoints.at<Point>( y, x );
if( pt.x > -1 )
{
*pelOutputPtr = downSampPelBuff[pt.y][pt.x];
outputMask.at<unsigned char>( y, x ) = 0;
}
pelOutputPtr++;
}
}
if( m_iInpaintMethod )
{
Mat outputFrameUnfilled, outputFrame;
m_pcOutputFrame->toMat( outputFrameUnfilled, true, false, ch );
cv::inpaint( outputFrameUnfilled, outputMask, outputFrame, 16, m_iInpaintMethod );
m_pcOutputFrame->fromMat( outputFrame, ch );
}
}
}
void ThreeSixtyDownsampling::upsamplingOperation( CalypFrame* pcInputFrame )
{
unsigned numBytes = m_pcDownsampled->getBitsPel() > 8 ? 2 : 1;
m_pcDownsampled->reset();
m_pcOutputFrame->reset();
for( unsigned ch = 0; ch < m_pcDownsampled->getNumberChannels(); ch++ )
{
Mat_<Point> reshapePoints = *( m_cvReshapePoints[ch] );
ClpPel** downSampPelBuff = m_pcDownsampled->getPelBufferYUV()[ch];
ClpPel* pelInputPtr = pcInputFrame->getPelBufferYUV()[ch][0];
for( unsigned y = 0; y < pcInputFrame->getHeight( ch ); y++ )
{
for( unsigned x = 0; x < pcInputFrame->getWidth( ch ); x++ )
{
Point pt = reshapePoints.at<Point>( y, x );
if( pt.x > -1 )
downSampPelBuff[pt.y][pt.x] = *( pelInputPtr );
pelInputPtr++;
}
}
Mat cvDownsampled;
m_pcDownsampled->toMat( cvDownsampled, true, false, ch );
int imH = m_pcOutputFrame->getHeight( ch );
Mat cvUpsample( 1, 2 * imH, numBytes > 2 ? CV_16UC1 : CV_8UC1 );
for( int y = 0; y < imH; y++ )
{
unsigned lineWidth = cv::countNonZero( m_cvDownsamplingMask[ch]->row( y ) );
unsigned int xpos = imH - lineWidth / 2; // N - WIDTH/2
// resize
cv::resize( cvDownsampled.row( y ).colRange( xpos, xpos + lineWidth ), cvUpsample, cvUpsample.size(), 0, 0, m_iInterpolation );
// copy to output
ClpPel* outpel = &m_pcOutputFrame->getPelBufferYUV()[ch][y][0];
unsigned char* cv_data = cvUpsample.data;
for( int i = 0; i < 2 * imH; i++ )
{
*outpel = 0;
for( unsigned b = 0; b < numBytes; b++ )
{
*outpel = ClpPel( *outpel + ( *cv_data++ << ( 8 * b ) ) );
}
outpel++;
}
}
}
//m_pcOutputFrame->copyFrom( m_pcDownsampled );
}
CalypFrame* ThreeSixtyDownsampling::process( std::vector<CalypFrame*> apcFrameList )
{
m_pcDownsampled->reset();
if( m_uiDownsampling == 1 || m_uiDownsampling == 2 )
{
downsamplingOperation( apcFrameList[0] );
}
if( m_uiDownsampling == 0 || m_uiDownsampling == 2 )
{
upsamplingOperation( m_uiDownsampling == 2 ? m_pcOutputFrame : apcFrameList[0] );
}
return m_pcOutputFrame;
}
bool ThreeSixtyDownsampling::keyPressed( enum Module_Key_Supported value )
{
bool bRet = false;
if( value == MODULE_KEY_LEFT )
{
m_uiX0 = m_uiX0 == 0 ? 0 : m_uiX0 - 1;
}
else if( value == MODULE_KEY_RIGHT )
{
m_uiX0 = m_uiX0 == 60 ? 60 : m_uiX0 + 1;
}
else if( value == MODULE_KEY_DOWN )
{
m_uiY0 = m_uiY0 == 0 ? 0 : m_uiY0 - 1;
}
else if( value == MODULE_KEY_UP )
{
m_uiY0 = m_uiY0 == 60 ? 60 : m_uiY0 + 1;
}
if( bRet )
{
bRet = createDownsamplingMask( m_pcDownsampled );
}
return bRet;
}
void ThreeSixtyDownsampling::destroy()
{
if( m_pcDownsampled )
delete m_pcDownsampled;
m_pcDownsampled = NULL;
if( m_pcOutputFrame )
delete m_pcOutputFrame;
m_pcOutputFrame = NULL;
}
modules/ThreeSixtyDownsampling.h
+75
View File
@@ -0,0 +1,75 @@
/* This file is a part of Calyp project
* Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com)
* Luis Lucas (luisfrlucas@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file ThreeSixtyDownsampling.h
* \brief Adaptive Latitude Downsampling
*/
#ifndef __THREESIXTYDOWNSAMPLING_H__
#define __THREESIXTYDOWNSAMPLING_H__
// CalypLib
#include "lib/CalypModuleIf.h"
#include "opencv2/core/mat.hpp"
#include "opencv2/core/types.hpp"
using cv::Mat;
using cv::Mat_;
using cv::Point;
class ThreeSixtyDownsampling : public CalypModuleIf
{
REGISTER_CLASS_FACTORY( ThreeSixtyDownsampling )
private:
unsigned m_uiDownsampling;
unsigned m_uiX0;
unsigned m_uiY0;
int m_iInterpolation;
int m_iInpaintMethod;
unsigned m_bForceIntSlope;
unsigned m_iRearrange;
bool m_isOdd;
int m_iWidth;
int m_iHeight;
double m_dPixelRatio;
CalypFrame* m_pcDownsampled;
// Values one per channel
std::vector<Mat*> m_cvDownsamplingMask;
std::vector<Mat_<Point>*> m_cvReshapePoints;
bool createDownsamplingMask( CalypFrame* pcFrame );
void downsamplingOperation( CalypFrame* pcInputFrame );
void upsamplingOperation( CalypFrame* pcInputFrame );
public:
ThreeSixtyDownsampling();
virtual ~ThreeSixtyDownsampling() {}
bool create( std::vector<CalypFrame*> apcFrameList );
CalypFrame* process( std::vector<CalypFrame*> apcFrameList );
ClpString moduleInfo();
bool keyPressed( enum Module_Key_Supported value );
void destroy();
};
#endif // __THREESIXTYDOWNSAMPLING_H__
modules/ThreeSixtyDynamicDownsampling.cpp
+155
View File
@@ -0,0 +1,155 @@
/* This file is a part of Calyp project
* Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com)
* Luis Lucas (luisfrlucas@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file ThreeSixtyDynamicDownsampling.cpp
*/
#include "ThreeSixtyDynamicDownsampling.h"
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc.hpp>
using cv::InputArray;
using cv::Mat;
using cv::OutputArray;
using cv::Point;
using cv::Rect;
using cv::Size;
ThreeSixtyDynamicDownsampling::ThreeSixtyDynamicDownsampling()
{
/* Module Definition */
m_iModuleAPI = CLP_MODULE_API_2;
m_iModuleType = CLP_FRAME_PROCESSING_MODULE;
m_pchModuleCategory = "360Video";
m_pchModuleLongName = "Dynamic Downsampling";
m_pchModuleName = "ThreeSixtyDynamicDownsampling";
m_pchModuleTooltip = "Dynamic downsampling of 360 videos";
m_uiNumberOfFrames = 1;
m_uiModuleRequirements = CLP_MODULE_REQUIRES_SKIP_WHILE_PLAY | CLP_MODULE_REQUIRES_OPTIONS;
m_cModuleOptions.addOptions() /**/
( "CodingFaceWidth", m_uiFaceWidth, "Coding Face Width [same as input]" ) /**/
( "CodingFaceHeight", m_uiFaceHeight, "Coding Face Height [same as input]" ) /**/
( "SourceFPStructure", m_arrayInputDefinitionString, "Input frame packing" ) /**/
( "CodingFPStructure", m_arrayOutputDefinitionString, "Coding frame packing" ); /**/
m_arrayInputDefinitionString = "6 2 3 0 0 0 0 100 1 0 1 0 100 2 0 2 0 100 0 1 3 0 100 1 1 4 0 100 2 1 5 0 100";
m_arrayOutputDefinitionString = "6 3 3 0 0 0 0 100 1 0 1 0 200 0 1 2 0 100 0 2 3 0 100 1 2 4 0 100 2 2 5 0 100 ";
m_uiFaceWidth = 0;
m_uiFaceHeight = 0;
}
void processFPSString( const ClpString& fps, ClpVideoFPStruct* pcFPSStruct )
{
std::istringstream ss( fps );
ss >> pcFPSStruct->numFaces;
ss >> pcFPSStruct->rows;
ss >> pcFPSStruct->cols;
for( int i = 0; i < pcFPSStruct->numFaces; i++ )
{
ss >> pcFPSStruct->faces[i].x;
ss >> pcFPSStruct->faces[i].y;
ss >> pcFPSStruct->faces[i].id;
ss >> pcFPSStruct->faces[i].rot;
ss >> pcFPSStruct->faces[i].ratio;
}
}
bool ThreeSixtyDynamicDownsampling::create( std::vector<CalypFrame*> apcFrameList )
{
_BASIC_MODULE_API_2_CHECK_
processFPSString( m_arrayInputDefinitionString, &m_cInputFPSStruct );
processFPSString( m_arrayOutputDefinitionString, &m_cOutputFPSStruct );
if( !m_uiFaceWidth && !m_uiFaceHeight )
{
m_uiFaceWidth = apcFrameList[0]->getWidth() / m_cInputFPSStruct.cols;
m_uiFaceHeight = apcFrameList[0]->getHeight() / m_cInputFPSStruct.rows;
}
else if( !m_uiFaceHeight )
{
m_uiFaceHeight = m_uiFaceWidth;
}
m_pcOutputFrame = new CalypFrame( m_uiFaceWidth * m_cOutputFPSStruct.cols,
m_uiFaceHeight * m_cOutputFPSStruct.rows,
apcFrameList[0]->getPelFormat(), apcFrameList[0]->getBitsPel() );
return true;
}
CalypFrame* ThreeSixtyDynamicDownsampling::process( std::vector<CalypFrame*> apcFrameList )
{
m_pcOutputFrame->reset();
for( unsigned ch = 0; ch < apcFrameList[0]->getNumberChannels(); ch++ )
{
Mat inputFrame;
apcFrameList[0]->toMat( inputFrame, true, false, ch );
unsigned inGridW = apcFrameList[0]->getWidth( ch ) / m_cInputFPSStruct.cols;
unsigned inGridH = apcFrameList[0]->getHeight( ch ) / m_cInputFPSStruct.rows;
unsigned outGridW = CHROMASHIFT( m_uiFaceWidth, ch > 0 ? m_pcOutputFrame->getChromaWidthRatio() : 0 );
unsigned outGridH = CHROMASHIFT( m_uiFaceHeight, ch > 0 ? m_pcOutputFrame->getChromaHeightRatio() : 0 );
Mat outputFrame;
m_pcOutputFrame->toMat( outputFrame, true, false, ch );
for( int face = 0; face < m_cOutputFPSStruct.numFaces; face++ )
{
ClpFaceProperty outFp = m_cOutputFPSStruct.faces[face];
Mat outputFace( outGridW * outFp.ratio / 100, outGridH * outFp.ratio / 100, CV_8UC1 );
Rect outRect( Point( outFp.x * outGridW, outFp.y * outGridH ), outputFace.size() );
int inFace = face;
for( int i = 0; i < m_cInputFPSStruct.numFaces; i++ )
{
if( m_cInputFPSStruct.faces[i].id == outFp.id )
{
inFace = i;
break;
}
}
ClpFaceProperty inFp = m_cInputFPSStruct.faces[inFace];
Mat inputFace( inGridW * inFp.ratio / 100, inGridH * inFp.ratio / 100, CV_8UC1 );
Rect inRect( Point( inFp.x * inGridW, inFp.y * inGridH ), inputFace.size() );
inputFrame( inRect ).copyTo( inputFace );
cv::resize( inputFace, outputFace, outputFace.size(), 0, 0, cv::INTER_LANCZOS4 );
outputFace.copyTo( outputFrame( outRect ) );
}
m_pcOutputFrame->fromMat( outputFrame, ch );
}
return m_pcOutputFrame;
}
void ThreeSixtyDynamicDownsampling::destroy()
{
if( m_pcOutputFrame )
delete m_pcOutputFrame;
m_pcOutputFrame = NULL;
}
modules/ThreeSixtyDynamicDownsampling.h
+66
View File
@@ -0,0 +1,66 @@
/* This file is a part of Calyp project
* Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com)
* Luis Lucas (luisfrlucas@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file ThreeSixtyDynamicDownsampling.h
*/
#ifndef __ThreeSixtyDynamicDownsampling_H__
#define __ThreeSixtyDynamicDownsampling_H__
// CalypLib
#include "lib/CalypModuleIf.h"
struct ClpFaceProperty
{
unsigned id;
unsigned x;
unsigned y;
int rot;
unsigned ratio;
};
struct ClpVideoFPStruct
{
unsigned rows;
unsigned cols;
int numFaces;
ClpFaceProperty faces[60];
};
class ThreeSixtyDynamicDownsampling : public CalypModuleIf
{
REGISTER_CLASS_FACTORY( ThreeSixtyDynamicDownsampling )
private:
ClpString m_arrayInputDefinitionString;
ClpString m_arrayOutputDefinitionString;
ClpVideoFPStruct m_cInputFPSStruct;
ClpVideoFPStruct m_cOutputFPSStruct;
unsigned m_uiFaceWidth;
unsigned m_uiFaceHeight;
public:
ThreeSixtyDynamicDownsampling();
virtual ~ThreeSixtyDynamicDownsampling() {}
bool create( std::vector<CalypFrame*> apcFrameList );
CalypFrame* process( std::vector<CalypFrame*> apcFrameList );
void destroy();
};
#endif // __ThreeSixtyDynamicDownsampling_H__
modules/ThreeSixtyNxN.cpp
+257
View File
@@ -0,0 +1,257 @@
/* This file is a part of Calyp project
* Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com)
* Luis Lucas (luisfrlucas@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file ThreeSixtyNxN.cpp
* \brief Downsampling ERP to NxN
*/
#include "ThreeSixtyNxN.h"
#include <opencv2/core/core.hpp>
#include "lib/LibMemory.h"
//#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/photo.hpp>
//using cv::InputArray;
//using cv::OutputArray;
using cv::Rect;
using cv::Scalar;
using cv::Size;
ThreeSixtyNxN::ThreeSixtyNxN()
{
/* Module Definition */
m_iModuleAPI = CLP_MODULE_API_2;
m_iModuleType = CLP_FRAME_PROCESSING_MODULE;
m_pchModuleCategory = "360Video";
m_pchModuleName = "ThreeSixtyNxN";
m_pchModuleLongName = "ERP to NxN";
m_pchModuleTooltip = "Downsampling ERP to NxN";
m_uiNumberOfFrames = 1;
m_uiModuleRequirements = CLP_MODULE_REQUIRES_SKIP_WHILE_PLAY |
CLP_MODULE_REQUIRES_OPTIONS;
m_cModuleOptions.addOptions() /**/
( "Downsampling", m_uiDownsampling, "ERP to NxN (0 for up, 1 for down and 2 for recon) [1]" ) /**/
;
m_uiDownsampling = 1;
m_pcOutputFrame = NULL;
}
bool ThreeSixtyNxN::create( std::vector<CalypFrame*> apcFrameList )
{
_BASIC_MODULE_API_2_CHECK_
m_uiDownsampling = m_uiDownsampling > 2 ? 1 : m_uiDownsampling;
if( m_uiDownsampling == 0 )
{
m_pcOutputFrame = new CalypFrame( 2 * apcFrameList[0]->getHeight(), apcFrameList[0]->getHeight(), apcFrameList[0]->getPelFormat(), apcFrameList[0]->getBitsPel() );
}
else
{
m_pcOutputFrame = new CalypFrame( apcFrameList[0]->getHeight(), apcFrameList[0]->getHeight(), apcFrameList[0]->getPelFormat(), apcFrameList[0]->getBitsPel() );
}
return true;
}
#define WIDTH_TRIM_START_POINT 0.0
#define WIDTH_FINAL 1.0
//#define CLAMP8BITS( in ) ( uint8_t )( ( in > 255 ? 255 : in < 0 ? 0 : in ) + 0.5 )
#define CLAMP_RANGE( X, MIN, MAX ) ( ( X ) < MIN ? MIN : ( ( X ) > MAX ? MAX : X ) )
#define CIRCULAR( a, b ) ( ( ( a < 0 ? b : 0 ) + ( a % b ) ) % b )
#define PI 3.14159265358979323846264338327950288419716939937510
#define s60 0.8660254037844386467637231707529361834714026269051
#define s45 0.7071067811865475244008443621048490392848359376884
//interpolação Lanczos3 360
static inline ClpPel interpol_lanczos3( ClpPel* image, double x, int N, int max_value )
{
static const double cs[][2] = {
{ 1, 0 },
{ -0.5, -s60 },
{ -0.5, s60 },
{ 1, 0 },
{ -0.5, -s60 },
{ -0.5, s60 },
};
double sum_coeffs = 0, sum = 0, coeffs[6];
int x_int = floor( x );
ClpPel xx[6];
x -= x_int;
bool n0 = x > 0;
xx[2] = *( image + CIRCULAR( x_int, N ) );
if( !n0 )
return xx[2];
xx[3] = *( image + CIRCULAR( x_int + n0, N ) );
xx[1] = *( image + CIRCULAR( x_int - 1, N ) );
xx[4] = *( image + CIRCULAR( x_int + n0 + 1, N ) );
xx[0] = *( image + CIRCULAR( x_int - 2, N ) );
xx[5] = *( image + CIRCULAR( x_int + n0 + 2, N ) );
double aux = -2 - x, s_aux = sin( aux * PI / 3 ), c_aux = cos( aux * PI / 3 );
for( int i = 0; i < 6; i++ )
sum_coeffs += coeffs[i] = ( cs[i][0] * s_aux + cs[i][1] * c_aux ) / ( i - 2 - x ) / ( i - 2 - x );
for( int i = 0; i < 6; i++ )
sum += xx[i] * coeffs[i] / sum_coeffs;
return CLAMP_RANGE( sum, 0, max_value );
}
//interpolação cúbica 360
static inline ClpPel interpol_cub( ClpPel* image, double x, int N, int max_value )
{
int x_int = floor( x );
x -= x_int;
bool n0 = x > 0;
ClpPel x1 = *( image + CIRCULAR( x_int, N ) );
if( !n0 )
return x1;
ClpPel x2 = *( image + CIRCULAR( x_int + n0, N ) );
ClpPel x0 = *( image + CIRCULAR( x_int - 1, N ) );
ClpPel x3 = *( image + CIRCULAR( x_int + n0 + 1, N ) );
ClpPel out = CLAMP_RANGE( x1 - x * ( x0 - x2 + x * ( 2 * x1 - 2 * x0 - x2 + x3 + x * ( x0 - x1 + x2 - x3 ) ) ), 0, max_value ); //cubic
return out;
}
void ThreeSixtyNxN::downsamplingOperation( const CalypFrame* pcInputFrame )
{
for( unsigned ch = 0; ch < m_pcOutputFrame->getNumberChannels(); ch++ )
{
int max_value = ( 1 << m_pcOutputFrame->getBitsPel() ) - 1;
int N = pcInputFrame->getHeight( ch );
ClpPel* in = &pcInputFrame->getPelBufferYUV()[ch][0][0];
ClpPel* out = &m_pcOutputFrame->getPelBufferYUV()[ch][0][0];
unsigned long nn = N * N;
int n_2 = N / 2;
int n1 = N - 1;
double dist_y;
double dist_x;
double novo_x;
double novo_y;
double tx_x;
ClpPel* tx_y;
ClpPel* rx;
bool valida;
for( int y = 0; y < N; y++ )
{
for( int x = 0; x < N; x++ )
{
dist_y = ( abs( y * 2 - n1 ) - 1 ) / 2;
dist_x = ( abs( x * 2 - n1 ) - 1 ) / 2;
valida = ( n_2 - dist_x - dist_y - 1 ) >= 0;
novo_y = ( valida ? y : ( y < n_2 ? dist_x : n1 - dist_x ) ) * 2;
novo_x = ( valida ? x : ( x < n_2 ? -dist_y + x * 2 : n1 + dist_y - ( n1 - x ) * 2 ) ) - n_2 + 1;
tx_x = ( N / ( N - fabs( novo_y - n1 ) ) ) * novo_x + n1;
tx_y = (int)novo_y * N + in;
rx = x + y * N + out;
*( rx ) = interpol_lanczos3( tx_y, tx_x, N * 2, max_value );
//*( rx ) = interpol_cub( tx_y, tx_x, N * 2, max_value );
}
}
}
}
void ThreeSixtyNxN::upsamplingOperation( const CalypFrame* pcInputFrame )
{
for( unsigned ch = 0; ch < m_pcOutputFrame->getNumberChannels(); ch++ )
{
int max_value = ( 1 << m_pcOutputFrame->getBitsPel() ) - 1;
int N = pcInputFrame->getHeight( ch );
ClpPel* in = &pcInputFrame->getPelBufferYUV()[ch][0][0];
ClpPel* out = &m_pcOutputFrame->getPelBufferYUV()[ch][0][0];
int x;
int y;
int n2 = 2 * N;
double dist_y;
double dist_x;
double dist_Lj;
double novo_x;
double novo_y;
double Lj;
double Dj;
double tx;
ClpPel* rx;
ClpPel* linhaY;
getMem1D( &linhaY, n2 );
for( y = 0; y < N; y++ )
{
Lj = ( N - abs( 2 * y - N + 1 ) ) * 2;
Dj = n2 / Lj;
for( x = 0; x < Lj; x++ )
{
dist_x = x - Lj / 2;
dist_y = ( N - abs( 2 * y - N + 1 ) - 1 ) / 2;
dist_Lj = ( Lj - abs( 2 * x - Lj + 1 ) - 1 ) / 2;
novo_x = ( dist_x > dist_y ? dist_y : ( dist_x + 1 < -dist_y ? -dist_y - 1 : dist_x ) ) + N / 2;
novo_y = abs( dist_x ) > dist_y ? ( y < N / 2 ? dist_Lj : N - dist_Lj - 1 ) : y;
*( linhaY + x ) = *( in + (int)novo_x + (int)novo_y * N );
}
for( x = 0; x < n2; x++ )
{
tx = ( x + 1 ) / Dj - 1;
rx = x + y * n2 + out;
*( rx ) = interpol_lanczos3( linhaY, tx, Lj, max_value );
}
}
freeMem1D( linhaY );
}
}
CalypFrame* ThreeSixtyNxN::process( std::vector<CalypFrame*> apcFrameList )
{
m_pcOutputFrame->reset();
if( m_uiDownsampling == 1 || m_uiDownsampling == 2 )
{
downsamplingOperation( apcFrameList[0] );
}
if( m_uiDownsampling == 0 || m_uiDownsampling == 2 )
{
upsamplingOperation( m_uiDownsampling == 2 ? m_pcOutputFrame : apcFrameList[0] );
}
return m_pcOutputFrame;
}
void ThreeSixtyNxN::destroy()
{
if( m_pcOutputFrame )
delete m_pcOutputFrame;
m_pcOutputFrame = NULL;
}
modules/ThreeSixtyNxN.h
+53
View File
@@ -0,0 +1,53 @@
/* This file is a part of Calyp project
* Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com)
* Luis Lucas (luisfrlucas@gmail.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
/**
* \file ThreeSixtyNxN.h
* \brief Downsampling ERP to NxN
*/
#ifndef __THREESIXTYNXN_H__
#define __THREESIXTYNXN_H__
// CalypLib
#include "lib/CalypModuleIf.h"
#include "opencv2/core/mat.hpp"
#include "opencv2/core/types.hpp"
using cv::Mat;
using cv::Mat_;
using cv::Point;
class ThreeSixtyNxN : public CalypModuleIf
{
REGISTER_CLASS_FACTORY( ThreeSixtyNxN )
private:
unsigned m_uiDownsampling;
void downsamplingOperation( const CalypFrame* pcInputFrame );
void upsamplingOperation( const CalypFrame* pcInputFrame );
public:
ThreeSixtyNxN();
virtual ~ThreeSixtyNxN() {}
bool create( std::vector<CalypFrame*> apcFrameList );
CalypFrame* process( std::vector<CalypFrame*> apcFrameList );
void destroy();
};
#endif // __THREESIXTYNXN_H__