From 3c85c967699702dbafb4169606b6b9a8d2fb85cd Mon Sep 17 00:00:00 2001 From: Joao Carreira Date: Sat, 8 May 2021 22:56:07 +0100 Subject: [PATCH] Add modules to calyp --- modules/CMakeLists.txt | 29 + modules/SaliencyBasedFiltering.cpp | 166 +++++ modules/SaliencyBasedFiltering.h | 48 ++ modules/ThreeSixtyDownsampling.cpp | 764 ++++++++++++++++++++++ modules/ThreeSixtyDownsampling.h | 75 +++ modules/ThreeSixtyDynamicDownsampling.cpp | 155 +++++ modules/ThreeSixtyDynamicDownsampling.h | 66 ++ modules/ThreeSixtyNxN.cpp | 257 ++++++++ modules/ThreeSixtyNxN.h | 53 ++ 9 files changed, 1613 insertions(+) create mode 100644 modules/SaliencyBasedFiltering.cpp create mode 100644 modules/SaliencyBasedFiltering.h create mode 100644 modules/ThreeSixtyDownsampling.cpp create mode 100644 modules/ThreeSixtyDownsampling.h create mode 100644 modules/ThreeSixtyDynamicDownsampling.cpp create mode 100644 modules/ThreeSixtyDynamicDownsampling.h create mode 100644 modules/ThreeSixtyNxN.cpp create mode 100644 modules/ThreeSixtyNxN.h diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index c8b112a..eefd381 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -76,6 +76,35 @@ ADD_MODULE_USE_OPENCV(FrameResampling "FrameResampling" VERSION 3.3.0 MODULES co ADD_MODULE_USE_OPENCV(ThreeSixtySpatialtoTemporal "ThreeSixtySpatialtoTemporal" VERSION 3.1.0 MODULES core) ADD_MODULE_USE_OPENCV(ThreeSixtyFaceExtraction "ThreeSixtyFaceExtraction" VERSION 3.1.0 MODULES core) ADD_MODULE_USE_OPENCV(ThreeSixtyFaceConcatenation "ThreeSixtyFaceConcatenation" VERSION 3.1.0 MODULES core) +ADD_MODULE_USE_OPENCV( + ThreeSixtyDownsampling + "ThreeSixtyDownsampling" + VERSION + 3.1.0 + MODULES + core + imgproc + photo +) +ADD_MODULE_USE_OPENCV( + ThreeSixtyDynamicDownsampling + "ThreeSixtyDynamicDownsampling" + VERSION + 3.1.0 + MODULES + core + imgproc +) +ADD_MODULE_USE_OPENCV( + ThreeSixtyNxN + "ThreeSixtyNxN" + VERSION + 3.1.0 + MODULES + core + imgproc + photo +) IF(EXISTS ${CALYP_EXTRA_MODULES_DIR}) ADD_EXTRA_MODULES(${CALYP_EXTRA_MODULES_DIR}) diff --git a/modules/SaliencyBasedFiltering.cpp b/modules/SaliencyBasedFiltering.cpp new file mode 100644 index 0000000..8879389 --- /dev/null +++ b/modules/SaliencyBasedFiltering.cpp @@ -0,0 +1,166 @@ +/* This file is a part of plaYUVer project + * Copyright (C) 2014-2015 by Luis Lucas (luisfrlucas@gmail.com) + * Joao Carreira (jfmcarreira@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file SaliencyBasedFiltering.cpp + */ + +#include "SaliencyBasedFiltering.h" + +#include +#include +#include + +#define CLAMP_RANGE( X, MIN, MAX ) ( ( X ) < MIN ? MIN : ( ( X ) > MAX ? MAX : X ) ) + +SaliencyBasedFiltering::SaliencyBasedFiltering() +{ + /* Module Definition */ + m_iModuleAPI = MODULE_API_2; + m_iModuleType = FRAME_PROCESSING_MODULE; + m_pchModuleCategory = "Quality"; + m_pchModuleName = "SaliencyBasedFiltering"; + m_pchModuleTooltip = "Image filter based on a map"; + m_uiModuleRequirements = MODULE_REQUIRES_NOTHING; + m_uiNumberOfFrames = MODULE_REQUIRES_TWO_FRAMES; + + m_pcProcessedFrame = NULL; +} + +Bool SaliencyBasedFiltering::create( std::vector apcFrameList ) +{ + _BASIC_MODULE_API_2_CHECK_ + + for( UInt i = 0; i < apcFrameList.size(); i++ ) + { + if( !apcFrameList[i]->haveSameFmt( apcFrameList[0], PlaYUVerFrame::MATCH_RESOLUTION ) ) + return false; + } + if( apcFrameList[1]->getNumberChannels() != 1 ) + { + return false; + } + m_pcProcessedFrame = new PlaYUVerFrame( apcFrameList[0] ); + return true; +} + +static cv::Mat* getCvFrame( Pel* pInputPel, UInt uiWidth, UInt uiHeight ) +{ + cv::Mat* cvInput = new cv::Mat( uiHeight, uiWidth, CV_MAKETYPE( CV_8U, 1 ) ); + UChar* pCvPel = cvInput->data; + for( UInt i = 0; i < uiWidth * uiHeight; i++ ) + { + *pCvPel++ = *pInputPel++; + } + return cvInput; +} + +static cv::Mat* filterframe( cv::Mat* cvFrame, UInt kernelSize ) +{ + cv::Mat* cvFiltered = new cv::Mat( cvFrame->rows, cvFrame->cols, CV_MAKETYPE( CV_8U, 1 ) ); + /// Initialize arguments for the filter + cv::Point anchor = cv::Point( -1, -1 ); + Double delta = 0; + Int ddepth = -1; + + /// Update kernel size for a normalized box filter + cv::Mat kernel = cv::Mat::ones( kernelSize, kernelSize, CV_32F ) / (Double)( kernelSize * kernelSize ); + + /// Apply filter + cv::filter2D( *cvFrame, *cvFiltered, ddepth, kernel, anchor, delta, cv::BORDER_DEFAULT ); + + return cvFiltered; +} + +static void filterComponent( PlaYUVerFrame* pInput, PlaYUVerFrame* pOutput, PlaYUVerFrame* Map, UInt uiComp ) +{ + Pel* pMapPel = Map->getPelBufferYUV()[0][0]; + Int iMapStep = 1; + Pel* pInputPel = pInput->getPelBufferYUV()[uiComp][0]; + Pel* pOutputPel = pOutput->getPelBufferYUV()[uiComp][0]; + Int iHeight = pInput->getHeight(); + Int iWidth = pInput->getWidth(); + + if( uiComp > 0 ) + { + iHeight = pInput->getChromaHeight(); + iWidth = pInput->getChromaWidth(); + iMapStep = 2; + } + + cv::Mat* cvInput = getCvFrame( pInputPel, iWidth, iHeight ); + + UInt numberFilters = 1; + UChar* pCvPel[3]; + cv::Mat* cvOutput[3]; + for( UInt i = 0; i < numberFilters; i++ ) + { + cvOutput[i] = filterframe( cvInput, ( 1 << ( 2 * i + 7 ) ) + 1 ); + pCvPel[i] = cvOutput[i]->data; + } + + Int iCheckNeighbour = 8; + UInt uiRelevance = 0; + for( Int y = 0; y < iHeight; y++ ) + { + for( Int x = 0; x < iWidth; x++ ) + { + uiRelevance = 0; + // uiRelevance = *pMapPel; + for( Int j = CLAMP_RANGE( y - iCheckNeighbour, 0, iHeight ); j < CLAMP_RANGE( y + iCheckNeighbour + 1, 0, iHeight ); j++ ) + { + for( Int i = CLAMP_RANGE( x - iCheckNeighbour, 0, iWidth ); i < CLAMP_RANGE( x + iCheckNeighbour + 1, 0, iWidth ); i++ ) + { + uiRelevance += *( pMapPel + ( ( j - y ) * iWidth ) + ( i - x ) ); + } + } + + if( uiRelevance > 0 ) + { + *pOutputPel = *pInputPel; + } + else + { + *pOutputPel = *pCvPel[0]; + //*pOutputPel = 0; + } + + for( UInt i = 0; i < numberFilters; i++ ) + pCvPel[i]++; + pInputPel++; + pOutputPel++; + pMapPel += iMapStep; + } + pMapPel += ( iMapStep - 1 ) * iWidth; + } +} + +PlaYUVerFrame* SaliencyBasedFiltering::process( std::vector apcFrameList ) +{ + PlaYUVerFrame* pcInputFrame = apcFrameList[0]; + + for( UInt c = 0; c < pcInputFrame->getNumberChannels(); c++ ) + filterComponent( pcInputFrame, m_pcProcessedFrame, apcFrameList[1], c ); + + return m_pcProcessedFrame; +} + +Void SaliencyBasedFiltering::destroy() +{ +} diff --git a/modules/SaliencyBasedFiltering.h b/modules/SaliencyBasedFiltering.h new file mode 100644 index 0000000..e2f2964 --- /dev/null +++ b/modules/SaliencyBasedFiltering.h @@ -0,0 +1,48 @@ +/* This file is a part of plaYUVer project + * Copyright (C) 2014-2015 by Luis Lucas (luisfrlucas@gmail.com) + * Joao Carreira (jfmcarreira@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file SaliencyBasedFiltering.h + */ + +#ifndef __SALIENCYBASEDFILTERING_H__ +#define __SALIENCYBASEDFILTERING_H__ + +#include + +#include "lib/PlaYUVerModuleIf.h" + +class SaliencyBasedFiltering : public PlaYUVerModuleIf +{ + REGISTER_CLASS_FACTORY( SaliencyBasedFiltering ) + +private: + PlaYUVerFrame* m_pcProcessedFrame; + +public: + SaliencyBasedFiltering(); + virtual ~SaliencyBasedFiltering() + { + } + Bool create( std::vector apcFrameList ); + PlaYUVerFrame* process( std::vector apcFrameList ); + Void destroy(); +}; + +#endif // __SALIENCYBASEDFILTERING_H__ diff --git a/modules/ThreeSixtyDownsampling.cpp b/modules/ThreeSixtyDownsampling.cpp new file mode 100644 index 0000000..14d31c3 --- /dev/null +++ b/modules/ThreeSixtyDownsampling.cpp @@ -0,0 +1,764 @@ +/* This file is a part of Calyp project + * Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com) + * Luis Lucas (luisfrlucas@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file ThreeSixtyDownsampling.cpp + * \brief Adaptive Latitude Downsampling + */ + +#include "ThreeSixtyDownsampling.h" + +#include +//#include +#include +#include + +//using cv::InputArray; +//using cv::OutputArray; +using cv::Rect; +using cv::Scalar; +using cv::Size; + +ThreeSixtyDownsampling::ThreeSixtyDownsampling() +{ + /* Module Definition */ + m_iModuleAPI = CLP_MODULE_API_2; + m_iModuleType = CLP_FRAME_PROCESSING_MODULE; + m_pchModuleCategory = "360Video"; + m_pchModuleName = "ThreeSixtyDownsampling"; + m_pchModuleLongName = "Adaptive Latitude Downsampling"; + m_pchModuleTooltip = "Adaptive Latitude Downsampling"; + m_uiNumberOfFrames = 1; + m_uiModuleRequirements = CLP_MODULE_REQUIRES_SKIP_WHILE_PLAY | + CLP_MODULE_REQUIRES_OPTIONS | + CLP_MODULES_HAS_INFO | + CLP_MODULE_USES_KEYS; + + m_cModuleOptions.addOptions() /**/ + ( "Downsampling", m_uiDownsampling, "ERP to NxN (0 for up, 1 for down and 2 for recon) [1]" ) /**/ + ( "y0", m_uiY0, "Vertical anchor point ratio (0-100) [0]" ) /**/ + ( "x0", m_uiX0, "Horizontal anchor point ratio (0-100) [25%]" ) /**/ + // INTER_NEAREST - a nearest-neighbor interpolation + // INTER_LINEAR - a bilinear interpolation (used by default) + // INTER_AREA - resampling using pixel area relation. It may be a preferred + // method for image decimation, as it gives moire’-free results. But when + // the image is zoomed, it is similar to the INTER_NEAREST method. + // INTER_CUBIC - a bicubic interpolation over 4x4 pixel neighborhood + // INTER_LANCZOS4 - a Lanczos interpolation over 8x8 pixel neighborhood + ( "Interpolation", m_iInterpolation, "Interpolation method (1-4) [2]" ) /**/ + ( "IntegerSlope", m_bForceIntSlope, "Force the downsampling slope to be integer (binary) [0]" ) /**/ + ( "Packing", m_iRearrange, "Rearrange image into a rectangle (0-3) [0]" ) /**/ + ( "Inpainting", m_iInpaintMethod, "Use inpainting to fill empty pixels (0-2) [0]" ) /**/ + ( "UpsampWidth", m_iWidth, "Image width after upsampling [-1]" ); + + m_uiDownsampling = 1; + m_uiY0 = 0; + m_uiX0 = 0; + m_iInterpolation = 2; + m_iInpaintMethod = 0; + m_bForceIntSlope = 0; + m_iRearrange = 0; + m_iWidth = -1; + m_pcOutputFrame = NULL; + m_pcDownsampled = NULL; + m_dPixelRatio = 0; + + //#ifdef DEBUG + // m_uiY0 = 25; + // m_uiX0 = 0; + // m_iInterpolation = 4; + // m_bForceIntSlope = 1; + // m_iRearrange = 1; + //#endif +} + +ClpString ThreeSixtyDownsampling::moduleInfo() +{ + ClpString info( "" ); + info += "Interpolation: "; + switch( m_iInterpolation ) + { + case cv::INTER_NEAREST: + info += "Nearest"; + break; + case cv::INTER_LINEAR: + info += "Linear"; + break; + case cv::INTER_AREA: + info += "Area"; + break; + case cv::INTER_CUBIC: + info += "Cubic"; + break; + case cv::INTER_LANCZOS4: + info += "Lanczos4"; + break; + default: + assert( 0 ); + } + info += "\ny0 = " + std::to_string( m_uiY0 ) + "\nx0 = " + std::to_string( m_uiX0 ); + if( m_bForceIntSlope == 0 ) + info += "\nNo integer slope"; + else + info += "\nUisng integer slope"; + info += "\nPixel ratio = " + std::to_string( m_dPixelRatio ); + return info; +} + +bool ThreeSixtyDownsampling::create( std::vector apcFrameList ) +{ + _BASIC_MODULE_API_2_CHECK_ + + m_uiDownsampling = m_uiDownsampling > 2 ? 1 : m_uiDownsampling; + + if( !createDownsamplingMask( apcFrameList[0] ) ) + { + return false; + } + + switch( m_iInterpolation ) + { + case 2: + m_iInterpolation = cv::INTER_LINEAR; + break; + case 3: + m_iInterpolation = cv::INTER_CUBIC; + break; + case 4: + m_iInterpolation = cv::INTER_LANCZOS4; + break; + default: + m_iInterpolation = cv::INTER_NEAREST; + } + + switch( m_iInpaintMethod ) + { + case 1: + m_iInterpolation = cv::INPAINT_NS; + break; + case 2: + m_iInpaintMethod = cv::INPAINT_TELEA; + break; + } + return true; +} + +#define WIDTH_TRIM_START_POINT 0.0 +#define WIDTH_FINAL 1.0 + +bool ThreeSixtyDownsampling::createDownsamplingMask( CalypFrame* pcInputFrame ) +{ + if( m_iRearrange == 3 ) + { + m_uiX0 = 0; + m_uiY0 = std::ceil( 100.0 / 3.0 ); + } + + if( m_iRearrange == 4 ) + { + m_uiX0 = 0; + m_uiY0 = m_uiY0 > 37 ? 40 : 34; + } + + m_iHeight = -1; + if( m_uiDownsampling == 0 ) + { + if( m_iWidth == -1 ) + { + // Guess the output dimensions when upsampling + switch( m_iRearrange ) + { + case 1: // Cannot get this - user input needed + break; + case 2: + m_iWidth = pcInputFrame->getWidth(); + break; + } + } + m_iHeight = m_iWidth / 2; + } + else + { + m_iWidth = pcInputFrame->getWidth(); + m_iHeight = pcInputFrame->getHeight(); + } + if( m_iWidth == -1 || m_iHeight == -1 ) // output dimensions are needed + return false; + + //m_pcDownsampled = new CalypFrame( m_iWidth, m_uiHeight, pcInputFrame->getPelFormat(), pcInputFrame->getBitsPel() ); + m_pcDownsampled = new CalypFrame( m_iWidth, m_iHeight, CalypFrame::findPixelFormat( "GRAY" ), pcInputFrame->getBitsPel() ); + + m_isOdd = !( m_iHeight % 2 ); + for( unsigned ch = 0; ch < m_pcDownsampled->getNumberChannels(); ch++ ) + { + Mat* downsamplingMaskPtr = new Mat( m_pcDownsampled->getHeight( ch ), m_pcDownsampled->getWidth( ch ), CV_8UC1, Scalar( 0 ) ); + Mat downsamplingMask = *downsamplingMaskPtr; + Mat_ initialReshapePoints( m_pcDownsampled->getHeight( ch ), m_pcDownsampled->getWidth( ch ), Point( -1, -1 ) ); + + int imH = m_pcDownsampled->getHeight( ch ); + int imN = imH / 2; + + // Find lines + double theta = atan2( imN, imH - WIDTH_TRIM_START_POINT - WIDTH_FINAL ); + double y0 = std::ceil( float( int( m_uiY0 ) * imH ) / 200.0 ); + double x0 = std::round( float( int( m_uiX0 ) * imH ) / 100.0 ); + double xl = 0; + double slope2 = 0; + + if( m_iRearrange == 3 || m_iRearrange == 4 ) + { + y0 = std::ceil( y0 / 64.0 ) * 64; + } + + Point p0; + Point p1; + Point p2; + + if( m_uiY0 > 0 ) + { + double x0min = ( 1 + y0 / tan( theta ) - imH + WIDTH_TRIM_START_POINT ) * double( imN - y0 ) / y0 + y0 / tan( theta ); + + x0 = std::max( x0, x0min ); + xl = double( imH - WIDTH_TRIM_START_POINT ) - y0 / tan( theta ) - y0 * y0 / double( imN - y0 ) * ( 1.0 / tan( theta ) - x0 / y0 ); + xl = std::floor( xl ); + if( m_iRearrange == 3 ) + xl = imH / 2; + else if( m_iRearrange == 4 && m_uiY0 == 34 ) + xl = imH / 2; + else if( m_iRearrange == 4 && m_uiY0 == 40 ) + xl = imH / 3; + + slope2 = xl / ( double( imN ) - y0 ); + if( m_bForceIntSlope ) + slope2 = std::round( slope2 ); + + // calculate the final coordinates of point corresponding to y0 + // p0 corresponds to a coordinate system with reference ate left-most equator + p0 = Point( int( ( double( y0 + 1 ) - double( imH ) / 2.0 ) * slope2 + double( imH - WIDTH_FINAL ) ), y0 ); + // give the y value of the triangle which can be used to fill the gap + if( m_iRearrange == 2 ) + { + int y1 = double( -p0.x + WIDTH_FINAL ) / slope2 + imN; + p1 = Point( int( ( double( y1 + 1 ) - double( imH ) / 2.0 ) * slope2 + double( imH - WIDTH_FINAL ) ), y1 ); + p2 = Point( -1, ( imH - p1.y ) / 2 ); + } + } + else + { + slope2 = 2; + p0 = Point( 0, 0 ); + p1 = Point( -1, imN ); + } + + // Reset mask + downsamplingMask = Scalar( 0 ); + + for( int line = 0; line < imN; line++ ) + { + for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 ) + { + // calculate destination sizes + int ypos = ( ( imH + ( m_isOdd ? hemisfer : 0 ) ) >> 1 ) + line * hemisfer; + int lineWidth = 0; + + if( m_uiY0 == 0 ) + { + lineWidth = ( 2 * imH - 2 ) - 4 * line; + } + else + { + double lp = 0; + if( line < y0 ) + lp = line * x0 / y0; + else + lp = ( line + 1 - imN ) * slope2 + double( imH - WIDTH_FINAL ); + lp = std::round( lp ); + assert( imH >= lp ); + lineWidth = 2 * imH - 2 * lp; + } + assert( lineWidth > 0 ); + int xpos = imH - lineWidth / 2; // N - WIDTH/2 + + // create mask and init reshape matrix + Mat downsamplingMaskRow = downsamplingMask.row( ypos ); + Mat_ reshapePointsRow = initialReshapePoints.row( ypos ); + for( int x = xpos; x < xpos + lineWidth; x++ ) + { + *( downsamplingMaskRow.data + x ) = ch == 0 ? 255 : 128; + reshapePointsRow( x ) = Point( x, ypos ); + } + } + } + if( m_iRearrange > 0 ) + { + // initialReshapePoints -> Original coordinates without re-arrange + // intermediateReshapePoints -> Coordinates with re-arrange <- Set this + + Mat_ intermediateReshapePoints; + + int imH = int( m_pcDownsampled->getHeight( ch ) ); + int imN = imH / 2; + + // Strecth into wide rectangle + if( m_iRearrange == 1 ) + { + intermediateReshapePoints = Mat_( initialReshapePoints.rows, initialReshapePoints.cols, Point( -1, -1 ) ); + Size maskSize( imH - p0.x, imN - p0.y ); + for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 ) + { + // copy center + Point r0_pt( 0, imN ); + Point r0_size( 2 * imH, hemisfer * p0.y ); + initialReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ).copyTo( intermediateReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ) ); + + // main triangle + Mat currMask; + downsamplingMask( Rect( Point( p0.x, 0 ), maskSize ) ).copyTo( currMask ); + if( hemisfer == 1 ) + cv::flip( currMask, currMask, 0 ); // Filp vertically + + initialReshapePoints( Rect( Point( p0.x, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ) + .copyTo( intermediateReshapePoints( Rect( Point( 1, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ), currMask ); + + // rotated triangle + Mat tmpTriangle; + initialReshapePoints( Rect( Point( imH, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ).copyTo( tmpTriangle ); + cv::flip( tmpTriangle, tmpTriangle, 0 ); + + cv::flip( currMask, currMask, 1 ); // Filp horizontaly + cv::flip( currMask, currMask, 0 ); + tmpTriangle.copyTo( intermediateReshapePoints( Rect( Point( 0, hemisfer == -1 ? 0 : imN + p0.y ), maskSize ) ), currMask ); + tmpTriangle.release(); + } + // At this point the two triangle were merged into a rectangle + if( ( ( imN - p0.y ) % p0.y ) == 0 ) // check if we can divide the rectangles into equally heighted slices + { + int numberRectSlices = ( imN - p0.y ) / p0.y; + int rectSlicesWidth = 2 * imN - p0.x + 1; + // create extended reshapePoints + Mat_ extendedReshapePoints( 2 * p0.y, intermediateReshapePoints.cols + rectSlicesWidth * numberRectSlices, Point( -1, -1 ) ); + intermediateReshapePoints( Rect( Point( 0, imN - p0.y ), Size( intermediateReshapePoints.cols, 2 * p0.y ) ) ) + .copyTo( extendedReshapePoints( Rect( Point( 0, 0 ), Size( intermediateReshapePoints.cols, 2 * p0.y ) ) ) ); + for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 ) + { + Point initSize( rectSlicesWidth, hemisfer * p0.y ); + Point finalPoint = Point( 0, p0.y ) + Point( 2 * imH, 0 ); + for( int i = 0; i < numberRectSlices; i++ ) + { + Point initPoint = Point( 0, imN + hemisfer * ( p0.y + i * p0.y ) ); + intermediateReshapePoints( Rect( initPoint, initPoint + initSize ) ).copyTo( extendedReshapePoints( Rect( finalPoint, finalPoint + initSize ) ) ); + finalPoint += Point( rectSlicesWidth, 0 ); + } + } + intermediateReshapePoints.release(); + extendedReshapePoints.copyTo( intermediateReshapePoints ); + extendedReshapePoints.release(); + } + } + // Rotate triangles into a rectangle and spread the top small triangles + else if( m_iRearrange == 2 ) + { + intermediateReshapePoints = Mat_( initialReshapePoints.rows, initialReshapePoints.cols, Point( -1, -1 ) ); + for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 ) + { + if( p0.x < imN ) + { + for( int side = -1; side < 2; side += 2 ) + { + // center + Point r0_pt( imH, imN ); + Point r0_size( side * imH, hemisfer * p0.y ); + initialReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ).copyTo( intermediateReshapePoints( Rect( r0_pt, r0_pt + r0_size ) ) ); + + // copy center unmodified + Point r1_pt = r0_pt + Point( 0, hemisfer * p0.y ); + Point r1_size( side * imH, hemisfer * ( p1.y - p0.y ) / 2 ); + Rect mainRoi( r1_pt, r1_pt + r1_size ); + initialReshapePoints( mainRoi ).copyTo( intermediateReshapePoints( mainRoi ), downsamplingMask( mainRoi ) ); + + // rotated polygon + Point r2_pt = r1_pt + Point( 0, hemisfer * ( p1.y - p0.y ) / 2 ); + Mat_ auxPointMap; + Mat auxMask; + initialReshapePoints( Rect( r2_pt, r2_pt + r1_size ) ).copyTo( auxPointMap ); + downsamplingMask( Rect( r2_pt, r2_pt + r1_size ) ).copyTo( auxMask ); + cv::flip( auxPointMap, auxPointMap, -1 ); + cv::flip( auxMask, auxMask, -1 ); + auxPointMap.copyTo( intermediateReshapePoints( mainRoi ), auxMask ); + auxPointMap.release(); + auxMask.release(); + + // top triangle + // the top triangle is converted to a rectangle and then reshape into and new rectangle with a larger width + Point r34_size( side * ( imH - p1.x + 1 ), hemisfer * ( imN - p1.y ) / 2 ); + Point r3_pt( imH, imN + hemisfer * p1.y ); + Point r4_pt = r3_pt + Point( 0, r34_size.y ); + + Point r3l_pt( r3_pt.x, r2_pt.y ); + + Mat_ topTriangleReshape; + initialReshapePoints( Rect( r3_pt, r3_pt + r34_size ) ).copyTo( topTriangleReshape ); + + // rotate the top of the top triangle to form a rectangle + initialReshapePoints( Rect( r4_pt, r4_pt + r34_size ) ).copyTo( auxPointMap ); + downsamplingMask( Rect( r4_pt, r4_pt + r34_size ) ).copyTo( auxMask ); + + cv::flip( auxPointMap, auxPointMap, -1 ); + cv::flip( auxMask, auxMask, -1 ); + + //cv::imwrite( "auxMask.png", auxMask ); + auxPointMap.copyTo( topTriangleReshape, auxMask ); + + long shape_area = std::abs( r34_size.x ) * std::abs( r34_size.y ); + int reshapedWidth = imH; + while( 1 ) + { + if( !( shape_area % reshapedWidth ) ) + break; + reshapedWidth--; + } + int reshaped_height = shape_area / reshapedWidth; + if( shape_area % reshapedWidth ) + { + return false; + } + topTriangleReshape = topTriangleReshape.reshape( 1, reshaped_height ); + topTriangleReshape.copyTo( intermediateReshapePoints( Rect( r3l_pt, r3l_pt + Point( side * topTriangleReshape.cols, hemisfer * topTriangleReshape.rows ) ) ) ); + } + } + else + { + return false; + } + } + } + else if( m_iRearrange == 3 ) + { + intermediateReshapePoints = Mat_( 8 * p0.y, initialReshapePoints.cols, Point( -1, -1 ) ); + for( int hemisfer = -1; hemisfer < 2; hemisfer += 2 ) + { + Point r_pt; + Point r_size; + Point r_pt_final; + Mat currMask; + + // copy half of rectangle 6 and 5 + r_pt = Point( p0.x, imN ); + r_size = Point( 2 * ( imH - p0.x ), hemisfer * p0.y ); + r_pt_final = Point( p0.x, 3 * p0.y ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) ); + + // copy triangle 2 or 4 + r_pt = Point( imH, imN + hemisfer * p0.y ); + r_size = Point( -( imH - p0.x ), hemisfer * ( imN - p0.y ) ); + r_pt_final = Point( imH, 3 * p0.y + hemisfer * p0.y ); + downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask ); + currMask.release(); + + // copy triangle 3 or 1 + r_pt = Point( imH, imN - hemisfer * p0.y ); + r_size = Point( imH - p0.x, -1 * hemisfer * ( imN - p0.y ) ); + r_pt_final = Point( p0.x, hemisfer == -1 ? 0 : 6 * p0.y ); + + downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask ); + + // copy rectangle 7 and 8 + r_pt = Point( imH - hemisfer * ( imH - p0.x ), imN - p0.y ); + r_size = Point( -hemisfer * p0.x, p0.y * 2 ); + if( hemisfer == -1 ) + r_pt_final = Point( imH, 0 ); + else + r_pt_final = Point( 2 * imH - p0.x, 4 * p0.y ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) ); + } + } + else if( m_iRearrange == 4 ) + { + intermediateReshapePoints = Mat_( 10 * p0.y, initialReshapePoints.cols, Point( -1, -1 ) ); + //unsigned hor_pad = 0; + unsigned ver_pad = 0; + if( m_uiY0 == 40 ) + { + if( unsigned( 4 * p0.y + ( imH - p0.y ) ) % 2 ) + { + ver_pad = 1; + } + } + else + { + if( unsigned( 6 * p0.y + ( imH - p0.y ) ) % 2 ) + { + ver_pad = 1; + } + } + for( int side = -1; side < 2; side += 2 ) + { + Point r_pt; + Point r_size; + Point r_pt_final; + Mat currMask; + + // copy half of rectangle 6 and 5 + r_pt = Point( p0.x, imN - p0.y ); + r_size = Point( 2 * ( imH - p0.x ), 2 * p0.y ); + r_pt_final = Point( p0.x, imN - p0.y ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) ); + + // copy triangle 2 or 1 + r_pt = Point( imH, imN - p0.y ); + r_size = Point( side * ( imH - p0.x ), -( imN - p0.y ) ); + r_pt_final = Point( imH, imN - p0.y ); + downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask ); + currMask.release(); + + // copy triangle 4 or 3 + r_pt = Point( imH, imN + p0.y ); + r_size = Point( side * ( imH - p0.x ), imN - p0.y ); + r_pt_final = Point( imH - side * ( imH - p0.x ), 0 ); + downsamplingMask( Rect( r_pt, r_pt + r_size ) ).copyTo( currMask ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ), currMask ); + + // copy rectangle 7 and 8 + if( m_uiY0 == 40 ) + { + r_pt = Point( imH + side * ( imH - p0.x ), imN - p0.y ); + r_size = Point( side * p0.x, p0.y * 2 ); + r_pt_final = Point( imH - side * ( imH - p0.x ), ver_pad + imN + ( side == -1 ? p0.y : 3 * p0.y ) ); + if( side == -1 ) + { + Mat flipRect; + cv::flip( initialReshapePoints( Rect( r_pt, r_pt + r_size ) ), flipRect, -1 ); + flipRect.copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) ); + } + else + { + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) ); + } + } + else + { + r_pt = Point( imH + side * ( imH - p0.x ), imN - p0.y ); + r_size = Point( side * p0.x, p0.y * 2 ); + r_pt_final = Point( imH - side * p0.x, imN + p0.y + ver_pad ); + initialReshapePoints( Rect( r_pt, r_pt + r_size ) ).copyTo( intermediateReshapePoints( Rect( r_pt_final, r_pt_final + r_size ) ) ); + } + } + } + initialReshapePoints.release(); + intermediateReshapePoints.copyTo( initialReshapePoints ); // copy back to the initial Mat + } + + // Create a mask with valid reshape points in order to create the final rectangular frame + Mat reshapePointsMask( initialReshapePoints.rows, initialReshapePoints.cols, CV_8UC1, Scalar( 0 ) ); + m_dPixelRatio = 0; + for( int y = 0; y < initialReshapePoints.rows; y++ ) + for( int x = 0; x < initialReshapePoints.cols; x++ ) + if( initialReshapePoints.at( y, x ).x != -1 ) + { + reshapePointsMask.at( y, x ) = 1; + m_dPixelRatio++; + } + m_dPixelRatio = m_dPixelRatio / double( pcInputFrame->getWidth() * pcInputFrame->getHeight() ); + + Rect reshapeArea = boundingRect( reshapePointsMask ); + Mat_* finalReshapePointsPtr = new Mat_( reshapeArea.height, reshapeArea.width ); + initialReshapePoints( reshapeArea ).copyTo( *finalReshapePointsPtr ); + + m_cvDownsamplingMask.push_back( downsamplingMaskPtr ); // Save Mat's for each channel + m_cvReshapePoints.push_back( finalReshapePointsPtr ); + } + switch( m_uiDownsampling ) + { + case 0: + m_pcOutputFrame = new CalypFrame( m_pcDownsampled->getWidth(), m_pcDownsampled->getHeight(), pcInputFrame->getPelFormat(), pcInputFrame->getBitsPel() ); + break; + case 1: + m_pcOutputFrame = new CalypFrame( m_cvReshapePoints[0]->cols, m_cvReshapePoints[0]->rows, pcInputFrame->getPelFormat(), pcInputFrame->getBitsPel() ); + break; + case 2: + m_pcOutputFrame = new CalypFrame( pcInputFrame ); + m_pcOutputFrame->reset(); + break; + } + return true; +} + +void ThreeSixtyDownsampling::downsamplingOperation( CalypFrame* pcInputFrame ) +{ + unsigned numBytes = m_pcDownsampled->getBitsPel() > 8 ? 2 : 1; + m_pcDownsampled->reset(); + m_pcOutputFrame->reset(); + for( unsigned ch = 0; ch < m_pcDownsampled->getNumberChannels(); ch++ ) + { + int imH = int( m_pcDownsampled->getHeight( ch ) ); + + Mat cvInputFrame; + pcInputFrame->toMat( cvInputFrame, true, false, ch ); + for( int y = 0; y < imH; y++ ) + { + // calculate destination sizes + int lineWidth = cv::countNonZero( m_cvDownsamplingMask[ch]->row( y ) ); + int xpos = imH - lineWidth / 2; // (2*HEIGHT - WIDTH)/2 + + Mat cvDownsample( 1, lineWidth, numBytes > 2 ? CV_16UC1 : CV_8UC1 ); + cv::resize( cvInputFrame.row( y ), cvDownsample, cvDownsample.size(), 0, 0, m_iInterpolation ); + + // copy to output + ClpPel* outpel = &m_pcDownsampled->getPelBufferYUV()[ch][y][xpos]; + unsigned char* cv_data = cvDownsample.data; + for( int i = 0; i < lineWidth; i++ ) + { + *outpel = 0; + for( unsigned b = 0; b < numBytes; b++ ) + { + *outpel = ClpPel( *outpel + ( *cv_data++ << ( 8 * b ) ) ); + } + outpel++; + } + } + Mat_ reshapePoints = *( m_cvReshapePoints[ch] ); + ClpPel** downSampPelBuff = m_pcDownsampled->getPelBufferYUV()[ch]; + ClpPel* pelOutputPtr = m_pcOutputFrame->getPelBufferYUV()[ch][0]; + + Mat outputMask( m_pcOutputFrame->getHeight( ch ), m_pcOutputFrame->getWidth( ch ), CV_8UC1, Scalar( 1 ) ); + for( unsigned y = 0; y < m_pcOutputFrame->getHeight( ch ); y++ ) + { + for( unsigned x = 0; x < m_pcOutputFrame->getWidth( ch ); x++ ) + { + Point pt = reshapePoints.at( y, x ); + if( pt.x > -1 ) + { + *pelOutputPtr = downSampPelBuff[pt.y][pt.x]; + outputMask.at( y, x ) = 0; + } + pelOutputPtr++; + } + } + if( m_iInpaintMethod ) + { + Mat outputFrameUnfilled, outputFrame; + m_pcOutputFrame->toMat( outputFrameUnfilled, true, false, ch ); + cv::inpaint( outputFrameUnfilled, outputMask, outputFrame, 16, m_iInpaintMethod ); + m_pcOutputFrame->fromMat( outputFrame, ch ); + } + } +} + +void ThreeSixtyDownsampling::upsamplingOperation( CalypFrame* pcInputFrame ) +{ + unsigned numBytes = m_pcDownsampled->getBitsPel() > 8 ? 2 : 1; + m_pcDownsampled->reset(); + m_pcOutputFrame->reset(); + for( unsigned ch = 0; ch < m_pcDownsampled->getNumberChannels(); ch++ ) + { + Mat_ reshapePoints = *( m_cvReshapePoints[ch] ); + ClpPel** downSampPelBuff = m_pcDownsampled->getPelBufferYUV()[ch]; + ClpPel* pelInputPtr = pcInputFrame->getPelBufferYUV()[ch][0]; + for( unsigned y = 0; y < pcInputFrame->getHeight( ch ); y++ ) + { + for( unsigned x = 0; x < pcInputFrame->getWidth( ch ); x++ ) + { + Point pt = reshapePoints.at( y, x ); + if( pt.x > -1 ) + downSampPelBuff[pt.y][pt.x] = *( pelInputPtr ); + pelInputPtr++; + } + } + + Mat cvDownsampled; + m_pcDownsampled->toMat( cvDownsampled, true, false, ch ); + int imH = m_pcOutputFrame->getHeight( ch ); + Mat cvUpsample( 1, 2 * imH, numBytes > 2 ? CV_16UC1 : CV_8UC1 ); + for( int y = 0; y < imH; y++ ) + { + unsigned lineWidth = cv::countNonZero( m_cvDownsamplingMask[ch]->row( y ) ); + unsigned int xpos = imH - lineWidth / 2; // N - WIDTH/2 + + // resize + cv::resize( cvDownsampled.row( y ).colRange( xpos, xpos + lineWidth ), cvUpsample, cvUpsample.size(), 0, 0, m_iInterpolation ); + + // copy to output + ClpPel* outpel = &m_pcOutputFrame->getPelBufferYUV()[ch][y][0]; + unsigned char* cv_data = cvUpsample.data; + for( int i = 0; i < 2 * imH; i++ ) + { + *outpel = 0; + for( unsigned b = 0; b < numBytes; b++ ) + { + *outpel = ClpPel( *outpel + ( *cv_data++ << ( 8 * b ) ) ); + } + outpel++; + } + } + } + //m_pcOutputFrame->copyFrom( m_pcDownsampled ); +} + +CalypFrame* ThreeSixtyDownsampling::process( std::vector apcFrameList ) +{ + m_pcDownsampled->reset(); + if( m_uiDownsampling == 1 || m_uiDownsampling == 2 ) + { + downsamplingOperation( apcFrameList[0] ); + } + if( m_uiDownsampling == 0 || m_uiDownsampling == 2 ) + { + upsamplingOperation( m_uiDownsampling == 2 ? m_pcOutputFrame : apcFrameList[0] ); + } + return m_pcOutputFrame; +} + +bool ThreeSixtyDownsampling::keyPressed( enum Module_Key_Supported value ) +{ + bool bRet = false; + if( value == MODULE_KEY_LEFT ) + { + m_uiX0 = m_uiX0 == 0 ? 0 : m_uiX0 - 1; + } + else if( value == MODULE_KEY_RIGHT ) + { + m_uiX0 = m_uiX0 == 60 ? 60 : m_uiX0 + 1; + } + else if( value == MODULE_KEY_DOWN ) + { + m_uiY0 = m_uiY0 == 0 ? 0 : m_uiY0 - 1; + } + else if( value == MODULE_KEY_UP ) + { + m_uiY0 = m_uiY0 == 60 ? 60 : m_uiY0 + 1; + } + if( bRet ) + { + bRet = createDownsamplingMask( m_pcDownsampled ); + } + return bRet; +} + +void ThreeSixtyDownsampling::destroy() +{ + if( m_pcDownsampled ) + delete m_pcDownsampled; + m_pcDownsampled = NULL; + if( m_pcOutputFrame ) + delete m_pcOutputFrame; + m_pcOutputFrame = NULL; +} diff --git a/modules/ThreeSixtyDownsampling.h b/modules/ThreeSixtyDownsampling.h new file mode 100644 index 0000000..fedf882 --- /dev/null +++ b/modules/ThreeSixtyDownsampling.h @@ -0,0 +1,75 @@ +/* This file is a part of Calyp project + * Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com) + * Luis Lucas (luisfrlucas@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file ThreeSixtyDownsampling.h + * \brief Adaptive Latitude Downsampling + */ + +#ifndef __THREESIXTYDOWNSAMPLING_H__ +#define __THREESIXTYDOWNSAMPLING_H__ + +// CalypLib +#include "lib/CalypModuleIf.h" +#include "opencv2/core/mat.hpp" +#include "opencv2/core/types.hpp" + +using cv::Mat; +using cv::Mat_; +using cv::Point; + +class ThreeSixtyDownsampling : public CalypModuleIf +{ + REGISTER_CLASS_FACTORY( ThreeSixtyDownsampling ) +private: + unsigned m_uiDownsampling; + unsigned m_uiX0; + unsigned m_uiY0; + int m_iInterpolation; + int m_iInpaintMethod; + unsigned m_bForceIntSlope; + unsigned m_iRearrange; + + bool m_isOdd; + int m_iWidth; + int m_iHeight; + + double m_dPixelRatio; + + CalypFrame* m_pcDownsampled; + + // Values one per channel + std::vector m_cvDownsamplingMask; + std::vector*> m_cvReshapePoints; + bool createDownsamplingMask( CalypFrame* pcFrame ); + + void downsamplingOperation( CalypFrame* pcInputFrame ); + void upsamplingOperation( CalypFrame* pcInputFrame ); + +public: + ThreeSixtyDownsampling(); + virtual ~ThreeSixtyDownsampling() {} + bool create( std::vector apcFrameList ); + CalypFrame* process( std::vector apcFrameList ); + ClpString moduleInfo(); + bool keyPressed( enum Module_Key_Supported value ); + void destroy(); +}; + +#endif // __THREESIXTYDOWNSAMPLING_H__ diff --git a/modules/ThreeSixtyDynamicDownsampling.cpp b/modules/ThreeSixtyDynamicDownsampling.cpp new file mode 100644 index 0000000..14a018a --- /dev/null +++ b/modules/ThreeSixtyDynamicDownsampling.cpp @@ -0,0 +1,155 @@ +/* This file is a part of Calyp project + * Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com) + * Luis Lucas (luisfrlucas@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file ThreeSixtyDynamicDownsampling.cpp + */ + +#include "ThreeSixtyDynamicDownsampling.h" + +#include +#include + +using cv::InputArray; +using cv::Mat; +using cv::OutputArray; +using cv::Point; +using cv::Rect; +using cv::Size; + +ThreeSixtyDynamicDownsampling::ThreeSixtyDynamicDownsampling() +{ + /* Module Definition */ + m_iModuleAPI = CLP_MODULE_API_2; + m_iModuleType = CLP_FRAME_PROCESSING_MODULE; + m_pchModuleCategory = "360Video"; + m_pchModuleLongName = "Dynamic Downsampling"; + m_pchModuleName = "ThreeSixtyDynamicDownsampling"; + m_pchModuleTooltip = "Dynamic downsampling of 360 videos"; + m_uiNumberOfFrames = 1; + m_uiModuleRequirements = CLP_MODULE_REQUIRES_SKIP_WHILE_PLAY | CLP_MODULE_REQUIRES_OPTIONS; + + m_cModuleOptions.addOptions() /**/ + ( "CodingFaceWidth", m_uiFaceWidth, "Coding Face Width [same as input]" ) /**/ + ( "CodingFaceHeight", m_uiFaceHeight, "Coding Face Height [same as input]" ) /**/ + ( "SourceFPStructure", m_arrayInputDefinitionString, "Input frame packing" ) /**/ + ( "CodingFPStructure", m_arrayOutputDefinitionString, "Coding frame packing" ); /**/ + + m_arrayInputDefinitionString = "6 2 3 0 0 0 0 100 1 0 1 0 100 2 0 2 0 100 0 1 3 0 100 1 1 4 0 100 2 1 5 0 100"; + + m_arrayOutputDefinitionString = "6 3 3 0 0 0 0 100 1 0 1 0 200 0 1 2 0 100 0 2 3 0 100 1 2 4 0 100 2 2 5 0 100 "; + + m_uiFaceWidth = 0; + m_uiFaceHeight = 0; +} + +void processFPSString( const ClpString& fps, ClpVideoFPStruct* pcFPSStruct ) +{ + std::istringstream ss( fps ); + ss >> pcFPSStruct->numFaces; + ss >> pcFPSStruct->rows; + ss >> pcFPSStruct->cols; + for( int i = 0; i < pcFPSStruct->numFaces; i++ ) + { + ss >> pcFPSStruct->faces[i].x; + ss >> pcFPSStruct->faces[i].y; + ss >> pcFPSStruct->faces[i].id; + ss >> pcFPSStruct->faces[i].rot; + ss >> pcFPSStruct->faces[i].ratio; + } +} + +bool ThreeSixtyDynamicDownsampling::create( std::vector apcFrameList ) +{ + _BASIC_MODULE_API_2_CHECK_ + + processFPSString( m_arrayInputDefinitionString, &m_cInputFPSStruct ); + processFPSString( m_arrayOutputDefinitionString, &m_cOutputFPSStruct ); + + if( !m_uiFaceWidth && !m_uiFaceHeight ) + { + m_uiFaceWidth = apcFrameList[0]->getWidth() / m_cInputFPSStruct.cols; + m_uiFaceHeight = apcFrameList[0]->getHeight() / m_cInputFPSStruct.rows; + } + else if( !m_uiFaceHeight ) + { + m_uiFaceHeight = m_uiFaceWidth; + } + + m_pcOutputFrame = new CalypFrame( m_uiFaceWidth * m_cOutputFPSStruct.cols, + m_uiFaceHeight * m_cOutputFPSStruct.rows, + apcFrameList[0]->getPelFormat(), apcFrameList[0]->getBitsPel() ); + + return true; +} + +CalypFrame* ThreeSixtyDynamicDownsampling::process( std::vector apcFrameList ) +{ + m_pcOutputFrame->reset(); + + for( unsigned ch = 0; ch < apcFrameList[0]->getNumberChannels(); ch++ ) + { + Mat inputFrame; + apcFrameList[0]->toMat( inputFrame, true, false, ch ); + + unsigned inGridW = apcFrameList[0]->getWidth( ch ) / m_cInputFPSStruct.cols; + unsigned inGridH = apcFrameList[0]->getHeight( ch ) / m_cInputFPSStruct.rows; + + unsigned outGridW = CHROMASHIFT( m_uiFaceWidth, ch > 0 ? m_pcOutputFrame->getChromaWidthRatio() : 0 ); + unsigned outGridH = CHROMASHIFT( m_uiFaceHeight, ch > 0 ? m_pcOutputFrame->getChromaHeightRatio() : 0 ); + + Mat outputFrame; + m_pcOutputFrame->toMat( outputFrame, true, false, ch ); + + for( int face = 0; face < m_cOutputFPSStruct.numFaces; face++ ) + { + ClpFaceProperty outFp = m_cOutputFPSStruct.faces[face]; + Mat outputFace( outGridW * outFp.ratio / 100, outGridH * outFp.ratio / 100, CV_8UC1 ); + Rect outRect( Point( outFp.x * outGridW, outFp.y * outGridH ), outputFace.size() ); + + int inFace = face; + for( int i = 0; i < m_cInputFPSStruct.numFaces; i++ ) + { + if( m_cInputFPSStruct.faces[i].id == outFp.id ) + { + inFace = i; + break; + } + } + ClpFaceProperty inFp = m_cInputFPSStruct.faces[inFace]; + Mat inputFace( inGridW * inFp.ratio / 100, inGridH * inFp.ratio / 100, CV_8UC1 ); + Rect inRect( Point( inFp.x * inGridW, inFp.y * inGridH ), inputFace.size() ); + inputFrame( inRect ).copyTo( inputFace ); + + cv::resize( inputFace, outputFace, outputFace.size(), 0, 0, cv::INTER_LANCZOS4 ); + + outputFace.copyTo( outputFrame( outRect ) ); + } + m_pcOutputFrame->fromMat( outputFrame, ch ); + } + + return m_pcOutputFrame; +} + +void ThreeSixtyDynamicDownsampling::destroy() +{ + if( m_pcOutputFrame ) + delete m_pcOutputFrame; + m_pcOutputFrame = NULL; +} diff --git a/modules/ThreeSixtyDynamicDownsampling.h b/modules/ThreeSixtyDynamicDownsampling.h new file mode 100644 index 0000000..4a97aa1 --- /dev/null +++ b/modules/ThreeSixtyDynamicDownsampling.h @@ -0,0 +1,66 @@ +/* This file is a part of Calyp project + * Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com) + * Luis Lucas (luisfrlucas@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file ThreeSixtyDynamicDownsampling.h + */ + +#ifndef __ThreeSixtyDynamicDownsampling_H__ +#define __ThreeSixtyDynamicDownsampling_H__ + +// CalypLib +#include "lib/CalypModuleIf.h" + +struct ClpFaceProperty +{ + unsigned id; + unsigned x; + unsigned y; + int rot; + unsigned ratio; +}; +struct ClpVideoFPStruct +{ + unsigned rows; + unsigned cols; + int numFaces; + ClpFaceProperty faces[60]; +}; + +class ThreeSixtyDynamicDownsampling : public CalypModuleIf +{ + REGISTER_CLASS_FACTORY( ThreeSixtyDynamicDownsampling ) + +private: + ClpString m_arrayInputDefinitionString; + ClpString m_arrayOutputDefinitionString; + ClpVideoFPStruct m_cInputFPSStruct; + ClpVideoFPStruct m_cOutputFPSStruct; + unsigned m_uiFaceWidth; + unsigned m_uiFaceHeight; + +public: + ThreeSixtyDynamicDownsampling(); + virtual ~ThreeSixtyDynamicDownsampling() {} + bool create( std::vector apcFrameList ); + CalypFrame* process( std::vector apcFrameList ); + void destroy(); +}; + +#endif // __ThreeSixtyDynamicDownsampling_H__ diff --git a/modules/ThreeSixtyNxN.cpp b/modules/ThreeSixtyNxN.cpp new file mode 100644 index 0000000..66a1aca --- /dev/null +++ b/modules/ThreeSixtyNxN.cpp @@ -0,0 +1,257 @@ +/* This file is a part of Calyp project + * Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com) + * Luis Lucas (luisfrlucas@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file ThreeSixtyNxN.cpp + * \brief Downsampling ERP to NxN + */ + +#include "ThreeSixtyNxN.h" + +#include + +#include "lib/LibMemory.h" +//#include +#include +#include + +//using cv::InputArray; +//using cv::OutputArray; +using cv::Rect; +using cv::Scalar; +using cv::Size; + +ThreeSixtyNxN::ThreeSixtyNxN() +{ + /* Module Definition */ + m_iModuleAPI = CLP_MODULE_API_2; + m_iModuleType = CLP_FRAME_PROCESSING_MODULE; + m_pchModuleCategory = "360Video"; + m_pchModuleName = "ThreeSixtyNxN"; + m_pchModuleLongName = "ERP to NxN"; + m_pchModuleTooltip = "Downsampling ERP to NxN"; + m_uiNumberOfFrames = 1; + m_uiModuleRequirements = CLP_MODULE_REQUIRES_SKIP_WHILE_PLAY | + CLP_MODULE_REQUIRES_OPTIONS; + + m_cModuleOptions.addOptions() /**/ + ( "Downsampling", m_uiDownsampling, "ERP to NxN (0 for up, 1 for down and 2 for recon) [1]" ) /**/ + ; + + m_uiDownsampling = 1; + m_pcOutputFrame = NULL; +} + +bool ThreeSixtyNxN::create( std::vector apcFrameList ) +{ + _BASIC_MODULE_API_2_CHECK_ + m_uiDownsampling = m_uiDownsampling > 2 ? 1 : m_uiDownsampling; + if( m_uiDownsampling == 0 ) + { + m_pcOutputFrame = new CalypFrame( 2 * apcFrameList[0]->getHeight(), apcFrameList[0]->getHeight(), apcFrameList[0]->getPelFormat(), apcFrameList[0]->getBitsPel() ); + } + else + { + m_pcOutputFrame = new CalypFrame( apcFrameList[0]->getHeight(), apcFrameList[0]->getHeight(), apcFrameList[0]->getPelFormat(), apcFrameList[0]->getBitsPel() ); + } + + return true; +} + +#define WIDTH_TRIM_START_POINT 0.0 +#define WIDTH_FINAL 1.0 + +//#define CLAMP8BITS( in ) ( uint8_t )( ( in > 255 ? 255 : in < 0 ? 0 : in ) + 0.5 ) +#define CLAMP_RANGE( X, MIN, MAX ) ( ( X ) < MIN ? MIN : ( ( X ) > MAX ? MAX : X ) ) +#define CIRCULAR( a, b ) ( ( ( a < 0 ? b : 0 ) + ( a % b ) ) % b ) +#define PI 3.14159265358979323846264338327950288419716939937510 +#define s60 0.8660254037844386467637231707529361834714026269051 +#define s45 0.7071067811865475244008443621048490392848359376884 + +//interpolação Lanczos3 360 +static inline ClpPel interpol_lanczos3( ClpPel* image, double x, int N, int max_value ) +{ + static const double cs[][2] = { + { 1, 0 }, + { -0.5, -s60 }, + { -0.5, s60 }, + { 1, 0 }, + { -0.5, -s60 }, + { -0.5, s60 }, + }; + + double sum_coeffs = 0, sum = 0, coeffs[6]; + int x_int = floor( x ); + ClpPel xx[6]; + x -= x_int; + bool n0 = x > 0; + xx[2] = *( image + CIRCULAR( x_int, N ) ); + if( !n0 ) + return xx[2]; + xx[3] = *( image + CIRCULAR( x_int + n0, N ) ); + xx[1] = *( image + CIRCULAR( x_int - 1, N ) ); + xx[4] = *( image + CIRCULAR( x_int + n0 + 1, N ) ); + xx[0] = *( image + CIRCULAR( x_int - 2, N ) ); + xx[5] = *( image + CIRCULAR( x_int + n0 + 2, N ) ); + + double aux = -2 - x, s_aux = sin( aux * PI / 3 ), c_aux = cos( aux * PI / 3 ); + + for( int i = 0; i < 6; i++ ) + sum_coeffs += coeffs[i] = ( cs[i][0] * s_aux + cs[i][1] * c_aux ) / ( i - 2 - x ) / ( i - 2 - x ); + + for( int i = 0; i < 6; i++ ) + sum += xx[i] * coeffs[i] / sum_coeffs; + + return CLAMP_RANGE( sum, 0, max_value ); +} + +//interpolação cúbica 360 +static inline ClpPel interpol_cub( ClpPel* image, double x, int N, int max_value ) +{ + int x_int = floor( x ); + x -= x_int; + bool n0 = x > 0; + ClpPel x1 = *( image + CIRCULAR( x_int, N ) ); + if( !n0 ) + return x1; + ClpPel x2 = *( image + CIRCULAR( x_int + n0, N ) ); + ClpPel x0 = *( image + CIRCULAR( x_int - 1, N ) ); + ClpPel x3 = *( image + CIRCULAR( x_int + n0 + 1, N ) ); + + ClpPel out = CLAMP_RANGE( x1 - x * ( x0 - x2 + x * ( 2 * x1 - 2 * x0 - x2 + x3 + x * ( x0 - x1 + x2 - x3 ) ) ), 0, max_value ); //cubic + return out; +} + +void ThreeSixtyNxN::downsamplingOperation( const CalypFrame* pcInputFrame ) +{ + for( unsigned ch = 0; ch < m_pcOutputFrame->getNumberChannels(); ch++ ) + { + int max_value = ( 1 << m_pcOutputFrame->getBitsPel() ) - 1; + int N = pcInputFrame->getHeight( ch ); + ClpPel* in = &pcInputFrame->getPelBufferYUV()[ch][0][0]; + ClpPel* out = &m_pcOutputFrame->getPelBufferYUV()[ch][0][0]; + + unsigned long nn = N * N; + int n_2 = N / 2; + int n1 = N - 1; + double dist_y; + double dist_x; + double novo_x; + double novo_y; + double tx_x; + ClpPel* tx_y; + ClpPel* rx; + bool valida; + + for( int y = 0; y < N; y++ ) + { + for( int x = 0; x < N; x++ ) + { + dist_y = ( abs( y * 2 - n1 ) - 1 ) / 2; + dist_x = ( abs( x * 2 - n1 ) - 1 ) / 2; + valida = ( n_2 - dist_x - dist_y - 1 ) >= 0; + + novo_y = ( valida ? y : ( y < n_2 ? dist_x : n1 - dist_x ) ) * 2; + novo_x = ( valida ? x : ( x < n_2 ? -dist_y + x * 2 : n1 + dist_y - ( n1 - x ) * 2 ) ) - n_2 + 1; + + tx_x = ( N / ( N - fabs( novo_y - n1 ) ) ) * novo_x + n1; + tx_y = (int)novo_y * N + in; + rx = x + y * N + out; + + *( rx ) = interpol_lanczos3( tx_y, tx_x, N * 2, max_value ); + //*( rx ) = interpol_cub( tx_y, tx_x, N * 2, max_value ); + } + } + } +} + +void ThreeSixtyNxN::upsamplingOperation( const CalypFrame* pcInputFrame ) +{ + for( unsigned ch = 0; ch < m_pcOutputFrame->getNumberChannels(); ch++ ) + { + int max_value = ( 1 << m_pcOutputFrame->getBitsPel() ) - 1; + int N = pcInputFrame->getHeight( ch ); + ClpPel* in = &pcInputFrame->getPelBufferYUV()[ch][0][0]; + ClpPel* out = &m_pcOutputFrame->getPelBufferYUV()[ch][0][0]; + + int x; + int y; + int n2 = 2 * N; + double dist_y; + double dist_x; + double dist_Lj; + double novo_x; + double novo_y; + double Lj; + double Dj; + double tx; + ClpPel* rx; + ClpPel* linhaY; + + getMem1D( &linhaY, n2 ); + + for( y = 0; y < N; y++ ) + { + Lj = ( N - abs( 2 * y - N + 1 ) ) * 2; + Dj = n2 / Lj; + + for( x = 0; x < Lj; x++ ) + { + dist_x = x - Lj / 2; + dist_y = ( N - abs( 2 * y - N + 1 ) - 1 ) / 2; + dist_Lj = ( Lj - abs( 2 * x - Lj + 1 ) - 1 ) / 2; + + novo_x = ( dist_x > dist_y ? dist_y : ( dist_x + 1 < -dist_y ? -dist_y - 1 : dist_x ) ) + N / 2; + novo_y = abs( dist_x ) > dist_y ? ( y < N / 2 ? dist_Lj : N - dist_Lj - 1 ) : y; + + *( linhaY + x ) = *( in + (int)novo_x + (int)novo_y * N ); + } + + for( x = 0; x < n2; x++ ) + { + tx = ( x + 1 ) / Dj - 1; + rx = x + y * n2 + out; + + *( rx ) = interpol_lanczos3( linhaY, tx, Lj, max_value ); + } + } + freeMem1D( linhaY ); + } +} + +CalypFrame* ThreeSixtyNxN::process( std::vector apcFrameList ) +{ + m_pcOutputFrame->reset(); + if( m_uiDownsampling == 1 || m_uiDownsampling == 2 ) + { + downsamplingOperation( apcFrameList[0] ); + } + if( m_uiDownsampling == 0 || m_uiDownsampling == 2 ) + { + upsamplingOperation( m_uiDownsampling == 2 ? m_pcOutputFrame : apcFrameList[0] ); + } + return m_pcOutputFrame; +} + +void ThreeSixtyNxN::destroy() +{ + if( m_pcOutputFrame ) + delete m_pcOutputFrame; + m_pcOutputFrame = NULL; +} diff --git a/modules/ThreeSixtyNxN.h b/modules/ThreeSixtyNxN.h new file mode 100644 index 0000000..e9a563c --- /dev/null +++ b/modules/ThreeSixtyNxN.h @@ -0,0 +1,53 @@ +/* This file is a part of Calyp project + * Copyright (C) 2014-2019 by Joao Carreira (jfmcarreira@gmail.com) + * Luis Lucas (luisfrlucas@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * \file ThreeSixtyNxN.h + * \brief Downsampling ERP to NxN + */ + +#ifndef __THREESIXTYNXN_H__ +#define __THREESIXTYNXN_H__ + +// CalypLib +#include "lib/CalypModuleIf.h" +#include "opencv2/core/mat.hpp" +#include "opencv2/core/types.hpp" + +using cv::Mat; +using cv::Mat_; +using cv::Point; + +class ThreeSixtyNxN : public CalypModuleIf +{ + REGISTER_CLASS_FACTORY( ThreeSixtyNxN ) +private: + unsigned m_uiDownsampling; + void downsamplingOperation( const CalypFrame* pcInputFrame ); + void upsamplingOperation( const CalypFrame* pcInputFrame ); + +public: + ThreeSixtyNxN(); + virtual ~ThreeSixtyNxN() {} + bool create( std::vector apcFrameList ); + CalypFrame* process( std::vector apcFrameList ); + void destroy(); +}; + +#endif // __THREESIXTYNXN_H__