From 127009b9ec3ddc28deac8a21d6bcc4d49567d38a Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Thu, 1 Sep 2022 11:21:51 +0200 Subject: [PATCH 01/26] added cropping of capture/background --- app/background.cc | 9 +++++--- app/deepseg.cc | 54 ++++++++++++++++++++++++++++++++++++++++----- lib/libbackscrub.cc | 25 ++++++++++++++++++++- lib/libbackscrub.h | 2 ++ 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/app/background.cc b/app/background.cc index 51266bd..4e72ac3 100644 --- a/app/background.cc +++ b/app/background.cc @@ -8,7 +8,7 @@ #include #include #include - +#include // Internal state of background processing struct background_t { int debug; @@ -183,11 +183,14 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::resize(pbkd->raw, out, cv::Size(width, height)); + cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width, height); + cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { // resize still image as requested into out - cv::resize(pbkd->raw, out, cv::Size(width, height)); + cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width, height); + cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); + out = pbkd->raw; frm = 1; } return frm; diff --git a/app/deepseg.cc b/app/deepseg.cc index 55733ea..051c590 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -372,6 +372,7 @@ int main(int argc, char* argv[]) try { bool flipVertical = false; int fourcc = 0; size_t blur_strength = 0; + cv::Rect_ crop_region(0,0,0,0); const char* modelname = "selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite"; @@ -568,6 +569,15 @@ int main(int argc, char* argv[]) try { if (expWidth != vidGeo.value().first) { fprintf(stderr, "Warning: virtual camera aspect ratio does not match capture device.\n"); } + // calculate crop region, only if result always smaller + if (expWidth != vidGeo.value().first && + vidGeo.value().first <= capGeo.value().first && + vidGeo.value().second <= capGeo.value().second) { + crop_region = calcCropping(capGeo.value().first, + capGeo.value().second, + vidGeo.value().first, + vidGeo.value().second); + } // dump settings.. printf("debug: %d\n", debug); @@ -600,7 +610,12 @@ int main(int argc, char* argv[]) try { } } // default green screen background (at capture true geometry) - cv::Mat bg = cv::Mat(capGeo.value().second, capGeo.value().first, CV_8UC3, cv::Scalar(0, 255, 0)); + cv::Mat bg; + if ( crop_region.height == 0) { + bg = cv::Mat(capGeo.value().second, capGeo.value().first, CV_8UC3, cv::Scalar(0, 255, 0)); + } else { + bg = cv::Mat(crop_region.width, crop_region.height, CV_8UC3, cv::Scalar(0, 255, 0)); + } // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); @@ -615,9 +630,26 @@ int main(int argc, char* argv[]) try { // Processing components, all at capture true geometry - cv::Mat mask(capGeo.value().second, capGeo.value().first, CV_8U); + cv::Mat mask; + if ( crop_region.height ) { + cv::Mat masktmp((int)(crop_region.height), (int)(crop_region.width), CV_8U); + mask = masktmp; + } else { + cv::Mat masktmp(capGeo.value().second, capGeo.value().first, CV_8U); + mask = masktmp; + } + cv::Mat raw; - CalcMask ai(s_model.value(), threads, capGeo.value().first, capGeo.value().second); + int aiw,aih; + if ( crop_region.width == 0) { + aiw=capGeo.value().first; + aih=capGeo.value().second; + } else { + aiw=crop_region.width; + aih=crop_region.height; + } + CalcMask ai(s_model.value(), threads, aiw, aih); + ti.lastns = timestamp(); printf("Startup: %ldns\n", diffnanosecs(ti.lastns,ti.bootns)); @@ -631,11 +663,15 @@ int main(int argc, char* argv[]) try { // copy new frame to buffer cap.retrieve(raw); ti.retrns = timestamp(); - ai.set_input_frame(raw); ti.copyns = timestamp(); if (raw.rows == 0 || raw.cols == 0) continue; // sanity check + if ( crop_region.height) { + raw((cv::Rect_)crop_region).copyTo(raw); + } + ai.set_input_frame(raw); + if (filterActive) { // do background detection magic ai.get_output_mask(mask); @@ -646,7 +682,15 @@ int main(int argc, char* argv[]) try { // - default green (initial value) bool canBlur = false; if (pbk) { - if (grab_background(pbk, capGeo.value().first, capGeo.value().second, bg)<0) + int tw,th; + if ( crop_region.height ) { + tw = crop_region.width; + th = crop_region.height; + } else { + tw = capGeo.value().first; + th = capGeo.value().second; + } + if (grab_background(pbk, tw, th, bg)<0) throw "Failed to read background frame"; canBlur = true; } else if (blur_strength) { diff --git a/lib/libbackscrub.cc b/lib/libbackscrub.cc index fb69be5..ccf4663 100644 --- a/lib/libbackscrub.cc +++ b/lib/libbackscrub.cc @@ -365,7 +365,11 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { // scale up into full-sized mask cv::Mat tmpbuf; - cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size()); + // with body-pix-float-050-8.tflite the size of ctx.ofinal is 33x33 + // and the wanted roi may be greater as 33x33 so we can crash with + // cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size()); + ctx.ofinal.copyTo(tmpbuf); + cv::resize(tmpbuf,tmpbuf,ctx.mroi.size()); // blur at full size for maximum smoothness cv::blur(tmpbuf,ctx.mroi,ctx.blur); @@ -375,3 +379,22 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { return true; } +cv::Rect calcCropping(int cw, int ch, int vw, int vh) +{ + // if the input and output aspect reatio are not the same + // we can crop the source image. For example if the + // input image has a 16:9 (1280x720) ratio and the output is 4:3 (960x720) + // we will return the cropRegion set as x=160, width=960, y=0, height=720 + // which the centered part ofe the original image + cv::Rect cropRegion = {0,0,0,0}; + float sc = (float)vw / cw; + float st = (float)vh / ch; + sc = st > sc ? st : sc; + int sz = (int)(vw / sc) - cw; + cropRegion.x = (sz < 0 ? -sz : sz)/2; + sz = (int)(vh / sc) - ch; + cropRegion.y = (sz < 0 ? -sz : sz)/2; + cropRegion.width = cw - cropRegion.x*2; + cropRegion.height = ch - cropRegion.y*2; + return cropRegion; +} diff --git a/lib/libbackscrub.h b/lib/libbackscrub.h index 8eed1f0..f718fc4 100644 --- a/lib/libbackscrub.h +++ b/lib/libbackscrub.h @@ -38,4 +38,6 @@ extern void bs_maskgen_delete(void *context); // Process a video frame into a mask extern bool bs_maskgen_process(void *context, cv::Mat& frame, cv::Mat &mask); +extern cv::Rect calcCropping(int inWidth, int inHeight, int targetWidth, int targetHight); + #endif From 65627e8743eb6ea717131b47d0dade37407029e0 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Thu, 1 Sep 2022 17:22:56 +0200 Subject: [PATCH 02/26] Suggestions mostly added, a little bugfix --- app/background.cc | 10 ++++++---- app/deepseg.cc | 22 +++++++++++----------- lib/libbackscrub.cc | 23 +++++++++++++---------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/app/background.cc b/app/background.cc index 4e72ac3..4c59776 100644 --- a/app/background.cc +++ b/app/background.cc @@ -8,7 +8,9 @@ #include #include #include + #include + // Internal state of background processing struct background_t { int debug; @@ -183,14 +185,14 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width, height); + cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width,height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { // resize still image as requested into out - cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width, height); - cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); - out = pbkd->raw; + cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width,height); + cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); + out = pbkd->raw; frm = 1; } return frm; diff --git a/app/deepseg.cc b/app/deepseg.cc index 051c590..da32006 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -372,7 +372,7 @@ int main(int argc, char* argv[]) try { bool flipVertical = false; int fourcc = 0; size_t blur_strength = 0; - cv::Rect_ crop_region(0,0,0,0); + cv::Rect_ crop_region(0, 0, 0, 0); const char* modelname = "selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite"; @@ -573,10 +573,9 @@ int main(int argc, char* argv[]) try { if (expWidth != vidGeo.value().first && vidGeo.value().first <= capGeo.value().first && vidGeo.value().second <= capGeo.value().second) { - crop_region = calcCropping(capGeo.value().first, - capGeo.value().second, - vidGeo.value().first, - vidGeo.value().second); + crop_region = calcCropping( + capGeo.value().first, capGeo.value().second, + vidGeo.value().first, vidGeo.value().second); } // dump settings.. @@ -612,11 +611,11 @@ int main(int argc, char* argv[]) try { // default green screen background (at capture true geometry) cv::Mat bg; if ( crop_region.height == 0) { - bg = cv::Mat(capGeo.value().second, capGeo.value().first, CV_8UC3, cv::Scalar(0, 255, 0)); + bg = cv::Mat(capGeo.value().first, capGeo.value().second, CV_8UC3, cv::Scalar(0, 255, 0)); } else { bg = cv::Mat(crop_region.width, crop_region.height, CV_8UC3, cv::Scalar(0, 255, 0)); } - + // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); if(lbfd < 0) { @@ -663,7 +662,6 @@ int main(int argc, char* argv[]) try { // copy new frame to buffer cap.retrieve(raw); ti.retrns = timestamp(); - ti.copyns = timestamp(); if (raw.rows == 0 || raw.cols == 0) continue; // sanity check @@ -671,11 +669,13 @@ int main(int argc, char* argv[]) try { raw((cv::Rect_)crop_region).copyTo(raw); } ai.set_input_frame(raw); + ti.copyns = timestamp(); - if (filterActive) { - // do background detection magic - ai.get_output_mask(mask); + // do background detection magic + ai.get_output_mask(mask); + ti.copyns = timestamp(); + if (filterActive) { // get background frame: // - specified source if set // - copy of input video if blur_strength != 0 diff --git a/lib/libbackscrub.cc b/lib/libbackscrub.cc index ccf4663..735ccae 100644 --- a/lib/libbackscrub.cc +++ b/lib/libbackscrub.cc @@ -369,7 +369,7 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { // and the wanted roi may be greater as 33x33 so we can crash with // cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size()); ctx.ofinal.copyTo(tmpbuf); - cv::resize(tmpbuf,tmpbuf,ctx.mroi.size()); + cv::resize(tmpbuf, tmpbuf, ctx.mroi.size()); // blur at full size for maximum smoothness cv::blur(tmpbuf,ctx.mroi,ctx.blur); @@ -379,22 +379,25 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { return true; } -cv::Rect calcCropping(int cw, int ch, int vw, int vh) -{ - // if the input and output aspect reatio are not the same +cv::Rect calcCropping(int cw, int ch, int vw, int vh) { + // if the input and output aspect ratio are not the same // we can crop the source image. For example if the // input image has a 16:9 (1280x720) ratio and the output is 4:3 (960x720) // we will return the cropRegion set as x=160, width=960, y=0, height=720 - // which the centered part ofe the original image - cv::Rect cropRegion = {0,0,0,0}; + // which is the centered part of the original image + cv::Rect cropRegion = {0, 0, 0, 0}; float sc = (float)vw / cw; float st = (float)vh / ch; sc = st > sc ? st : sc; - int sz = (int)(vw / sc) - cw; - cropRegion.x = (sz < 0 ? -sz : sz)/2; - sz = (int)(vh / sc) - ch; - cropRegion.y = (sz < 0 ? -sz : sz)/2; + + int sx = (int)(vw / sc) - cw; + cropRegion.x = (sx < 0 ? -sx : sx)/2; + + int sy = (int)(vh / sc) - ch; + cropRegion.y = (sy < 0 ? -sy : sy)/2; + cropRegion.width = cw - cropRegion.x*2; cropRegion.height = ch - cropRegion.y*2; + return cropRegion; } From 7b416728be83e5be490fed0a4104c6a1ffa3f5fa Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 07:24:03 +0200 Subject: [PATCH 03/26] Suggestions mostly added, a little bugfix --- app/deepseg.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index da32006..c15553d 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -629,8 +629,8 @@ int main(int argc, char* argv[]) try { // Processing components, all at capture true geometry - cv::Mat mask; - if ( crop_region.height ) { + cv::Mat mask; + if ( crop_region.height ) { cv::Mat masktmp((int)(crop_region.height), (int)(crop_region.width), CV_8U); mask = masktmp; } else { From cb0fe4f2f02ca0ebfd409f9f6720e7893b73df24 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 08:13:28 +0200 Subject: [PATCH 04/26] fix typo --- lib/libbackscrub.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/libbackscrub.cc b/lib/libbackscrub.cc index 735ccae..7e3603e 100644 --- a/lib/libbackscrub.cc +++ b/lib/libbackscrub.cc @@ -391,13 +391,13 @@ cv::Rect calcCropping(int cw, int ch, int vw, int vh) { sc = st > sc ? st : sc; int sx = (int)(vw / sc) - cw; - cropRegion.x = (sx < 0 ? -sx : sx)/2; + cropRegion.x = (sx < 0 ? -sx : sx) / 2; int sy = (int)(vh / sc) - ch; - cropRegion.y = (sy < 0 ? -sy : sy)/2; + cropRegion.y = (sy < 0 ? -sy : sy) / 2; - cropRegion.width = cw - cropRegion.x*2; - cropRegion.height = ch - cropRegion.y*2; + cropRegion.width = cw - cropRegion.x * 2; + cropRegion.height = ch - cropRegion.y * 2; return cropRegion; } From 3eeb58bbfa46d716403334becef05ccbb8166321 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 09:02:34 +0200 Subject: [PATCH 05/26] typo correction --- app/background.cc | 2 +- app/deepseg.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/background.cc b/app/background.cc index 4c59776..e7bbcb7 100644 --- a/app/background.cc +++ b/app/background.cc @@ -185,7 +185,7 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width,height); + cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width,height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { diff --git a/app/deepseg.cc b/app/deepseg.cc index c15553d..aa63f55 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -615,7 +615,7 @@ int main(int argc, char* argv[]) try { } else { bg = cv::Mat(crop_region.width, crop_region.height, CV_8UC3, cv::Scalar(0, 255, 0)); } - + // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); if(lbfd < 0) { @@ -690,7 +690,7 @@ int main(int argc, char* argv[]) try { tw = capGeo.value().first; th = capGeo.value().second; } - if (grab_background(pbk, tw, th, bg)<0) + if (grab_background(pbk, tw, th, bg) < 0) throw "Failed to read background frame"; canBlur = true; } else if (blur_strength) { From 01e9a755f058becd7884e7b90f66a572946b0cf4 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 15:39:25 +0200 Subject: [PATCH 06/26] fix typo --- app/background.cc | 4 ++-- app/deepseg.cc | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/background.cc b/app/background.cc index e7bbcb7..52a4dce 100644 --- a/app/background.cc +++ b/app/background.cc @@ -185,12 +185,12 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width,height); + cv::Rect_ crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { // resize still image as requested into out - cv::Rect_ crop = calcCropping(pbkd->raw.cols,pbkd->raw.rows,width,height); + cv::Rect_ crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); out = pbkd->raw; frm = 1; diff --git a/app/deepseg.cc b/app/deepseg.cc index aa63f55..b5e6c0f 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -627,7 +627,6 @@ int main(int argc, char* argv[]) try { loopback_free(lbfd); }); - // Processing components, all at capture true geometry cv::Mat mask; if ( crop_region.height ) { From f81e26e1585164d9e4a4a8b748a5bb1fc4cf43a2 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 20:31:04 +0200 Subject: [PATCH 07/26] Syntax modified --- app/deepseg.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index b5e6c0f..7eb44bf 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -609,12 +609,11 @@ int main(int argc, char* argv[]) try { } } // default green screen background (at capture true geometry) - cv::Mat bg; + std::optional> bg_dim = capGeo; if ( crop_region.height == 0) { - bg = cv::Mat(capGeo.value().first, capGeo.value().second, CV_8UC3, cv::Scalar(0, 255, 0)); - } else { - bg = cv::Mat(crop_region.width, crop_region.height, CV_8UC3, cv::Scalar(0, 255, 0)); + bg_dim = {crop_region.width, crop_region.height}; } + cv::Mat bg(bg_dim.value().first, bg_dim.value().second, CV_8UC3, cv::Scalar(0, 255, 0)); // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); @@ -628,14 +627,11 @@ int main(int argc, char* argv[]) try { }); // Processing components, all at capture true geometry - cv::Mat mask; + std::optional> mask_dim = capGeo; if ( crop_region.height ) { - cv::Mat masktmp((int)(crop_region.height), (int)(crop_region.width), CV_8U); - mask = masktmp; - } else { - cv::Mat masktmp(capGeo.value().second, capGeo.value().first, CV_8U); - mask = masktmp; + mask_dim = {crop_region.width, crop_region.height}; } + cv::Mat mask(mask_dim.value().second, mask_dim.value().first, CV_8U); cv::Mat raw; int aiw,aih; From 112e979702b784349703d410e0b5b35c50876a50 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 20:35:49 +0200 Subject: [PATCH 08/26] Syntax modified --- app/deepseg.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index 7eb44bf..6df0c5f 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -613,7 +613,7 @@ int main(int argc, char* argv[]) try { if ( crop_region.height == 0) { bg_dim = {crop_region.width, crop_region.height}; } - cv::Mat bg(bg_dim.value().first, bg_dim.value().second, CV_8UC3, cv::Scalar(0, 255, 0)); + cv::Mat bg(bg_dim.value().second, bg_dim.value().first, CV_8UC3, cv::Scalar(0, 255, 0)); // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); From 22472ad7135870f3a04d4df5054779f173495154 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 20:48:52 +0200 Subject: [PATCH 09/26] correction --- app/deepseg.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index 6df0c5f..ae28af4 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -372,7 +372,7 @@ int main(int argc, char* argv[]) try { bool flipVertical = false; int fourcc = 0; size_t blur_strength = 0; - cv::Rect_ crop_region(0, 0, 0, 0); + cv::Rect crop_region(0, 0, 0, 0); const char* modelname = "selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite"; From 7dc9bccfe4fe1dc8ce33c3495bc21f62aca77a64 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 21:15:08 +0200 Subject: [PATCH 10/26] coorection as suggested --- app/deepseg.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index ae28af4..a653797 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -661,7 +661,7 @@ int main(int argc, char* argv[]) try { if (raw.rows == 0 || raw.cols == 0) continue; // sanity check if ( crop_region.height) { - raw((cv::Rect_)crop_region).copyTo(raw); + raw(crop_region).copyTo(raw); } ai.set_input_frame(raw); ti.copyns = timestamp(); From 6894b3e21930ec4ffc70156c29b8285ba74e2dff Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Fri, 2 Sep 2022 21:17:17 +0200 Subject: [PATCH 11/26] coorection as suggested --- app/background.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/background.cc b/app/background.cc index 52a4dce..b283d3a 100644 --- a/app/background.cc +++ b/app/background.cc @@ -185,12 +185,12 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::Rect_ crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { // resize still image as requested into out - cv::Rect_ crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); out = pbkd->raw; frm = 1; From 1769f7e3de8e7ea3b6de447976b5c30f4f50fcb8 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sat, 3 Sep 2022 13:13:43 +0200 Subject: [PATCH 12/26] suggestion processed --- app/deepseg.cc | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index a653797..9590ce5 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -570,12 +570,12 @@ int main(int argc, char* argv[]) try { fprintf(stderr, "Warning: virtual camera aspect ratio does not match capture device.\n"); } // calculate crop region, only if result always smaller - if (expWidth != vidGeo.value().first && - vidGeo.value().first <= capGeo.value().first && - vidGeo.value().second <= capGeo.value().second) { + if (expWidth != vidGeo->first && + vidGeo->first <= capGeo->first && + vidGeo->second <= capGeo->second) { crop_region = calcCropping( - capGeo.value().first, capGeo.value().second, - vidGeo.value().first, vidGeo.value().second); + capGeo->first, capGeo->second, + vidGeo->first, vidGeo->second); } // dump settings.. @@ -609,11 +609,11 @@ int main(int argc, char* argv[]) try { } } // default green screen background (at capture true geometry) - std::optional> bg_dim = capGeo; - if ( crop_region.height == 0) { + std::pair bg_dim = *capGeo; + if (!crop_region.height) { bg_dim = {crop_region.width, crop_region.height}; } - cv::Mat bg(bg_dim.value().second, bg_dim.value().first, CV_8UC3, cv::Scalar(0, 255, 0)); + cv::Mat bg(bg_dim.second, bg_dim.first, CV_8UC3, cv::Scalar(0, 255, 0)); // Virtual camera (at specified geometry) int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug); @@ -627,22 +627,22 @@ int main(int argc, char* argv[]) try { }); // Processing components, all at capture true geometry - std::optional> mask_dim = capGeo; - if ( crop_region.height ) { + std::pair mask_dim = *capGeo; + if (crop_region.height) { mask_dim = {crop_region.width, crop_region.height}; } - cv::Mat mask(mask_dim.value().second, mask_dim.value().first, CV_8U); + cv::Mat mask(mask_dim.second, mask_dim.first, CV_8U); cv::Mat raw; int aiw,aih; - if ( crop_region.width == 0) { - aiw=capGeo.value().first; - aih=capGeo.value().second; + if (!crop_region.width) { + aiw=capGeo->first; + aih=capGeo->second; } else { aiw=crop_region.width; aih=crop_region.height; } - CalcMask ai(s_model.value(), threads, aiw, aih); + CalcMask ai(*s_model, threads, aiw, aih); ti.lastns = timestamp(); printf("Startup: %ldns\n", diffnanosecs(ti.lastns,ti.bootns)); @@ -660,7 +660,7 @@ int main(int argc, char* argv[]) try { if (raw.rows == 0 || raw.cols == 0) continue; // sanity check - if ( crop_region.height) { + if (crop_region.height) { raw(crop_region).copyTo(raw); } ai.set_input_frame(raw); @@ -682,8 +682,8 @@ int main(int argc, char* argv[]) try { tw = crop_region.width; th = crop_region.height; } else { - tw = capGeo.value().first; - th = capGeo.value().second; + tw = capGeo->first; + th = capGeo->second; } if (grab_background(pbk, tw, th, bg) < 0) throw "Failed to read background frame"; From e9a5242dc4242bfd8c82f338a209c9822d9136f9 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 09:16:51 +0200 Subject: [PATCH 13/26] bug fixed --- app/background.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/background.cc b/app/background.cc index b283d3a..9d88b55 100644 --- a/app/background.cc +++ b/app/background.cc @@ -192,7 +192,7 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c // resize still image as requested into out cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); - out = pbkd->raw; + pbkd->raw = out; frm = 1; } return frm; From 0f62a5728fcf45e942f49a4434c531f1d97d4fc8 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 09:35:31 +0200 Subject: [PATCH 14/26] bug fixed --- app/background.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/background.cc b/app/background.cc index 9d88b55..e44dd8a 100644 --- a/app/background.cc +++ b/app/background.cc @@ -191,8 +191,7 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c } else { // resize still image as requested into out cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); - pbkd->raw = out; + cv::resize(pbkd->raw(crop), pbkd->raw, cv::Size(width, height)); frm = 1; } return frm; From 81c1c839f7433ff597fc40de258026c6d4ea61fc Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 13:11:59 +0200 Subject: [PATCH 15/26] bug fix --- app/background.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/background.cc b/app/background.cc index e44dd8a..3d49d16 100644 --- a/app/background.cc +++ b/app/background.cc @@ -191,7 +191,10 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c } else { // resize still image as requested into out cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - cv::resize(pbkd->raw(crop), pbkd->raw, cv::Size(width, height)); + // under some circumstances we must do the job with 2 step! + // without this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. + cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); + pbkd->raw = out; frm = 1; } return frm; From 4c5f72c7e301fd2c6c8330a59bcfa6378764117b Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 13:24:45 +0200 Subject: [PATCH 16/26] bug fix --- app/background.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/background.cc b/app/background.cc index 3d49d16..9d88b55 100644 --- a/app/background.cc +++ b/app/background.cc @@ -191,8 +191,6 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c } else { // resize still image as requested into out cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - // under some circumstances we must do the job with 2 step! - // without this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); pbkd->raw = out; frm = 1; From ffb98833a0880d39de0b55340dcc97769ec68202 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 13:50:35 +0200 Subject: [PATCH 17/26] type --- app/background.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/background.cc b/app/background.cc index 9d88b55..3b5d8ac 100644 --- a/app/background.cc +++ b/app/background.cc @@ -191,6 +191,8 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c } else { // resize still image as requested into out cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + // Under some circumstances we must do the job in two steps! + // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); pbkd->raw = out; frm = 1; From 7353ff8673f9c528f1f34aa5d44e505d26ca1d46 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 13:53:07 +0200 Subject: [PATCH 18/26] typo --- app/background.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/background.cc b/app/background.cc index 3b5d8ac..9adab6b 100644 --- a/app/background.cc +++ b/app/background.cc @@ -191,8 +191,8 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c } else { // resize still image as requested into out cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - // Under some circumstances we must do the job in two steps! - // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. + // Under some circumstances we must do the job in two steps! + // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); pbkd->raw = out; frm = 1; From 5a92b93d3e57a49325fa09f9422c0fa8ee6f7efe Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 4 Sep 2022 15:48:47 +0200 Subject: [PATCH 19/26] typo --- app/background.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/background.cc b/app/background.cc index 9adab6b..a701e4d 100644 --- a/app/background.cc +++ b/app/background.cc @@ -193,7 +193,8 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); // Under some circumstances we must do the job in two steps! // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. - cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); + pbkd->raw(crop).copyTo(out); + cv::resize(out, out, cv::Size(width, height)); pbkd->raw = out; frm = 1; } From 5ee0d4a6486f800450d31632eb82d8bb86a26f6b Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Thu, 8 Sep 2022 20:25:14 +0200 Subject: [PATCH 20/26] bug fixed --- app/deepseg.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/deepseg.cc b/app/deepseg.cc index 9590ce5..c996330 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -610,7 +610,7 @@ int main(int argc, char* argv[]) try { } // default green screen background (at capture true geometry) std::pair bg_dim = *capGeo; - if (!crop_region.height) { + if (crop_region.height) { bg_dim = {crop_region.width, crop_region.height}; } cv::Mat bg(bg_dim.second, bg_dim.first, CV_8UC3, cv::Scalar(0, 255, 0)); @@ -678,7 +678,7 @@ int main(int argc, char* argv[]) try { bool canBlur = false; if (pbk) { int tw,th; - if ( crop_region.height ) { + if (crop_region.height) { tw = crop_region.width; th = crop_region.height; } else { From 44dbb6aa34e8c30a813152504a6838e79dc25a63 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sat, 10 Sep 2022 18:30:26 +0200 Subject: [PATCH 21/26] Fix and suggestion considered --- app/background.cc | 30 +++++++++++++++++++----------- app/deepseg.cc | 12 +++++------- lib/libbackscrub.cc | 15 +++++++-------- lib/libbackscrub.h | 2 +- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/app/background.cc b/app/background.cc index a701e4d..2e0948f 100644 --- a/app/background.cc +++ b/app/background.cc @@ -20,6 +20,7 @@ struct background_t { int frame; double fps; cv::Mat raw; + int bg_stored; std::mutex rawmux; cv::Mat thumb; std::mutex thumbmux; @@ -132,6 +133,7 @@ std::shared_ptr load_background(const std::string& path, int debug pbkd->debug = debug; pbkd->video = false; pbkd->run = false; + pbkd->bg_stored = false; pbkd->cap.open(path, cv::CAP_ANY); // explicitly ask for auto-detection of backend if (!pbkd->cap.isOpened()) { if (pbkd->debug) fprintf(stderr, "background: cap cannot open: %s\n", path.c_str()); @@ -145,7 +147,9 @@ std::shared_ptr load_background(const std::string& path, int debug // if: can read 2 video frames => it's a video // else: is loaded as an image => it's an image // else: it's not usable. - if (pbkd->cap.read(pbkd->raw) && pbkd->cap.read(pbkd->raw)) { +// if (pbkd->cap.read(pbkd->raw) && pbkd->cap.read(pbkd->raw)) { +printf("pbkd->fps %g cnt %d\n", pbkd->fps,cnt); + if (cnt > -1) { // it's a video, try a reset and start reader thread.. if (pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0)) pbkd->frame = 0; @@ -156,6 +160,7 @@ std::shared_ptr load_background(const std::string& path, int debug pbkd->thread = std::thread(read_thread, std::weak_ptr(pbkd)); } else { // static image file, try loading.. +printf("Read image\n"); pbkd->cap.release(); pbkd->raw = cv::imread(path); if (pbkd->raw.empty()) { @@ -185,19 +190,22 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { - // resize still image as requested into out - cv::Rect crop = calcCropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - // Under some circumstances we must do the job in two steps! - // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. - pbkd->raw(crop).copyTo(out); - cv::resize(out, out, cv::Size(width, height)); - pbkd->raw = out; - frm = 1; - } + if (!pbkd->bg_stored) { + // resize still image as requested into out + cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + // Under some circumstances we must do the job in two steps! + // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. + pbkd->raw(crop).copyTo(pbkd->raw); + cv::resize(pbkd->raw, pbkd->raw, cv::Size(width, height)); + pbkd->bg_stored = true; + } + out = pbkd->raw ; + frm = 1; + } return frm; } diff --git a/app/deepseg.cc b/app/deepseg.cc index c996330..e8e2288 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -84,7 +84,7 @@ std::optional> geometryFromString(const std::string& i } // OpenCV helper functions -cv::Mat convert_rgb_to_yuyv( cv::Mat input ) { +cv::Mat convert_rgb_to_yuyv(cv::Mat input) { cv::Mat tmp; cv::cvtColor(input, tmp, cv::COLOR_RGB2YUV); std::vector yuv; @@ -570,12 +570,10 @@ int main(int argc, char* argv[]) try { fprintf(stderr, "Warning: virtual camera aspect ratio does not match capture device.\n"); } // calculate crop region, only if result always smaller - if (expWidth != vidGeo->first && - vidGeo->first <= capGeo->first && - vidGeo->second <= capGeo->second) { - crop_region = calcCropping( - capGeo->first, capGeo->second, - vidGeo->first, vidGeo->second); + if (expWidth != vidGeo->first) { + crop_region = bs_calc_cropping( + capGeo->first, capGeo->second, + vidGeo->first, vidGeo->second); } // dump settings.. diff --git a/lib/libbackscrub.cc b/lib/libbackscrub.cc index 7e3603e..7acf82d 100644 --- a/lib/libbackscrub.cc +++ b/lib/libbackscrub.cc @@ -379,25 +379,24 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) { return true; } -cv::Rect calcCropping(int cw, int ch, int vw, int vh) { +cv::Rect bs_calc_cropping(int inWidth, int inHeight, int targetWidth, int targetHigh) { // if the input and output aspect ratio are not the same // we can crop the source image. For example if the // input image has a 16:9 (1280x720) ratio and the output is 4:3 (960x720) // we will return the cropRegion set as x=160, width=960, y=0, height=720 // which is the centered part of the original image cv::Rect cropRegion = {0, 0, 0, 0}; - float sc = (float)vw / cw; - float st = (float)vh / ch; + float sc = (float)targetWidth / inWidth; + float st = (float)targetHigh / inHeight; sc = st > sc ? st : sc; - int sx = (int)(vw / sc) - cw; + int sx = (int)(targetWidth / sc) - inWidth; cropRegion.x = (sx < 0 ? -sx : sx) / 2; - int sy = (int)(vh / sc) - ch; + int sy = (int)(targetHigh / sc) - inHeight; cropRegion.y = (sy < 0 ? -sy : sy) / 2; - cropRegion.width = cw - cropRegion.x * 2; - cropRegion.height = ch - cropRegion.y * 2; - + cropRegion.width = inWidth - cropRegion.x * 2; + cropRegion.height = inHeight - cropRegion.y * 2; return cropRegion; } diff --git a/lib/libbackscrub.h b/lib/libbackscrub.h index f718fc4..08a4e13 100644 --- a/lib/libbackscrub.h +++ b/lib/libbackscrub.h @@ -38,6 +38,6 @@ extern void bs_maskgen_delete(void *context); // Process a video frame into a mask extern bool bs_maskgen_process(void *context, cv::Mat& frame, cv::Mat &mask); -extern cv::Rect calcCropping(int inWidth, int inHeight, int targetWidth, int targetHight); +extern cv::Rect bs_calc_cropping(int inWidth, int inHeight, int targetWidth, int targetHight); #endif From 4c6431ec74978d8a79fe29ec19aa7de9b8db9b7c Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sat, 10 Sep 2022 19:00:22 +0200 Subject: [PATCH 22/26] little cleanup --- app/background.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/background.cc b/app/background.cc index 2e0948f..6d86bf5 100644 --- a/app/background.cc +++ b/app/background.cc @@ -147,8 +147,6 @@ std::shared_ptr load_background(const std::string& path, int debug // if: can read 2 video frames => it's a video // else: is loaded as an image => it's an image // else: it's not usable. -// if (pbkd->cap.read(pbkd->raw) && pbkd->cap.read(pbkd->raw)) { -printf("pbkd->fps %g cnt %d\n", pbkd->fps,cnt); if (cnt > -1) { // it's a video, try a reset and start reader thread.. if (pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0)) @@ -160,7 +158,6 @@ printf("pbkd->fps %g cnt %d\n", pbkd->fps,cnt); pbkd->thread = std::thread(read_thread, std::weak_ptr(pbkd)); } else { // static image file, try loading.. -printf("Read image\n"); pbkd->cap.release(); pbkd->raw = cv::imread(path); if (pbkd->raw.empty()) { From 975b96fac7d09ad15be998af65a2e4c12b102189 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 11 Sep 2022 20:06:32 +0200 Subject: [PATCH 23/26] little corrections --- app/background.cc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/app/background.cc b/app/background.cc index 6d86bf5..963e36f 100644 --- a/app/background.cc +++ b/app/background.cc @@ -20,7 +20,7 @@ struct background_t { int frame; double fps; cv::Mat raw; - int bg_stored; + bool bg_stored; std::mutex rawmux; cv::Mat thumb; std::mutex thumbmux; @@ -147,7 +147,7 @@ std::shared_ptr load_background(const std::string& path, int debug // if: can read 2 video frames => it's a video // else: is loaded as an image => it's an image // else: it's not usable. - if (cnt > -1) { + if (pbkd->fps > 0) { // it's a video, try a reset and start reader thread.. if (pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0)) pbkd->frame = 0; @@ -191,18 +191,18 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); frm = pbkd->frame; } else { - if (!pbkd->bg_stored) { - // resize still image as requested into out - cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - // Under some circumstances we must do the job in two steps! - // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. - pbkd->raw(crop).copyTo(pbkd->raw); - cv::resize(pbkd->raw, pbkd->raw, cv::Size(width, height)); - pbkd->bg_stored = true; - } - out = pbkd->raw ; - frm = 1; - } + if (!pbkd->bg_stored) { + // resize still image as requested into out + cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + // Under some circumstances we must do the job in two steps! + // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail. + pbkd->raw(crop).copyTo(pbkd->raw); + cv::resize(pbkd->raw, pbkd->raw, cv::Size(width, height)); + pbkd->bg_stored = true; + } + out = pbkd->raw ; + frm = 1; + } return frm; } From 481cb47e2017a34dace54cbb22f53fd675ffc9a9 Mon Sep 17 00:00:00 2001 From: Jean-Jacques Date: Sun, 11 Sep 2022 20:50:51 +0200 Subject: [PATCH 24/26] little corrections --- app/background.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/background.cc b/app/background.cc index 963e36f..4c65590 100644 --- a/app/background.cc +++ b/app/background.cc @@ -200,7 +200,7 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c cv::resize(pbkd->raw, pbkd->raw, cv::Size(width, height)); pbkd->bg_stored = true; } - out = pbkd->raw ; + out = pbkd->raw; frm = 1; } return frm; From f80a8a6a73eb4e56b0d9ded439e3a5921c9b4d02 Mon Sep 17 00:00:00 2001 From: "jj.sarton@jjsarton.de" Date: Sat, 1 Oct 2022 13:03:20 +0200 Subject: [PATCH 25/26] bettet video/image detection, gstreamer support foranimated gif --- app/background.cc | 54 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/app/background.cc b/app/background.cc index 4c65590..40fc50a 100644 --- a/app/background.cc +++ b/app/background.cc @@ -11,6 +11,14 @@ #include +#if CV_VERSION_MAJOR < 4 ||\ + CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR < 5 ||\ + CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR == 5 && CV_VERSION_REVISION < 3 +# define HAVE_IMCOUNT 0 +#else +# define HAVE_IMCOUNT 1 +#endif + // Internal state of background processing struct background_t { int debug; @@ -18,6 +26,7 @@ struct background_t { volatile bool run; cv::VideoCapture cap; int frame; + int cnt; double fps; cv::Mat raw; bool bg_stored; @@ -61,6 +70,11 @@ static void read_thread(std::weak_ptr weak) { grab.copyTo(pbkd->raw); pbkd->frame += 1; } + // if last frame reached (gstreamer and GIF file) set to frame 0 + if (pbkd->frame == pbkd->cnt) { + pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0); + pbkd->frame = 0; + } // grab timing point auto now = std::chrono::steady_clock::now(); // generate thumbnail frame with overlay info if double debug enabled @@ -134,6 +148,12 @@ std::shared_ptr load_background(const std::string& path, int debug pbkd->video = false; pbkd->run = false; pbkd->bg_stored = false; + +#if HAVE_IMCOUNT + size_t images = cv::imcount(path.c_str()); +#else + size_t images = 0; +#endif pbkd->cap.open(path, cv::CAP_ANY); // explicitly ask for auto-detection of backend if (!pbkd->cap.isOpened()) { if (pbkd->debug) fprintf(stderr, "background: cap cannot open: %s\n", path.c_str()); @@ -143,23 +163,39 @@ std::shared_ptr load_background(const std::string& path, int debug int fcc = (int)pbkd->cap.get(cv::CAP_PROP_FOURCC); pbkd->fps = pbkd->cap.get(cv::CAP_PROP_FPS); int cnt = (int)pbkd->cap.get(cv::CAP_PROP_FRAME_COUNT); + pbkd->cnt = cnt; +#if !HAVE_IMCOUNT + if ( cnt < 0 ) + images = 1; +#endif // Here be the logic... - // if: can read 2 video frames => it's a video - // else: is loaded as an image => it's an image + // if: count > 1 it's a video + // else: if cnt < 1 it´s is a still image + // else: if cnt = 1 it´s is a static gif file // else: it's not usable. - if (pbkd->fps > 0) { + if (cnt > 1) { + if (!pbkd->cap.read(pbkd->raw)) { + if (pbkd->debug) fprintf(stderr, "background: read failed %s\n", path.c_str()); + return nullptr; + } // it's a video, try a reset and start reader thread.. if (pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0)) pbkd->frame = 0; else - pbkd->frame = 2; // unable to reset, so we're 2 frames in + pbkd->frame = 1; // unable to reset, so we're 1 frames in pbkd->video = true; pbkd->run = true; pbkd->thread = std::thread(read_thread, std::weak_ptr(pbkd)); } else { // static image file, try loading.. - pbkd->cap.release(); - pbkd->raw = cv::imread(path); + if (cnt < 0) { + pbkd->cap.release(); + pbkd->raw = cv::imread(path); + } else if (cnt == 1) { + // this is a static gif file with 1 frame + pbkd->cap.read(pbkd->raw); + pbkd->cap.release(); + } if (pbkd->raw.empty()) { if (pbkd->debug) fprintf(stderr, "background: imread cannot open: %s\n", path.c_str()); return nullptr; @@ -183,12 +219,14 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (!pbkd) return -1; // static image or video? - int frm ; + int frm; if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - cv::resize(pbkd->raw(crop), out, cv::Size(width, height)); + cv::Mat tmp; + pbkd->raw(crop).copyTo(tmp); + cv::resize(tmp, out, cv::Size(width, height)); frm = pbkd->frame; } else { if (!pbkd->bg_stored) { From ea1e35027c0756cdaa8f812eacc9e62f6e674d94 Mon Sep 17 00:00:00 2001 From: "jj.sarton@jjsarton.de" Date: Mon, 10 Oct 2022 16:08:23 +0200 Subject: [PATCH 26/26] better crop handling --- app/background.cc | 17 +++++++++++++---- app/deepseg.cc | 16 ++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/app/background.cc b/app/background.cc index 40fc50a..d3cc8b0 100644 --- a/app/background.cc +++ b/app/background.cc @@ -28,6 +28,7 @@ struct background_t { int frame; int cnt; double fps; + cv::Rect cropRegion; cv::Mat raw; bool bg_stored; std::mutex rawmux; @@ -148,6 +149,7 @@ std::shared_ptr load_background(const std::string& path, int debug pbkd->video = false; pbkd->run = false; pbkd->bg_stored = false; + pbkd->cropRegion = {-1, -1, -1, -1}; #if HAVE_IMCOUNT size_t images = cv::imcount(path.c_str()); @@ -223,10 +225,17 @@ int grab_background(std::shared_ptr pbkd, int width, int height, c if (pbkd->video) { // grab frame & frame no. under mutex std::unique_lock hold(pbkd->rawmux); - cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); - cv::Mat tmp; - pbkd->raw(crop).copyTo(tmp); - cv::resize(tmp, out, cv::Size(width, height)); + if ( pbkd->cropRegion.x < 0 ) { // calculate cropping if not already done + pbkd->cropRegion = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height); + } + // Perform cropping if necessary + if ( pbkd->cropRegion.x || pbkd->cropRegion.y ) { + cv::Mat tmp; + pbkd->raw(pbkd->cropRegion).copyTo(tmp); + cv::resize(tmp, out, cv::Size(width, height)); + } else { + cv::resize(pbkd->raw, out, cv::Size(width, height)); + } frm = pbkd->frame; } else { if (!pbkd->bg_stored) { diff --git a/app/deepseg.cc b/app/deepseg.cc index e8e2288..1a2d195 100644 --- a/app/deepseg.cc +++ b/app/deepseg.cc @@ -570,10 +570,10 @@ int main(int argc, char* argv[]) try { fprintf(stderr, "Warning: virtual camera aspect ratio does not match capture device.\n"); } // calculate crop region, only if result always smaller - if (expWidth != vidGeo->first) { + if (capGeo != vidGeo) { crop_region = bs_calc_cropping( - capGeo->first, capGeo->second, - vidGeo->first, vidGeo->second); + capGeo->first, capGeo->second, + vidGeo->first, vidGeo->second); } // dump settings.. @@ -608,7 +608,7 @@ int main(int argc, char* argv[]) try { } // default green screen background (at capture true geometry) std::pair bg_dim = *capGeo; - if (crop_region.height) { + if (crop_region.x || crop_region.y) { bg_dim = {crop_region.width, crop_region.height}; } cv::Mat bg(bg_dim.second, bg_dim.first, CV_8UC3, cv::Scalar(0, 255, 0)); @@ -626,14 +626,14 @@ int main(int argc, char* argv[]) try { // Processing components, all at capture true geometry std::pair mask_dim = *capGeo; - if (crop_region.height) { + if (crop_region.x || crop_region.y) { mask_dim = {crop_region.width, crop_region.height}; } cv::Mat mask(mask_dim.second, mask_dim.first, CV_8U); cv::Mat raw; int aiw,aih; - if (!crop_region.width) { + if (!(crop_region.x || crop_region.y)) { aiw=capGeo->first; aih=capGeo->second; } else { @@ -658,7 +658,7 @@ int main(int argc, char* argv[]) try { if (raw.rows == 0 || raw.cols == 0) continue; // sanity check - if (crop_region.height) { + if (crop_region.x || crop_region.y) { raw(crop_region).copyTo(raw); } ai.set_input_frame(raw); @@ -676,7 +676,7 @@ int main(int argc, char* argv[]) try { bool canBlur = false; if (pbk) { int tw,th; - if (crop_region.height) { + if (crop_region.x || crop_region.y) { tw = crop_region.width; th = crop_region.height; } else {