Facing issues while giving out Color Image

[ashfaquekhan]

I was using the "cvapp_hello_world_cmsis_cv" and trying to output a color image and change resolution, but I am not able to do that, I modifies the program as below, a bit to do so.

I am a beginner and trying to learn, and have to say this code base is really overwhelming for me, but I look upto it as I am learning a lot about Embedded system , and programming in general from this code base ( mad respect for the devs who have built this product)

I just have these questions:

• What should I start following[like a doc or something] to understand the methods, api's or programs and structure, and should I use the HiMax Wiseye 2 Datasheet ?
• How to just simply capture and send color images to the local web app [provided by you guys] ?
• I am kind of understanding the DSP libs and other Libs and how to implement them in your program and build a firmware , BUT what would be your suggestions: as I am working to implement a visual odometry/ optical flow using a camera, are there any libs other than DSPlib[cmsis] for feature tracking and etc as the cvlib[cmsis] has only for edge and not for pattern matching/block matching ?

I would really appreciate any help you provide 🙏🙏

`#include <cstdio>
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include "WE2_device.h"
#include "board.h"
#include "cvapp_hello_world_cmsis_cv.h"
#include "cisdp_sensor.h"

#include "WE2_core.h"

#include "img_proc_helium.h"


#include "xprintf.h"
#include "spi_master_protocol.h"
#include "cisdp_cfg.h"
#include "memory_manage.h"
#include "send_result.h"

#include "arm_cv.h"
#include "dsp/support_functions.h"
#include "dsp/complex_math_functions.h"
#include "spi_master_protocol.h"
#define ImageOut_WIDTH_SIZE 320
#define ImageOut_HEIGHT_SIZE 240
#define ImageOut_BUFSIZE  (ImageOut_WIDTH_SIZE*ImageOut_HEIGHT_SIZE) 
#define SRAM2_START_ADDR 0x36000000
/**
 * RUN_CANNY_SOBEL_ALGO
 * 1: run the canny sobel algorithm and send the canny sobel result jpeg image out
 * 0: send the guassion filter result jpeg image out
 * **/
#define RUN_CANNY_SOBEL_ALGO 0

#include "JPEGENC.h"




__attribute__(( section(".bss.NoInit"))) uint8_t o_uint8_buf[ImageOut_BUFSIZE] __ALIGNED(32);
__attribute__(( section(".bss.NoInit"))) uint8_t o_jpeg_uint8_buf[ImageOut_BUFSIZE] __ALIGNED(32);


uint32_t o_canny_sobel_buff = (uint32_t)SRAM2_START_ADDR; 
uint32_t str_assign_buf = (uint32_t)(o_canny_sobel_buff + ImageOut_BUFSIZE*sizeof(uint8_t));


#define  EACH_STEP_TICK 0
#define TOTAL_STEP_TICK 1



using namespace std;

namespace {


};

#define CPU_CLK	0xffffff+1
uint32_t systick_1, systick_2;
uint32_t loop_cnt_1, loop_cnt_2;
static uint32_t capture_image_tick = 0;

static q15_t *gaussian_tmp_buffer;
static q15_t *canny_tmp_buffer;

void cv_hello_world_cmsis_cv_init() {
	// //set memory allocation
    gaussian_tmp_buffer = (q15_t*)malloc(arm_get_linear_scratch_size_buffer_15(ImageOut_WIDTH_SIZE));

#if RUN_CANNY_SOBEL_ALGO
    canny_tmp_buffer = (q15_t*)malloc(arm_cv_get_scratch_size_canny_sobel(ImageOut_WIDTH_SIZE));
#endif
}

int cv_hello_world_cmsis_cv_run() {
	int ercode = 0;


	// arm_cv_image_gray8_t guass_ImageIn;
	// guass_ImageIn.width = ImageOut_WIDTH_SIZE;
	// guass_ImageIn.height = ImageOut_HEIGHT_SIZE;
	// guass_ImageIn.pData = (uint8_t *)app_get_raw_addr();

	// arm_cv_image_gray8_t guass_ImageOut;
	// guass_ImageOut.width= ImageOut_WIDTH_SIZE;
	// guass_ImageOut.height= ImageOut_HEIGHT_SIZE;

	arm_cv_image_rgb24_t img;
	img.width = ImageOut_WIDTH_SIZE;
	img.height = ImageOut_HEIGHT_SIZE;
	img.pData = (uint8_t *)app_get_raw_addr();



	// guass_ImageOut.pData = (uint8_t *)o_uint8_buf;

	// arm_gaussian_filter_5x5_fixp(&guass_ImageIn,
	// 	                         &guass_ImageOut,
	// 	                         gaussian_tmp_buffer,
	// 	                         ARM_CV_BORDER_NEAREST);

	hx_CleanDCache_by_Addr((volatile void*)img.pData, sizeof(uint8_t) *ImageOut_BUFSIZE );


#if RUN_CANNY_SOBEL_ALGO
	arm_cv_image_gray8_t canny_sobel_ImageOut;
	uint8_t low_threshold = 50;
	uint8_t high_threshold = 33;

	canny_sobel_ImageOut.width = ImageOut_WIDTH_SIZE;
	canny_sobel_ImageOut.height = ImageOut_HEIGHT_SIZE;
	canny_sobel_ImageOut.pData = (uint8_t *)o_canny_sobel_buff;

	arm_cv_canny_edge_sobel(&guass_ImageOut, &canny_sobel_ImageOut, canny_tmp_buffer,low_threshold, high_threshold);

	hx_CleanDCache_by_Addr((volatile void*)canny_sobel_ImageOut.pData, sizeof(uint8_t) *ImageOut_BUFSIZE );


	el_img_t temp_el_raw_img = el_img_t{};
	temp_el_raw_img.data = (uint8_t *)canny_sobel_ImageOut.pData;
	temp_el_raw_img.size = ImageOut_BUFSIZE;
	temp_el_raw_img.width = ImageOut_WIDTH_SIZE;
	temp_el_raw_img.height = ImageOut_HEIGHT_SIZE;
	temp_el_raw_img.format = EL_PIXEL_FORMAT_GRAYSCALE;
	temp_el_raw_img.rotate = EL_PIXEL_ROTATE_0;

	///////////////////////send raw image
	// img_2_json_str_assign_buf(&temp_el_raw_img,(char *)str_assign_buf);

	hx_drv_spi_mst_protocol_write_sp((uint32_t )temp_el_raw_img.data, ImageOut_BUFSIZE, DATA_TYPE_RAW_IMG);

#else

	//////////////////////////////////////////////////
	el_img_t temp_el_raw_img = el_img_t{};
	temp_el_raw_img.data = (uint8_t *)img.pData;
	temp_el_raw_img.size = ImageOut_BUFSIZE;
	temp_el_raw_img.width = ImageOut_WIDTH_SIZE;
	temp_el_raw_img.height = ImageOut_HEIGHT_SIZE;
	temp_el_raw_img.format = EL_PIXEL_FORMAT_RGB888;
	temp_el_raw_img.rotate = EL_PIXEL_ROTATE_0;
	///////////////
	///////////////////////send raw image
	// img_2_json_str_assign_buf(&temp_el_raw_img,(char *)str_assign_buf);
	
	hx_drv_spi_mst_protocol_write_sp((uint32_t )temp_el_raw_img.data, ImageOut_BUFSIZE, DATA_TYPE_RAW_IMG);

#endif

	////////////////////////////////////////
	///////////////////////send jpeg image
	el_img_t temp_el_jpg_img_dst = el_img_t{};
	temp_el_jpg_img_dst.data = (uint8_t *)o_jpeg_uint8_buf;
	temp_el_jpg_img_dst.size = ImageOut_BUFSIZE;
	temp_el_jpg_img_dst.width = ImageOut_WIDTH_SIZE;
	temp_el_jpg_img_dst.height = ImageOut_HEIGHT_SIZE;
	temp_el_jpg_img_dst.format = EL_PIXEL_FORMAT_RGB888;
	temp_el_jpg_img_dst.rotate = EL_PIXEL_ROTATE_0;
	rgb_to_jpeg(&temp_el_raw_img,&temp_el_jpg_img_dst);
	hx_CleanDCache_by_Addr((volatile void*)temp_el_jpg_img_dst.data, sizeof(uint8_t) *ImageOut_BUFSIZE );
	send_device_id();
	event_reply(concat_strings(", ", img_2_json_str(&temp_el_jpg_img_dst)));
	//////////////////////////////////////


	SystemGetTick(&systick_1, &loop_cnt_1);
	//recapture image
	sensordplib_retrigger_capture();

	SystemGetTick(&systick_2, &loop_cnt_2);
	capture_image_tick = (loop_cnt_2-loop_cnt_1)*CPU_CLK+(systick_1-systick_2);	
	
	return ercode;
}

[kris-himax]

Hi @ashfaquekhan ,

The scenario app, cvapp_hello_world_cmsis_cv , is not a color image example.
If you want to use color image example, you can reference yolov8_pose example which including the readme at the scenario app folder.

Thanks,
Kris

[ashfaquekhan]

Thanks for the reply @kris-himax ,
can you just tell me why does the image is not in color as I have changed it to "EL_PIXEL_FORMAT_RGB888" format .
Also the CV type , here:

	arm_cv_image_rgb24_t img;
	img.width = ImageOut_WIDTH_SIZE;
	img.height = ImageOut_HEIGHT_SIZE;
	img.pData = (uint8_t *)app_get_raw_addr();

And Respectively here.

	el_img_t temp_el_raw_img = el_img_t{};
	temp_el_raw_img.data = (uint8_t *)img.pData;
	temp_el_raw_img.size = ImageOut_BUFSIZE;
	temp_el_raw_img.width = ImageOut_WIDTH_SIZE;
	temp_el_raw_img.height = ImageOut_HEIGHT_SIZE;
	temp_el_raw_img.format = EL_PIXEL_FORMAT_RGB888;
	temp_el_raw_img.rotate = EL_PIXEL_ROTATE_0;
	
	hx_drv_spi_mst_protocol_write_sp((uint32_t )temp_el_raw_img.data, ImageOut_BUFSIZE, DATA_TYPE_RAW_IMG);

	el_img_t temp_el_jpg_img_dst = el_img_t{};
	temp_el_jpg_img_dst.data = (uint8_t *)o_jpeg_uint8_buf;
	temp_el_jpg_img_dst.size = ImageOut_BUFSIZE;
	temp_el_jpg_img_dst.width = ImageOut_WIDTH_SIZE;
	temp_el_jpg_img_dst.height = ImageOut_HEIGHT_SIZE;
	temp_el_jpg_img_dst.format = EL_PIXEL_FORMAT_RGB888;
	temp_el_jpg_img_dst.rotate = EL_PIXEL_ROTATE_0;
	rgb_to_jpeg(&temp_el_raw_img,&temp_el_jpg_img_dst);
	hx_CleanDCache_by_Addr((volatile void*)temp_el_jpg_img_dst.data, sizeof(uint8_t) *ImageOut_BUFSIZE );
	send_device_id();
	event_reply(concat_strings(", ", img_2_json_str(&temp_el_jpg_img_dst)));

	SystemGetTick(&systick_1, &loop_cnt_1);
	//recapture image
	sensordplib_retrigger_capture();

	SystemGetTick(&systick_2, &loop_cnt_2);
	capture_image_tick = (loop_cnt_2-loop_cnt_1)*CPU_CLK+(systick_1-systick_2);

Also how does this part work, what is happening here :

	SystemGetTick(&systick_1, &loop_cnt_1);
	//recapture image
	sensordplib_retrigger_capture();

	SystemGetTick(&systick_2, &loop_cnt_2);
	capture_image_tick = (loop_cnt_2-loop_cnt_1)*CPU_CLK+(systick_1-systick_2);

[kris-himax]

Hi @ashfaquekhan ,
Because you set the sensor to be YUV output https://github.com/HimaxWiseEyePlus/Seeed_Grove_Vision_AI_Module_V2/blob/main/EPII_CM55M_APP_S/app/scenario_app/hello_world_cmsis_cv/hello_world_cmsis_cv.c#L777.

The code what you mentioned above is capture the new picture.

Thanks,
Kris

[ashfaquekhan]

@kris-himax thanks man