From 01a30a550573d1c32a28e4f47a9a0627594ffa42 Mon Sep 17 00:00:00 2001 From: kurte Date: Wed, 6 Aug 2025 06:43:08 -0700 Subject: [PATCH 1/2] WIP/Experiment - Camera Viewport and Snapshot Note: this all uses the Zephyr updates from my PR https://github.com/zephyrproject-rtos/zephyr/pull/93797 Which I added to the STM dcmi driver the ability to have the camera work in snapshot mode instead of in continuous video mode. This allows for example that we start the camera it grabs a frame and stops, we then take the buffer and process it, and repeat this. This helps minimize how much the SDRAM gets used concurrently. In addition, I added to the VIDEO_STM32_DCMI and GC2145 the ability to use some of the new set_selection and get_selection code that was added for the DCMIPP. In particular the DCMI simply forwards these messages to the camera if it defined thise apis... And with this it allows you to setup a viewport into the frame. For example: You can setup the frame on the GC2145 to be 800x600 and then define a view port to be 480x320 to fill an ST7796/ILI9486 tft display or you could do it 400x240 to half fill the GIGA display. You can also move that view port around within the frame (pan) I have examples that do this on Portenta H7 on the ST7796 display and another one that does this on the GIGA display shield. Still WIP as we probably need to refine the APIS and the like --- libraries/Camera/src/camera.cpp | 89 +++++++++++++++++-- libraries/Camera/src/camera.h | 72 ++++++++++++++- .../arduino_giga_r1_stm32h747xx_m7.conf | 5 +- 3 files changed, 156 insertions(+), 10 deletions(-) diff --git a/libraries/Camera/src/camera.cpp b/libraries/Camera/src/camera.cpp index fd4ac2ac..6c65ca0a 100644 --- a/libraries/Camera/src/camera.cpp +++ b/libraries/Camera/src/camera.cpp @@ -46,7 +46,12 @@ Camera::Camera() : vdev(NULL), byte_swap(false), yuv_to_gray(false) { } } + bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byte_swap) { + return begin(width, height, width, height, pixformat, byte_swap); +} + +bool Camera::begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_t crop_height, uint32_t pixformat, bool byte_swap) { #if DT_HAS_CHOSEN(zephyr_camera) this->vdev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); #endif @@ -75,14 +80,14 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byt return false; } - for (size_t i=0; caps.format_caps[i].pixelformat != NULL; i++) { + for (size_t i=0; caps.format_caps[i].pixelformat != 0; i++) { const struct video_format_cap *fcap = &caps.format_caps[i]; - if (fcap->width_min == width && - fcap->height_min == height && + if (fcap->width_min <= width && fcap->width_max >= width && + fcap->height_min <= height && fcap->height_max >= height && fcap->pixelformat == pixformat) { break; } - if (caps.format_caps[i+1].pixelformat == NULL) { + if (caps.format_caps[i+1].pixelformat == 0) { Serial.println("The specified format is not supported"); return false; } @@ -101,6 +106,24 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byt return false; } + // optionally set the crop values + if (width != crop_width || height != crop_height) { + struct video_selection vselCrop; + vselCrop.type = VIDEO_BUF_TYPE_OUTPUT; + vselCrop.target = VIDEO_SEL_TGT_CROP; + vselCrop.rect.left = (width - crop_width) / 2; + vselCrop.rect.top = (height - crop_height) / 2; + vselCrop.rect.width = crop_width; + vselCrop.rect.height = crop_height;; + + int ret; + if ((ret = setSelection(&vselCrop)) != 0) { + printk("ERROR: %d\n", ret); + } + } + // this should compute the sizes needed. + video_get_format(this->vdev, &fmt); + // Allocate video buffers. for (size_t i = 0; i < ARRAY_SIZE(this->vbuf); i++) { this->vbuf[i] = video_buffer_aligned_alloc(fmt.pitch * fmt.height, @@ -126,11 +149,11 @@ bool Camera::grabFrame(FrameBuffer &fb, uint32_t timeout) { if (this->vdev == NULL) { return false; } - + //printk("Camera::grabFrame called\n"); if (video_dequeue(this->vdev, &fb.vbuf, K_MSEC(timeout))) { return false; } - + //printk("video_dequeue returned :%p\n", fb.vbuf->buffer); if (this->byte_swap) { uint16_t *pixels = (uint16_t *) fb.vbuf->buffer; for (size_t i=0; ibytesused / 2; i++) { @@ -154,7 +177,10 @@ bool Camera::releaseFrame(FrameBuffer &fb) { return false; } - if (video_enqueue(this->vdev, fb.vbuf)) { + int ret; + //printk("Camera::ReleaseFrame called\n"); + if (ret = video_enqueue(this->vdev, fb.vbuf)) { + printk("Failed to enqueue buffer %d\n", ret); return false; } @@ -170,3 +196,52 @@ bool Camera::setHorizontalMirror(bool mirror_enable) { struct video_control ctrl = {.id = VIDEO_CID_HFLIP, .val = mirror_enable}; return video_set_ctrl(this->vdev, &ctrl) == 0; } + +int Camera::setSelection(struct video_selection *sel) { + return video_set_selection(vdev, sel); +} + +/** + * @brief Get video selection (crop/compose). + * + * Retrieve the current settings related to the crop and compose of the video device. + * This can also be used to read the native size of the input stream of the video + * device. + * This function can be used to read crop / compose capabilities of the device prior + * to performing configuration via the @ref video_set_selection api. + * + * @param sel Pointer to a video selection structure, @c type and @c target set by the caller + * + * @retval 0 Is successful. + * @retval -EINVAL If parameters are invalid. + * @retval -ENOTSUP If format is not supported. + * @retval -EIO General input / output error. + */ +int Camera::getSelection(struct video_selection *sel) { + return video_get_selection(vdev, sel); +} + +/** + * @brief returns if snapshot mode is turned on or off. + * + * @param snapshot_mode pointer to Turn Snaphsot mode on or off.. + */ +int Camera::getSnapshotMode(bool *snapshot_mode) { + #if DT_HAS_CHOSEN(zephyr_camera) + this->vdev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + #endif + return video_get_snapshot_mode(vdev, snapshot_mode); +} + +/** + * @brief Function pointer type for video_set_snapshot_mode() + * + * @param snapshot_mode Turn Snaphsot mode on or off.. + */ +int Camera::setSnapshotMode(bool snapshot_mode) { + #if DT_HAS_CHOSEN(zephyr_camera) + this->vdev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + #endif + return video_set_snapshot_mode(vdev, snapshot_mode); +} + diff --git a/libraries/Camera/src/camera.h b/libraries/Camera/src/camera.h index c456c5ff..e2a83d5b 100644 --- a/libraries/Camera/src/camera.h +++ b/libraries/Camera/src/camera.h @@ -62,15 +62,19 @@ class FrameBuffer { friend class Camera; }; + +// bugbug temporary +#include + /** * @class Camera * @brief The main class for controlling a camera. */ class Camera { private: + const struct device *vdev; bool byte_swap; bool yuv_to_gray; - const struct device *vdev; struct video_buffer *vbuf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; public: @@ -90,6 +94,19 @@ class Camera { */ bool begin(uint32_t width, uint32_t height, uint32_t pixformat = CAMERA_RGB565, bool byte_swap = false); + /** + * @brief Initialize the camera. + * + * @param width Frame width in pixels. + * @param height Frame height in pixels. + * @param crop_width crop width in pixels. + * @param crop_height crop height in pixels. + * @param pixformat Initial pixel format (default: CAMERA_RGB565). + * @param byte_swap Enable byte swapping (default: false). + * @return true if the camera is successfully initialized, otherwise false. + */ + bool begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_t crop_height, uint32_t pixformat = CAMERA_RGB565, bool byte_swap = false); + /** * @brief Capture a frame. * @@ -122,6 +139,59 @@ class Camera { * @return true on success, false on failure. */ bool setHorizontalMirror(bool mirror_enable); + + + /* Experiments to be able to set crop and the like */ + /** + * @brief Set video selection (crop/compose). + * + * Configure the optional crop and compose feature of a video device. + * Crop is first applied on the input frame, and the result of that crop is applied + * to the compose. The result of the compose (width/height) is equal to the format + * width/height given to the @ref video_set_format function. + * + * Some targets are inter-dependents. For instance, setting a @ref VIDEO_SEL_TGT_CROP will + * reset @ref VIDEO_SEL_TGT_COMPOSE to the same size. + * + * @param sel Pointer to a video selection structure + * + * @retval 0 Is successful. + * @retval -EINVAL If parameters are invalid. + * @retval -ENOTSUP If format is not supported. + * @retval -EIO General input / output error. + */ + int setSelection(struct video_selection *sel); + /** + * @brief Get video selection (crop/compose). + * + * Retrieve the current settings related to the crop and compose of the video device. + * This can also be used to read the native size of the input stream of the video + * device. + * This function can be used to read crop / compose capabilities of the device prior + * to performing configuration via the @ref video_set_selection api. + * + * @param sel Pointer to a video selection structure, @c type and @c target set by the caller + * + * @retval 0 Is successful. + * @retval -EINVAL If parameters are invalid. + * @retval -ENOTSUP If format is not supported. + * @retval -EIO General input / output error. + */ + int getSelection(struct video_selection *sel); + + /** + * @brief returns if snapshot mode is turned on or off. + * + * @param snapshot_mode pointer to Turn Snaphsot mode on or off.. + */ + int getSnapshotMode(bool *snapshot_mode); + + /** + * @brief Function pointer type for video_set_snapshot_mode() + * + * @param snapshot_mode Turn Snaphsot mode on or off.. + */ + int setSnapshotMode(bool snapshot_mode); }; #endif // __CAMERA_H__ diff --git a/variants/arduino_giga_r1_stm32h747xx_m7/arduino_giga_r1_stm32h747xx_m7.conf b/variants/arduino_giga_r1_stm32h747xx_m7/arduino_giga_r1_stm32h747xx_m7.conf index 8ad58579..280065bd 100644 --- a/variants/arduino_giga_r1_stm32h747xx_m7/arduino_giga_r1_stm32h747xx_m7.conf +++ b/variants/arduino_giga_r1_stm32h747xx_m7/arduino_giga_r1_stm32h747xx_m7.conf @@ -35,9 +35,9 @@ CONFIG_ENTROPY_GENERATOR=y CONFIG_TEST_RANDOM_GENERATOR=y CONFIG_VIDEO=y -CONFIG_VIDEO_LOG_LEVEL_DBG=y +CONFIG_VIDEO_LOG_LEVEL_DBG=n CONFIG_VIDEO_STM32_DCMI=y -CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=3 +CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=1 CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=614400 CONFIG_VIDEO_BUFFER_POOL_ALIGN=32 CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP=y @@ -61,6 +61,7 @@ CONFIG_BT_CTLR_ADV_EXT=y CONFIG_BT_CTLR_ADV_PERIODIC=y CONFIG_BT_CTLR_DTM_HCI=y CONFIG_CYW4343W_MURATA_1DX=y +CONFIG_BT_HCI_DRIVER_LOG_LEVEL_DBG=n CONFIG_SYSTEM_WORKQUEUE_STACK_SIZE=2048 CONFIG_BT_RX_STACK_SIZE=4096 CONFIG_BT_HCI_TX_STACK_SIZE=4096 From 5698e1eab3f206e8f30b9e14a48a1ea5712a8a92 Mon Sep 17 00:00:00 2001 From: kurte Date: Fri, 8 Aug 2025 11:18:08 -0700 Subject: [PATCH 2/2] Updated with first set of CR changes on the Zephyr side --- libraries/Camera/src/camera.cpp | 61 ++++++++++++++++++++++----------- libraries/Camera/src/camera.h | 15 +++++--- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/libraries/Camera/src/camera.cpp b/libraries/Camera/src/camera.cpp index 6c65ca0a..125b8955 100644 --- a/libraries/Camera/src/camera.cpp +++ b/libraries/Camera/src/camera.cpp @@ -32,15 +32,18 @@ uint32_t FrameBuffer::getBufferSize() { if (this->vbuf) { return this->vbuf->bytesused; } + return 0; } uint8_t* FrameBuffer::getBuffer() { if (this->vbuf) { return this->vbuf->buffer; } + return nullptr; } -Camera::Camera() : vdev(NULL), byte_swap(false), yuv_to_gray(false) { +Camera::Camera() : vdev(NULL), byte_swap(false), yuv_to_gray(false), + snapshot_mode(CONFIG_VIDEO_BUFFER_POOL_NUM_MAX <= 1) { for (size_t i = 0; i < ARRAY_SIZE(this->vbuf); i++) { this->vbuf[i] = NULL; } @@ -95,6 +98,7 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_ // Set format. static struct video_format fmt = { + .type = VIDEO_BUF_TYPE_OUTPUT, .pixelformat = pixformat, .width = width, .height = height, @@ -124,6 +128,16 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_ // this should compute the sizes needed. video_get_format(this->vdev, &fmt); + + // If we are in snapshot mode, try starting the video stream with no buffers + // to tell it that we want snapshot... + if (snapshot_mode) { + if (video_stream_start(this->vdev, VIDEO_BUF_TYPE_OUTPUT)) { + Serial.println("Snapshot mode Failed to start capture"); + //return false; + } + } + // Allocate video buffers. for (size_t i = 0; i < ARRAY_SIZE(this->vbuf); i++) { this->vbuf[i] = video_buffer_aligned_alloc(fmt.pitch * fmt.height, @@ -137,11 +151,12 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_ } // Start video capture - if (video_stream_start(this->vdev, VIDEO_BUF_TYPE_OUTPUT)) { - Serial.println("Failed to start capture"); - return false; - } - + if (!snapshot_mode) { + if (video_stream_start(this->vdev, VIDEO_BUF_TYPE_OUTPUT)) { + Serial.println("Failed to start capture"); + return false; + } + } return true; } @@ -179,7 +194,7 @@ bool Camera::releaseFrame(FrameBuffer &fb) { int ret; //printk("Camera::ReleaseFrame called\n"); - if (ret = video_enqueue(this->vdev, fb.vbuf)) { + if ((ret = video_enqueue(this->vdev, fb.vbuf)) != 0) { printk("Failed to enqueue buffer %d\n", ret); return false; } @@ -226,22 +241,28 @@ int Camera::getSelection(struct video_selection *sel) { * * @param snapshot_mode pointer to Turn Snaphsot mode on or off.. */ -int Camera::getSnapshotMode(bool *snapshot_mode) { - #if DT_HAS_CHOSEN(zephyr_camera) - this->vdev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); - #endif - return video_get_snapshot_mode(vdev, snapshot_mode); +bool Camera::getSnapshotMode() { + return snapshot_mode; } /** - * @brief Function pointer type for video_set_snapshot_mode() + * @brief returns if snapshot mode is turned on or off. + * + * Must be called before begin to take effect. + * + * @param snap_shot mode if true. * - * @param snapshot_mode Turn Snaphsot mode on or off.. + * @retval 0 is successful. */ -int Camera::setSnapshotMode(bool snapshot_mode) { - #if DT_HAS_CHOSEN(zephyr_camera) - this->vdev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); - #endif - return video_set_snapshot_mode(vdev, snapshot_mode); +int Camera::setSnapshotMode(bool snap_shot) { + if (snap_shot) { + snapshot_mode = snap_shot; + return 0; + } else { + #if CONFIG_VIDEO_BUFFER_POOL_NUM_MAX <= 1 + return -EINVAL; + #endif + snapshot_mode = snap_shot; + return 0; + } } - diff --git a/libraries/Camera/src/camera.h b/libraries/Camera/src/camera.h index e2a83d5b..35ae666c 100644 --- a/libraries/Camera/src/camera.h +++ b/libraries/Camera/src/camera.h @@ -75,6 +75,7 @@ class Camera { const struct device *vdev; bool byte_swap; bool yuv_to_gray; + bool snapshot_mode; struct video_buffer *vbuf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; public: @@ -182,16 +183,20 @@ class Camera { /** * @brief returns if snapshot mode is turned on or off. * - * @param snapshot_mode pointer to Turn Snaphsot mode on or off.. + * @retval true if in snapshot mode false otherwise. */ - int getSnapshotMode(bool *snapshot_mode); + bool getSnapshotMode(); /** - * @brief Function pointer type for video_set_snapshot_mode() + * @brief returns if snapshot mode is turned on or off. + * + * Must be called before begin to take effect. + * + * @param snap_shot mode if true. * - * @param snapshot_mode Turn Snaphsot mode on or off.. + * @retval 0 is successful. */ - int setSnapshotMode(bool snapshot_mode); + int setSnapshotMode(bool snap_shot); }; #endif // __CAMERA_H__