Camera: Viewport and Snapshot

Note: this all uses the Zephyr updates from my PR
zephyrproject-rtos/zephyr#93797

Which I added to the STM dcmi driver the ability to have the camera work in snapshot mode
instead of in continuous video mode.  This allows for example that we start the camera
it grabs a frame and stops, we then take the buffer and process it, and repeat this.
This helps minimize how much the SDRAM gets used concurrently.

In addition, I added to the VIDEO_STM32_DCMI and GC2145 the ability to use some of the
new set_selection and get_selection code that was added for the DCMIPP.  In particular
the DCMI simply forwards these messages to the camera if it defined thise apis...

And with this it allows you to setup a viewport into the frame.  For example:
You can setup the frame on the GC2145 to be 800x600 and then define a view port
to be 480x320 to fill an ST7796/ILI9486 tft display or you could do it 400x240 to half fill
the GIGA display.  You can also move that view port around within the frame (pan)
I have examples that do this on Portenta H7 on the ST7796 display and another one
that does this on the GIGA display shield.

Still WIP as we probably need to refine the APIS and the like

Updated with first set of CR changes on the Zephyr side

Author: KurtE

libraries/Camera/src/camera.cpp

@@ -32,21 +32,29 @@ uint32_t FrameBuffer::getBufferSize() {
     if (this->vbuf) {
         return this->vbuf->bytesused;
     }
+    return 0;
 }
 
 uint8_t* FrameBuffer::getBuffer() {
     if (this->vbuf) {
         return this->vbuf->buffer;
     }
+    return nullptr;
 }
 
-Camera::Camera() : vdev(NULL), byte_swap(false), yuv_to_gray(false) {
+Camera::Camera() : vdev(NULL), byte_swap(false), yuv_to_gray(false),
+         snapshot_mode(CONFIG_VIDEO_BUFFER_POOL_NUM_MAX <= 1) {
 	for (size_t i = 0; i < ARRAY_SIZE(this->vbuf); i++) {
 		this->vbuf[i] = NULL;
     }
 }
 
+
 bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byte_swap) {
+    return begin(width, height, width, height, pixformat, byte_swap);
+}
+
+bool Camera::begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_t crop_height, uint32_t pixformat, bool byte_swap) {
     #if DT_HAS_CHOSEN(zephyr_camera)
     this->vdev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera));
     #endif
@@ -75,21 +83,22 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byt
         return false;
     }
 
-    for (size_t i=0; caps.format_caps[i].pixelformat != NULL; i++) {
+    for (size_t i=0; caps.format_caps[i].pixelformat != 0; i++) {
         const struct video_format_cap *fcap = &caps.format_caps[i];
-        if (fcap->width_min == width &&
-            fcap->height_min == height &&
+        if (fcap->width_min <= width && fcap->width_max >= width &&
+            fcap->height_min <= height && fcap->height_max >= height &&
             fcap->pixelformat == pixformat) {
             break;
         }
-        if (caps.format_caps[i+1].pixelformat == NULL) {
+        if (caps.format_caps[i+1].pixelformat == 0) {
             Serial.println("The specified format is not supported");
             return false;
         }
     }
 
     // Set format.
     static struct video_format fmt = {
+        .type = VIDEO_BUF_TYPE_OUTPUT,
         .pixelformat = pixformat,
         .width = width,
         .height = height,
@@ -101,6 +110,34 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byt
 		return false;
 	}
 
+    // optionally set the crop values
+    if (width != crop_width || height != crop_height) {
+          struct video_selection vselCrop;
+          vselCrop.type = VIDEO_BUF_TYPE_OUTPUT;
+          vselCrop.target = VIDEO_SEL_TGT_CROP;
+          vselCrop.rect.left = (width - crop_width) / 2;
+          vselCrop.rect.top = (height - crop_height) / 2;
+          vselCrop.rect.width = crop_width;
+          vselCrop.rect.height = crop_height;;
+
+          int ret;
+          if ((ret = setSelection(&vselCrop)) != 0) {
+            printk("ERROR: %d\n", ret);
+          }
+    }
+    // this should compute the sizes needed.
+    video_get_format(this->vdev, &fmt);
+
+
+    // If we are in snapshot mode, try starting the video stream with no buffers
+    // to tell it that we want snapshot...
+    if (snapshot_mode) {
+        if (video_stream_start(this->vdev, VIDEO_BUF_TYPE_OUTPUT)) {
+            Serial.println("Snapshot mode Failed to start capture");
+            //return false;
+        }
+    }
+
 	// Allocate video buffers.
 	for (size_t i = 0; i < ARRAY_SIZE(this->vbuf); i++) {
 		this->vbuf[i] = video_buffer_aligned_alloc(fmt.pitch * fmt.height,
@@ -114,23 +151,24 @@ bool Camera::begin(uint32_t width, uint32_t height, uint32_t pixformat, bool byt
 	}
 
 	// Start video capture
-	if (video_stream_start(this->vdev, VIDEO_BUF_TYPE_OUTPUT)) {
-        Serial.println("Failed to start capture");
-		return false;
-	}
-
+    if (!snapshot_mode) {
+    	if (video_stream_start(this->vdev, VIDEO_BUF_TYPE_OUTPUT)) {
+            Serial.println("Failed to start capture");
+    		return false;
+    	}
+    }
     return true;
 }
 
 bool Camera::grabFrame(FrameBuffer &fb, uint32_t timeout) {
     if (this->vdev == NULL) {
         return false;
     }
-
+    //printk("Camera::grabFrame called\n");
     if (video_dequeue(this->vdev, &fb.vbuf, K_MSEC(timeout))) {
         return false;
     }
-
+    //printk("video_dequeue returned :%p\n", fb.vbuf->buffer);
     if (this->byte_swap) {
         uint16_t *pixels = (uint16_t *) fb.vbuf->buffer;
         for (size_t i=0; i<fb.vbuf->bytesused / 2; i++) {
@@ -154,7 +192,10 @@ bool Camera::releaseFrame(FrameBuffer &fb) {
         return false;
     }
 
-    if (video_enqueue(this->vdev, fb.vbuf)) {
+    int ret;
+    //printk("Camera::ReleaseFrame called\n");
+    if ((ret = video_enqueue(this->vdev, fb.vbuf)) != 0) {
+        printk("Failed to enqueue buffer %d\n", ret);
         return false;
 	}
 
@@ -170,3 +211,58 @@ bool Camera::setHorizontalMirror(bool mirror_enable) {
     struct video_control ctrl = {.id = VIDEO_CID_HFLIP, .val = mirror_enable};
     return video_set_ctrl(this->vdev, &ctrl) == 0;
 }
+
+int Camera::setSelection(struct video_selection *sel) {
+    return video_set_selection(vdev, sel);
+}
+
+/**
+ * @brief Get video selection (crop/compose).
+ *
+ * Retrieve the current settings related to the crop and compose of the video device.
+ * This can also be used to read the native size of the input stream of the video
+ * device.
+ * This function can be used to read crop / compose capabilities of the device prior
+ * to performing configuration via the @ref video_set_selection api.
+ *
+ * @param sel Pointer to a video selection structure, @c type and @c target set by the caller
+ *
+ * @retval 0 Is successful.
+ * @retval -EINVAL If parameters are invalid.
+ * @retval -ENOTSUP If format is not supported.
+ * @retval -EIO General input / output error.
+ */
+int Camera::getSelection(struct video_selection *sel) {
+    return video_get_selection(vdev, sel);
+}
+
+/**
+ * @brief returns if snapshot mode is turned on or off.
+ *
+ * @param snapshot_mode pointer to Turn Snaphsot mode on or off..
+ */
+bool Camera::getSnapshotMode() {
+    return snapshot_mode;
+}
+
+/**
+ * @brief returns if snapshot mode is turned on or off.
+ * 
+ * Must be called before begin to take effect.
+ * 
+ * @param snap_shot mode if true.
+ *
+ * @retval 0 is successful.
+ */
+int Camera::setSnapshotMode(bool snap_shot) {
+    if (snap_shot) {
+        snapshot_mode = snap_shot;
+        return 0;
+    } else {
+        #if CONFIG_VIDEO_BUFFER_POOL_NUM_MAX <= 1
+        return -EINVAL;
+        #endif
+        snapshot_mode = snap_shot;
+        return 0;
+    }
+}

libraries/Camera/src/camera.h

@@ -62,15 +62,20 @@ class FrameBuffer {
     friend class Camera;
 };
 
+
+// bugbug temporary
+#include <zephyr/drivers/video.h>
+
 /**
  * @class Camera
  * @brief The main class for controlling a camera.
  */
 class Camera {
     private:
+        const struct device *vdev;
         bool byte_swap;
         bool yuv_to_gray;
-        const struct device *vdev;
+        bool snapshot_mode;
         struct video_buffer *vbuf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX];
 
     public:
@@ -90,6 +95,19 @@ class Camera {
          */
         bool begin(uint32_t width, uint32_t height, uint32_t pixformat = CAMERA_RGB565, bool byte_swap = false);
 
+        /**
+         * @brief Initialize the camera.
+         * 
+         * @param width Frame width in pixels.
+         * @param height Frame height in pixels.
+         * @param crop_width crop width in pixels.
+         * @param crop_height crop height in pixels.
+         * @param pixformat Initial pixel format (default: CAMERA_RGB565).
+         * @param byte_swap Enable byte swapping (default: false).
+         * @return true if the camera is successfully initialized, otherwise false.
+         */
+        bool begin(uint32_t width, uint32_t height, uint32_t crop_width, uint32_t crop_height, uint32_t pixformat = CAMERA_RGB565, bool byte_swap = false);
+
         /**
          * @brief Capture a frame.
          * 
@@ -122,6 +140,63 @@ class Camera {
          * @return true on success, false on failure.
          */
         bool setHorizontalMirror(bool mirror_enable);
+
+
+        /* Experiments to be able to set crop and the like */
+        /**
+         * @brief Set video selection (crop/compose).
+         *
+         * Configure the optional crop and compose feature of a video device.
+         * Crop is first applied on the input frame, and the result of that crop is applied
+         * to the compose. The result of the compose (width/height) is equal to the format
+         * width/height given to the @ref video_set_format function.
+         *
+         * Some targets are inter-dependents. For instance, setting a @ref VIDEO_SEL_TGT_CROP will
+         * reset @ref VIDEO_SEL_TGT_COMPOSE to the same size.
+         *
+         * @param sel Pointer to a video selection structure
+         *
+         * @retval 0 Is successful.
+         * @retval -EINVAL If parameters are invalid.
+         * @retval -ENOTSUP If format is not supported.
+         * @retval -EIO General input / output error.
+         */
+        int setSelection(struct video_selection *sel);
+        /**
+         * @brief Get video selection (crop/compose).
+         *
+         * Retrieve the current settings related to the crop and compose of the video device.
+         * This can also be used to read the native size of the input stream of the video
+         * device.
+         * This function can be used to read crop / compose capabilities of the device prior
+         * to performing configuration via the @ref video_set_selection api.
+         *
+         * @param sel Pointer to a video selection structure, @c type and @c target set by the caller
+         *
+         * @retval 0 Is successful.
+         * @retval -EINVAL If parameters are invalid.
+         * @retval -ENOTSUP If format is not supported.
+         * @retval -EIO General input / output error.
+         */
+        int getSelection(struct video_selection *sel);
+
+        /**
+         * @brief returns if snapshot mode is turned on or off.
+         *
+         * @retval true if in snapshot mode false otherwise.
+         */
+        bool getSnapshotMode();
+
+        /**
+         * @brief returns if snapshot mode is turned on or off.
+         * 
+         * Must be called before begin to take effect.
+         * 
+         * @param snap_shot mode if true.
+         *
+         * @retval 0 is successful.
+         */
+        int setSnapshotMode(bool snap_shot);
 };
 
 #endif // __CAMERA_H__

variants/arduino_giga_r1_stm32h747xx_m7/arduino_giga_r1_stm32h747xx_m7.conf

@@ -35,9 +35,9 @@ CONFIG_ENTROPY_GENERATOR=y
 CONFIG_TEST_RANDOM_GENERATOR=y
 
 CONFIG_VIDEO=y
-CONFIG_VIDEO_LOG_LEVEL_DBG=y
+CONFIG_VIDEO_LOG_LEVEL_DBG=n
 CONFIG_VIDEO_STM32_DCMI=y
-CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=3
+CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=1
 CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=614400
 CONFIG_VIDEO_BUFFER_POOL_ALIGN=32
 CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP=y
@@ -61,6 +61,7 @@ CONFIG_BT_CTLR_ADV_EXT=y
 CONFIG_BT_CTLR_ADV_PERIODIC=y
 CONFIG_BT_CTLR_DTM_HCI=y
 CONFIG_CYW4343W_MURATA_1DX=y
+CONFIG_BT_HCI_DRIVER_LOG_LEVEL_DBG=n
 CONFIG_SYSTEM_WORKQUEUE_STACK_SIZE=2048
 CONFIG_BT_RX_STACK_SIZE=4096
 CONFIG_BT_HCI_TX_STACK_SIZE=4096