From 399d98dd87d6cbfe198c24aa8a60b1461873aad4 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Thu, 5 Feb 2026 16:16:39 +0000 Subject: [PATCH 1/8] libpisp: pisp_be_config: Add missing include from UAPI This is needed to compile correcly without earlier header inclusion. Signed-off-by: Naushir Patuck --- src/libpisp/backend/pisp_be_config.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libpisp/backend/pisp_be_config.h b/src/libpisp/backend/pisp_be_config.h index a3b2055..b1b5c0a 100644 --- a/src/libpisp/backend/pisp_be_config.h +++ b/src/libpisp/backend/pisp_be_config.h @@ -8,6 +8,7 @@ #ifndef _PISP_BE_CONFIG_H_ #define _PISP_BE_CONFIG_H_ +#include #include #include "common/pisp_common.h" From 98b95b6e714020181af3bc024cbb0e0327c7d331 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Thu, 5 Feb 2026 16:16:49 +0000 Subject: [PATCH 2/8] helpers: Rework buffer handling and add dmabuf support - Add Buffer and BufferRef classes for managing V4L2 buffers - Add DmaHeap helper for dmabuf allocation - Add caching for v4l2_format to reduce ioctl calls - Improve buffer queue tracking - Add dmabuf allocator support to BackendDevice - Optimize stream on/off handling - Update convert example for new buffer API Signed-off-by: Naushir Patuck --- src/examples/convert.cpp | 37 +++--- src/helpers/backend_device.cpp | 106 ++++++++------- src/helpers/backend_device.hpp | 16 ++- src/helpers/buffer.cpp | 200 +++++++++++++++++++++++++++++ src/helpers/buffer.hpp | 65 ++++++++++ src/helpers/device_fd.hpp | 9 +- src/helpers/dma_heap.hpp | 93 ++++++++++++++ src/helpers/media_device.cpp | 10 +- src/helpers/media_device.hpp | 1 - src/helpers/meson.build | 3 + src/helpers/v4l2_device.cpp | 227 +++++++++++++++++++-------------- src/helpers/v4l2_device.hpp | 71 ++++++----- 12 files changed, 633 insertions(+), 205 deletions(-) create mode 100644 src/helpers/buffer.cpp create mode 100644 src/helpers/buffer.hpp create mode 100644 src/helpers/dma_heap.hpp diff --git a/src/examples/convert.cpp b/src/examples/convert.cpp index a42cb67..1b669ee 100644 --- a/src/examples/convert.cpp +++ b/src/examples/convert.cpp @@ -28,6 +28,8 @@ #include "libpisp/common/utils.hpp" #include "libpisp/variants/variant.hpp" +using Buffer = libpisp::helpers::Buffer; + void read_plane(uint8_t *mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { @@ -53,7 +55,7 @@ void write_plane(std::ofstream &out, uint8_t *mem, unsigned int width, unsigned } } -void read_rgb888(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_rgb888(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { read_plane((uint8_t *)mem[0], in, width * 3, height, file_stride, buffer_stride); @@ -65,7 +67,7 @@ void write_rgb888(std::ofstream &out, std::array &mem, unsigned in write_plane(out, (uint8_t *)mem[0], width * 3, height, file_stride, buffer_stride); } -void read_32(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_32(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { read_plane((uint8_t *)mem[0], in, width * 4, height, file_stride, buffer_stride); @@ -77,7 +79,7 @@ void write_32(std::ofstream &out, std::array &mem, unsigned int wi write_plane(out, (uint8_t *)mem[0], width * 4, height, file_stride, buffer_stride); } -void read_yuv(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_yuv(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride, unsigned int ss_x, unsigned int ss_y) { uint8_t *dst = mem[0]; @@ -105,25 +107,25 @@ void write_yuv(std::ofstream &out, std::array &mem, unsigned int w write_plane(out, src, width / ss_x, height / ss_y, file_stride / ss_x, buffer_stride / ss_x); } -void read_yuv420(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_yuv420(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { read_yuv(mem, in, width, height, file_stride, buffer_stride, 2, 2); } -void read_yuv422p(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_yuv422p(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { read_yuv(mem, in, width, height, file_stride, buffer_stride, 2, 1); } -void read_yuv444p(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_yuv444p(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { read_yuv(mem, in, width, height, file_stride, buffer_stride, 1, 1); } -void read_yuv422i(std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, +void read_yuv422i(const std::array &mem, std::ifstream &in, unsigned int width, unsigned int height, unsigned int file_stride, unsigned int buffer_stride) { read_plane(mem[0], in, width * 2, height, file_stride, buffer_stride); @@ -155,7 +157,7 @@ void write_yuv422i(std::ofstream &out, std::array &mem, unsigned i struct FormatFuncs { - std::function &, std::ifstream &, unsigned int, unsigned int, unsigned int, + std::function &, std::ifstream &, unsigned int, unsigned int, unsigned int, unsigned int)> read_file; std::function &, unsigned int, unsigned int, unsigned int, unsigned int)> write_file; @@ -364,7 +366,7 @@ int main(int argc, char *argv[]) be.Prepare(&config); backend_device.Setup(config); - auto buffers = backend_device.AcquireBuffers(); + auto buffers = backend_device.GetBufferSlice(); std::string input_filename = args["input"].as(); std::ifstream in(input_filename, std::ios::binary); @@ -377,10 +379,11 @@ int main(int argc, char *argv[]) std::cerr << "Reading " << input_filename << " " << in_file.width << ":" << in_file.height << ":" << in_file.stride << ":" << in_file.format << std::endl; - Formats.at(in_file.format) - .read_file(buffers["pispbe-input"].mem, in, in_file.width, in_file.height, in_file.stride, - i.stride); - in.close(); + { + Buffer::Sync input(buffers.at("pispbe-input"), Buffer::Sync::Access::ReadWrite); + Formats.at(in_file.format).read_file(input.Get(), in, in_file.width, in_file.height, in_file.stride, i.stride); + in.close(); + } int ret = backend_device.Run(buffers); if (ret) @@ -397,13 +400,11 @@ int main(int argc, char *argv[]) exit(-1); } - Formats.at(out_file.format) - .write_file(out, buffers["pispbe-output0"].mem, out_file.width, out_file.height, out_file.stride, - o.image.stride); + Buffer::Sync output(buffers.at("pispbe-output0"), Buffer::Sync::Access::Read); + Formats.at(out_file.format).write_file(out, const_cast &>(output.Get()), out_file.width, out_file.height, out_file.stride, + o.image.stride); out.close(); - backend_device.ReleaseBuffer(buffers); - std::cerr << "Writing " << output_file << " " << out_file.width << ":" << out_file.height << ":" << out_file.stride << ":" << out_file.format << std::endl; diff --git a/src/helpers/backend_device.cpp b/src/helpers/backend_device.cpp index 4c638e7..6b85e7e 100644 --- a/src/helpers/backend_device.cpp +++ b/src/helpers/backend_device.cpp @@ -5,131 +5,149 @@ * backend_device.cpp - PiSP Backend device helper */ -#include -#include - #include "backend_device.hpp" +#include + using namespace libpisp::helpers; -BackendDevice::BackendDevice(const std::string &device) - : valid_(true) +namespace +{ + +const Buffer &AsBuffer(BufferRef ref) +{ + return ref.get(); +} +const Buffer &AsBuffer(const Buffer &b) +{ + return b; +} + +} // namespace + +BackendDevice::BackendDevice(const std::string &device) : valid_(true) { nodes_ = MediaDevice().OpenV4l2Nodes(device); if (nodes_.empty()) + { valid_ = false; + return; + } // Allocate a config buffer to persist. - nodes_.at("pispbe-config").RequestBuffers(1); + nodes_.at("pispbe-config").AllocateBuffers(1); nodes_.at("pispbe-config").StreamOn(); - config_buffer_ = nodes_.at("pispbe-config").AcquireBuffer().value(); } BackendDevice::~BackendDevice() { nodes_.at("pispbe-config").StreamOff(); + + for (auto const &n : nodes_enabled_) + nodes_.at(n).StreamOff(); } void BackendDevice::Setup(const pisp_be_tiles_config &config, unsigned int buffer_count, bool use_opaque_format) { + for (auto const &n : nodes_enabled_) + nodes_.at(n).StreamOff(); + nodes_enabled_.clear(); if ((config.config.global.rgb_enables & PISP_BE_RGB_ENABLE_INPUT) || (config.config.global.bayer_enables & PISP_BE_BAYER_ENABLE_INPUT)) { nodes_.at("pispbe-input").SetFormat(config.config.input_format, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-input").ReturnBuffers(); - nodes_.at("pispbe-input").RequestBuffers(buffer_count); + nodes_.at("pispbe-input").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-input"); } if (config.config.global.rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT0) { nodes_.at("pispbe-output0").SetFormat(config.config.output_format[0].image, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-output0").ReturnBuffers(); - nodes_.at("pispbe-output0").RequestBuffers(buffer_count); + nodes_.at("pispbe-output0").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-output0"); } if (config.config.global.rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT1) { nodes_.at("pispbe-output1").SetFormat(config.config.output_format[1].image, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-output1").ReturnBuffers(); - nodes_.at("pispbe-output1").RequestBuffers(buffer_count); + nodes_.at("pispbe-output1").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-output1"); } if (config.config.global.bayer_enables & PISP_BE_BAYER_ENABLE_TDN_INPUT) { nodes_.at("pispbe-tdn_input").SetFormat(config.config.tdn_input_format, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-tdn_input").ReturnBuffers(); - nodes_.at("pispbe-tdn_input").RequestBuffers(buffer_count); + nodes_.at("pispbe-tdn_input").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-tdn_input"); } if (config.config.global.bayer_enables & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) { nodes_.at("pispbe-tdn_output").SetFormat(config.config.tdn_output_format, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-tdn_output").ReturnBuffers(); - nodes_.at("pispbe-tdn_output").RequestBuffers(buffer_count); + nodes_.at("pispbe-tdn_output").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-tdn_output"); } if (config.config.global.bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_INPUT) { nodes_.at("pispbe-stitch_input").SetFormat(config.config.stitch_input_format, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-stitch_input").ReturnBuffers(); - nodes_.at("pispbe-stitch_input").RequestBuffers(buffer_count); + nodes_.at("pispbe-stitch_input").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-stitch_input"); } if (config.config.global.bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) { nodes_.at("pispbe-stitch_output").SetFormat(config.config.stitch_output_format, use_opaque_format); - // Release old/allocate a single buffer. - nodes_.at("pispbe-stitch_output").ReturnBuffers(); - nodes_.at("pispbe-stitch_output").RequestBuffers(buffer_count); + nodes_.at("pispbe-stitch_output").AllocateBuffers(buffer_count); nodes_enabled_.emplace("pispbe-stitch_output"); } - std::memcpy(reinterpret_cast(config_buffer_.mem[0]), &config, sizeof(config)); + for (auto const &n : nodes_enabled_) + nodes_.at(n).StreamOn(); + + auto config_buffer = nodes_.at("pispbe-config").Buffers()[0]; + Buffer::Sync s(config_buffer, Buffer::Sync::Access::ReadWrite); + std::memcpy(reinterpret_cast(s.Get()[0]), &config, sizeof(config)); } -std::map BackendDevice::AcquireBuffers() +std::map> BackendDevice::GetBuffers() { - std::map buffers; + std::map> buffers; for (auto const &n : nodes_enabled_) - buffers[n] = nodes_.at(n).AcquireBuffer().value(); + { + for (auto &ref : nodes_.at(n).Buffers()) + buffers[n].push_back(ref); + } return buffers; } -void BackendDevice::ReleaseBuffer(const std::map &buffers) +std::map BackendDevice::GetBufferSlice() const { - for (auto const &[n, b] : buffers) - nodes_.at(n).ReleaseBuffer(b); + std::map buffers; + + for (auto const &n : nodes_enabled_) + buffers.emplace(n, nodes_.at(n).Buffers()[0]); + + return buffers; } -int BackendDevice::Run(const std::map &buffers) +template +int BackendDevice::Run(const T &buffers) { int ret = 0; for (auto const &n : nodes_enabled_) { - nodes_.at(n).StreamOn(); - if (nodes_.at(n).QueueBuffer(buffers.at(n).buffer.index)) + if (nodes_.at(n).QueueBuffer(AsBuffer(buffers.at(n)))) ret = -1; } - // Triggers the HW job. - if (nodes_.at("pispbe-config").QueueBuffer(config_buffer_.buffer.index)) + auto config_buffer = nodes_.at("pispbe-config").Buffers()[0]; + if (nodes_.at("pispbe-config").QueueBuffer(config_buffer)) ret = -1; for (auto const &n : nodes_enabled_) @@ -138,12 +156,12 @@ int BackendDevice::Run(const std::map &buffers) ret = -1; } - // Must dequeue the config buffer in case it's used again. + /* Must dequeue the config buffer in case it's used again. */ if (nodes_.at("pispbe-config").DequeueBuffer(1000) < 0) ret = -1; - for (auto const &n : nodes_enabled_) - nodes_.at(n).StreamOff(); - return ret; } + +template int BackendDevice::Run(const std::map &); +template int BackendDevice::Run(const std::map &); diff --git a/src/helpers/backend_device.hpp b/src/helpers/backend_device.hpp index 01c9d3e..6a7c634 100644 --- a/src/helpers/backend_device.hpp +++ b/src/helpers/backend_device.hpp @@ -9,7 +9,8 @@ #include #include -#include "libpisp/backend/pisp_be_config.h" +#include "backend/pisp_be_config.h" +#include "buffer.hpp" #include "media_device.hpp" #include "v4l2_device.hpp" @@ -23,7 +24,9 @@ class BackendDevice ~BackendDevice(); void Setup(const pisp_be_tiles_config &config, unsigned int buffer_count = 1, bool use_opaque_format = false); - int Run(const std::map &buffers); + + template + int Run(const T &buffers); bool Valid() const { @@ -35,11 +38,11 @@ class BackendDevice return nodes_.at(node); } - std::map AcquireBuffers(); - void ReleaseBuffer(const std::map &buffers); - V4l2Device::Buffer &ConfigBuffer() + std::map> GetBuffers(); + std::map GetBufferSlice() const; + BufferRef ConfigBuffer() { - return config_buffer_; + return nodes_.at("pispbe-config").Buffers()[0]; } private: @@ -47,7 +50,6 @@ class BackendDevice V4l2DevMap nodes_; MediaDevice devices_; std::unordered_set nodes_enabled_; - V4l2Device::Buffer config_buffer_; }; } // namespace libpisp diff --git a/src/helpers/buffer.cpp b/src/helpers/buffer.cpp new file mode 100644 index 0000000..8d357d3 --- /dev/null +++ b/src/helpers/buffer.cpp @@ -0,0 +1,200 @@ + +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2025 Raspberry Pi Ltd + * + * buffer.cpp - PiSP V4L2 Buffer and Sync implementation + */ + +#include "buffer.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +using namespace libpisp::helpers; + +Buffer::Sync::Sync(BufferRef buffer, Access access) + : buffer_(buffer), access_(access) +{ + struct dma_buf_sync dma_sync {}; + + dma_sync.flags = DMA_BUF_SYNC_START; + if (access_ == Access::Read || access_ == Access::ReadWrite) + dma_sync.flags |= DMA_BUF_SYNC_READ; + if (access_ == Access::Write || access_ == Access::ReadWrite) + dma_sync.flags |= DMA_BUF_SYNC_WRITE; + + for (unsigned int p = 0; p < 3; p++) + { + if (buffer_.get().fd[p] >= 0) + ioctl(buffer_.get().fd[p], DMA_BUF_IOCTL_SYNC, &dma_sync); + } +} + +Buffer::Sync::~Sync() +{ + struct dma_buf_sync dma_sync {}; + + dma_sync.flags = DMA_BUF_SYNC_END; + if (access_ == Access::Read || access_ == Access::ReadWrite) + dma_sync.flags |= DMA_BUF_SYNC_READ; + if (access_ == Access::Write || access_ == Access::ReadWrite) + dma_sync.flags |= DMA_BUF_SYNC_WRITE; + + for (unsigned int p = 0; p < 3; p++) + { + if (buffer_.get().fd[p] >= 0) + ioctl(buffer_.get().fd[p], DMA_BUF_IOCTL_SYNC, &dma_sync); + } +} + +const std::array &Buffer::Sync::Get() const +{ + if (!buffer_.get().mem[0]) + buffer_.get().mmap(); + + return buffer_.get().mem; +} + +Buffer::Buffer() + : size(), mem(), fd({ -1, -1, -1 }) +{ +} + +Buffer::Buffer(const Buffer &other) + : size(other.size), mem(), fd({ -1, -1, -1 }) +{ + for (unsigned int p = 0; p < 3; p++) + { + if (other.fd[p] < 0) + break; + + fd[p] = dup(other.fd[p]); + if (fd[p] < 0) + { + // Close any fds we already dup'd before throwing. + for (unsigned int q = 0; q < p; q++) + close(fd[q]); + + throw std::runtime_error("Unable to dup fd: " + std::string(strerror(errno))); + } + } +} + +Buffer::Buffer(Buffer &&other) + : size(other.size), mem(other.mem), fd(other.fd) +{ + // Clear other without closing: we took ownership of its fds. + other.size = {}; + other.fd = { -1, -1, -1 }; + other.mem = {}; +} + +Buffer::Buffer(const std::array &fd, const std::array &size) + : size(size), mem(), fd(fd) +{ +} + +Buffer::~Buffer() +{ + release(); +} + +bool Buffer::operator==(const Buffer &other) const +{ + for (unsigned int p = 0; p < 3; p++) + { + if (fd[p] != other.fd[p] || size[p] != other.size[p]) + return false; + } + return true; +} + +Buffer &Buffer::operator=(const Buffer &other) +{ + if (this == &other) + return *this; + + std::array new_fd = { -1, -1, -1 }; + for (unsigned int p = 0; p < 3; p++) + { + if (other.fd[p] < 0) + break; + + new_fd[p] = dup(other.fd[p]); + if (new_fd[p] < 0) + { + // Close any fds we already dup'd before throwing. + for (unsigned int q = 0; q < p; q++) + close(new_fd[q]); + + throw std::runtime_error("Unable to dup fd: " + std::string(strerror(errno))); + } + } + + release(); + fd = new_fd; + size = other.size; + return *this; +} + +Buffer &Buffer::operator=(Buffer &&other) +{ + release(); + + size = other.size; + fd = other.fd; + mem = other.mem; + + // Clear other without closing: we took ownership of its fds. + other.size = {}; + other.fd = { -1, -1, -1 }; + other.mem = {}; + return *this; +} + +void Buffer::release() +{ + for (unsigned int p = 0; p < 3; p++) + { + if (mem[p] && size[p]) + munmap(mem[p], size[p]); + + if (fd[p] >= 0) + close(fd[p]); + + mem[p] = nullptr; + fd[p] = -1; + size[p] = 0; + } +} + +void Buffer::mmap() const +{ + for (unsigned int p = 0; p < 3; p++) + { + if (fd[p] < 0) + break; + + void *m = ::mmap(0, size[p], PROT_READ | PROT_WRITE, MAP_SHARED, fd[p], 0); + if (m == MAP_FAILED) + { + // Unmap any regions we already mapped before throwing. + for (unsigned int q = 0; q < p; q++) + { + munmap(mem[q], size[q]); + mem[q] = nullptr; + } + + throw std::runtime_error("Unable to mmap buffer"); + } + + mem[p] = (uint8_t *)m; + } +} diff --git a/src/helpers/buffer.hpp b/src/helpers/buffer.hpp new file mode 100644 index 0000000..8942670 --- /dev/null +++ b/src/helpers/buffer.hpp @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2025 Raspberry Pi Ltd + * + * buffer.hpp - PiSP V4L2 Buffer and Sync (included by v4l2_device.hpp) + * No includes - must be included after , , . + */ +#pragma once + +#include +#include +#include + +namespace libpisp::helpers +{ + +struct Buffer +{ +public: + Buffer(); + Buffer(const Buffer &other); + Buffer(Buffer &&other); + Buffer(const std::array &fd, const std::array &size); + ~Buffer(); + + bool operator==(const Buffer &other) const; + Buffer &operator=(const Buffer &other); + Buffer &operator=(Buffer &&); + + const std::array &Size() const { return size; } + const std::array &Fd() const { return fd; } + + struct Sync; + +private: + void release(); + void mmap() const; + + std::array size; + mutable std::array mem; + std::array fd; +}; + +using BufferRef = std::reference_wrapper; + +struct Buffer::Sync +{ + enum class Access + { + Read, + Write, + ReadWrite + }; + + Sync(BufferRef buffer, Access access); + ~Sync(); + + const std::array &Get() const; + +private: + BufferRef buffer_; + Access access_; +}; + +} // namespace libpisp::helpers diff --git a/src/helpers/device_fd.hpp b/src/helpers/device_fd.hpp index 35c7dd3..a45361a 100644 --- a/src/helpers/device_fd.hpp +++ b/src/helpers/device_fd.hpp @@ -20,6 +20,11 @@ class DeviceFd DeviceFd(DeviceFd const &) = delete; void operator=(DeviceFd const &) = delete; + DeviceFd() + : deviceFd_(-1) + { + } + DeviceFd(const std::string &file, mode_t mode) : deviceFd_(-1) { @@ -50,7 +55,7 @@ class DeviceFd return *this; } - int Get() + int Get() const { return deviceFd_; } @@ -63,7 +68,7 @@ class DeviceFd deviceFd_ = -1; } - bool Valid() + bool Valid() const { return deviceFd_ >= 0; } diff --git a/src/helpers/dma_heap.hpp b/src/helpers/dma_heap.hpp new file mode 100644 index 0000000..2510d56 --- /dev/null +++ b/src/helpers/dma_heap.hpp @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2026, Raspberry Pi Ltd + * + * dma_heap.hpp - Helper class for dma-heap allocations. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/logging.hpp" +#include "device_fd.hpp" + +namespace libpisp::helpers +{ + +class DmaHeap +{ +public: + DmaHeap() + { + // /dev/dma-heap/vidbuf_cached sym links to either the system heap (Pi 5) or the + // CMA allocator (Pi 4 and below). If missing, fallback to the system allocator. + static const std::vector heapNames { + "/dev/dma_heap/vidbuf_cached", + "/dev/dma_heap/system", + }; + + for (const char *name : heapNames) + { + dmaHeapHandle_ = DeviceFd(name, O_RDWR | O_CLOEXEC); + if (!dmaHeapHandle_.Valid()) + { + PISP_LOG(debug, "Failed to open " << name); + continue; + } + break; + } + + if (!dmaHeapHandle_.Valid()) + PISP_LOG(warning, "Could not open any dmaHeap device"); + } + + ~DmaHeap() + { + } + + DmaHeap(DmaHeap &&other) = default; + + bool Valid() const + { + return dmaHeapHandle_.Valid(); + } + + int alloc(const char *name, std::size_t size) const + { + int ret; + + if (!name) + return {}; + + struct dma_heap_allocation_data alloc = {}; + + alloc.len = size; + alloc.fd_flags = O_CLOEXEC | O_RDWR; + + ret = ::ioctl(dmaHeapHandle_.Get(), DMA_HEAP_IOCTL_ALLOC, &alloc); + if (ret < 0) + { + PISP_LOG(warning, "dmaHeap allocation failure for " << name); + return -1; + } + + ret = ::ioctl(alloc.fd, DMA_BUF_SET_NAME, name); + if (ret < 0) + { + PISP_LOG(warning, "dmaHeap naming failure for " << name); + return -1; + } + + return alloc.fd; + } + +private: + DeviceFd dmaHeapHandle_; +}; + +} //libpisp::helpers diff --git a/src/helpers/media_device.cpp b/src/helpers/media_device.cpp index 8388e8c..deeb7d4 100644 --- a/src/helpers/media_device.cpp +++ b/src/helpers/media_device.cpp @@ -43,13 +43,15 @@ class MediaEnumerator public: using MediaDevList = std::vector; - MediaEnumerator(MediaEnumerator &other) = delete; - void operator=(const MediaEnumerator &other) = delete; + MediaEnumerator(const MediaEnumerator &) = delete; + MediaEnumerator(MediaEnumerator &&) = delete; + MediaEnumerator &operator=(const MediaEnumerator &) = delete; + MediaEnumerator &operator=(MediaEnumerator &&) = delete; static const MediaEnumerator *Get() { - static std::unique_ptr mdev(new MediaEnumerator); - return mdev.get(); + static MediaEnumerator mdev; + return &mdev; } const MediaDevList &MediaDeviceList() const diff --git a/src/helpers/media_device.hpp b/src/helpers/media_device.hpp index 6cba36f..1ea6d44 100644 --- a/src/helpers/media_device.hpp +++ b/src/helpers/media_device.hpp @@ -9,7 +9,6 @@ #include #include -#include #include diff --git a/src/helpers/meson.build b/src/helpers/meson.build index 5fa71c9..242b944 100644 --- a/src/helpers/meson.build +++ b/src/helpers/meson.build @@ -3,13 +3,16 @@ pisp_sources += files([ 'backend_device.cpp', + 'buffer.cpp', 'media_device.cpp', 'v4l2_device.cpp' ]) helper_headers = files([ 'backend_device.hpp', + 'buffer.hpp', 'device_fd.hpp', + 'dma_heap.hpp', 'media_device.hpp', 'v4l2_device.hpp', ]) diff --git a/src/helpers/v4l2_device.cpp b/src/helpers/v4l2_device.cpp index 34d4d08..e726679 100644 --- a/src/helpers/v4l2_device.cpp +++ b/src/helpers/v4l2_device.cpp @@ -6,21 +6,20 @@ * v4l2_device.cpp - PiSP V4L2 device helper */ +#include "v4l2_device.hpp" + #include -#include #include #include #include #include #include #include -#include #include +#include #include "libpisp/common/utils.hpp" -#include "v4l2_device.hpp" - using namespace libpisp::helpers; namespace { @@ -41,6 +40,8 @@ static FormatInfo get_v4l2_format(const std::string &format) { "YUV444P", { V4L2_PIX_FMT_YUV444M, 3 } }, { "YUYV", { V4L2_PIX_FMT_YUYV, 1 } }, { "UYVY", { V4L2_PIX_FMT_UYVY, 1 } }, + { "NV12", { V4L2_PIX_FMT_NV12M, 2 } }, + { "YUV420SP_COL128", { V4L2_PIX_FMT_NV12MT_COL128, 2 } }, }; auto it = formats.find(format); @@ -53,7 +54,7 @@ static FormatInfo get_v4l2_format(const std::string &format) } // namespace V4l2Device::V4l2Device(const std::string &device) - : fd_(device, O_RDWR | O_NONBLOCK | O_CLOEXEC), num_memory_planes_(1) + : fd_(device, O_RDWR | O_NONBLOCK | O_CLOEXEC), num_memory_planes_(1), max_slots_(0), v4l2_format_() { struct v4l2_capability caps; @@ -73,127 +74,161 @@ V4l2Device::V4l2Device(const std::string &device) V4l2Device::~V4l2Device() { - ReturnBuffers(); + ReleaseBuffers(); Close(); } -int V4l2Device::RequestBuffers(unsigned int count) +int V4l2Device::AllocateBuffers(unsigned int count) { + struct v4l2_format f = {}; int ret; - ReturnBuffers(); - - v4l2_requestbuffers req_bufs {}; - - req_bufs.count = count; - req_bufs.type = buf_type_; - req_bufs.memory = V4L2_MEMORY_MMAP; - - ret = ioctl(fd_.Get(), VIDIOC_REQBUFS, &req_bufs); - if (ret < 0) - throw std::runtime_error("VIDIOC_REQBUFS failed: " + std::to_string(ret)); + f.type = buf_type_; + ret = ioctl(fd_.Get(), VIDIOC_G_FMT, &f); + if (ret) + throw std::runtime_error("VIDIOC_G_FMT failed: " + std::to_string(ret)); - for (unsigned int i = 0; i < req_bufs.count; i++) + for (unsigned int i = 0; i < count; i++) { - v4l2_plane planes[VIDEO_MAX_PLANES] = {}; - v4l2_buffer buffer {}; + std::array fds = { -1, -1, -1 }; + std::array sizes = { 0, 0, 0 }; - buffer.index = i; - buffer.type = buf_type_; - buffer.memory = V4L2_MEMORY_MMAP; - if (!isMeta()) + for (unsigned int p = 0; p < num_memory_planes_; p++) { - buffer.m.planes = planes; - buffer.length = num_memory_planes_; + const size_t size = isMeta() ? f.fmt.meta.buffersize : f.fmt.pix_mp.plane_fmt[p].sizeimage; + int fd = dma_heap_.alloc("v4l2_device_buf", size); + + if (fd < 0) + throw std::runtime_error("DMABUF allocation failed"); + + fds[p] = fd; + sizes[p] = size; } - ret = ioctl(fd_.Get(), VIDIOC_QUERYBUF, &buffer); + const Buffer &b = buffer_allocs_.emplace_back(fds, sizes); + + // May as well, and this also calls REQBUFS. + ImportBuffer(b); + } + + return buffer_allocs_.size(); +} + +int V4l2Device::ImportBuffer(BufferRef buffer) +{ + std::vector::iterator cache_it; + + if (!max_slots_) + { + v4l2_requestbuffers req_bufs {}; + req_bufs.count = 64; + req_bufs.type = buf_type_; + req_bufs.memory = V4L2_MEMORY_DMABUF; + + int ret = ioctl(fd_.Get(), VIDIOC_REQBUFS, &req_bufs); if (ret < 0) - throw std::runtime_error("VIDIOC_QUERYBUF failed: " + std::to_string(ret)); + throw std::runtime_error("VIDIOC_REQBUFS failed: " + std::to_string(ret)); - // Don't keep this pointer dangling when putting into v4l2_buffers_. - buffer.m.planes = NULL; - v4l2_buffers_.emplace_back(buffer); - available_buffers_.push(i); + max_slots_ = req_bufs.count; + buffer_cache_.reserve(max_slots_); + } - for (unsigned int p = 0; p < num_memory_planes_; p++) - { - size_t size = !isMeta() ? planes[p].length : buffer.length; - unsigned int offset = !isMeta() ? planes[p].m.mem_offset : buffer.m.offset; + // Check if buffer with matching fd and size already exists and is not queued - reuse it. + cache_it = std::find_if(buffer_cache_.begin(), buffer_cache_.end(), + [&buffer](const auto &b) { return b == buffer && !b.queued; }); - void *mem = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_.Get(), offset); - if (mem == MAP_FAILED) - throw std::runtime_error("Unable to mmap buffer"); + if (cache_it != buffer_cache_.end()) + return cache_it - buffer_cache_.begin(); - v4l2_buffers_.back().size[p] = size; - v4l2_buffers_.back().mem[p] = (uint8_t *)mem; - } + for (unsigned int p = 0; p < num_memory_planes_; p++) + { + const size_t size = isMeta() ? v4l2_format_.fmt.meta.buffersize + : v4l2_format_.fmt.pix_mp.plane_fmt[p].sizeimage; + + if (buffer.get().Fd()[p] < 0 || buffer.get().Size()[p] < size) + throw std::runtime_error("Plane " + std::to_string(p) + " buffer is invalid."); + } + + if (buffer_cache_.size() == max_slots_) + { + // Find and replace the first buffer that is not queued. + cache_it = std::find_if(buffer_cache_.begin(), buffer_cache_.end(), + [](const auto &buf) { return !buf.queued; }); + if (cache_it == buffer_cache_.end()) + throw std::runtime_error("Unable to import buffer, run out of slots."); + + *cache_it = BufferCache(buffer.get().Fd(), buffer.get().Size(), cache_it->id); + } + else + { + cache_it = buffer_cache_.emplace(buffer_cache_.end(), + buffer.get().Fd(), buffer.get().Size(), buffer_cache_.size()); } - return v4l2_buffers_.size(); + return cache_it - buffer_cache_.begin(); } -void V4l2Device::ReturnBuffers() +void V4l2Device::ReleaseBuffers() { - v4l2_requestbuffers req_bufs {}; - - if (!v4l2_buffers_.size()) + if (!buffer_cache_.size()) return; - for (auto const &b : v4l2_buffers_) - { - for (unsigned int p = 0; p < num_memory_planes_; p++) - munmap(b.mem[p], b.size[p]); - } + v4l2_requestbuffers req_bufs {}; req_bufs.type = buf_type_; req_bufs.count = 0; - req_bufs.memory = V4L2_MEMORY_MMAP; + req_bufs.memory = V4L2_MEMORY_DMABUF; ioctl(fd_.Get(), VIDIOC_REQBUFS, &req_bufs); - v4l2_buffers_.clear(); + buffer_allocs_ = {}; + buffer_cache_ = {}; + max_slots_ = 0; } -std::optional V4l2Device::AcquireBuffer() +std::vector V4l2Device::Buffers() const { - if (available_buffers_.empty()) - return {}; + std::vector refs; - unsigned int index = available_buffers_.front(); - available_buffers_.pop(); - return findBuffer(index); -} + for (const Buffer &b : buffer_allocs_) + refs.push_back(b); -void V4l2Device::ReleaseBuffer(const Buffer &buffer) -{ - available_buffers_.push(buffer.buffer.index); + return refs; } -int V4l2Device::QueueBuffer(unsigned int index) +int V4l2Device::QueueBuffer(const Buffer &buffer) { - std::optional buf = findBuffer(index); - if (!buf) - return -1; - v4l2_plane planes[VIDEO_MAX_PLANES] = {}; + v4l2_buffer buf {}; + + int idx = ImportBuffer(buffer); + buffer_cache_[idx].queued = true; + + buf.index = buffer_cache_[idx].id; + buf.type = buf_type_; + buf.memory = V4L2_MEMORY_DMABUF; + if (!isMeta()) { - buf->buffer.m.planes = planes; - buf->buffer.length = num_memory_planes_; + buf.m.planes = planes; + buf.length = num_memory_planes_; for (unsigned int p = 0; p < num_memory_planes_; p++) { - buf->buffer.m.planes[p].bytesused = buf->size[p]; - buf->buffer.m.planes[p].length = buf->size[p]; + buf.m.planes[p].bytesused = buffer.Size()[p]; + buf.m.planes[p].length = buffer.Size()[p]; + buf.m.planes[p].m.fd = buffer.Fd()[p]; } } else - buf->buffer.bytesused = buf->size[0]; + { + buf.bytesused = buffer.Size()[0]; + buf.m.fd = buffer.Fd()[0]; + } - buf->buffer.timestamp.tv_sec = time(NULL); - buf->buffer.field = V4L2_FIELD_NONE; - buf->buffer.flags = 0; + buf.timestamp.tv_sec = time(NULL); + buf.field = V4L2_FIELD_NONE; + buf.flags = 0; - int ret = ioctl(fd_.Get(), VIDIOC_QBUF, &buf->buffer); + int ret = ioctl(fd_.Get(), VIDIOC_QBUF, &buf); if (ret < 0) throw std::runtime_error("Unable to queue buffer: " + std::string(strerror(errno))); @@ -216,7 +251,7 @@ int V4l2Device::DequeueBuffer(unsigned int timeout_ms) v4l2_plane planes[VIDEO_MAX_PLANES] = {}; buf.type = buf_type_; - buf.memory = V4L2_MEMORY_MMAP; + buf.memory = V4L2_MEMORY_DMABUF; if (!isMeta()) { buf.m.planes = planes; @@ -227,11 +262,20 @@ int V4l2Device::DequeueBuffer(unsigned int timeout_ms) if (ret) return -1; - return buf.index; + // Find the buffer in cache with matching id and set queued to false + auto cache_it = std::find_if(buffer_cache_.begin(), buffer_cache_.end(), + [&buf](const auto &b) { return b.id == buf.index; }); + if (cache_it != buffer_cache_.end()) + cache_it->queued = false; + + return 0; } void V4l2Device::SetFormat(const pisp_image_format_config &format, bool use_opaque_format) { + // Release old buffers before setting the new format. + ReleaseBuffers(); + struct v4l2_format f = {}; FormatInfo info = get_v4l2_format(libpisp::get_pisp_image_format(format.format)); @@ -270,7 +314,11 @@ void V4l2Device::SetFormat(const pisp_image_format_config &format, bool use_opaq { const unsigned int stride = p == 0 ? format.stride : format.stride2; // Wallpaper stride is not something the V4L2 kernel knows about! - f.fmt.pix_mp.plane_fmt[p].bytesperline = stride; + // Do NOT use the column strides that have been computed in compute_stride_align + if (PISP_IMAGE_FORMAT_WALLPAPER(format.format)) + f.fmt.pix_mp.plane_fmt[p].bytesperline = (format.width + 127) & ~127; + else + f.fmt.pix_mp.plane_fmt[p].bytesperline = stride; f.fmt.pix_mp.plane_fmt[p].sizeimage = libpisp::get_plane_size(format, p); } @@ -281,6 +329,8 @@ void V4l2Device::SetFormat(const pisp_image_format_config &format, bool use_opaq int ret = ioctl(fd_.Get(), VIDIOC_S_FMT, &f); if (ret) throw std::runtime_error("Unable to set format: " + std::string(strerror(errno))); + + v4l2_format_ = f; } void V4l2Device::StreamOn() @@ -296,16 +346,3 @@ void V4l2Device::StreamOff() if (ret < 0) throw std::runtime_error("Stream off failed: " + std::string(strerror(errno))); } - -std::optional V4l2Device::findBuffer(unsigned int index) const -{ - auto it = std::find_if(v4l2_buffers_.begin(), v4l2_buffers_.end(), - [index](auto const &b) { return b.buffer.index == index; }); - if (it == v4l2_buffers_.end()) - { - throw std::runtime_error("find buffers failed"); - return {}; - } - - return *it; -} diff --git a/src/helpers/v4l2_device.hpp b/src/helpers/v4l2_device.hpp index fe53bbf..7f7db78 100644 --- a/src/helpers/v4l2_device.hpp +++ b/src/helpers/v4l2_device.hpp @@ -8,16 +8,22 @@ #pragma once #include -#include -#include -#include #include #include -#include "libpisp/backend/pisp_be_config.h" +#ifndef V4L2_PIX_FMT_NV12MT_COL128 +#define V4L2_PIX_FMT_NV12MT_COL128 v4l2_fourcc('N', 'c', '1', '2') /* 12 Y/CbCr 4:2:0 128 pixel wide column */ +#endif +#ifndef V4L2_PIX_FMT_NV12MT_10_COL128 +#define V4L2_PIX_FMT_NV12MT_10_COL128 v4l2_fourcc('N', 'c', '3', '0') +#endif +#include "backend/pisp_be_config.h" + +#include "buffer.hpp" #include "device_fd.hpp" +#include "dma_heap.hpp" namespace libpisp::helpers { @@ -50,33 +56,11 @@ class V4l2Device fd_.Close(); } - struct Buffer - { - Buffer() - { - } - - Buffer(const v4l2_buffer& buf) - : buffer(buf), size({}), mem({}) - { - } - - v4l2_buffer buffer; - std::array size; - std::array mem; - }; - - int RequestBuffers(unsigned int count = 1); - void ReturnBuffers(); - - std::optional AcquireBuffer(); - void ReleaseBuffer(const Buffer &buffer); - const std::vector &Buffers() const - { - return v4l2_buffers_; - }; - - int QueueBuffer(unsigned int index); + int AllocateBuffers(unsigned int count = 1); + int ImportBuffer(BufferRef buffer); + void ReleaseBuffers(); + std::vector Buffers() const; + int QueueBuffer(const Buffer &buffer); int DequeueBuffer(unsigned int timeout_ms = 500); void SetFormat(const pisp_image_format_config &format, bool use_opaque_format = false); @@ -100,13 +84,32 @@ class V4l2Device return !isCapture(); } - std::optional findBuffer(unsigned int index) const; + struct BufferCache + { + BufferCache(const std::array &fd, const std::array &size, unsigned int id) + : fd(fd), size(size), id(id), queued(false) + { + } + + std::array fd; + std::array size; + unsigned int id; + bool queued; + + bool operator==(const Buffer &other) const + { + return fd == other.Fd() && size == other.Size(); + } + }; - std::queue available_buffers_; - std::vector v4l2_buffers_; + std::vector buffer_cache_; + std::vector buffer_allocs_; DeviceFd fd_; enum v4l2_buf_type buf_type_; unsigned int num_memory_planes_; + DmaHeap dma_heap_; + unsigned int max_slots_; + v4l2_format v4l2_format_; }; } // namespace libpisp From 1a864e3813ccb038ceda3ee568505397791d3110 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Thu, 5 Feb 2026 16:16:43 +0000 Subject: [PATCH 3/8] libpisp: utils: Fix support for wallpaper and semiplanar formats Correct the stride calucation and add YUV420SP, YUV420SP_COL128, and YUV420SP10_COL128 identifiers. Signed-off-by: Naushir Patuck --- src/helpers/v4l2_device.cpp | 1 + src/libpisp/common/pisp_utils.cpp | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/helpers/v4l2_device.cpp b/src/helpers/v4l2_device.cpp index e726679..557bcfa 100644 --- a/src/helpers/v4l2_device.cpp +++ b/src/helpers/v4l2_device.cpp @@ -42,6 +42,7 @@ static FormatInfo get_v4l2_format(const std::string &format) { "UYVY", { V4L2_PIX_FMT_UYVY, 1 } }, { "NV12", { V4L2_PIX_FMT_NV12M, 2 } }, { "YUV420SP_COL128", { V4L2_PIX_FMT_NV12MT_COL128, 2 } }, + { "YUV420SP10_COL128", { V4L2_PIX_FMT_NV12MT_10_COL128, 2 } }, }; auto it = formats.find(format); diff --git a/src/libpisp/common/pisp_utils.cpp b/src/libpisp/common/pisp_utils.cpp index 9ec594e..fd8f814 100644 --- a/src/libpisp/common/pisp_utils.cpp +++ b/src/libpisp/common/pisp_utils.cpp @@ -64,7 +64,7 @@ void compute_stride_align(pisp_image_format_config &config, int align, bool pres { if (PISP_IMAGE_FORMAT_WALLPAPER(config.format)) { - config.stride2 = config.stride = config.height * PISP_WALLPAPER_WIDTH; + config.stride2 = config.stride = ((config.height + 7) & ~7) * PISP_WALLPAPER_WIDTH; if (PISP_IMAGE_FORMAT_SAMPLING_420(config.format)) config.stride2 /= 2; return; @@ -227,6 +227,14 @@ static const std::map &formats_table() PISP_IMAGE_FORMAT_PLANARITY_PLANAR }, { "YUV420P", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_8 + PISP_IMAGE_FORMAT_SAMPLING_420 + PISP_IMAGE_FORMAT_PLANARITY_PLANAR }, + { "YUV420SP", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_8 + PISP_IMAGE_FORMAT_SAMPLING_420 + + PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR }, + { "YUV420SP_COL128", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_8 + + PISP_IMAGE_FORMAT_SAMPLING_420 + PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR + + PISP_IMAGE_FORMAT_WALLPAPER_ROLL }, + { "YUV420SP10_COL128", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_10 + + PISP_IMAGE_FORMAT_SAMPLING_420 + PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR + + PISP_IMAGE_FORMAT_WALLPAPER_ROLL }, { "NV12", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_8 + PISP_IMAGE_FORMAT_SAMPLING_420 + PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR }, { "NV21", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_8 + PISP_IMAGE_FORMAT_SAMPLING_420 + From ea7ce6aad4f7d192be4642eef9518f623cca0051 Mon Sep 17 00:00:00 2001 From: Dave Stevenson Date: Tue, 17 Feb 2026 13:13:31 +0000 Subject: [PATCH 4/8] libpisp: utils: Add support for RGBX8888 Signed-off-by: Dave Stevenson --- src/helpers/v4l2_device.cpp | 1 + src/libpisp/common/pisp_utils.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/helpers/v4l2_device.cpp b/src/helpers/v4l2_device.cpp index 557bcfa..9be1098 100644 --- a/src/helpers/v4l2_device.cpp +++ b/src/helpers/v4l2_device.cpp @@ -35,6 +35,7 @@ static FormatInfo get_v4l2_format(const std::string &format) std::map formats { { "RGB888", { V4L2_PIX_FMT_RGB24, 1 } }, { "RGBX8888", { V4L2_PIX_FMT_RGBX32, 1 } }, + { "XRGB8888", { V4L2_PIX_FMT_XRGB32, 1 } }, { "YUV420P", { V4L2_PIX_FMT_YUV420, 1 } }, { "YUV422P", { V4L2_PIX_FMT_YUV422P, 1 } }, { "YUV444P", { V4L2_PIX_FMT_YUV444M, 3 } }, diff --git a/src/libpisp/common/pisp_utils.cpp b/src/libpisp/common/pisp_utils.cpp index fd8f814..92c74d6 100644 --- a/src/libpisp/common/pisp_utils.cpp +++ b/src/libpisp/common/pisp_utils.cpp @@ -249,6 +249,7 @@ static const std::map &formats_table() PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR + PISP_IMAGE_FORMAT_ORDER_SWAPPED }, { "RGB888", PISP_IMAGE_FORMAT_THREE_CHANNEL }, { "RGBX8888", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPP_32 }, + { "XRGB8888", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPP_32 + PISP_IMAGE_FORMAT_ORDER_SWAPPED }, { "RGB161616", PISP_IMAGE_FORMAT_THREE_CHANNEL + PISP_IMAGE_FORMAT_BPS_16 }, { "BAYER16", PISP_IMAGE_FORMAT_BPS_16 + PISP_IMAGE_FORMAT_UNCOMPRESSED }, { "PISP_COMP1", PISP_IMAGE_FORMAT_COMPRESSION_MODE_1 }, From 2404991795c3482c4f5eeb25dbc48946b5651163 Mon Sep 17 00:00:00 2001 From: Dave Stevenson Date: Tue, 17 Feb 2026 13:16:38 +0000 Subject: [PATCH 5/8] libpisp: helpers: Unify on the multiplanar YUV V4L2 formats Signed-off-by: Dave Stevenson --- src/helpers/v4l2_device.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/v4l2_device.cpp b/src/helpers/v4l2_device.cpp index 9be1098..417e15d 100644 --- a/src/helpers/v4l2_device.cpp +++ b/src/helpers/v4l2_device.cpp @@ -36,8 +36,8 @@ static FormatInfo get_v4l2_format(const std::string &format) { "RGB888", { V4L2_PIX_FMT_RGB24, 1 } }, { "RGBX8888", { V4L2_PIX_FMT_RGBX32, 1 } }, { "XRGB8888", { V4L2_PIX_FMT_XRGB32, 1 } }, - { "YUV420P", { V4L2_PIX_FMT_YUV420, 1 } }, - { "YUV422P", { V4L2_PIX_FMT_YUV422P, 1 } }, + { "YUV420P", { V4L2_PIX_FMT_YUV420M, 3 } }, + { "YUV422P", { V4L2_PIX_FMT_YUV422M, 3 } }, { "YUV444P", { V4L2_PIX_FMT_YUV444M, 3 } }, { "YUYV", { V4L2_PIX_FMT_YUYV, 1 } }, { "UYVY", { V4L2_PIX_FMT_UYVY, 1 } }, From 90c698e95345e8c393a024d55f6fcd5fa680c395 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Thu, 5 Feb 2026 16:19:35 +0000 Subject: [PATCH 6/8] utils: test_convert: Update for new buffer API Signed-off-by: Naushir Patuck --- utils/test_convert.py | 155 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 138 insertions(+), 17 deletions(-) diff --git a/utils/test_convert.py b/utils/test_convert.py index 3effbf1..edabf89 100644 --- a/utils/test_convert.py +++ b/utils/test_convert.py @@ -18,13 +18,16 @@ class ConvertTester: - def __init__(self, convert_binary, output_dir=None, input_dir=None, reference_dir=None): + def __init__(self, convert_binary, output_dir=None, input_dir=None, reference_dir=None, use_gstreamer=False, gst_plugin_path=None): """Initialize the tester with the path to the convert binary.""" self.convert_binary = convert_binary self.output_dir = output_dir self.input_dir = input_dir self.reference_dir = reference_dir - if not os.path.exists(convert_binary): + self.use_gstreamer = use_gstreamer + self.gst_plugin_path = gst_plugin_path + + if not use_gstreamer and not os.path.exists(convert_binary): raise FileNotFoundError(f"Convert binary not found: {convert_binary}") # Test cases: (input_file, output_file, input_format, output_format, reference_file) @@ -34,25 +37,104 @@ def __init__(self, convert_binary, output_dir=None, input_dir=None, reference_di "output_file": "out_4056x3050_12168s_rgb888.rgb", "input_format": "4056:3040:4056:YUV420P", "output_format": "4056:3040:12168:RGB888", - "reference_file": "ref_4056x3050_12168s_rgb888.rgb" + "reference_file": "ref_4056x3050_12168s_rgb888.rgb", + "skip_gst": False }, { "input_file": "conv_800x600_1200s_422_yuyv.yuv", "output_file": "out_1600x1200_422p.yuv", "input_format": "800:600:1600:YUYV", "output_format": "1600:1200:1600:YUV422P", - "reference_file": "ref_1600x1200_422p.yuv" + "reference_file": "ref_1600x1200_422p.yuv", + "skip_gst": False }, { "input_file": "conv_rgb888_800x600_2432s.rgb", "output_file": "out_4000x3000_4032s.yuv", "input_format": "800:600:2432:RGB888", - "output_format": "4000:3000:0:YUV444P", - "reference_file": "ref_4000x3000_4032s.yuv" + "output_format": "4000:3000:4032:YUV444P", + "reference_file": "ref_4000x3000_4032s.yuv", + "skip_gst": True }, # Add more test cases here as needed ] + def _parse_format(self, format_str): + """Parse format string like '4056:3040:4056:YUV420P' into components.""" + parts = format_str.split(':') + if len(parts) != 4: + raise ValueError(f"Invalid format string: {format_str}") + return { + 'width': int(parts[0]), + 'height': int(parts[1]), + 'stride': int(parts[2]), + 'format': parts[3] + } + + def _pisp_to_gst_format(self, pisp_format): + """Convert PiSP format to GStreamer format string.""" + format_map = { + 'YUV420P': 'I420', + 'YVU420P': 'YV12', + 'YUV422P': 'Y42B', + 'YUV444P': 'Y444', + 'YUYV': 'YUY2', + 'UYVY': 'UYVY', + 'RGB888': 'RGB', + } + return format_map.get(pisp_format, pisp_format) + + def run_gstreamer(self, input_file, output_file, input_format, output_format): + """Run GStreamer pipeline with pispconvert.""" + # Use input directory if specified + if self.input_dir: + input_file = os.path.join(self.input_dir, input_file) + + # Use output directory if specified + if self.output_dir: + output_file = os.path.join(self.output_dir, output_file) + + # Parse format strings + in_fmt = self._parse_format(input_format) + out_fmt = self._parse_format(output_format) + + # Convert to GStreamer format names + gst_in_format = self._pisp_to_gst_format(in_fmt['format']) + gst_out_format = self._pisp_to_gst_format(out_fmt['format']) + + # Build GStreamer pipeline + pipeline = [ + 'gst-launch-1.0', + 'filesrc', f'location={input_file}', '!', + 'rawvideoparse', + f'width={in_fmt["width"]}', + f'height={in_fmt["height"]}', + f'format={gst_in_format.lower()}', + 'framerate=30/1', '!', + 'pispconvert', '!', + f'video/x-raw,format={gst_out_format},width={out_fmt["width"]},height={out_fmt["height"]}', '!', + 'filesink', f'location={output_file}' + ] + + print(f"Running GStreamer pipeline:") + print(' '.join(pipeline)) + + # Set GST_PLUGIN_PATH environment variable if specified + env = os.environ.copy() + if self.gst_plugin_path: + env['GST_PLUGIN_PATH'] = self.gst_plugin_path + print(f"GST_PLUGIN_PATH={self.gst_plugin_path}") + + try: + result = subprocess.run(pipeline, capture_output=True, text=True, check=True, env=env) + print("GStreamer pipeline completed successfully") + return True + except subprocess.CalledProcessError as e: + print(f"GStreamer pipeline failed with exit code {e.returncode}") + print(f"stdout: {e.stdout}") + print(f"stderr: {e.stderr}") + return False + def run_convert(self, input_file, output_file, input_format, output_format): """Run the convert utility with the specified parameters.""" # Use input directory if specified @@ -137,13 +219,26 @@ def run_test_case(self, test_case): print(f"Error: Input file {input_file} does not exist") return False - # Run the convert utility - success = self.run_convert( - test_case['input_file'], - test_case['output_file'], - test_case['input_format'], - test_case['output_format'] - ) + # Skip GStreamer test if marked to skip + if self.use_gstreamer and test_case.get('skip_gst', False): + print(f"SKIPPED: Test case marked as skip_gst=True") + return None # Return None to indicate skipped + + # Run the convert utility or GStreamer pipeline + if self.use_gstreamer: + success = self.run_gstreamer( + test_case['input_file'], + test_case['output_file'], + test_case['input_format'], + test_case['output_format'] + ) + else: + success = self.run_convert( + test_case['input_file'], + test_case['output_file'], + test_case['input_format'], + test_case['output_format'] + ) if not success: return False @@ -166,7 +261,12 @@ def run_test_case(self, test_case): def run_all_tests(self): """Run all test cases.""" - print(f"Testing convert utility: {self.convert_binary}") + if self.use_gstreamer: + print("Testing with GStreamer pispconvert plugin") + if self.gst_plugin_path: + print(f"GST_PLUGIN_PATH: {self.gst_plugin_path}") + else: + print(f"Testing convert utility: {self.convert_binary}") if self.input_dir: print(f"Input directory: {self.input_dir}") if self.output_dir: @@ -177,11 +277,16 @@ def run_all_tests(self): passed = 0 failed = 0 + skipped = 0 for i, test_case in enumerate(self.test_cases, 1): print(f"\n--- Test case {i}/{len(self.test_cases)} ---") - if self.run_test_case(test_case): + result = self.run_test_case(test_case) + if result is None: + skipped += 1 + print("⊘ Test SKIPPED") + elif result: passed += 1 print("✓ Test PASSED") else: @@ -191,6 +296,7 @@ def run_all_tests(self): print(f"\n=== Test Summary ===") print(f"Passed: {passed}") print(f"Failed: {failed}") + print(f"Skipped: {skipped}") print(f"Total: {len(self.test_cases)}") return failed == 0 @@ -198,16 +304,31 @@ def run_all_tests(self): def main(): parser = argparse.ArgumentParser(description="Test script for libpisp convert utility") - parser.add_argument("convert_binary", help="Path to the convert binary") + parser.add_argument("convert_binary", nargs='?', default=None, help="Path to the convert binary (not needed with --gst-plugin-path)") parser.add_argument("--test-dir", help="Directory containing test files") parser.add_argument("--in", dest="input_dir", help="Directory containing input files") parser.add_argument("--out", help="Directory where output files will be written") parser.add_argument("--ref", help="Directory containing reference files") + parser.add_argument("--gst-plugin-path", help="Path to GStreamer plugin directory (enables GStreamer testing)") args = parser.parse_args() try: - tester = ConvertTester(args.convert_binary, args.out, args.input_dir, args.ref) + # Determine if using GStreamer based on --gst-plugin-path + use_gstreamer = args.gst_plugin_path is not None + + # Validate arguments + if not use_gstreamer and not args.convert_binary: + parser.error("convert_binary is required unless --gst-plugin-path is specified") + + tester = ConvertTester( + args.convert_binary, + args.out, + args.input_dir, + args.ref, + use_gstreamer=use_gstreamer, + gst_plugin_path=args.gst_plugin_path + ) # Change to test directory if specified if args.test_dir: From 983b912dc96ba9f60425d6d4e648a021692e27db Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 10 Feb 2026 09:04:35 +0000 Subject: [PATCH 7/8] Make importbuffer private --- src/helpers/v4l2_device.cpp | 14 +++++++------- src/helpers/v4l2_device.hpp | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/helpers/v4l2_device.cpp b/src/helpers/v4l2_device.cpp index 417e15d..fdb49ce 100644 --- a/src/helpers/v4l2_device.cpp +++ b/src/helpers/v4l2_device.cpp @@ -110,13 +110,13 @@ int V4l2Device::AllocateBuffers(unsigned int count) const Buffer &b = buffer_allocs_.emplace_back(fds, sizes); // May as well, and this also calls REQBUFS. - ImportBuffer(b); + importBuffer(b); } return buffer_allocs_.size(); } -int V4l2Device::ImportBuffer(BufferRef buffer) +std::vector::iterator V4l2Device::importBuffer(BufferRef buffer) { std::vector::iterator cache_it; @@ -140,7 +140,7 @@ int V4l2Device::ImportBuffer(BufferRef buffer) [&buffer](const auto &b) { return b == buffer && !b.queued; }); if (cache_it != buffer_cache_.end()) - return cache_it - buffer_cache_.begin(); + return cache_it; for (unsigned int p = 0; p < num_memory_planes_; p++) { @@ -167,7 +167,7 @@ int V4l2Device::ImportBuffer(BufferRef buffer) buffer.get().Fd(), buffer.get().Size(), buffer_cache_.size()); } - return cache_it - buffer_cache_.begin(); + return cache_it; } void V4l2Device::ReleaseBuffers() @@ -202,10 +202,10 @@ int V4l2Device::QueueBuffer(const Buffer &buffer) v4l2_plane planes[VIDEO_MAX_PLANES] = {}; v4l2_buffer buf {}; - int idx = ImportBuffer(buffer); - buffer_cache_[idx].queued = true; + auto cache_it = importBuffer(buffer); + cache_it->queued = true; - buf.index = buffer_cache_[idx].id; + buf.index = cache_it->id; buf.type = buf_type_; buf.memory = V4L2_MEMORY_DMABUF; diff --git a/src/helpers/v4l2_device.hpp b/src/helpers/v4l2_device.hpp index 7f7db78..7853f4c 100644 --- a/src/helpers/v4l2_device.hpp +++ b/src/helpers/v4l2_device.hpp @@ -57,7 +57,6 @@ class V4l2Device } int AllocateBuffers(unsigned int count = 1); - int ImportBuffer(BufferRef buffer); void ReleaseBuffers(); std::vector Buffers() const; int QueueBuffer(const Buffer &buffer); @@ -102,6 +101,8 @@ class V4l2Device } }; + std::vector::iterator importBuffer(BufferRef buffer); + std::vector buffer_cache_; std::vector buffer_allocs_; DeviceFd fd_; From ab4eeaf2c0d6f25bfee3deb8eb6cb702145134ac Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Thu, 5 Feb 2026 16:16:55 +0000 Subject: [PATCH 8/8] gst: Add pispconvert GStreamer element Add a GStreamer element for hardware-accelerated image scaling and format conversion using the PiSP Backend. Features: - Single and dual output support - DMABuf input/output for zero-copy pipelines - Software buffer fallback - Crop support with per-output configuration - Format conversion between RGB, YUV420, YUV422, etc. Signed-off-by: Naushir Patuck --- README.md | 45 ++ meson_options.txt | 1 + src/gst/gstpispconvert.cpp | 1512 ++++++++++++++++++++++++++++++++++++ src/gst/gstpispconvert.h | 110 +++ src/gst/meson.build | 29 + src/gst/usage.md | 145 ++++ src/meson.build | 4 +- 7 files changed, 1845 insertions(+), 1 deletion(-) create mode 100644 src/gst/gstpispconvert.cpp create mode 100644 src/gst/gstpispconvert.h create mode 100644 src/gst/meson.build create mode 100644 src/gst/usage.md diff --git a/README.md b/README.md index 371549b..3c11506 100644 --- a/README.md +++ b/README.md @@ -24,5 +24,50 @@ libpisp_dep = dependency('libpisp', fallback : ['libpisp', 'libpisp_dep']) Alternatively [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) can be used to locate ``libpisp.so`` installed in of the system directories for other build environments. +## Command-Line Tools + +### convert + +A simple command-line image converter that uses the PiSP Backend for hardware-accelerated format conversion and scaling. + +```sh +pisp_convert input.yuv output.rgb \ + --input-format 1920:1080:1920:YUV420P \ + --output-format 1280:720:3840:RGB888 +``` + +Format strings use the form `width:height:stride:format`. Use `--formats` to list available formats, or `--list` to enumerate available PiSP devices. + +## GStreamer Element + +libpisp includes a GStreamer element (`pispconvert`) that provides hardware-accelerated image scaling and format conversion using the PiSP Backend. + +### Building with GStreamer Support + +GStreamer support is enabled by default if the required dependencies are found. To explicitly enable or disable: + +```sh +meson setup -Dgstreamer=enabled # require GStreamer support +meson setup -Dgstreamer=disabled # disable GStreamer support +``` + +### Using the Element + +After installation, the element will be available as `pispconvert`: + +```sh +gst-inspect-1.0 pispconvert +``` + +For usage examples and supported formats, see [src/gst/usage.md](src/gst/usage.md). + +### Testing Without Installing + +To test the plugin without installing: + +```sh +GST_PLUGIN_PATH=/src/gst gst-inspect-1.0 pispconvert +``` + ## License Copyright © 2023, Raspberry Pi Ltd. Released under the BSD-2-Clause License. diff --git a/meson_options.txt b/meson_options.txt index 0ce2b17..3079a35 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -3,3 +3,4 @@ option('logging', type : 'feature', value : 'auto') option('examples', type : 'boolean', value : false) +option('gstreamer', type : 'feature', value : 'auto', description : 'Build GStreamer plugin') diff --git a/src/gst/gstpispconvert.cpp b/src/gst/gstpispconvert.cpp new file mode 100644 index 0000000..b4de165 --- /dev/null +++ b/src/gst/gstpispconvert.cpp @@ -0,0 +1,1512 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2026 Raspberry Pi Ltd + * + * gstpispconvert.cpp - GStreamer element for PiSP hardware conversion + */ + +#include "gstpispconvert.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "common/logging.hpp" +#include "common/utils.hpp" +#include "helpers/v4l2_device.hpp" +#include "variants/variant.hpp" + +using BufferRef = libpisp::helpers::BufferRef; +using Buffer = libpisp::helpers::Buffer; + +GST_DEBUG_CATEGORY_STATIC(gst_pisp_convert_debug); +#define GST_CAT_DEFAULT gst_pisp_convert_debug + +/* Supported GStreamer formats */ +#define PISP_FORMATS "{ RGB, RGBx, BGRx, I420, YV12, Y42B, Y444, YUY2, UYVY, NV12, NV12_128C8, NV12_10LE32_128C8 }" +/* Supported DRM fourccs */ +#define PISP_DRM_FORMATS "{ RG24, XB24, XR24, YU12, YV12, YU16, YU24, YUYV, UYVY, NV12, NV12:0x0700000000000004, P030:0x0700000000000004 }" + +#define PISP_SRC_CAPS \ + "video/x-raw(memory:DMABuf), format=(string)DMA_DRM, drm-format=(string)" PISP_DRM_FORMATS \ + ", width=(int)[1,32768], height=(int)[1,32768], framerate=(fraction)[0/1,2147483647/1]" \ + ";" \ + GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_DMABUF, PISP_FORMATS) ";" \ + GST_VIDEO_CAPS_MAKE(PISP_FORMATS) + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE( + "sink", GST_PAD_SINK, GST_PAD_ALWAYS, + GST_STATIC_CAPS( + /* DMA-DRM format (GStreamer 1.24+) */ + "video/x-raw(memory:DMABuf), format=(string)DMA_DRM, drm-format=(string)" PISP_DRM_FORMATS + ", width=(int)[1,32768], height=(int)[1,32768], framerate=(fraction)[0/1,2147483647/1]" + ";" /* Regular dmabuf with standard formats */ + /* System memory */ + GST_VIDEO_CAPS_MAKE(PISP_FORMATS))); + +static GstStaticPadTemplate src0_template = GST_STATIC_PAD_TEMPLATE( + "src0", GST_PAD_SRC, GST_PAD_ALWAYS, + GST_STATIC_CAPS(PISP_SRC_CAPS)); + +static GstStaticPadTemplate src1_template = GST_STATIC_PAD_TEMPLATE( + "src1", GST_PAD_SRC, GST_PAD_ALWAYS, + GST_STATIC_CAPS(PISP_SRC_CAPS)); + +#define gst_pisp_convert_parent_class parent_class +G_DEFINE_TYPE(GstPispConvert, gst_pisp_convert, GST_TYPE_ELEMENT); +GST_ELEMENT_REGISTER_DEFINE(pispconvert, "pispconvert", GST_RANK_PRIMARY, GST_TYPE_PISP_CONVERT); + +/* Bidirectional mapping between GstVideoFormat and PiSP format strings */ +static const std::map gst_pisp_format_map = { + { GST_VIDEO_FORMAT_RGB, "RGB888" }, + { GST_VIDEO_FORMAT_RGBx, "RGBX8888" }, + { GST_VIDEO_FORMAT_BGRx, "XRGB8888" }, + { GST_VIDEO_FORMAT_I420, "YUV420P" }, + { GST_VIDEO_FORMAT_YV12, "YVU420P" }, + { GST_VIDEO_FORMAT_Y42B, "YUV422P" }, + { GST_VIDEO_FORMAT_Y444, "YUV444P" }, + { GST_VIDEO_FORMAT_YUY2, "YUYV" }, + { GST_VIDEO_FORMAT_UYVY, "UYVY" }, + { GST_VIDEO_FORMAT_NV12, "YUV420SP" }, + { GST_VIDEO_FORMAT_NV12_128C8, "YUV420SP_COL128" }, + { GST_VIDEO_FORMAT_NV12_10LE32_128C8, "YUV420SP10_COL128" }, +}; + +/* Bidirectional mapping between DRM fourcc and PiSP format strings */ +static const std::map drm_pisp_format_map = { + { "RG24", "RGB888" }, + { "BG24", "RGB888" }, + { "XB24", "RGBX8888" }, + { "XR24", "XRGB8888" }, + { "YU12", "YUV420P" }, + { "YV12", "YVU420P" }, + { "YU16", "YUV422P" }, + { "YU24", "YUV444P" }, + { "YUYV", "YUYV" }, + { "UYVY", "UYVY" }, + { "NV12", "YUV420SP" }, + { "NV12:0x0700000000000004", "YUV420SP_COL128" }, + { "P030:0x0700000000000004", "YUV420SP10_COL128" }, +}; + +static const char *gst_format_to_pisp(GstVideoFormat format) +{ + auto it = gst_pisp_format_map.find(format); + return it != gst_pisp_format_map.end() ? it->second.c_str() : nullptr; +} + +static GstVideoFormat pisp_to_gst_video_format(const char *pisp_format) +{ + if (!pisp_format) + return GST_VIDEO_FORMAT_UNKNOWN; + + for (const auto &[gst_fmt, pisp_fmt] : gst_pisp_format_map) + { + if (g_str_equal(pisp_format, pisp_fmt.c_str())) + return gst_fmt; + } + return GST_VIDEO_FORMAT_UNKNOWN; +} + +static const char *drm_format_to_pisp(const gchar *drm_format) +{ + if (!drm_format) + return nullptr; + + auto it = drm_pisp_format_map.find(drm_format); + return it != drm_pisp_format_map.end() ? it->second.c_str() : nullptr; +} + +/* Helper function to check if a PiSP format string is YUV */ +static bool is_yuv_format(const char *format) +{ + if (!format) + return false; + return format[0] == 'Y' || format[0] == 'U'; +} + +/* Map GStreamer colorimetry to PiSP colour space string. + * Returns nullptr if the colorimetry is unknown/unspecified. */ +static const char *colorimetry_to_pisp(const GstVideoColorimetry *colorimetry) +{ + if (!colorimetry || colorimetry->matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN) + return nullptr; + + bool full_range = colorimetry->range == GST_VIDEO_COLOR_RANGE_0_255; + + if (colorimetry->matrix == GST_VIDEO_COLOR_MATRIX_BT2020) + return full_range ? "bt2020_full" : "bt2020"; + if (colorimetry->matrix == GST_VIDEO_COLOR_MATRIX_BT709) + return full_range ? "rec709_full" : "rec709"; + if (colorimetry->matrix == GST_VIDEO_COLOR_MATRIX_BT601) + return full_range ? "jpeg" : "smpte170m"; + + return nullptr; +} + +/* Configure colour space conversion blocks for the backend */ +static uint32_t configure_colour_conversion(libpisp::BackEnd *backend, const char *in_format, + const char *in_colorspace, const char *out_format, + const char *out_colorspace, unsigned int output_index) +{ + uint32_t rgb_enables = 0; + + /* YUV->RGB conversion on input */ + if (is_yuv_format(in_format)) + { + pisp_be_ccm_config csc; + backend->InitialiseYcbcrInverse(csc, in_colorspace); + backend->SetCcm(csc); + rgb_enables |= PISP_BE_RGB_ENABLE_CCM; + } + + /* RGB->YUV conversion on output */ + if (is_yuv_format(out_format)) + { + pisp_be_ccm_config csc; + backend->InitialiseYcbcr(csc, out_colorspace); + backend->SetCsc(output_index, csc); + rgb_enables |= PISP_BE_RGB_ENABLE_CSC(output_index); + } + else if (g_str_equal(out_format, "RGB888") || + g_str_equal(out_format, "RGBX8888") || + g_str_equal(out_format, "XRGB8888")) + { + /* R/B channel swap to match GStreamer/DRM byte ordering */ + pisp_be_ccm_config csc = {}; + csc.coeffs[2] = csc.coeffs[4] = csc.coeffs[6] = 1 << 10; + backend->SetCsc(output_index, csc); + rgb_enables |= PISP_BE_RGB_ENABLE_CSC(output_index); + } + + return rgb_enables; +} + +/* GObject vmethod implementations */ +static void gst_pisp_convert_finalize(GObject *object); + +/* GstElement vmethod implementations */ +static GstStateChangeReturn gst_pisp_convert_change_state(GstElement *element, GstStateChange transition); + +/* Pad functions */ +static gboolean gst_pisp_convert_sink_event(GstPad *pad, GstObject *parent, GstEvent *event); +static GstFlowReturn gst_pisp_convert_chain(GstPad *pad, GstObject *parent, GstBuffer *buf); +static gboolean gst_pisp_convert_src_event(GstPad *pad, GstObject *parent, GstEvent *event); +static gboolean gst_pisp_convert_src_query(GstPad *pad, GstObject *parent, GstQuery *query); +static gboolean gst_pisp_convert_sink_query(GstPad *pad, GstObject *parent, GstQuery *query); + +/* Internal functions */ +static gboolean gst_pisp_convert_start(GstPispConvert *self); +static gboolean gst_pisp_convert_stop(GstPispConvert *self); +static gboolean gst_pisp_convert_configure(GstPispConvert *self); +static gboolean gst_pisp_convert_setup_output_pool(GstPispConvert *self, guint index); + +enum +{ + PROP_0, + PROP_OUTPUT_BUFFER_COUNT, + PROP_CROP, + PROP_CROP0, + PROP_CROP1, + N_PROPERTIES +}; + +static void gst_pisp_convert_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); +static void gst_pisp_convert_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); + +/* Initialize the class */ +static void gst_pisp_convert_class_init(GstPispConvertClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS(klass); + GstElementClass *element_class = GST_ELEMENT_CLASS(klass); + + gobject_class->finalize = gst_pisp_convert_finalize; + gobject_class->set_property = gst_pisp_convert_set_property; + gobject_class->get_property = gst_pisp_convert_get_property; + + gst_element_class_set_static_metadata(element_class, "PiSP Hardware Image Converter", + "Filter/Converter/Video/Scaler", + "Hardware accelerated format conversion and scaling using libpisp", + "Raspberry Pi"); + + gst_element_class_add_static_pad_template(element_class, &sink_template); + gst_element_class_add_static_pad_template(element_class, &src0_template); + gst_element_class_add_static_pad_template(element_class, &src1_template); + + g_object_class_install_property( + gobject_class, PROP_OUTPUT_BUFFER_COUNT, + g_param_spec_uint("output-buffer-count", "Output buffer count", + "Number of backend buffers to allocate (round-robin)", 1, 32, 4, + static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property( + gobject_class, PROP_CROP, + g_param_spec_string("crop", "Crop region for all outputs", + "Crop region as 'x,y,width,height' applied to all outputs (0,0,0,0 = full input)", + "0,0,0,0", + static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property( + gobject_class, PROP_CROP0, + g_param_spec_string("crop0", "Crop region for output 0", + "Crop region as 'x,y,width,height' (0,0,0,0 = no crop / full input)", + "0,0,0,0", + static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property( + gobject_class, PROP_CROP1, + g_param_spec_string("crop1", "Crop region for output 1", + "Crop region as 'x,y,width,height' (0,0,0,0 = no crop / full input)", + "0,0,0,0", + static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + element_class->change_state = GST_DEBUG_FUNCPTR(gst_pisp_convert_change_state); + + GST_DEBUG_CATEGORY_INIT(gst_pisp_convert_debug, "pispconvert", 0, "PiSP hardware converter"); +} + +static void gst_pisp_convert_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) +{ + GstPispConvert *self = GST_PISP_CONVERT(object); + + switch (prop_id) + { + case PROP_OUTPUT_BUFFER_COUNT: + self->priv->output_buffer_count = g_value_get_uint(value); + break; + case PROP_CROP: + case PROP_CROP0: + case PROP_CROP1: + { + const gchar *str = g_value_get_string(value); + guint x, y, w, h; + if (str && sscanf(str, "%u,%u,%u,%u", &x, &y, &w, &h) == 4) + { + pisp_be_crop_config crop_cfg = { (uint16_t)x, (uint16_t)y, (uint16_t)w, (uint16_t)h }; + if (prop_id == PROP_CROP) + { + self->priv->crop[0] = crop_cfg; + self->priv->crop[1] = crop_cfg; + } + else + { + guint idx = (prop_id == PROP_CROP0) ? 0 : 1; + self->priv->crop[idx] = crop_cfg; + } + } + else + GST_WARNING_OBJECT(self, "Invalid %s format '%s', expected 'x,y,width,height'", pspec->name, str); + + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_pisp_convert_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec) +{ + GstPispConvert *self = GST_PISP_CONVERT(object); + + switch (prop_id) + { + case PROP_OUTPUT_BUFFER_COUNT: + g_value_set_uint(value, self->priv->output_buffer_count); + break; + case PROP_CROP: + case PROP_CROP0: + case PROP_CROP1: + { + guint idx = (prop_id == PROP_CROP1) ? 1 : 0; + gchar *str = g_strdup_printf("%u,%u,%u,%u", self->priv->crop[idx].offset_x, self->priv->crop[idx].offset_y, + self->priv->crop[idx].width, self->priv->crop[idx].height); + g_value_take_string(value, str); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +/* Initialize the element instance */ +static void gst_pisp_convert_init(GstPispConvert *self) +{ + /* Create and configure sink pad */ + self->sinkpad = gst_pad_new_from_static_template(&sink_template, "sink"); + gst_pad_set_chain_function(self->sinkpad, GST_DEBUG_FUNCPTR(gst_pisp_convert_chain)); + gst_pad_set_event_function(self->sinkpad, GST_DEBUG_FUNCPTR(gst_pisp_convert_sink_event)); + gst_pad_set_query_function(self->sinkpad, GST_DEBUG_FUNCPTR(gst_pisp_convert_sink_query)); + gst_element_add_pad(GST_ELEMENT(self), self->sinkpad); + + /* Create and configure src0 pad (primary output) */ + self->srcpad[0] = gst_pad_new_from_static_template(&src0_template, "src0"); + gst_pad_set_event_function(self->srcpad[0], GST_DEBUG_FUNCPTR(gst_pisp_convert_src_event)); + gst_pad_set_query_function(self->srcpad[0], GST_DEBUG_FUNCPTR(gst_pisp_convert_src_query)); + gst_element_add_pad(GST_ELEMENT(self), self->srcpad[0]); + + /* Create and configure src1 pad (secondary output) */ + self->srcpad[1] = gst_pad_new_from_static_template(&src1_template, "src1"); + gst_pad_set_event_function(self->srcpad[1], GST_DEBUG_FUNCPTR(gst_pisp_convert_src_event)); + gst_pad_set_query_function(self->srcpad[1], GST_DEBUG_FUNCPTR(gst_pisp_convert_src_query)); + gst_element_add_pad(GST_ELEMENT(self), self->srcpad[1]); + + /* Initialize private data */ + self->priv = new GstPispConvertPrivate(); + self->priv->backend_device = nullptr; + self->priv->backend = nullptr; + self->priv->media_dev_path = nullptr; + self->priv->configured = FALSE; + self->priv->dmabuf_allocator = gst_dmabuf_allocator_new(); + self->priv->use_dmabuf_input = FALSE; + + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + self->priv->out_width[i] = 0; + self->priv->out_height[i] = 0; + self->priv->out_stride[i] = 0; + self->priv->out_hw_stride[i] = 0; + self->priv->out_format[i] = nullptr; + self->priv->output_enabled[i] = FALSE; + self->priv->use_dmabuf_output[i] = FALSE; + self->priv->output_pool[i] = nullptr; + self->priv->src_caps[i] = nullptr; + } + + self->priv->pending_segment = nullptr; + self->priv->output_buffer_count = 4; + self->priv->buffer_slice_index = 0; +} + +static void gst_pisp_convert_finalize(GObject *object) +{ + GstPispConvert *self = GST_PISP_CONVERT(object); + + if (self->priv) + { + if (self->priv->dmabuf_allocator) + { + gst_object_unref(self->priv->dmabuf_allocator); + self->priv->dmabuf_allocator = nullptr; + } + + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (self->priv->output_pool[i]) + { + gst_object_unref(self->priv->output_pool[i]); + self->priv->output_pool[i] = nullptr; + } + if (self->priv->src_caps[i]) + { + gst_caps_unref(self->priv->src_caps[i]); + self->priv->src_caps[i] = nullptr; + } + } + + if (self->priv->pending_segment) + { + gst_event_unref(self->priv->pending_segment); + self->priv->pending_segment = nullptr; + } + + g_free(self->priv->media_dev_path); + delete self->priv; + self->priv = nullptr; + } + + G_OBJECT_CLASS(parent_class)->finalize(object); +} + +/* + * Parse output caps for a specific output index. + * Returns TRUE if caps are valid and were parsed successfully. + */ +static gboolean parse_output_caps(GstPispConvert *self, guint index, GstCaps *caps) +{ + GstVideoInfo out_info; + + if (!caps || gst_caps_is_empty(caps)) + return FALSE; + + GstCapsFeatures *out_features = gst_caps_get_features(caps, 0); + self->priv->use_dmabuf_output[index] = out_features && + gst_caps_features_contains(out_features, GST_CAPS_FEATURE_MEMORY_DMABUF); + + GstStructure *out_structure = gst_caps_get_structure(caps, 0); + const gchar *out_format_str = gst_structure_get_string(out_structure, "format"); + gboolean out_is_drm = out_format_str && g_str_equal(out_format_str, "DMA_DRM"); + + if (out_is_drm) + { + const gchar *drm_format = gst_structure_get_string(out_structure, "drm-format"); + gst_structure_get_int(out_structure, "width", (gint *)&self->priv->out_width[index]); + gst_structure_get_int(out_structure, "height", (gint *)&self->priv->out_height[index]); + self->priv->out_format[index] = drm_format_to_pisp(drm_format); + self->priv->out_stride[index] = 0; + + GstVideoColorimetry colorimetry = {}; + const gchar *colorimetry_str = gst_structure_get_string(out_structure, "colorimetry"); + if (colorimetry_str) + gst_video_colorimetry_from_string(&colorimetry, colorimetry_str); + self->priv->out_colorspace[index] = colorimetry_to_pisp(&colorimetry); + + GST_INFO_OBJECT(self, "Output%u DMA-DRM format: drm-format=%s, pisp=%s, colorspace=%s (matrix=%d, range=%d)", + index, drm_format, self->priv->out_format[index], self->priv->out_colorspace[index], + colorimetry.matrix, colorimetry.range); + } + else + { + if (!gst_video_info_from_caps(&out_info, caps)) + { + GST_ERROR_OBJECT(self, "Failed to parse output%u caps", index); + return FALSE; + } + self->priv->out_width[index] = GST_VIDEO_INFO_WIDTH(&out_info); + self->priv->out_height[index] = GST_VIDEO_INFO_HEIGHT(&out_info); + self->priv->out_stride[index] = GST_VIDEO_INFO_PLANE_STRIDE(&out_info, 0); + self->priv->out_format[index] = gst_format_to_pisp(GST_VIDEO_INFO_FORMAT(&out_info)); + self->priv->out_colorspace[index] = colorimetry_to_pisp(&GST_VIDEO_INFO_COLORIMETRY(&out_info)); + GST_INFO_OBJECT(self, "Output%u format: pisp=%s, colorspace=%s (matrix=%d, range=%d)", index, + self->priv->out_format[index], self->priv->out_colorspace[index], + GST_VIDEO_INFO_COLORIMETRY(&out_info).matrix, GST_VIDEO_INFO_COLORIMETRY(&out_info).range); + } + + if (!self->priv->out_format[index]) + { + GST_ERROR_OBJECT(self, "Unsupported output%u format", index); + return FALSE; + } + + self->priv->output_enabled[index] = TRUE; + return TRUE; +} + +/* + * Parse input caps from the sink pad. + */ +static gboolean parse_input_caps(GstPispConvert *self, GstCaps *caps) +{ + GstVideoInfo in_info; + + GST_DEBUG_OBJECT(self, "parse_input_caps: caps=%" GST_PTR_FORMAT, caps); + + GstCapsFeatures *in_features = gst_caps_get_features(caps, 0); + self->priv->use_dmabuf_input = in_features && + gst_caps_features_contains(in_features, GST_CAPS_FEATURE_MEMORY_DMABUF); + GST_DEBUG_OBJECT(self, "in_features=%p, use_dmabuf_input=%d", in_features, self->priv->use_dmabuf_input); + + GstStructure *in_structure = gst_caps_get_structure(caps, 0); + const gchar *in_format_str = gst_structure_get_string(in_structure, "format"); + gboolean in_is_drm = in_format_str && g_str_equal(in_format_str, "DMA_DRM"); + + if (in_is_drm) + { + const gchar *drm_format = gst_structure_get_string(in_structure, "drm-format"); + gst_structure_get_int(in_structure, "width", (gint *)&self->priv->in_width); + gst_structure_get_int(in_structure, "height", (gint *)&self->priv->in_height); + self->priv->in_format = drm_format_to_pisp(drm_format); + self->priv->in_stride = 0; + + GstVideoColorimetry colorimetry = {}; + const gchar *colorimetry_str = gst_structure_get_string(in_structure, "colorimetry"); + if (colorimetry_str) + gst_video_colorimetry_from_string(&colorimetry, colorimetry_str); + self->priv->in_colorspace = colorimetry_to_pisp(&colorimetry); + + GST_INFO_OBJECT(self, "Input DMA-DRM format: drm-format=%s, pisp=%s, colorspace=%s (matrix=%d, range=%d)", + drm_format, self->priv->in_format, self->priv->in_colorspace, + colorimetry.matrix, colorimetry.range); + } + else + { + if (!gst_video_info_from_caps(&in_info, caps)) + { + GST_ERROR_OBJECT(self, "Failed to parse input caps"); + return FALSE; + } + self->priv->in_width = GST_VIDEO_INFO_WIDTH(&in_info); + self->priv->in_height = GST_VIDEO_INFO_HEIGHT(&in_info); + self->priv->in_stride = GST_VIDEO_INFO_PLANE_STRIDE(&in_info, 0); + self->priv->in_format = gst_format_to_pisp(GST_VIDEO_INFO_FORMAT(&in_info)); + self->priv->in_colorspace = colorimetry_to_pisp(&GST_VIDEO_INFO_COLORIMETRY(&in_info)); + GST_INFO_OBJECT(self, "Input format: pisp=%s, colorspace=%s (matrix=%d, range=%d)", + self->priv->in_format, self->priv->in_colorspace, + GST_VIDEO_INFO_COLORIMETRY(&in_info).matrix, GST_VIDEO_INFO_COLORIMETRY(&in_info).range); + } + + if (!self->priv->in_format) + { + GST_ERROR_OBJECT(self, "Unsupported input format"); + return FALSE; + } + + return TRUE; +} + +/* Helper functions for dmabuf support */ +static gboolean gst_buffer_is_dmabuf(GstBuffer *buffer) +{ + GstMemory *mem; + + if (gst_buffer_n_memory(buffer) == 0) + return FALSE; + + mem = gst_buffer_peek_memory(buffer, 0); + return gst_is_dmabuf_memory(mem); +} + +/* Helper function to create Buffer object from GStreamer dmabuf buffer */ +static std::optional gst_to_libpisp_buffer(GstBuffer *buffer) +{ + std::array fds = { -1, -1, -1 }; + std::array sizes = { 0, 0, 0 }; + guint n_mem = gst_buffer_n_memory(buffer); + + for (guint i = 0; i < std::min(n_mem, 3u); i++) + { + GstMemory *mem = gst_buffer_peek_memory(buffer, i); + if (!gst_is_dmabuf_memory(mem)) + return std::nullopt; + + int raw = gst_dmabuf_memory_get_fd(mem); + fds[i] = dup(raw); + if (fds[i] < 0) + { + for (guint j = 0; j < i; j++) + { + if (fds[j] >= 0) + close(fds[j]); + } + return std::nullopt; + } + sizes[i] = mem->size; + } + + return libpisp::helpers::Buffer(fds, sizes); +} + +/* Create GstBuffer (dmabuf) from libpisp backend Buffer; dups FDs so backend keeps its ref */ +static GstBuffer *libpisp_to_gst_dmabuf(const Buffer &buffer, GstAllocator *dmabuf_allocator) +{ + GstBuffer *gstbuf = gst_buffer_new(); + if (!gstbuf) + return nullptr; + + for (unsigned int p = 0; p < 3; p++) + { + int fd = buffer.Fd()[p]; + size_t size = buffer.Size()[p]; + if (fd < 0 || size == 0) + continue; + + int dup_fd = dup(fd); + if (dup_fd < 0) + { + GST_WARNING("dup(plane %u) failed: %s", p, strerror(errno)); + gst_buffer_unref(gstbuf); + return nullptr; + } + + GstMemory *mem = gst_dmabuf_allocator_alloc(dmabuf_allocator, dup_fd, size); + if (!mem) + { + close(dup_fd); + gst_buffer_unref(gstbuf); + return nullptr; + } + gst_buffer_append_memory(gstbuf, mem); + } + + return gstbuf; +} + +/* Attach GstVideoMeta with the correct hardware stride to a dmabuf output buffer */ +static void add_video_meta(GstBuffer *buffer, const char *pisp_format, guint width, guint height, guint hw_stride) +{ + GstVideoFormat gst_fmt = pisp_to_gst_video_format(pisp_format); + if (gst_fmt == GST_VIDEO_FORMAT_UNKNOWN) + return; + + GstVideoInfo vinfo; + gst_video_info_set_format(&vinfo, gst_fmt, width, height); + + gsize offsets[GST_VIDEO_MAX_PLANES] = {}; + gint strides[GST_VIDEO_MAX_PLANES] = {}; + gsize offset = 0; + guint n_planes = GST_VIDEO_INFO_N_PLANES(&vinfo); + GstMemory *mem; + + for (guint p = 0; p < n_planes; p++) + { + offsets[p] = offset; + strides[p] = hw_stride * GST_VIDEO_INFO_PLANE_STRIDE(&vinfo, p) / + GST_VIDEO_INFO_PLANE_STRIDE(&vinfo, 0); + mem = gst_buffer_peek_memory (buffer, p); + offset += mem->size; + } + + gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE, gst_fmt, + width, height, n_planes, offsets, strides); +} + +static void copy_planes(std::array src, guint src_stride, std::array dst, guint dst_stride, + guint width, guint height, const char *format) +{ + GST_DEBUG("copy_planes: %ux%u, src_stride=%u, dst_stride=%u, format=%s", width, height, src_stride, dst_stride, + format); + + /* YUV420SP_COL128 (NV12 column 128) - special tiled format */ + if (strncmp(format, "YUV420SP_COL128", 15) == 0 || strncmp(format, "YUV420SP10_COL128", 17) == 0) + { + guint y_size = GST_VIDEO_TILE_X_TILES(src_stride) * 128 * GST_VIDEO_TILE_Y_TILES(src_stride) * 8; + memcpy(dst[0], src[0], y_size); + + uint8_t *src_uv = src[1] ? src[1] : src[0] + y_size; + uint8_t *dst_uv = dst[1] ? dst[1] : dst[0] + y_size; + memcpy(dst_uv, src_uv, y_size / 2); + return; + } + + /* Planar YUV formats: YUV420P, YVU420P, YUV422P, YUV444P */ + if (is_yuv_format(format) && strstr(format, "P") != nullptr) + { + /* Copy Y plane line by line */ + for (guint y = 0; y < height; ++y) + memcpy(dst[0] + y * dst_stride, src[0] + y * src_stride, width); + + /* Determine UV subsampling */ + guint uv_width, uv_height; + if (strstr(format, "420") != nullptr) + { + uv_width = width / 2; + uv_height = height / 2; + } + else if (strstr(format, "422") != nullptr) + { + uv_width = width / 2; + uv_height = height; + } + else /* 444 */ + { + uv_width = width; + uv_height = height; + } + + guint src_uv_stride = (uv_width == width) ? src_stride : src_stride / 2; + guint dst_uv_stride = (uv_width == width) ? dst_stride : dst_stride / 2; + + /* Calculate plane pointers if not explicitly provided (single contiguous buffer) */ + uint8_t *src_u = src[1] ? src[1] : src[0] + src_stride * height; + uint8_t *src_v = src[2] ? src[2] : src_u + src_uv_stride * uv_height; + uint8_t *dst_u = dst[1] ? dst[1] : dst[0] + dst_stride * height; + uint8_t *dst_v = dst[2] ? dst[2] : dst_u + dst_uv_stride * uv_height; + + /* Copy U and V planes */ + for (guint y = 0; y < uv_height; ++y) + { + memcpy(dst_u + y * dst_uv_stride, src_u + y * src_uv_stride, uv_width); + memcpy(dst_v + y * dst_uv_stride, src_v + y * src_uv_stride, uv_width); + } + return; + } + + /* Packed formats: YUYV, UYVY (2 bpp), RGB888 (3 bpp), RGBX (4 bpp) */ + guint bytes_per_pixel = is_yuv_format(format) ? 2 : (strstr(format, "RGB888") != nullptr) ? 3 : 4; + guint line_stride = width * bytes_per_pixel; + + for (guint y = 0; y < height; ++y) + memcpy(dst[0] + y * dst_stride, src[0] + y * src_stride, line_stride); +} + +static void copy_buffer_to_pisp(GstBuffer *gstbuf, std::array &mem, guint width, guint height, + guint gst_stride, guint hw_stride, const char *format) +{ + GstMapInfo map; + gst_buffer_map(gstbuf, &map, GST_MAP_READ); + + /* GstBuffer is always contiguous - planes calculated from offsets */ + std::array src = { map.data, nullptr, nullptr }; + + copy_planes(src, gst_stride, mem, hw_stride, width, height, format); + + gst_buffer_unmap(gstbuf, &map); +} + +static void copy_pisp_to_buffer(const std::array &mem, GstBuffer *gstbuf, guint width, guint height, + guint gst_stride, guint hw_stride, const char *format) +{ + GstMapInfo map; + gst_buffer_map(gstbuf, &map, GST_MAP_WRITE); + + /* GstBuffer is always contiguous - planes calculated from offsets */ + std::array dst = { map.data, nullptr, nullptr }; + + copy_planes(const_cast &>(mem), hw_stride, dst, gst_stride, width, height, format); + + gst_buffer_unmap(gstbuf, &map); +} + +/* + * Configure the PiSP backend for the current input/output settings. + */ +static gboolean gst_pisp_convert_configure(GstPispConvert *self) +{ + if (!self->priv->backend_device || !self->priv->backend) + { + GST_ERROR_OBJECT(self, "Backend not initialized"); + return FALSE; + } + + /* Check which outputs are enabled */ + gboolean any_output = FALSE; + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + self->priv->output_enabled[i] = gst_pad_is_linked(self->srcpad[i]); + if (self->priv->output_enabled[i]) + any_output = TRUE; + } + + if (!any_output) + { + GST_ERROR_OBJECT(self, "No outputs are linked"); + return FALSE; + } + + /* Get output caps from linked pads */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i]) + continue; + + GstCaps *peer_caps = gst_pad_peer_query_caps(self->srcpad[i], nullptr); + if (!peer_caps || gst_caps_is_empty(peer_caps)) + { + if (peer_caps) + gst_caps_unref(peer_caps); + GST_ERROR_OBJECT(self, "Failed to get caps for output%d", i); + return FALSE; + } + + GstCaps *fixed_caps = gst_caps_fixate(peer_caps); + if (!parse_output_caps(self, i, fixed_caps)) + { + gst_caps_unref(fixed_caps); + return FALSE; + } + + /* Store caps and send downstream */ + if (self->priv->src_caps[i]) + gst_caps_unref(self->priv->src_caps[i]); + self->priv->src_caps[i] = fixed_caps; + + gst_pad_push_event(self->srcpad[i], gst_event_new_caps(fixed_caps)); + } + + /* Forward pending segment event now that caps have been sent */ + if (self->priv->pending_segment) + { + GST_DEBUG_OBJECT(self, "Forwarding stored segment event"); + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (self->priv->output_enabled[i]) + gst_pad_push_event(self->srcpad[i], gst_event_ref(self->priv->pending_segment)); + } + gst_event_unref(self->priv->pending_segment); + self->priv->pending_segment = nullptr; + } + + try + { + pisp_be_global_config global; + self->priv->backend->GetGlobal(global); + global.bayer_enables = 0; + global.rgb_enables = PISP_BE_RGB_ENABLE_INPUT; + + /* Configure input format */ + pisp_image_format_config input_cfg = {}; + input_cfg.width = self->priv->in_width; + input_cfg.height = self->priv->in_height; + input_cfg.format = libpisp::get_pisp_image_format(self->priv->in_format); + if (!input_cfg.format) + { + GST_ERROR_OBJECT(self, "Failed to get input format"); + return FALSE; + } + libpisp::compute_stride(input_cfg); + self->priv->in_hw_stride = input_cfg.stride; + self->priv->backend->SetInputFormat(input_cfg); + + GST_INFO_OBJECT(self, "Input: %ux%u %s (stride: gst=%u hw=%u) colorspace %s", self->priv->in_width, self->priv->in_height, + self->priv->in_format, self->priv->in_stride, self->priv->in_hw_stride, self->priv->in_colorspace); + + if (!self->priv->in_colorspace) + self->priv->in_colorspace = "jpeg"; + + /* Configure each enabled output - first pass: formats and enables */ + pisp_be_output_format_config output_cfg[PISP_NUM_OUTPUTS] = {}; + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i]) + continue; + + global.rgb_enables |= PISP_BE_RGB_ENABLE_OUTPUT(i); + + output_cfg[i].image.width = self->priv->out_width[i]; + output_cfg[i].image.height = self->priv->out_height[i]; + output_cfg[i].image.format = libpisp::get_pisp_image_format(self->priv->out_format[i]); + if (!output_cfg[i].image.format) + { + GST_ERROR_OBJECT(self, "Failed to get output%d format", i); + return FALSE; + } + libpisp::compute_optimal_stride(output_cfg[i].image, true); + self->priv->out_hw_stride[i] = output_cfg[i].image.stride; + self->priv->backend->SetOutputFormat(i, output_cfg[i]); + + if ((g_str_equal(self->priv->out_format[i], "RGBX8888") || + g_str_equal(self->priv->out_format[i], "XRGB8888")) && !self->priv->variant->BackendRGB32Supported(0)) + GST_WARNING_OBJECT(self, "pisp_be HW does not support 32-bit RGB output, the image will be corrupt."); + + if (!self->priv->out_colorspace[i]) + self->priv->out_colorspace[i] = self->priv->in_colorspace; + + global.rgb_enables |= configure_colour_conversion(self->priv->backend.get(), + self->priv->in_format, self->priv->in_colorspace, + self->priv->out_format[i], self->priv->out_colorspace[i], i); + + GST_INFO_OBJECT(self, "Output%d: %ux%u %s (stride: gst=%u hw=%u) colorspace %s", i, self->priv->out_width[i], + self->priv->out_height[i], self->priv->out_format[i], self->priv->out_stride[i], + self->priv->out_hw_stride[i], self->priv->out_colorspace[i]); + } + + self->priv->backend->SetGlobal(global); + + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i]) + continue; + + pisp_be_crop_config crop = self->priv->crop[i]; + + /* Default to full input if width/height not specified */ + if (!crop.width) + crop.width = input_cfg.width; + if (!crop.height) + crop.height = input_cfg.height; + + /* Clip crop region to fit within input */ + if (crop.offset_x >= input_cfg.width) + crop.offset_x = 0; + if (crop.offset_y >= input_cfg.height) + crop.offset_y = 0; + if (crop.offset_x + crop.width > input_cfg.width) + crop.width = input_cfg.width - crop.offset_x; + if (crop.offset_y + crop.height > input_cfg.height) + crop.height = input_cfg.height - crop.offset_y; + + GST_INFO_OBJECT(self, "Crop%u: offset=(%u,%u) size=%ux%u", i, crop.offset_x, crop.offset_y, crop.width, crop.height); + + /* Only call SetCrop if parameters changed */ + if (memcmp(&crop, &self->priv->applied_crop[i], sizeof(crop)) != 0) + { + self->priv->backend->SetCrop(i, crop); + self->priv->applied_crop[i] = crop; + } + + /* Only call SetSmartResize if parameters changed */ + libpisp::BackEnd::SmartResize smart_resize = { (uint16_t)output_cfg[i].image.width, + (uint16_t)output_cfg[i].image.height }; + if (memcmp(&smart_resize, &self->priv->applied_smart_resize[i], sizeof(smart_resize)) != 0) + { + self->priv->backend->SetSmartResize(i, smart_resize); + self->priv->applied_smart_resize[i] = smart_resize; + } + } + + /* Prepare the hardware configuration */ + pisp_be_tiles_config config = {}; + self->priv->backend->Prepare(&config); + self->priv->backend_device->Setup(config, self->priv->output_buffer_count); + + self->priv->pisp_buffers = self->priv->backend_device->GetBuffers(); + self->priv->buffer_slice_index = 0; + self->priv->configured = TRUE; + + GST_INFO_OBJECT(self, "Backend configured successfully with %u output(s), %u buffer(s)", + self->priv->output_enabled[1] ? 2 : 1, self->priv->output_buffer_count); + } + catch (const std::exception &e) + { + GST_ERROR_OBJECT(self, "Failed to configure backend: %s", e.what()); + return FALSE; + } + + return TRUE; +} + +/* + * Setup output buffer pool for the specified output index. + */ +static gboolean gst_pisp_convert_setup_output_pool(GstPispConvert *self, guint index) +{ + if (self->priv->output_pool[index]) + { + gst_buffer_pool_set_active(self->priv->output_pool[index], FALSE); + gst_object_unref(self->priv->output_pool[index]); + self->priv->output_pool[index] = nullptr; + } + + if (!self->priv->src_caps[index]) + return FALSE; + + GstBufferPool *pool = gst_video_buffer_pool_new(); + GstStructure *config = gst_buffer_pool_get_config(pool); + + /* Calculate buffer size */ + gsize size; + GstStructure *structure = gst_caps_get_structure(self->priv->src_caps[index], 0); + const gchar *format_str = gst_structure_get_string(structure, "format"); + + if (format_str && g_str_equal(format_str, "DMA_DRM")) + { + size = self->priv->out_width[index] * self->priv->out_height[index] * 4; + } + else + { + GstVideoInfo vinfo; + if (!gst_video_info_from_caps(&vinfo, self->priv->src_caps[index])) + { + gst_object_unref(pool); + return FALSE; + } + size = GST_VIDEO_INFO_SIZE(&vinfo); + } + + gst_buffer_pool_config_set_params(config, self->priv->src_caps[index], size, 4, 0); + gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + if (!gst_buffer_pool_set_config(pool, config)) + { + GST_ERROR_OBJECT(self, "Failed to set buffer pool config for output%u", index); + gst_object_unref(pool); + return FALSE; + } + + if (!gst_buffer_pool_set_active(pool, TRUE)) + { + GST_ERROR_OBJECT(self, "Failed to activate buffer pool for output%u", index); + gst_object_unref(pool); + return FALSE; + } + + self->priv->output_pool[index] = pool; + GST_DEBUG_OBJECT(self, "Created buffer pool for output%u", index); + return TRUE; +} + +/* + * Chain function - process incoming buffer and push to output pads. + */ +static GstFlowReturn gst_pisp_convert_chain(GstPad *pad [[maybe_unused]], GstObject *parent, GstBuffer *inbuf) +{ + GstPispConvert *self = GST_PISP_CONVERT(parent); + GstFlowReturn ret = GST_FLOW_OK; + GstBuffer *outbuf[PISP_NUM_OUTPUTS] = { nullptr, nullptr }; + + /* Configure on first buffer if not already configured */ + if (!self->priv->configured) + { + if (!gst_pisp_convert_configure(self)) + { + gst_buffer_unref(inbuf); + return GST_FLOW_ERROR; + } + + /* Setup output buffer pools (only for non-dmabuf outputs) */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (self->priv->output_enabled[i] && !self->priv->use_dmabuf_output[i]) + { + if (!gst_pisp_convert_setup_output_pool(self, i)) + { + gst_buffer_unref(inbuf); + return GST_FLOW_ERROR; + } + } + } + } + + /* Acquire output buffers from pool only for non-dmabuf outputs */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i] || self->priv->use_dmabuf_output[i]) + continue; + + GstFlowReturn acquire_ret = gst_buffer_pool_acquire_buffer(self->priv->output_pool[i], &outbuf[i], nullptr); + if (acquire_ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(self, "Failed to acquire buffer for output%d", i); + gst_buffer_unref(inbuf); + for (unsigned int j = 0; j < i; j++) + { + if (outbuf[j]) + gst_buffer_unref(outbuf[j]); + } + return acquire_ret; + } + } + + gboolean input_is_dmabuf = gst_buffer_is_dmabuf(inbuf); + GST_DEBUG_OBJECT(self, "input_is_dmabuf=%d, use_dmabuf_input=%d", input_is_dmabuf, self->priv->use_dmabuf_input); + + /* Round-robin slice from allocated backend buffers */ + guint index = self->priv->output_buffer_count ? (self->priv->buffer_slice_index % self->priv->output_buffer_count) + : 0; + self->priv->buffer_slice_index++; + + /* Create the buffer slize to send to the hardware */ + std::map slice; + for (const auto &[node_name, buffers] : self->priv->pisp_buffers) + slice.emplace(node_name, buffers[index]); + + /* Prepare input: copy to slice buffer (memcpy path) or get dmabuf (zero-copy path) */ + if (input_is_dmabuf && self->priv->use_dmabuf_input) + { + std::optional dmabuf_input = gst_to_libpisp_buffer(inbuf); + if (!dmabuf_input) + { + ret = GST_FLOW_ERROR; + goto cleanup; + } + /* Move so slice takes ownership of the dupped fds; no second dup in copy assignment. */ + slice.at("pispbe-input") = std::move(*dmabuf_input); + GST_DEBUG_OBJECT(self, "Using zero-copy input path"); + } + else + { + Buffer::Sync s(slice.at("pispbe-input"), Buffer::Sync::Access::ReadWrite); + const auto &mem = s.Get(); + copy_buffer_to_pisp(inbuf, const_cast &>(mem), self->priv->in_width, + self->priv->in_height, self->priv->in_stride, self->priv->in_hw_stride, + self->priv->in_format); + GST_DEBUG_OBJECT(self, "Using memcpy input path"); + } + + if (self->priv->backend_device->Run(slice)) + { + GST_ERROR_OBJECT(self, "Hardware conversion failed"); + ret = GST_FLOW_ERROR; + goto cleanup; + } + + /* Output: wrap backend DMA as GstBuffer (dmabuf) or copy to pool buffer (memcpy) */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i]) + continue; + + const std::string node_name("pispbe-output" + std::to_string(i)); + + if (self->priv->use_dmabuf_output[i]) + { + outbuf[i] = libpisp_to_gst_dmabuf(slice.at(node_name), self->priv->dmabuf_allocator); + if (!outbuf[i]) + { + GST_ERROR_OBJECT(self, "Failed to wrap backend buffer as dmabuf for output%d", i); + ret = GST_FLOW_ERROR; + goto cleanup; + } + + add_video_meta(outbuf[i], self->priv->out_format[i], self->priv->out_width[i], + self->priv->out_height[i], self->priv->out_hw_stride[i]); + + GST_DEBUG_OBJECT(self, "Using zero-copy output%d path", i); + } + else + { + Buffer::Sync s(slice.at(node_name), Buffer::Sync::Access::Read); + copy_pisp_to_buffer(s.Get(), outbuf[i], self->priv->out_width[i], self->priv->out_height[i], + self->priv->out_stride[i], self->priv->out_hw_stride[i], self->priv->out_format[i]); + GST_DEBUG_OBJECT(self, "Using memcpy output%d path", i); + } + } + + /* Copy timestamp and duration from input */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i] || !outbuf[i]) + continue; + + GST_BUFFER_PTS(outbuf[i]) = GST_BUFFER_PTS(inbuf); + GST_BUFFER_DTS(outbuf[i]) = GST_BUFFER_DTS(inbuf); + GST_BUFFER_DURATION(outbuf[i]) = GST_BUFFER_DURATION(inbuf); + } + + /* Push buffers to output pads */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (!self->priv->output_enabled[i] || !outbuf[i]) + continue; + + GstFlowReturn push_ret = gst_pad_push(self->srcpad[i], outbuf[i]); + outbuf[i] = nullptr; /* Buffer ownership transferred */ + + if (push_ret != GST_FLOW_OK && ret == GST_FLOW_OK) + ret = push_ret; + } + + gst_buffer_unref(inbuf); + return ret; + +cleanup: + gst_buffer_unref(inbuf); + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (outbuf[i]) + gst_buffer_unref(outbuf[i]); + } + return ret; +} + +/* + * Start the element - acquire backend device. + */ +static gboolean gst_pisp_convert_start(GstPispConvert *self) +{ + GST_INFO_OBJECT(self, "Starting pispconvert element"); + + libpisp::logging_init(); + + try + { + libpisp::helpers::MediaDevice devices; + + std::string media_dev = devices.Acquire(); + if (media_dev.empty()) + { + GST_ERROR_OBJECT(self, "Unable to acquire any pisp_be device!"); + return FALSE; + } + + self->priv->media_dev_path = g_strdup(media_dev.c_str()); + self->priv->backend_device = std::make_unique(media_dev); + + if (!self->priv->backend_device->Valid()) + { + GST_ERROR_OBJECT(self, "Failed to create backend device"); + return FALSE; + } + + const std::vector &variants = libpisp::get_variants(); + const media_device_info info = devices.DeviceInfo(media_dev); + + auto variant_it = std::find_if(variants.begin(), variants.end(), + [&info](const auto &v) { return v.BackEndVersion() == info.hw_revision; }); + + if (variant_it == variants.end()) + { + GST_ERROR_OBJECT(self, "Backend hardware could not be identified: %u", info.hw_revision); + return FALSE; + } + + self->priv->variant = &(*variant_it); + self->priv->backend = std::make_unique(libpisp::BackEnd::Config({}), *variant_it); + + GST_INFO_OBJECT(self, "Acquired device %s", media_dev.c_str()); + } + catch (const std::exception &e) + { + GST_ERROR_OBJECT(self, "Failed to start: %s", e.what()); + return FALSE; + } + + return TRUE; +} + +/* + * Stop the element - release backend device. + */ +static gboolean gst_pisp_convert_stop(GstPispConvert *self) +{ + GST_INFO_OBJECT(self, "Stopping pispconvert element"); + + /* Deactivate and release buffer pools */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (self->priv->output_pool[i]) + { + gst_buffer_pool_set_active(self->priv->output_pool[i], FALSE); + gst_object_unref(self->priv->output_pool[i]); + self->priv->output_pool[i] = nullptr; + } + if (self->priv->src_caps[i]) + { + gst_caps_unref(self->priv->src_caps[i]); + self->priv->src_caps[i] = nullptr; + } + self->priv->output_enabled[i] = FALSE; + self->priv->use_dmabuf_output[i] = FALSE; + } + + if (self->priv->pending_segment) + { + gst_event_unref(self->priv->pending_segment); + self->priv->pending_segment = nullptr; + } + + self->priv->backend.reset(); + self->priv->backend_device.reset(); + + g_free(self->priv->media_dev_path); + self->priv->media_dev_path = nullptr; + self->priv->configured = FALSE; + self->priv->use_dmabuf_input = FALSE; + + return TRUE; +} + +/* + * State change handler. + */ +static GstStateChangeReturn gst_pisp_convert_change_state(GstElement *element, GstStateChange transition) +{ + GstPispConvert *self = GST_PISP_CONVERT(element); + GstStateChangeReturn ret; + + switch (transition) + { + case GST_STATE_CHANGE_NULL_TO_READY: + if (!gst_pisp_convert_start(self)) + return GST_STATE_CHANGE_FAILURE; + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) + { + case GST_STATE_CHANGE_READY_TO_NULL: + gst_pisp_convert_stop(self); + break; + default: + break; + } + + return ret; +} + +/* + * Sink pad event handler. + */ +static gboolean gst_pisp_convert_sink_event(GstPad *pad, GstObject *parent, GstEvent *event) +{ + GstPispConvert *self = GST_PISP_CONVERT(parent); + gboolean ret = TRUE; + + GST_TRACE_OBJECT(self, "Received sink event: %s", GST_EVENT_TYPE_NAME(event)); + + switch (GST_EVENT_TYPE(event)) + { + case GST_EVENT_CAPS: + { + GstCaps *caps; + gst_event_parse_caps(event, &caps); + + gchar *caps_str = gst_caps_to_string(caps); + GST_INFO_OBJECT(self, "Received input caps: %s", caps_str); + g_free(caps_str); + + if (!parse_input_caps(self, caps)) + { + GST_ERROR_OBJECT(self, "Failed to parse input caps"); + ret = FALSE; + } + + /* Mark as needing reconfiguration */ + self->priv->configured = FALSE; + + gst_event_unref(event); + break; + } + case GST_EVENT_SEGMENT: + /* Store segment to forward after caps are sent */ + if (self->priv->pending_segment) + gst_event_unref(self->priv->pending_segment); + self->priv->pending_segment = event; + GST_DEBUG_OBJECT(self, "Stored segment event for later forwarding"); + break; + case GST_EVENT_EOS: + case GST_EVENT_FLUSH_START: + case GST_EVENT_FLUSH_STOP: + case GST_EVENT_STREAM_START: + /* Forward to all linked src pads */ + for (unsigned int i = 0; i < PISP_NUM_OUTPUTS; i++) + { + if (gst_pad_is_linked(self->srcpad[i])) + gst_pad_push_event(self->srcpad[i], gst_event_ref(event)); + } + gst_event_unref(event); + break; + default: + ret = gst_pad_event_default(pad, parent, event); + break; + } + + return ret; +} + +/* + * Source pad event handler. + */ +static gboolean gst_pisp_convert_src_event(GstPad *pad [[maybe_unused]], GstObject *parent, GstEvent *event) +{ + GstPispConvert *self = GST_PISP_CONVERT(parent); + + GST_TRACE_OBJECT(self, "Received src event: %s", GST_EVENT_TYPE_NAME(event)); + + /* Forward upstream events to sink pad */ + return gst_pad_push_event(self->sinkpad, event); +} + +/* + * Sink pad query handler. + */ +static gboolean gst_pisp_convert_sink_query(GstPad *pad, GstObject *parent, GstQuery *query) +{ + GstPispConvert *self = GST_PISP_CONVERT(parent); + gboolean ret = TRUE; + + GST_TRACE_OBJECT(self, "Received sink query: %s", GST_QUERY_TYPE_NAME(query)); + + switch (GST_QUERY_TYPE(query)) + { + case GST_QUERY_ALLOCATION: + { + /* Indicate support for VideoMeta (required for DMABuf) */ + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr); + /* Offer dmabuf allocator */ + if (self->priv->dmabuf_allocator) + gst_query_add_allocation_param(query, self->priv->dmabuf_allocator, nullptr); + ret = TRUE; + break; + } + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + gst_query_parse_caps(query, &filter); + + caps = gst_pad_get_pad_template_caps(pad); + if (filter) + { + GstCaps *intersection = gst_caps_intersect_full(filter, caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref(caps); + caps = intersection; + } + + gst_query_set_caps_result(query, caps); + gst_caps_unref(caps); + ret = TRUE; + break; + } + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + gst_query_parse_accept_caps(query, &caps); + + GstCaps *template_caps = gst_pad_get_pad_template_caps(pad); + gboolean accept = gst_caps_can_intersect(caps, template_caps); + gst_caps_unref(template_caps); + + gst_query_set_accept_caps_result(query, accept); + ret = TRUE; + break; + } + default: + ret = gst_pad_query_default(pad, parent, query); + break; + } + + return ret; +} + +/* + * Source pad query handler. + */ +static gboolean gst_pisp_convert_src_query(GstPad *pad, GstObject *parent, GstQuery *query) +{ + GstPispConvert *self = GST_PISP_CONVERT(parent); + gboolean ret = TRUE; + + GST_TRACE_OBJECT(self, "Received src query: %s", GST_QUERY_TYPE_NAME(query)); + + switch (GST_QUERY_TYPE(query)) + { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + gst_query_parse_caps(query, &filter); + + caps = gst_pad_get_pad_template_caps(pad); + if (filter) + { + GstCaps *intersection = gst_caps_intersect_full(filter, caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref(caps); + caps = intersection; + } + + gst_query_set_caps_result(query, caps); + gst_caps_unref(caps); + ret = TRUE; + break; + } + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + gst_query_parse_accept_caps(query, &caps); + + GstCaps *template_caps = gst_pad_get_pad_template_caps(pad); + gboolean accept = gst_caps_can_intersect(caps, template_caps); + gst_caps_unref(template_caps); + + gst_query_set_accept_caps_result(query, accept); + ret = TRUE; + break; + } + default: + ret = gst_pad_query_default(pad, parent, query); + break; + } + + return ret; +} + +/* Plugin initialization */ +static gboolean plugin_init(GstPlugin *plugin) +{ + return GST_ELEMENT_REGISTER(pispconvert, plugin); +} + +#define PACKAGE "pispconvert" +#define VERSION "1.3.0" +#define GST_PACKAGE_NAME "GStreamer PiSP Plugin" +#define GST_PACKAGE_ORIGIN "https://github.com/raspberrypi/libpisp" +#define GST_LICENSE "BSD" + +GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, pispconvert, + "PiSP hardware accelerated format conversion and scaling", plugin_init, VERSION, GST_LICENSE, + GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/src/gst/gstpispconvert.h b/src/gst/gstpispconvert.h new file mode 100644 index 0000000..6700a92 --- /dev/null +++ b/src/gst/gstpispconvert.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2026 Raspberry Pi Ltd + * + * gstpispconvert.h - GStreamer element for PiSP hardware conversion + */ + +#pragma once + +#include +#include +#include +#include + +#include "backend/backend.hpp" +#include "helpers/backend_device.hpp" + +G_BEGIN_DECLS + +#define PISP_NUM_OUTPUTS 2 + +#define GST_TYPE_PISP_CONVERT (gst_pisp_convert_get_type()) +#define GST_PISP_CONVERT(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_PISP_CONVERT, GstPispConvert)) +#define GST_PISP_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_PISP_CONVERT, GstPispConvertClass)) +#define GST_IS_PISP_CONVERT(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_PISP_CONVERT)) +#define GST_IS_PISP_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_PISP_CONVERT)) + +typedef struct _GstPispConvert GstPispConvert; +typedef struct _GstPispConvertClass GstPispConvertClass; +typedef struct _GstPispConvertPrivate GstPispConvertPrivate; + +struct _GstPispConvert +{ + GstElement base_element; + + /* Pads */ + GstPad *sinkpad; + GstPad *srcpad[PISP_NUM_OUTPUTS]; /* src0 and src1 */ + + /* Private data */ + GstPispConvertPrivate *priv; +}; + +struct _GstPispConvertClass +{ + GstElementClass parent_class; +}; + +struct _GstPispConvertPrivate +{ + /* C++ objects */ + std::unique_ptr backend_device; + std::unique_ptr backend; + const libpisp::PiSPVariant *variant; + + /* Device info */ + char *media_dev_path; + + /* Configuration */ + gboolean configured; + + /* Crop settings per output */ + pisp_be_crop_config crop[PISP_NUM_OUTPUTS]; + + /* Cached crop/resize settings (to avoid redundant API calls) */ + pisp_be_crop_config applied_crop[PISP_NUM_OUTPUTS]; + libpisp::BackEnd::SmartResize applied_smart_resize[PISP_NUM_OUTPUTS]; + + /* Input format info */ + guint in_width; + guint in_height; + guint in_stride; // GStreamer buffer stride + guint in_hw_stride; // Hardware buffer stride + const char *in_format; + const char *in_colorspace; + + /* Output format info - arrays for dual outputs */ + guint out_width[PISP_NUM_OUTPUTS]; + guint out_height[PISP_NUM_OUTPUTS]; + guint out_stride[PISP_NUM_OUTPUTS]; // GStreamer buffer stride + guint out_hw_stride[PISP_NUM_OUTPUTS]; // Hardware buffer stride + const char *out_format[PISP_NUM_OUTPUTS]; + const char *out_colorspace[PISP_NUM_OUTPUTS]; + gboolean output_enabled[PISP_NUM_OUTPUTS]; // Track which outputs are active + + /* dmabuf support */ + GstAllocator *dmabuf_allocator; + gboolean use_dmabuf_input; + gboolean use_dmabuf_output[PISP_NUM_OUTPUTS]; + + /* Buffer pools for outputs */ + GstBufferPool *output_pool[PISP_NUM_OUTPUTS]; + + /* Backend buffers: all refs per node (from GetBuffers), round-robin slice index */ + std::map> pisp_buffers; + guint buffer_slice_index; + guint output_buffer_count; + + /* Caps tracking */ + GstCaps *src_caps[PISP_NUM_OUTPUTS]; + + /* Pending segment event (to be forwarded after caps) */ + GstEvent *pending_segment; +}; + +GType gst_pisp_convert_get_type(void); +GST_ELEMENT_REGISTER_DECLARE(pispconvert); + +G_END_DECLS + diff --git a/src/gst/meson.build b/src/gst/meson.build new file mode 100644 index 0000000..2aa2789 --- /dev/null +++ b/src/gst/meson.build @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: CC0-1.0 +# Copyright (C) 2026, Raspberry Pi Ltd + +gst_dep = dependency('gstreamer-1.0', version : '>= 1.14', required : get_option('gstreamer')) +gst_base_dep = dependency('gstreamer-base-1.0', version : '>= 1.14', required : get_option('gstreamer')) +gst_video_dep = dependency('gstreamer-video-1.0', version : '>= 1.14', required : get_option('gstreamer')) +gst_allocators_dep = dependency('gstreamer-allocators-1.0', version : '>= 1.14', required : get_option('gstreamer')) + +if gst_dep.found() and gst_base_dep.found() and gst_video_dep.found() and gst_allocators_dep.found() + gst_sources = [ + 'gstpispconvert.cpp', + ] + + gst_deps = [ + gst_dep, + gst_base_dep, + gst_video_dep, + gst_allocators_dep, + libpisp_dep, + ] + + library('gstpispconvert', + gst_sources, + dependencies : gst_deps, + include_directories : include_directories('..'), + install : true, + install_dir : get_option('libdir') / 'gstreamer-1.0', + ) +endif diff --git a/src/gst/usage.md b/src/gst/usage.md new file mode 100644 index 0000000..e910dff --- /dev/null +++ b/src/gst/usage.md @@ -0,0 +1,145 @@ +# PiSP Convert GStreamer Element + +`pispconvert` is a GStreamer element that provides hardware-accelerated image scaling and format conversion using the Raspberry Pi's PiSP Backend. + +## Properties + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `output-buffer-count` | uint | 4 | Number of backend buffers to allocate (1-32) | +| `crop` | string | "0,0,0,0" | Crop region for all outputs as "x,y,width,height" | +| `crop0` | string | "0,0,0,0" | Crop region for output 0 as "x,y,width,height" | +| `crop1` | string | "0,0,0,0" | Crop region for output 1 as "x,y,width,height" | + +### Crop Parameters + +Crop values of `0,0,0,0` (default) means no cropping - the full input is used. If width or height is 0, it defaults to the full input dimension. Values are automatically clipped to fit within the input. + +- `crop` - Sets the same crop region for both outputs +- `crop0` - Sets crop region for output 0 only +- `crop1` - Sets crop region for output 1 only + + +## Supported Formats + +### GStreamer Formats +`RGB`, `RGBx`, `BGRx`, `I420`, `YV12`, `Y42B`, `Y444`, `YUY2`, `UYVY`, `NV12_128C8`, `NV12_10LE32_128C8` + +### DRM Formats +`RG24`, `XB24`, `XR24`, `YU12`, `YV12`, `YU16`, `YU24`, `YUYV`, `UYVY`, `NV12`, `NV12:0x0700000000000004`, `P030:0x0700000000000004` + +## Colorimetry + +`pispconvert` uses the colorimetry reported in the input caps to select the correct +YCbCr conversion matrix. The supported colour spaces are: `jpeg` (full-range BT.601), +`smpte170m` (limited-range BT.601), `rec709`, `rec709_full`, `bt2020`, and `bt2020_full`. + +When no output colorimetry is specified, it defaults to matching the input. + +Some webcams and V4L2 sources report incorrect colorimetry (e.g. limited-range when the +sensor actually produces full-range data), which can result in incorrect colours. You can +override the input colorimetry by specifying it explicitly in the caps filter. The +colorimetry string format is `range:matrix:transfer:primaries`. + +For example, to force full-range BT.601 (jpeg) on a webcam: + +```bash +gst-launch-1.0 \ + v4l2src device=/dev/video0 io-mode=dmabuf ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=YUYV,width=640,height=480,colorimetry=1:4:0:1" ! \ + pispconvert ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=YUYV,width=800,height=600" ! \ + waylandsink +``` + +You can check the actual colorimetry of your camera with: + +```bash +v4l2-ctl -d /dev/video0 --get-fmt-video +``` + +## Examples + +### Single Output + +Basic scaling and format conversion: + +```bash +gst-launch-1.0 filesrc location=input.yuv ! \ + rawvideoparse width=4056 height=3040 format=i420 framerate=30/1 ! \ + pispconvert ! \ + video/x-raw,format=RGB,width=1920,height=1080 ! \ + filesink location=output.rgb +``` + +### Dual Output + +Simultaneous scaling to two different resolutions/formats: + +```bash +gst-launch-1.0 filesrc location=input.yuv ! \ + rawvideoparse width=4056 height=3040 format=i420 framerate=30/1 ! \ + pispconvert name=p \ + p.src0 ! queue ! video/x-raw,format=RGB,width=1920,height=1080 ! filesink location=output0.rgb \ + p.src1 ! queue ! video/x-raw,format=I420,width=640,height=480 ! filesink location=output1.yuv +``` + +### With Cropping + +Crop the input before scaling: + +```bash +gst-launch-1.0 filesrc location=input.yuv ! \ + rawvideoparse width=4056 height=3040 format=i420 framerate=30/1 ! \ + pispconvert crop="500,400,3000,2200" ! \ + video/x-raw,format=RGB,width=1920,height=1080 ! \ + filesink location=output.rgb +``` + +### Camera with DMABuf + +Using a camera source with DMABuf for zero-copy processing: + +```bash +gst-launch-1.0 \ + v4l2src device=/dev/video16 io-mode=dmabuf num-buffers=100 ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=YUYV,width=640,height=480" ! \ + pispconvert ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=YUYV,width=4096,height=1080" ! \ + waylandsink sync=false +``` + +### Camera with Software Buffers + +Using a camera source with standard memory buffers: + +```bash +gst-launch-1.0 \ + v4l2src device=/dev/video16 num-buffers=100 ! \ + "video/x-raw,width=640,height=480" ! \ + pispconvert ! \ + video/x-raw,format=RGB,width=4096,height=1080 ! \ + waylandsink +``` + +### Decode H.265 Video and Reformat + +Using hardware H.265 decoder with DMABuf passthrough: + +```bash +gst-launch-1.0 filesrc location=video.mkv ! matroskademux ! h265parse ! v4l2slh265dec ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=NV12:0x0700000000000004" ! \ + pispconvert ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=YUYV,width=4096,height=1080" ! \ + waylandsink +``` + +Using software buffer input instead of DMABuf: + +```bash +gst-launch-1.0 filesrc location=video.mkv ! matroskademux ! h265parse ! v4l2slh265dec ! \ + "video/x-raw,format=NV12_128C8" ! \ + pispconvert ! \ + "video/x-raw(memory:DMABuf),format=DMA_DRM,drm-format=YUYV,width=4096,height=1080" ! \ + waylandsink +``` diff --git a/src/meson.build b/src/meson.build index 6ace2ff..15b9a93 100644 --- a/src/meson.build +++ b/src/meson.build @@ -69,4 +69,6 @@ pkg_mod.generate(libpisp, if get_option('examples') subdir('examples') -endif \ No newline at end of file +endif + +subdir('gst')