h264 zero-copy

Makefile

@@ -46,7 +46,7 @@ endef
 
 
 ifneq ($(call optbool,$(WITH_OMX)),)
-_USTR_LIBS += -lbcm_host -lvcos -lopenmaxil -lmmal -lmmal_core -lmmal_util -lmmal_vc_client -lmmal_components -L$(RPI_VC_LIBS)
+_USTR_LIBS += -lbcm_host -lvcos -lvcsm -lopenmaxil -lmmal -lmmal_core -lmmal_util -lmmal_vc_client -lmmal_components -L$(RPI_VC_LIBS)
 override CFLAGS += -DWITH_OMX -DOMX_SKIP64BIT -I$(RPI_VC_HEADERS)
 _USTR_SRCS += $(shell ls \
 	src/ustreamer/encoders/omx/*.c \

src/ustreamer/device.c

@@ -178,6 +178,17 @@ void device_close(device_s *dev) {
 		for (unsigned index = 0; index < RUN(n_bufs); ++index) {
 #			define HW(_next) RUN(hw_bufs)[index]._next
 
+#			ifdef WITH_OMX
+			if (HW(vcsm_handle) > 0) {
+				vcsm_free(HW(vcsm_handle));
+				HW(vcsm_handle) = -1;
+			}
+			if (HW(dma_fd) >= 0) {
+				close(HW(dma_fd));
+				HW(dma_fd) = -1;
+			}
+#			endif
+
 			if (dev->io_method == V4L2_MEMORY_MMAP) {
 				if (HW(raw.allocated) > 0 && HW(raw.data) != MAP_FAILED) {
 					if (munmap(HW(raw.data), HW(raw.allocated)) < 0) {
@@ -209,6 +220,53 @@ void device_close(device_s *dev) {
 	}
 }
 
+#ifdef WITH_OMX
+int device_export_to_vcsm(device_s *dev) {
+#	define DMA_FD		RUN(hw_bufs[index].dma_fd)
+#	define VCSM_HANDLE	RUN(hw_bufs[index].vcsm_handle)
+
+	for (unsigned index = 0; index < RUN(n_bufs); ++index) {
+		struct v4l2_exportbuffer exp;
+		MEMSET_ZERO(exp);
+		exp.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+		exp.index = index;
+
+		LOG_DEBUG("Calling ioctl(VIDIOC_EXPBUF) for buffer index=%u ...", index);
+		if (xioctl(RUN(fd), VIDIOC_EXPBUF, &exp) < 0) {
+			LOG_PERROR("Unable to export device buffer index=%u", index);
+			goto error;
+		}
+		DMA_FD = exp.fd;
+
+		LOG_DEBUG("Importing DMA buffer fd=%d into VCSM ...", DMA_FD);
+		int vcsm_handle = vcsm_import_dmabuf(DMA_FD, "v4l2_buf");
+		if (vcsm_handle <= 0) {
+			LOG_PERROR("Unable to import DMA buffer fd=%d into VCSM", DMA_FD);
+			goto error;
+		}
+		VCSM_HANDLE = vcsm_handle;
+	}
+
+	return 0;
+
+	error:
+		for (unsigned index = 0; index < RUN(n_bufs); ++index) {
+			if (VCSM_HANDLE > 0) {
+				vcsm_free(VCSM_HANDLE);
+				VCSM_HANDLE = -1;
+			}
+			if (DMA_FD >= 0) {
+				close(DMA_FD);
+				DMA_FD = -1;
+			}
+		}
+		return -1;
+
+#	undef VCSM_HANDLE
+#	undef DMA_FD
+}
+#endif
+
 int device_switch_capturing(device_s *dev, bool enable) {
 	if (enable != RUN(capturing)) {
 		enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
@@ -648,6 +706,11 @@ static int _device_open_io_method_mmap(device_s *dev) {
 
 #		define HW(_next) RUN(hw_bufs)[RUN(n_bufs)]._next
 
+#		ifdef WITH_OMX
+		HW(dma_fd) = -1;
+		HW(vcsm_handle) = -1;
+#		endif
+
 		A_MUTEX_INIT(&HW(grabbed_mutex));
 
 		LOG_DEBUG("Mapping device buffer %u ...", RUN(n_bufs));

src/ustreamer/device.h

@@ -40,6 +40,9 @@
 #include <pthread.h>
 #include <linux/videodev2.h>
 #include <linux/v4l2-controls.h>
+#ifdef WITH_OMX
+#	include <interface/vcsm/user-vcsm.h>
+#endif
 
 #include "../libs/tools.h"
 #include "../libs/logging.h"
@@ -72,6 +75,11 @@ typedef struct {
 
 	struct v4l2_buffer buf_info;
 
+#	ifdef WITH_OMX
+	int dma_fd;
+	int vcsm_handle;
+#	endif
+
 	pthread_mutex_t grabbed_mutex;
 	bool			grabbed;
 } hw_buffer_s;
@@ -150,6 +158,9 @@ int device_parse_io_method(const char *str);
 int device_open(device_s *dev);
 void device_close(device_s *dev);
 
+#ifdef WITH_OMX
+int device_export_to_vcsm(device_s *dev);
+#endif
 int device_switch_capturing(device_s *dev, bool enable);
 int device_select(device_s *dev, bool *has_read, bool *has_write, bool *has_error);
 int device_grab_buffer(device_s *dev, hw_buffer_s **hw);

src/ustreamer/h264/encoder.c

@@ -24,7 +24,7 @@
 
 
 static void _h264_encoder_cleanup(h264_encoder_s *enc);
-static int _h264_encoder_compress_raw(h264_encoder_s *enc, const frame_s *src, frame_s *dest, bool force_key);
+static int _h264_encoder_compress_raw(h264_encoder_s *enc, const frame_s *src, int src_vcsm_handle, frame_s *dest, bool force_key);
 
 static void _mmal_callback(MMAL_WRAPPER_T *wrapper);
 static const char *_mmal_error_to_string(MMAL_STATUS_T error);
@@ -89,13 +89,13 @@ void h264_encoder_destroy(h264_encoder_s *enc) {
 	free(enc);
 }
 
-bool h264_encoder_is_prepared_for(h264_encoder_s *enc, const frame_s *frame) {
+bool h264_encoder_is_prepared_for(h264_encoder_s *enc, const frame_s *frame, bool zero_copy) {
-#	define EQ(_field) (frame->_field == enc->_field)
+#	define EQ(_field) (enc->_field == frame->_field)
-	return (EQ(width) && EQ(height) && EQ(format) && EQ(stride));
+	return (EQ(width) && EQ(height) && EQ(format) && EQ(stride) && (enc->zero_copy == zero_copy));
 #	undef EQ
 }
 
-int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame) {
+int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame, bool zero_copy) {
 	LOG_INFO("H264: Configuring MMAL encoder ...");
 
 	_h264_encoder_cleanup(enc);
@@ -104,6 +104,7 @@ int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame) {
 	enc->height = frame->height;
 	enc->format = frame->format;
 	enc->stride = frame->stride;
+	enc->zero_copy = zero_copy;
 
 	if (align_size(frame->width, 32) != frame->width && frame_get_padding(frame) == 0) {
 		LOG_ERROR("H264: MMAL encoder can't handle unaligned width");
@@ -167,7 +168,7 @@ int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame) {
 #		undef IFMT
 
 		COMMIT_PORT(input);
-		SET_PORT_PARAM(input, boolean, ZERO_COPY, MMAL_FALSE);
+		SET_PORT_PARAM(input, boolean, ZERO_COPY, zero_copy);
 	}
 
 	{
@@ -204,7 +205,7 @@ int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame) {
 		SET_PORT_PARAM(output, uint32,	NALUNITFORMAT,				MMAL_VIDEO_NALUNITFORMAT_STARTCODES);
 		SET_PORT_PARAM(output, boolean,	MINIMISE_FRAGMENTATION,		MMAL_TRUE);
 		SET_PORT_PARAM(output, uint32,	MB_ROWS_PER_SLICE,			0);
-		SET_PORT_PARAM(output, boolean,	VIDEO_IMMUTABLE_INPUT,		MMAL_FALSE);
+		SET_PORT_PARAM(output, boolean,	VIDEO_IMMUTABLE_INPUT,		MMAL_TRUE);
 		SET_PORT_PARAM(output, boolean,	VIDEO_DROPPABLE_PFRAMES,	MMAL_FALSE);
 		SET_PORT_PARAM(output, boolean,	VIDEO_ENCODE_INLINE_HEADER,	MMAL_TRUE); // SPS/PPS: https://github.com/raspberrypi/userland/issues/443
 		SET_PORT_PARAM(output, uint32,	VIDEO_BIT_RATE,				enc->bitrate * 1000);
@@ -254,13 +255,13 @@ static void _h264_encoder_cleanup(h264_encoder_s *enc) {
 	enc->ready = false;
 }
 
-int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, frame_s *dest) {
+int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, int src_vcsm_handle, frame_s *dest) {
 	assert(enc->ready);
 	assert(src->used > 0);
-	assert(src->width == enc->width);
+	assert(enc->width == src->width);
-	assert(src->height == enc->height);
+	assert(enc->height == src->height);
-	assert(src->format == enc->format);
+	assert(enc->format == src->format);
-	assert(src->stride == enc->stride);
+	assert(enc->stride == src->stride);
 
 	frame_copy_meta(src, dest);
 	dest->encode_begin_ts = get_now_monotonic();
@@ -269,7 +270,7 @@ int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, frame_s *dest
 
 	bool force_key = (enc->last_online != src->online);
 
-	if (_h264_encoder_compress_raw(enc, src, dest, force_key) < 0) {
+	if (_h264_encoder_compress_raw(enc, src, src_vcsm_handle, dest, force_key) < 0) {
 		_h264_encoder_cleanup(enc);
 		return -1;
 	}
@@ -282,7 +283,7 @@ int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, frame_s *dest
 	return 0;
 }
 
-static int _h264_encoder_compress_raw(h264_encoder_s *enc, const frame_s *src, frame_s *dest, bool force_key) {
+static int _h264_encoder_compress_raw(h264_encoder_s *enc, const frame_s *src, int src_vcsm_handle, frame_s *dest, bool force_key) {
 	LOG_DEBUG("H264: Compressing new frame; force_key=%d ...", force_key);
 
 	MMAL_STATUS_T error;
@@ -316,7 +317,12 @@ static int _h264_encoder_compress_raw(h264_encoder_s *enc, const frame_s *src, f
 
 		in = NULL;
 		if (!sent && mmal_wrapper_buffer_get_empty(enc->input_port, &in, 0) == MMAL_SUCCESS) {
-			in->data = src->data;
+			if (enc->zero_copy && src_vcsm_handle > 0) {
+				in->data = (uint8_t *)vcsm_vc_hdl_from_hdl(src_vcsm_handle);
+				in->alloc_size = src->used;
+			} else {
+				in->data = src->data;
+			}
 			in->length = src->used;
 			in->offset = 0;
 			in->flags = MMAL_BUFFER_HEADER_FLAG_EOS;

src/ustreamer/h264/encoder.h

@@ -33,6 +33,7 @@
 #include <interface/mmal/util/mmal_default_components.h>
 #include <interface/mmal/util/mmal_component_wrapper.h>
 #include <interface/mmal/util/mmal_util_params.h>
+#include <interface/vcsm/user-vcsm.h>
 
 #include "../../libs/tools.h"
 #include "../../libs/logging.h"
@@ -56,6 +57,7 @@ typedef struct {
 	unsigned	height;
 	unsigned	format;
 	unsigned	stride;
+	bool		zero_copy;
 	bool		ready;
 } h264_encoder_s;
 
@@ -63,6 +65,6 @@ typedef struct {
 h264_encoder_s *h264_encoder_init(unsigned bitrate, unsigned gop, unsigned fps);
 void h264_encoder_destroy(h264_encoder_s *enc);
 
-bool h264_encoder_is_prepared_for(h264_encoder_s *enc, const frame_s *frame);
+bool h264_encoder_is_prepared_for(h264_encoder_s *enc, const frame_s *frame, bool zero_copy);
-int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame);
+int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame, bool zero_copy);
-int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, frame_s *dest);
+int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, int src_vcsm_handle, frame_s *dest);

src/ustreamer/stream.c

@@ -30,7 +30,7 @@ static bool _stream_expose_frame(stream_s *stream, frame_s *frame, unsigned capt
 #ifdef WITH_OMX
 static h264_stream_s *_h264_stream_init(memsink_s *sink, unsigned bitrate, unsigned gop);
 static void _h264_stream_destroy(h264_stream_s *h264);
-static void _h264_stream_process(h264_stream_s *h264, const frame_s *frame);
+static void _h264_stream_process(h264_stream_s *h264, const frame_s *frame, int vcsm_handle);
 #endif
 
 
@@ -186,7 +186,7 @@ void stream_loop(stream_s *stream) {
 
 #							ifdef WITH_OMX
 							if (RUN(h264)) {
-								_h264_stream_process(RUN(h264), &hw->raw);
+								_h264_stream_process(RUN(h264), &hw->raw, hw->vcsm_handle);
 							}
 #							endif
 						}
@@ -247,7 +247,7 @@ static workers_pool_s *_stream_init_loop(stream_s *stream) {
 			}
 #			ifdef WITH_OMX
 			if (RUN(h264)) {
-				_h264_stream_process(RUN(h264), stream->blank);
+				_h264_stream_process(RUN(h264), stream->blank, -1);
 			}
 #			endif
 		}
@@ -280,6 +280,11 @@ static workers_pool_s *_stream_init_one(stream_s *stream) {
 	if (device_open(stream->dev) < 0) {
 		goto error;
 	}
+#	ifdef WITH_OMX
+	if (RUN(h264) && !is_jpeg(stream->dev->run->format)) {
+		device_export_to_vcsm(stream->dev);
+	}
+#	endif
 	if (device_switch_capturing(stream->dev, true) < 0) {
 		goto error;
 	}
@@ -375,26 +380,33 @@ static void _h264_stream_destroy(h264_stream_s *h264) {
 	free(h264);
 }
 
-static void _h264_stream_process(h264_stream_s *h264, const frame_s *frame) {
+static void _h264_stream_process(h264_stream_s *h264, const frame_s *frame, int vcsm_handle) {
 	long double now = get_now_monotonic();
+	bool zero_copy = false;
+
 	if (is_jpeg(frame->format)) {
 		LOG_DEBUG("H264: Input frame is JPEG; decoding ...");
 		if (unjpeg(frame, h264->tmp_src, true) < 0) {
 			return;
 		}
+		frame = h264->tmp_src;
 		LOG_VERBOSE("H264: JPEG decoded; time=%.3Lf", get_now_monotonic() - now);
+	} else if (vcsm_handle > 0) {
+		LOG_DEBUG("H264: Zero-copy available for the input");
+		zero_copy = true;
 	} else {
 		LOG_DEBUG("H264: Copying source to tmp buffer ...");
 		frame_copy(frame, h264->tmp_src);
+		frame = h264->tmp_src;
 		LOG_VERBOSE("H264: Source copied; time=%.3Lf", get_now_monotonic() - now);
 	}
 
-	if (!h264_encoder_is_prepared_for(h264->enc, h264->tmp_src)) {
+	if (!h264_encoder_is_prepared_for(h264->enc, frame, zero_copy)) {
-		h264_encoder_prepare(h264->enc, h264->tmp_src);
+		h264_encoder_prepare(h264->enc, frame, zero_copy);
 	}
 
 	if (h264->enc->ready) {
-		if (h264_encoder_compress(h264->enc, h264->tmp_src, h264->dest) == 0) {
+		if (h264_encoder_compress(h264->enc, frame, vcsm_handle, h264->dest) == 0) {
 			memsink_server_put(h264->sink, h264->dest);
 		}
 	}