Bump version: v6.62 -> v6.63

cpu encoder: fix chroma row advance timing in YUV420/YVU420 planar path

.bumpversion.cfg

@@ -1,39 +0,0 @@
-[bumpversion]
-commit = True
-tag = True
-current_version = 6.49
-parse = (?P<major>\d+)\.(?P<minor>\d+)
-serialize = 
-	{major}.{minor}
-
-[bumpversion:file:src/libs/const.h]
-parse = (?P<major>\d+)
-serialize = {major}
-search = VERSION_MAJOR {current_version}
-replace = VERSION_MAJOR {new_version}
-
-[bumpversion:file:./src/libs/const.h]
-parse = <major>\d+\.(?P<minor>\d+)
-serialize = {minor}
-search = VERSION_MINOR {current_version}
-replace = VERSION_MINOR {new_version}
-
-[bumpversion:file:python/setup.py]
-search = version="{current_version}"
-replace = version="{new_version}"
-
-[bumpversion:file:pkg/arch/PKGBUILD]
-search = pkgver={current_version}
-replace = pkgver={new_version}
-
-[bumpversion:file:pkg/openwrt/Makefile]
-search = PKG_VERSION:={current_version}
-replace = PKG_VERSION:={new_version}
-
-[bumpversion:file:man/ustreamer.1]
-search = "version {current_version}"
-replace = "version {new_version}"
-
-[bumpversion:file:man/ustreamer-dump.1]
-search = "version {current_version}"
-replace = "version {new_version}"

.gitignore

@@ -6,6 +6,7 @@
 /python/dist/
 /python/ustreamer.egg-info/
 /python/root/
+/python/build/
 /janus/build/
 /ustreamer
 /ustreamer-*

Makefile

@@ -72,7 +72,7 @@ apps:
 
 python:
 	$(MAKE) -C python
-	$(ECHO) ln -sf python/root/usr/lib/python*/site-packages/*.so .
+	$(ECHO) ln -sf python/build/lib.linux-*/*.so -t .
 
 
 janus:
@@ -132,7 +132,7 @@ linters:
 
 
 bump:
-	bumpversion $(if $(V),$(V),minor)
+	./bump.py $(if $(V),$(V),minor)
 
 
 push:

README.md

@@ -35,13 +35,14 @@ If you're going to live-stream from your backyard webcam and need to control it,
 # Installation
 
 ## Building
-You need to download the µStreamer onto your system and build it from the sources.
+You need to download the µStreamer onto your system and build it from the sources, or use a package:
 
-* AUR has a package for Arch Linux: https://aur.archlinux.org/packages/ustreamer.
-* Fedora: https://src.fedoraproject.org/rpms/ustreamer.
-* Ubuntu: https://packages.ubuntu.com/jammy/ustreamer.
+* Arch Linux: https://aur.archlinux.org/packages/ustreamer
+* Fedora: https://src.fedoraproject.org/rpms/ustreamer
+* Ubuntu: https://packages.ubuntu.com/jammy/ustreamer
 * Debian: https://packages.debian.org/sid/ustreamer
-* FreeBSD port: https://www.freshports.org/multimedia/ustreamer.
+* OpenWRT: https://github.com/openwrt/packages/tree/master/multimedia/ustreamer
+* FreeBSD port: https://www.freshports.org/multimedia/ustreamer
 
 ### Preconditions
 You'll need  ```make```, ```gcc```, ```pkg-config```, ```libevent``` with ```pthreads``` support, ```libjpeg9```/```libjpeg-turbo``` and ```libbsd``` (only for Linux).

bump.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# ========================================================================== #
+#                                                                            #
+#    KVMD - The main PiKVM daemon.                                           #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+# ========================================================================== #
+
+
+import sys
+import os
+import re
+import subprocess
+
+
+# =====
+def _run(cmd: list[str]) -> str:
+    return str(subprocess.run(cmd, stdout=subprocess.PIPE, check=True).stdout.decode())
+
+
+def _get_version() -> tuple[int, int]:
+    ver = _run(["/usr/bin/git", "describe", "--tags", "--abbrev=0"]).strip()
+    assert (m := re.match(r"v(\d+)\.(\d+)", ver)), ver
+    return (int(m.group(1)), int(m.group(2)))
+
+
+def _make_version(major: int, minor: int) -> str:
+    return f"v{major}.{minor}"
+
+
+def _patch_file(path: str, replace: str, old: tuple[int, int], new: tuple[int, int]) -> None:
+    with open(path, "r+") as f:
+        old_text = f.read()
+        new_text = old_text.replace(
+            replace.format(major=old[0], minor=old[1]),
+            replace.format(major=new[0], minor=new[1]),
+        )
+        # assert old_text != new_text, path
+        f.seek(0, os.SEEK_SET)
+        f.write(new_text)
+
+
+def _bump(do_major: bool, items: list[tuple[str, str]]) -> None:
+    _run(["/usr/bin/git", "diff-index", "--quiet", "HEAD", "--"])
+
+    old = _get_version()
+    new = ((old[0] + 1, old[1]) if do_major else (old[0], old[1] + 1))
+
+    for (path, replace) in items:
+        _patch_file(path, replace, old, new)
+        _run(["/usr/bin/git", "add", path])
+
+    ver_new = _make_version(*new)
+    _run(["/usr/bin/git", "commit", "-m", f"Bump version: {_make_version(*old)} -> {ver_new}"])
+    _run(["/usr/bin/git", "tag", ver_new])
+
+
+# =====
+if __name__ == "__main__":
+    _bump(
+        do_major=(len(sys.argv) == 2 and sys.argv[1] == "major"),
+        items=[
+            ("src/libs/const.h",     "define US_VERSION_MAJOR {major}"),
+            ("src/libs/const.h",     "define US_VERSION_MINOR {minor}"),
+            ("python/setup.py",      "version=\"{major}.{minor}\""),
+            ("man/ustreamer.1",      "version {major}.{minor}"),
+            ("man/ustreamer-dump.1", "version {major}.{minor}"),
+            ("pkg/arch/PKGBUILD",    "pkgver={major}.{minor}"),
+            ("pkg/openwrt/Makefile", "PKG_VERSION:={major}.{minor}"),
+        ],
+    )

janus/Makefile

@@ -19,7 +19,7 @@ _BUILD = build
 
 
 # =====
-ifneq ($(shell sh -c 'uname 2>/dev/null || echo Unknown'),FreeBSD)
+ifeq ($(findstring bsd,$(shell $(CC) -dumpmachine)),)
 override _LDFLAGS += -latomic
 endif
 

janus/src/acap.c

@@ -24,7 +24,6 @@
 
 #include <stdlib.h>
 #include <stdatomic.h>
-#include <assert.h>
 
 #include <pthread.h>
 #include <alsa/asoundlib.h>
@@ -34,13 +33,13 @@
 #include "uslibs/types.h"
 #include "uslibs/errors.h"
 #include "uslibs/tools.h"
+#include "uslibs/threading.h"
+#include "uslibs/logging.h"
 #include "uslibs/array.h"
 #include "uslibs/ring.h"
-#include "uslibs/threading.h"
 
 #include "rtp.h"
 #include "au.h"
-#include "logging.h"
 
 
 static void *_pcm_thread(void *v_acap);
@@ -60,14 +59,14 @@ us_acap_s *us_acap_init(const char *name, uint pcm_hz) {
 	{
 		if ((err = snd_pcm_open(&acap->dev, name, SND_PCM_STREAM_CAPTURE, 0)) < 0) {
 			acap->dev = NULL;
-			US_JLOG_PERROR_ALSA(err, "acap", "Can't open PCM capture");
+			US_LOG_PERROR_ALSA(err, "Can't open PCM capture");
 			goto error;
 		}
-		assert(!snd_pcm_hw_params_malloc(&acap->dev_params));
+		US_A(!snd_pcm_hw_params_malloc(&acap->dev_params));
 
 #		define SET_PARAM(_msg, _func, ...) { \
 				if ((err = _func(acap->dev, acap->dev_params, ##__VA_ARGS__)) < 0) { \
-					US_JLOG_PERROR_ALSA(err, "acap", _msg); \
+					US_LOG_PERROR_ALSA(err, _msg); \
 					goto error; \
 				} \
 			}
@@ -78,7 +77,7 @@ us_acap_s *us_acap_init(const char *name, uint pcm_hz) {
 		SET_PARAM("Can't set PCM sampling format",	snd_pcm_hw_params_set_format, SND_PCM_FORMAT_S16_LE);
 		SET_PARAM("Can't set PCM sampling rate",	snd_pcm_hw_params_set_rate_near, &acap->pcm_hz, 0);
 		if (acap->pcm_hz < US_AU_MIN_PCM_HZ || acap->pcm_hz > US_AU_MAX_PCM_HZ) {
-			US_JLOG_ERROR("acap", "Unsupported PCM freq: %u; should be: %u <= F <= %u",
+			US_LOG_ERROR("Unsupported PCM freq: %u; should be: %u <= F <= %u",
 				acap->pcm_hz, US_AU_MIN_PCM_HZ, US_AU_MAX_PCM_HZ);
 			goto error;
 		}
@@ -93,7 +92,7 @@ us_acap_s *us_acap_init(const char *name, uint pcm_hz) {
 		acap->res = speex_resampler_init(US_RTP_OPUS_CH, acap->pcm_hz, US_RTP_OPUS_HZ, SPEEX_RESAMPLER_QUALITY_DESKTOP, &err);
 		if (err < 0) {
 			acap->res = NULL;
-			US_JLOG_PERROR_RES(err, "acap", "Can't create resampler");
+			US_LOG_PERROR_RES(err, "Can't create resampler");
 			goto error;
 		}
 	}
@@ -101,16 +100,16 @@ us_acap_s *us_acap_init(const char *name, uint pcm_hz) {
 	{
 		// OPUS_APPLICATION_VOIP, OPUS_APPLICATION_RESTRICTED_LOWDELAY
 		acap->enc = opus_encoder_create(US_RTP_OPUS_HZ, US_RTP_OPUS_CH, OPUS_APPLICATION_AUDIO, &err);
-		assert(err == 0);
+		US_A(err == 0);
 		// https://github.com/meetecho/janus-gateway/blob/3cdd6ff/src/plugins/janus_audiobridge.c#L2272
 		// https://datatracker.ietf.org/doc/html/rfc7587#section-3.1.1
-		assert(!opus_encoder_ctl(acap->enc, OPUS_SET_BITRATE(128000)));
-		assert(!opus_encoder_ctl(acap->enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)));
-		assert(!opus_encoder_ctl(acap->enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)));
+		US_A(!opus_encoder_ctl(acap->enc, OPUS_SET_BITRATE(128000)));
+		US_A(!opus_encoder_ctl(acap->enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)));
+		US_A(!opus_encoder_ctl(acap->enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)));
 		// OPUS_SET_INBAND_FEC(1), OPUS_SET_PACKET_LOSS_PERC(10): see rtpa.c
 	}
 
-	US_JLOG_INFO("acap", "Capture configured on %uHz; capturing ...", acap->pcm_hz);
+	US_LOG_INFO("Capture configured on %uHz; capturing ...", acap->pcm_hz);
 	acap->tids_created = true;
 	US_THREAD_CREATE(acap->enc_tid, _encoder_thread, acap);
 	US_THREAD_CREATE(acap->pcm_tid, _pcm_thread, acap);
@@ -135,7 +134,7 @@ void us_acap_destroy(us_acap_s *acap) {
 	US_RING_DELETE_WITH_ITEMS(acap->enc_ring, us_au_encoded_destroy);
 	US_RING_DELETE_WITH_ITEMS(acap->pcm_ring, us_au_pcm_destroy);
 	if (acap->tids_created) {
-		US_JLOG_INFO("acap", "Capture closed");
+		US_LOG_INFO("Capture closed");
 	}
 	free(acap);
 }
@@ -161,7 +160,7 @@ int us_acap_get_encoded(us_acap_s *acap, u8 *data, uz *size, u64 *pts) {
 }
 
 static void *_pcm_thread(void *v_acap) {
-	US_THREAD_SETTLE("us_ac_pcm");
+	US_THREAD_SETTLE("us_acap_pcm");
 
 	us_acap_s *const acap = v_acap;
 	u8 in[US_AU_MAX_BUF8];
@@ -169,10 +168,10 @@ static void *_pcm_thread(void *v_acap) {
 	while (!atomic_load(&acap->stop)) {
 		const int frames = snd_pcm_readi(acap->dev, in, acap->pcm_frames);
 		if (frames < 0) {
-			US_JLOG_PERROR_ALSA(frames, "acap", "Fatal: Can't capture PCM frames");
+			US_LOG_PERROR_ALSA(frames, "Fatal: Can't capture PCM frames");
 			break;
 		} else if (frames < (int)acap->pcm_frames) {
-			US_JLOG_ERROR("acap", "Fatal: Too few PCM frames captured");
+			US_LOG_ERROR("Fatal: Too few PCM frames captured");
 			break;
 		}
 
@@ -182,7 +181,7 @@ static void *_pcm_thread(void *v_acap) {
 			memcpy(out->data, in, acap->pcm_size);
 			us_ring_producer_release(acap->pcm_ring, ri);
 		} else {
-			US_JLOG_ERROR("acap", "PCM ring is full");
+			US_LOG_ERROR("PCM ring is full");
 		}
 	}
 
@@ -191,7 +190,7 @@ static void *_pcm_thread(void *v_acap) {
 }
 
 static void *_encoder_thread(void *v_acap) {
-	US_THREAD_SETTLE("us_ac_enc");
+	US_THREAD_SETTLE("us_acap_enc");
 
 	us_acap_s *const acap = v_acap;
 	s16 in_res[US_AU_MAX_BUF16];
@@ -205,19 +204,19 @@ static void *_encoder_thread(void *v_acap) {
 
 		s16 *in_ptr;
 		if (acap->res != NULL) {
-			assert(acap->pcm_hz != US_RTP_OPUS_HZ);
+			US_A(acap->pcm_hz != US_RTP_OPUS_HZ);
 			u32 in_count = acap->pcm_frames;
 			u32 out_count = US_AU_HZ_TO_FRAMES(US_RTP_OPUS_HZ);
 			speex_resampler_process_interleaved_int(acap->res, in->data, &in_count, in_res, &out_count);
 			in_ptr = in_res;
 		} else {
-			assert(acap->pcm_hz == US_RTP_OPUS_HZ);
+			US_A(acap->pcm_hz == US_RTP_OPUS_HZ);
 			in_ptr = in->data;
 		}
 
 		const int out_ri = us_ring_producer_acquire(acap->enc_ring, 0);
 		if (out_ri < 0) {
-			US_JLOG_ERROR("acap", "OPUS encoder queue is full");
+			US_LOG_ERROR("OPUS encoder queue is full");
 			us_ring_consumer_release(acap->pcm_ring, in_ri);
 			continue;
 		}
@@ -233,7 +232,7 @@ static void *_encoder_thread(void *v_acap) {
 			acap->pts += US_AU_HZ_TO_FRAMES(US_RTP_OPUS_HZ);
 		} else {
 			out->used = 0;
-			US_JLOG_PERROR_OPUS(size, "acap", "Fatal: Can't encode PCM frame to OPUS");
+			US_LOG_PERROR_OPUS(size, "Fatal: Can't encode PCM frame to OPUS");
 		}
 		us_ring_producer_release(acap->enc_ring, out_ri);
 	}

janus/src/au.c

@@ -38,7 +38,7 @@ bool us_au_probe(const char *name) {
 	// the existence of it in /proc/asound/.
 	// It's enough for our case.
 
-	if (name == NULL) {
+	if (!us_str_is_ok(name)) {
 		return false;
 	}
 

janus/src/client.c

@@ -25,7 +25,6 @@
 #include <stdlib.h>
 #include <stdatomic.h>
 #include <string.h>
-#include <assert.h>
 
 #include <pthread.h>
 #include <janus/plugins/plugin.h>
@@ -35,15 +34,18 @@
 #include "uslibs/types.h"
 #include "uslibs/tools.h"
 #include "uslibs/threading.h"
+#include "uslibs/logging.h"
 #include "uslibs/array.h"
 #include "uslibs/list.h"
 #include "uslibs/ring.h"
+#include "uslibs/frame.h"
 
-#include "logging.h"
 #include "au.h"
 #include "rtp.h"
 
 
+static us_frame_s *_add_vplay_frame_to_ring(us_frame_s *frame, void *v_client);
+
 static void *_video_thread(void *v_client);
 static void *_acap_thread(void *v_client);
 static void *_video_or_acap_thread(void *v_client, bool video);
@@ -55,9 +57,13 @@ us_janus_client_s *us_janus_client_init(janus_callbacks *gw, janus_plugin_sessio
 	US_CALLOC(client, 1);
 	client->gw = gw;
 	client->session = session;
+	client->video_ssrc = us_get_ssrc();
+	client->audio_ssrc = us_get_ssrc();
+
 	atomic_init(&client->transmit, false);
 	atomic_init(&client->transmit_acap, false);
 	atomic_init(&client->transmit_aplay, false);
+	atomic_init(&client->transmit_vplay, false);
 	atomic_init(&client->video_orient, 0);
 
 	atomic_init(&client->stop, false);
@@ -72,6 +78,8 @@ us_janus_client_s *us_janus_client_init(janus_callbacks *gw, janus_plugin_sessio
 	US_RING_INIT_WITH_ITEMS(client->aplay_pcm_ring, 64, us_au_pcm_init);
 	US_THREAD_CREATE(client->aplay_tid, _aplay_thread, client);
 
+	client->rtpc = us_rtpc_init(_add_vplay_frame_to_ring, client);
+
 	return client;
 }
 
@@ -88,6 +96,9 @@ void us_janus_client_destroy(us_janus_client_s *client) {
 	US_RING_DELETE_WITH_ITEMS(client->aplay_enc_ring, us_au_encoded_destroy);
 	US_RING_DELETE_WITH_ITEMS(client->aplay_pcm_ring, us_au_pcm_destroy);
 
+	us_rtpc_destroy(client->rtpc);
+	// us_janus_client_stop_vplay(client);
+
 	free(client);
 }
 
@@ -99,7 +110,7 @@ void us_janus_client_send(us_janus_client_s *client, const us_rtp_s *rtp) {
 		us_ring_s *const ring = (rtp->video ? client->video_ring : client->acap_ring);
 		const int ri = us_ring_producer_acquire(ring, 0);
 		if (ri < 0) {
-			US_JLOG_ERROR("client", "Session %p %s ring is full",
+			US_LOG_ERROR("Session %p %s ring is full",
 				client->session, (rtp->video ? "video" : "acap"));
 			return;
 		}
@@ -108,65 +119,114 @@ void us_janus_client_send(us_janus_client_s *client, const us_rtp_s *rtp) {
 	}
 }
 
+static us_frame_s *_add_vplay_frame_to_ring(us_frame_s *ready_frame, void *v_client) {
+	us_janus_client_s *const client = v_client;
+
+	us_ring_s *const ring = client->vplay_enc_ring;
+	if (!ring) {
+		US_LOG_ERROR("Session %p has no vplay ring", client->session);
+		return ready_frame;
+	}
+
+	const int ri = us_ring_producer_acquire(ring, 0);
+	if (ri < 0) {
+		US_LOG_ERROR("Session %p vplay ring is full", client->session);
+		return ready_frame;
+	}
+
+	us_frame_s *const next_frame = ring->items[ri];
+	ring->items[ri] = ready_frame;
+	us_ring_producer_release(ring, ri);
+
+	return next_frame;
+}
+
 void us_janus_client_recv(us_janus_client_s *client, janus_plugin_rtp *packet) {
-	if (
-		packet->video
-		|| packet->length < sizeof(janus_rtp_header)
-		|| !atomic_load(&client->transmit)
-		|| !atomic_load(&client->transmit_aplay)
-	) {
+	if (packet->length < sizeof(janus_rtp_header) || !atomic_load(&client->transmit)) {
 		return;
 	}
 
+	int payload_size = 0;
+	const u8 *const payload = (u8 *)janus_rtp_payload(packet->buffer, packet->length, &payload_size);
+	if (payload == NULL || payload_size < 1) {
+		return;
+	}
 	const janus_rtp_header *const header = (janus_rtp_header*)packet->buffer;
-	if (header->type != US_RTP_OPUS_PAYLOAD) {
-		return;
-	}
-
 	const u16 seq = ntohs(header->seq_number);
-	if (
-		seq >= client->aplay_seq_next // In order or missing
-		|| (client->aplay_seq_next - seq) > 50 // In late sequence or sequence wrapped
-	) {
-		client->aplay_seq_next = seq + 1;
 
-		int size = 0;
-		const char *const data = janus_rtp_payload(packet->buffer, packet->length, &size);
-		if (data == NULL || size <= 0) {
-			return;
+	if (packet->video) {
+		if (header->type == US_RTP_H264_PAYLOAD && atomic_load(&client->transmit_vplay)) {
+			const u32 rts = ntohl(header->timestamp); // RTS == RTP TimeStamp
+			bool retry;
+			do {
+				retry = false;
+				switch (us_rtpc_unwrap(client->rtpc, payload, payload_size, seq, rts)) {
+					case US_RUR_SUCCESS:
+					case US_RUR_BAD_PACKET:
+					case US_RUR_INVALID_SEQ:
+					case US_RUR_UNSUPPORTED_UNIT_TYPE:
+					case US_RUR_DEPACKETIZATION_SKIPPED:
+						break;
+					case US_RUR_DEPACKETIZATION_FAILED_RETRY:
+						retry = true;
+						// fall through
+					case US_RUR_DEPACKETIZATION_FAILED:
+						client->gw->send_pli(client->session);
+						break;
+				}
+			} while (retry);
 		}
-
-		us_ring_s *const ring = client->aplay_enc_ring;
-		const int ri = us_ring_producer_acquire(ring, 0);
-		if (ri < 0) {
-			// US_JLOG_ERROR("client", "Session %p aplay ring is full", client->session);
-			return;
+	} else {
+		if (header->type == US_RTP_OPUS_PAYLOAD && atomic_load(&client->transmit_aplay)) {
+			if (seq >= client->aplay_seq_next || (client->aplay_seq_next - seq) > 50) {
+				// ^^^ In order or missing   |OR| ^^^ In late sequence or sequence wrapped
+				client->aplay_seq_next = seq + 1;
+				us_ring_s *const ring = client->aplay_enc_ring;
+				const int ri = us_ring_producer_acquire(ring, 0);
+				if (ri < 0) {
+					// US_LOG_ERROR("Session %p aplay ring is full", client->session);
+					return;
+				}
+				us_au_encoded_s *enc = ring->items[ri];
+				if ((uz)payload_size < US_ARRAY_LEN(enc->data)) {
+					memcpy(enc->data, payload, payload_size);
+					enc->used = payload_size;
+				} else {
+					enc->used = 0;
+				}
+				us_ring_producer_release(ring, ri);
+			}
 		}
-		us_au_encoded_s *enc = ring->items[ri];
-		if ((uz)size < US_ARRAY_LEN(enc->data)) {
-			memcpy(enc->data, data, size);
-			enc->used = size;
-		} else {
-			enc->used = 0;
-		}
-		us_ring_producer_release(ring, ri);
 	}
 }
 
+void us_janus_client_start_vplay(us_janus_client_s *client, us_ring_s* vplay_enc_ring) {
+	if (atomic_exchange(&client->transmit_vplay, true)) {
+		return;
+	}
+	US_A(client->vplay_enc_ring == NULL);
+	client->vplay_enc_ring = vplay_enc_ring;
+}
+
+/*void us_janus_client_stop_vplay(us_janus_client_s *client) {
+	atomic_store(&client->transmit_vplay, false);
+	client->vplay_enc_ring = NULL;
+}*/
+
 static void *_video_thread(void *v_client) {
-	US_THREAD_SETTLE("us_cx_vid");
+	US_THREAD_SETTLE("us_cx_vcap");
 	return _video_or_acap_thread(v_client, true);
 }
 
 static void *_acap_thread(void *v_client) {
-	US_THREAD_SETTLE("us_cx_ac");
+	US_THREAD_SETTLE("us_cx_acap");
 	return _video_or_acap_thread(v_client, false);
 }
 
 static void *_video_or_acap_thread(void *v_client, bool video) {
 	us_janus_client_s *const client = v_client;
 	us_ring_s *const ring = (video ? client->video_ring : client->acap_ring);
-	assert(ring != NULL);
+	US_A(ring != NULL);
 
 	while (!atomic_load(&client->stop)) {
 		const int ri = us_ring_consumer_acquire(ring, 0.1);
@@ -181,6 +241,8 @@ static void *_video_or_acap_thread(void *v_client, bool video) {
 			atomic_load(&client->transmit)
 			&& (video || atomic_load(&client->transmit_acap))
 		) {
+			us_rtp_write_ssrc(&rtp, (rtp.video ? client->video_ssrc : client->audio_ssrc));
+
 			janus_plugin_rtp packet = {
 				.video = rtp.video,
 				.buffer = (char*)rtp.datagram,
@@ -231,13 +293,13 @@ static void *_video_or_acap_thread(void *v_client, bool video) {
 }
 
 static void *_aplay_thread(void *v_client) {
-	US_THREAD_SETTLE("us_cx_ap");
+	US_THREAD_SETTLE("us_cx_aplay");
 
 	us_janus_client_s *const client = v_client;
 
 	int err;
 	OpusDecoder *dec = opus_decoder_create(US_RTP_OPUS_HZ, US_RTP_OPUS_CH, &err);
-	assert(err == 0);
+	US_A(err == 0);
 
 	while (!atomic_load(&client->stop)) {
 		const int in_ri = us_ring_consumer_acquire(client->aplay_enc_ring, 0.1);
@@ -253,7 +315,7 @@ static void *_aplay_thread(void *v_client) {
 
 		const int out_ri = us_ring_producer_acquire(client->aplay_pcm_ring, 0);
 		if (out_ri < 0) {
-			US_JLOG_ERROR("aplay", "OPUS decoder queue is full");
+			US_LOG_ERROR("OPUS decoder queue is full");
 			us_ring_consumer_release(client->aplay_enc_ring, in_ri);
 			continue;
 		}
@@ -266,7 +328,7 @@ static void *_aplay_thread(void *v_client) {
 			out->frames = frames;
 		} else {
 			out->frames = 0;
-			US_JLOG_PERROR_OPUS(frames, "aplay", "Fatal: Can't decode OPUS to PCM frame");
+			US_LOG_PERROR_OPUS(frames, "Fatal: Can't decode OPUS to PCM frame");
 		}
 		us_ring_producer_release(client->aplay_pcm_ring, out_ri);
 	}

janus/src/client.h

@@ -32,29 +32,39 @@
 #include "uslibs/ring.h"
 
 #include "rtp.h"
+#include "rtpv.h"
+#include "rtpc.h"
 
 
 typedef struct {
 	janus_callbacks			*gw;
 	janus_plugin_session	*session;
-	atomic_bool				transmit;
-	atomic_bool				transmit_acap;
-	atomic_bool				transmit_aplay;
-	atomic_uint				video_orient;
 
-	pthread_t				video_tid;
-	pthread_t				acap_tid;
-	pthread_t				aplay_tid;
-	atomic_bool				stop;
+	u32			video_ssrc;
+	u32			audio_ssrc;
 
-	us_ring_s				*video_ring;
-	us_ring_s				*acap_ring;
+	atomic_bool	transmit;
+	atomic_bool	transmit_acap;
+	atomic_bool	transmit_vplay;
+	atomic_bool	transmit_aplay;
+	atomic_uint	video_orient;
 
-	us_ring_s				*aplay_enc_ring;
-	u16						aplay_seq_next;
-	us_ring_s				*aplay_pcm_ring;
+	pthread_t	video_tid;
+	pthread_t	acap_tid;
+	pthread_t	aplay_tid;
+	atomic_bool	stop;
 
-    US_LIST_DECLARE;
+	us_ring_s	*video_ring;
+	us_ring_s	*acap_ring;
+
+	us_ring_s	*aplay_enc_ring;
+	u16			aplay_seq_next;
+	us_ring_s	*aplay_pcm_ring;
+
+	us_rtpc_s	*rtpc;
+	us_ring_s	*vplay_enc_ring;
+
+	US_LIST_DECLARE;
 } us_janus_client_s;
 
 
@@ -63,3 +73,6 @@ void us_janus_client_destroy(us_janus_client_s *client);
 
 void us_janus_client_send(us_janus_client_s *client, const us_rtp_s *rtp);
 void us_janus_client_recv(us_janus_client_s *client, janus_plugin_rtp *packet);
+
+void us_janus_client_start_vplay(us_janus_client_s *client, us_ring_s* vplay_enc_ring);
+// void us_janus_client_stop_vplay(us_janus_client_s *client);

janus/src/config.c

@@ -30,13 +30,13 @@
 
 #include "uslibs/types.h"
 #include "uslibs/tools.h"
+#include "uslibs/logging.h"
 
 #include "const.h"
-#include "logging.h"
 
 
 static char *_get_value(janus_config *jcfg, const char *section, const char *option);
-static uint _get_uint(janus_config *jcfg, const char *section, const char *option, uint def);
+static uint _get_uint(janus_config *jcfg, const char *section, const char *option, uint def, uint base);
 // static bool _get_bool(janus_config *jcfg, const char *section, const char *option, bool def);
 
 
@@ -48,29 +48,37 @@ us_config_s *us_config_init(const char *config_dir_path) {
 	janus_config *jcfg = NULL;
 
 	US_ASPRINTF(config_file_path, "%s/%s.jcfg", config_dir_path, US_PLUGIN_PACKAGE);
-	US_JLOG_INFO("config", "Reading config file '%s' ...", config_file_path);
+	US_LOG_INFO("Reading config file '%s' ...", config_file_path);
 
 	jcfg = janus_config_parse(config_file_path);
 	if (jcfg == NULL) {
-		US_JLOG_ERROR("config", "Can't read config");
+		US_LOG_ERROR("Can't read config");
 		goto error;
 	}
 	janus_config_print(jcfg);
 
-	if ((config->video_sink_name = _get_value(jcfg, "video", "sink")) == NULL) {
-		US_JLOG_ERROR("config", "Missing config value: video.sink");
+	config->video_sink_name = _get_value(jcfg, "video", "sink");
+	if (!us_str_is_ok(config->video_sink_name)) {
+		US_LOG_ERROR("Missing config value: video.sink");
 		goto error;
 	}
-	if ((config->acap_dev_name = _get_value(jcfg, "acap", "device")) != NULL) {
-		config->acap_hz = _get_uint(jcfg, "acap", "sampling_rate", 0);
+
+	config->acap_dev_name = _get_value(jcfg, "acap", "device");
+	if (us_str_is_ok(config->acap_dev_name)) {
+		config->acap_hz = _get_uint(jcfg, "acap", "sampling_rate", 0, 10);
 		config->tc358743_dev_path = _get_value(jcfg, "acap", "tc358743");
-		if (config->acap_hz == 0 && config->tc358743_dev_path == NULL) {
-			US_JLOG_ERROR("config", "Either acap.sampling_rate or acap.tc358743 required");
+		if (config->acap_hz == 0 && !us_str_is_ok(config->tc358743_dev_path)) {
+			US_LOG_ERROR("Either acap.sampling_rate or acap.tc358743 required");
 			goto error;
 		}
-		config->aplay_dev_name = _get_value(jcfg, "aplay", "device");
 	}
 
+	config->aplay_dev_name = _get_value(jcfg, "aplay", "device");
+
+	config->vplay_sink_name = _get_value(jcfg, "vplay", "sink");
+	config->vplay_sink_mode = _get_uint(jcfg, "vplay", "sink_mode", 0660, 8);
+	config->vplay_dev_path = _get_value(jcfg, "vplay", "device");
+
 	goto ok;
 
 error:
@@ -87,24 +95,26 @@ void us_config_destroy(us_config_s *config) {
 	US_DELETE(config->acap_dev_name, free);
 	US_DELETE(config->tc358743_dev_path, free);
 	US_DELETE(config->aplay_dev_name, free);
+	US_DELETE(config->vplay_sink_name, free);
+	US_DELETE(config->vplay_dev_path, free);
 	free(config);
 }
 
 static char *_get_value(janus_config *jcfg, const char *section, const char *option) {
 	janus_config_category *section_obj = janus_config_get_create(jcfg, NULL, janus_config_type_category, section);
 	janus_config_item *option_obj = janus_config_get(jcfg, section_obj, janus_config_type_item, option);
-	if (option_obj == NULL || option_obj->value == NULL || option_obj->value[0] == '\0') {
+	if (option_obj == NULL || !us_str_is_ok(option_obj->value)) {
 		return NULL;
 	}
 	return us_strdup(option_obj->value);
 }
 
-static uint _get_uint(janus_config *jcfg, const char *section, const char *option, uint def) {
+static uint _get_uint(janus_config *jcfg, const char *section, const char *option, uint def, uint base) {
 	char *const tmp = _get_value(jcfg, section, option);
 	uint value = def;
 	if (tmp != NULL) {
 		errno = 0;
-		value = (uint)strtoul(tmp, NULL, 10);
+		value = (uint)strtoul(tmp, NULL, base);
 		if (errno != 0) {
 			value = def;
 		}

janus/src/config.h

@@ -34,6 +34,10 @@ typedef struct {
 	char	*tc358743_dev_path;
 
 	char	*aplay_dev_name;
+
+	char	*vplay_sink_name;
+	uint	vplay_sink_mode;
+	char	*vplay_dev_path;
 } us_config_s;
 
 

janus/src/h264.c

@@ -0,0 +1,269 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    This source file based on code of Janys-Gateway.                        #
+#                                                                            #
+#    Copyright (C)  Lorenzo Miniero <lorenzo@meetecho.com>                   #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#include "h264.h"
+
+#include "uslibs/types.h"
+#include "uslibs/tools.h"
+
+
+static INLINE u32 _eg_get_bit(const u8 *buf, u32 bit_offset) {
+	return ((*(buf + (bit_offset >> 3))) >> (7 - (bit_offset & 0x07))) & 0x01;
+}
+
+static INLINE bool _is_valid_eg_offset(u32 bit_offset, uz size) {
+	return (bit_offset >> 3) + (bit_offset % 8 ? 1 : 0) < size;
+}
+
+static bool _eg_skip_u1(const u8 *buf, u32 *bit_offset, uz size) {
+	(void)buf;
+	if (!_is_valid_eg_offset(*bit_offset, size)) {
+		return false;
+	}
+	++(*bit_offset);
+	return true;
+}
+
+static bool _eg_decode_u1(const u8 *buf, u32 *bit_offset, uz size, bool *out) {
+	if (!_is_valid_eg_offset(*bit_offset, size)) {
+		return false;
+	}
+	*out = (_eg_get_bit(buf, *bit_offset) != 0);
+	++(*bit_offset);
+	return true;
+}
+
+static bool _eg_skip_ev(const u8 *buf, u32 *bit_offset, uz size) {
+	u32 zeros = 0;
+	while (true) {
+		if (!_is_valid_eg_offset(*bit_offset, size)) {
+			return false;
+		}
+		if (_eg_get_bit(buf, (*bit_offset)++) != 0) {
+			break;
+		}
+		++zeros;
+	}
+	for (; zeros; --zeros) {
+		if (!_is_valid_eg_offset((*bit_offset)++, size)) {
+			return false;
+		}
+	}
+	return true;
+}
+
+static bool _eg_decode_uev(const u8 *buf, u32 *bit_offset, uint size, u32 *out) {
+	u32 zeros = 0;
+	while (true) {
+		if (!_is_valid_eg_offset(*bit_offset, size)) {
+			return false;
+		}
+		if (_eg_get_bit(buf, (*bit_offset)++) != 0) {
+			break;
+		}
+		++zeros;
+	}
+	u32 res = (1 << zeros);
+	for (; zeros; --zeros) {
+		if (!_is_valid_eg_offset(*bit_offset, size)) {
+			return false;
+		}
+		res |= (_eg_get_bit(buf, *bit_offset) << (zeros - 1));
+		++(*bit_offset);
+	}
+	*out = res - 1;
+	return true;
+}
+
+int us_h264_parse_sps_nalu(const u8 *buf, uz size, uint *width, uint *height) {
+	// NAL header + profile_idc + constraint_setX_flag, reserved_zero_2bits + level_idc
+	if (size < 4) {
+		return -1;
+	}
+
+	// NAL header
+	++buf;
+	--size;
+
+	// profile_idc u(8) - Baseline profile
+	const u8 profile_idc = *buf;
+	if (profile_idc != 66) {
+		return -1;
+	}
+	++buf;
+	--size;
+
+	// constraint_set0_flag u(1)
+	// constraint_set1_flag u(1)
+	// constraint_set2_flag u(1)
+	// constraint_set3_flag u(1)
+	// constraint_set4_flag u(1)
+	// constraint_set5_flag u(1)
+	// reserved_zero_2bits u(2)
+	++buf;
+	--size;
+
+	// level_idc u(8)
+	++buf;
+	--size;
+
+	u32 bit_offset = 0;
+
+	// seq_parameter_set_id ue(v)
+	if (!_eg_skip_ev(buf, &bit_offset, size)) {
+		return -1;
+	}
+
+	// log2_max_frame_num_minus4 ue(v)
+	u32 log2_max_frame_num_minus4;
+	if (!_eg_decode_uev(buf, &bit_offset, size, &log2_max_frame_num_minus4)) {
+		return -1;
+	}
+
+	// pic_order_cnt_type ue(v)
+	u32 pic_order_cnt_type;
+	if (!_eg_decode_uev(buf, &bit_offset, size, &pic_order_cnt_type)) {
+		return -1;
+	}
+
+	if (pic_order_cnt_type == 0) {
+		// log2_max_pic_order_cnt_lsb_minus4 ue(v)
+		if (!_eg_skip_ev(buf, &bit_offset, size)) {
+			return -1;
+		}
+	} else if (pic_order_cnt_type == 1) {
+		// delta_pic_order_always_zero_flag u(1)
+		if (!_eg_skip_u1(buf, &bit_offset, size)) {
+			return -1;
+		}
+
+		// offset_for_non_ref_pic se(v)
+		if (!_eg_skip_ev(buf, &bit_offset, size)) {
+			return -1;
+		}
+
+		// offset_for_top_to_bottom_field se(v)
+		if (!_eg_skip_ev(buf, &bit_offset, size)) {
+			return -1;
+		}
+
+		// num_ref_frames_in_pic_order_cnt_cycle ue(v)
+		u32 num_ref_frames_in_pic_order_cnt_cycle;
+		if (!_eg_decode_uev(buf, &bit_offset, size, &num_ref_frames_in_pic_order_cnt_cycle)) {
+			return -1;
+		}
+
+		for (u32 i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; ++i) {
+			// offset_for_ref_frame[i] se(v)
+			if (!_eg_skip_ev(buf, &bit_offset, size)) {
+				return -1;
+			}
+		}
+	}
+
+	// max_num_ref_frames ue(v)
+	if (!_eg_skip_ev(buf, &bit_offset, size)) {
+		return -1;
+	}
+
+	// gaps_in_frame_num_value_allowed_flag u(1)
+	if (!_eg_skip_u1(buf, &bit_offset, size)) {
+		return -1;
+	}
+
+	// pic_width_in_mbs_minus1 ue(v)
+	u32 pic_width_in_mbs_minus1;
+	if (!_eg_decode_uev(buf, &bit_offset, size, &pic_width_in_mbs_minus1)) {
+		return -1;
+	}
+
+	//pic_height_in_map_units_minus1 ue(v)
+	u32 pic_height_in_map_units_minus1;
+	if (!_eg_decode_uev(buf, &bit_offset, size, &pic_height_in_map_units_minus1)) {
+		return -1;
+	}
+
+	// frame_mbs_only_flag u(1)
+	bool frame_mbs_only_flag;
+	if (!_eg_decode_u1(buf, &bit_offset, size, &frame_mbs_only_flag)) {
+		return -1;
+	}
+
+	if (!frame_mbs_only_flag) {
+		// mb_adaptive_frame_field_flag u(1)
+		if (!_eg_skip_u1(buf, &bit_offset, size)) {
+			return -1;
+		}
+	}
+
+	//direct_8x8_inference_flag u(1)
+	if (!_eg_skip_u1(buf, &bit_offset, size)) {
+		return -1;
+	}
+
+	// frame_cropping_flag u(1)
+	bool frame_cropping_flag;
+	if (!_eg_decode_u1(buf, &bit_offset, size, &frame_cropping_flag)) {
+		return -1;
+	}
+
+	u32 frame_crop_left_offset = 0;
+	u32 frame_crop_right_offset = 0;
+	u32 frame_crop_top_offset = 0;
+	u32 frame_crop_bottom_offset = 0;
+	if (frame_cropping_flag) {
+		// frame_crop_left_offset ue(v)
+		if (!_eg_decode_uev(buf, &bit_offset, size, &frame_crop_left_offset)) {
+			return -1;
+		}
+
+		// frame_crop_right_offset ue(v)
+		if (!_eg_decode_uev(buf, &bit_offset, size, &frame_crop_right_offset)) {
+			return -1;
+		}
+
+		// frame_crop_top_offset ue(v)
+		if (!_eg_decode_uev(buf, &bit_offset, size, &frame_crop_top_offset)) {
+			return -1;
+		}
+
+		// frame_crop_bottom_offset ue(v)
+		if (!_eg_decode_uev(buf, &bit_offset, size, &frame_crop_bottom_offset)) {
+			return -1;
+		}
+	}
+
+	*width = (
+		(pic_width_in_mbs_minus1 + 1) * 16
+		- frame_crop_left_offset * 2
+		- frame_crop_right_offset * 2
+	);
+	*height = (
+		(pic_height_in_map_units_minus1 + 1) * (2 - frame_mbs_only_flag) * 16
+		- frame_crop_top_offset * 2
+		- frame_crop_bottom_offset * 2
+	);
+	return 0;
+}

janus/src/h264.h

@@ -23,9 +23,6 @@
 #pragma once
 
 #include "uslibs/types.h"
-#include "uslibs/frame.h"
-#include "uslibs/memsinksh.h"
 
 
-int us_memsink_fd_wait_frame(int fd, us_memsink_shared_s *mem, u64 last_id);
-int us_memsink_fd_get_frame(int fd, us_memsink_shared_s *mem, us_frame_s *frame, u64 *frame_id, bool key_required);
+int us_h264_parse_sps_nalu(const u8 *buf, uz size, uint *width, uint *height);

janus/src/plugin.c

@@ -22,14 +22,9 @@
 
 #include <stdatomic.h>
 #include <stdlib.h>
-#include <inttypes.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <errno.h>
-#include <assert.h>
-
-#include <sys/mman.h>
-#include <sys/stat.h>
 
 #include <pthread.h>
 #include <jansson.h>
@@ -37,30 +32,32 @@
 #include <janus/rtp.h>
 #include <janus/rtcp.h>
 #include <alsa/asoundlib.h>
+#include <linux/videodev2.h>
 
 #include "uslibs/types.h"
 #include "uslibs/const.h"
 #include "uslibs/errors.h"
 #include "uslibs/tools.h"
 #include "uslibs/threading.h"
+#include "uslibs/logging.h"
 #include "uslibs/list.h"
 #include "uslibs/ring.h"
-#include "uslibs/memsinksh.h"
-#include "uslibs/tc358743.h"
+#include "uslibs/memsink.h"
+#include "uslibs/chip.h"
+#include "uslibs/memsink.h"
 
 #include "const.h"
-#include "logging.h"
 #include "client.h"
 #include "au.h"
 #include "acap.h"
 #include "rtp.h"
 #include "rtpv.h"
 #include "rtpa.h"
-#include "memsinkfd.h"
+#include "sdp.h"
 #include "config.h"
 
 
-static const char *const	default_ice_url = "stun:stun.l.google.com:19302";
+static const char *const	_g_default_ice_url = "stun:stun.l.google.com:19302";
 
 static us_config_s		*_g_config = NULL;
 static const useconds_t	_g_watchers_polling = 100000;
@@ -68,6 +65,7 @@ static const useconds_t	_g_watchers_polling = 100000;
 static us_janus_client_s	*_g_clients = NULL;
 static janus_callbacks		*_g_gw = NULL;
 static us_ring_s			*_g_video_ring = NULL;
+static us_ring_s			*_g_vplay_ring = NULL;
 static us_rtpv_s			*_g_rtpv = NULL;
 static us_rtpa_s			*_g_rtpa = NULL;
 
@@ -75,12 +73,15 @@ static pthread_t		_g_video_rtp_tid;
 static atomic_bool		_g_video_rtp_tid_created = false;
 static pthread_t		_g_video_sink_tid;
 static atomic_bool		_g_video_sink_tid_created = false;
+static pthread_t		_g_vplay_tid;
+static atomic_bool		_g_vplay_tid_created = false;
 static pthread_t		_g_acap_tid;
 static atomic_bool		_g_acap_tid_created = false;
 static pthread_t		_g_aplay_tid;
 static atomic_bool		_g_aplay_tid_created = false;
 
 static pthread_mutex_t	_g_video_lock = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t	_g_vplay_lock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t	_g_acap_lock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t	_g_aplay_lock = PTHREAD_MUTEX_INITIALIZER;
 static atomic_bool		_g_ready = false;
@@ -90,18 +91,29 @@ static atomic_bool		_g_has_listeners = false;
 static atomic_bool		_g_has_speakers = false;
 static atomic_bool		_g_key_required = false;
 
+static struct {
+	bool	active;
+	us_janus_client_s *client;
+	uint	width;
+	uint	height;
+	uint	fps;
+} _g_camera = {0}; // Access should be protected by _g_vplay_lock
+
 
 #define _LOCK_VIDEO		US_MUTEX_LOCK(_g_video_lock)
 #define _UNLOCK_VIDEO	US_MUTEX_UNLOCK(_g_video_lock)
 
+#define _LOCK_VPLAY		US_MUTEX_LOCK(_g_vplay_lock)
+#define _UNLOCK_VPLAY	US_MUTEX_UNLOCK(_g_vplay_lock)
+
 #define _LOCK_ACAP		US_MUTEX_LOCK(_g_acap_lock)
 #define _UNLOCK_ACAP	US_MUTEX_UNLOCK(_g_acap_lock)
 
 #define _LOCK_APLAY		US_MUTEX_LOCK(_g_aplay_lock)
 #define _UNLOCK_APLAY	US_MUTEX_UNLOCK(_g_aplay_lock)
 
-#define _LOCK_ALL		{ _LOCK_VIDEO; _LOCK_ACAP; _LOCK_APLAY; }
-#define _UNLOCK_ALL		{ _UNLOCK_APLAY; _UNLOCK_ACAP; _UNLOCK_VIDEO; }
+#define _LOCK_ALL		{ _LOCK_VIDEO; _LOCK_VPLAY; _LOCK_ACAP; _LOCK_APLAY; }
+#define _UNLOCK_ALL		{ _UNLOCK_APLAY; _UNLOCK_ACAP; _UNLOCK_VPLAY; _UNLOCK_VIDEO; }
 
 #define _READY			atomic_load(&_g_ready)
 #define _STOP			atomic_load(&_g_stop)
@@ -136,115 +148,66 @@ static void *_video_rtp_thread(void *arg) {
 
 static void *_video_sink_thread(void *arg) {
 	(void)arg;
-	US_THREAD_SETTLE("us_p_vsink");
+	US_THREAD_SETTLE("us_p_vcap");
 	atomic_store(&_g_video_sink_tid_created, true);
 
-	us_frame_s *drop = us_frame_init();
-	u64 frame_id = 0;
+	us_frame_s *tmp = us_frame_init();
 	int once = 0;
 
 	while (!_STOP) {
 		if (!_HAS_WATCHERS) {
-			US_ONCE({ US_JLOG_INFO("video", "No active watchers, memsink disconnected"); });
+			US_ONCE({ US_LOG_INFO("No active watchers, memsink disconnected"); });
 			usleep(_g_watchers_polling);
 			continue;
 		}
 
-		int fd = -1;
-		us_memsink_shared_s *mem = NULL;
-
-		const uz data_size = us_memsink_calculate_size(_g_config->video_sink_name);
-		if (data_size == 0) {
-			US_ONCE({ US_JLOG_ERROR("video", "Invalid memsink object suffix"); });
-			goto close_memsink;
-		}
-
-		if ((fd = shm_open(_g_config->video_sink_name, O_RDWR, 0)) <= 0) {
-			US_ONCE({ US_JLOG_PERROR("video", "Can't open memsink"); });
-			goto close_memsink;
-		}
-
-		if ((mem = us_memsink_shared_map(fd, data_size)) == NULL) {
-			US_ONCE({ US_JLOG_PERROR("video", "Can't map memsink"); });
+		us_memsink_s *sink = us_memsink_init_opened("vcap", _g_config->video_sink_name, false, 0, false, 0, 1);
+		if (sink == NULL) {
 			goto close_memsink;
 		}
 
 		once = 0;
 
-		US_JLOG_INFO("video", "Memsink opened; reading frames ...");
+		US_LOG_INFO("Memsink opened; reading frames ...");
 		while (!_STOP && _HAS_WATCHERS) {
-			const int waited = us_memsink_fd_wait_frame(fd, mem, frame_id);
-			if (waited == 0) {
+			const us_memsink_wants_s w_put = {.key = atomic_load(&_g_key_required)};
+			const int got = us_memsink_client_get(sink, tmp, NULL, &w_put);
+			if (got == 0) {
 				const int ri = us_ring_producer_acquire(_g_video_ring, 0);
-				us_frame_s *frame;
-				if (ri >= 0) {
-					frame = _g_video_ring->items[ri];
-				} else {
-					US_ONCE({ US_JLOG_PERROR("video", "Video ring is full"); });
-					frame = drop;
-				}
-
-				const int got = us_memsink_fd_get_frame(fd, mem, frame, &frame_id, atomic_load(&_g_key_required));
 				if (ri >= 0) {
+					us_frame_s *dest = _g_video_ring->items[ri];
+					us_frame_copy(tmp, dest);
 					us_ring_producer_release(_g_video_ring, ri);
+					if (tmp->key) {
+						atomic_store(&_g_key_required, false);
+					}
+				} else {
+					US_ONCE({ US_LOG_ERROR("Video ring is full"); });
 				}
-				if (got < 0) {
-					goto close_memsink;
-				}
-
-				if (ri >= 0 && frame->key) {
-					atomic_store(&_g_key_required, false);
-				}
-			} else if (waited != US_ERROR_NO_DATA) {
+			} else if (got == US_ERROR_NO_DATA) {
+				usleep(1000);
+			} else {
 				goto close_memsink;
 			}
 		}
 
 	close_memsink:
-		if (mem != NULL) {
-			us_memsink_shared_unmap(mem, data_size);
-			mem = NULL;
-		}
-		US_CLOSE_FD(fd);
-		US_JLOG_INFO("video", "Memsink closed");
+		US_DELETE(sink, us_memsink_destroy);
+		US_LOG_INFO("Memsink closed");
 		sleep(1); // error_delay
 	}
 
-	us_frame_destroy(drop);
+	us_frame_destroy(tmp);
 	return NULL;
 }
 
-static int _get_acap_hz(uint *hz) {
-	if (_g_config->acap_hz != 0) {
-		*hz = _g_config->acap_hz;
-		return 0;
-	}
-	if (_g_config->tc358743_dev_path == NULL) {
-		US_JLOG_ERROR("acap", "No configured sampling rate");
-		return -1;
-	}
-	int fd;
-	if ((fd = open(_g_config->tc358743_dev_path, O_RDWR)) < 0) {
-		US_JLOG_PERROR("acap", "Can't open TC358743 V4L2 device");
-		return -1;
-	}
-	const int checked = us_tc358743_xioctl_get_audio_hz(fd, hz);
-	if (checked < 0) {
-		US_JLOG_PERROR("acap", "Can't check TC358743 audio state (%d)", checked);
-		close(fd);
-		return -1;
-	}
-	close(fd);
-	return 0;
-}
-
 static void *_acap_thread(void *arg) {
 	(void)arg;
-	US_THREAD_SETTLE("us_p_ac");
+	US_THREAD_SETTLE("us_p_acap");
 	atomic_store(&_g_acap_tid_created, true);
 
-	assert(_g_config->acap_dev_name != NULL);
-	assert(_g_rtpa != NULL);
+	US_A(us_str_is_ok(_g_config->acap_dev_name));
+	US_A(_g_rtpa != NULL);
 
 	int once = 0;
 
@@ -254,21 +217,34 @@ static void *_acap_thread(void *arg) {
 			continue;
 		}
 
-		uint hz = 0;
+		int chip_fd = -1;
 		us_acap_s *acap = NULL;
 
 		if (!us_au_probe(_g_config->acap_dev_name)) {
-			US_ONCE({ US_JLOG_ERROR("acap", "No PCM capture device"); });
+			US_ONCE({ US_LOG_ERROR("No PCM capture device"); });
 			goto close_acap;
 		}
-		if (_get_acap_hz(&hz) < 0) {
-			goto close_acap;
+
+		int hz = _g_config->acap_hz;
+		if (hz <= 0) {
+			US_A(us_str_is_ok(_g_config->tc358743_dev_path));
+			chip_fd = open(_g_config->tc358743_dev_path, O_RDWR);
+			if (chip_fd < 0) {
+				US_ONCE({ US_LOG_PERROR("Can't open TC358743 for the audio request"); });
+				goto close_acap;
+			}
+
+			hz = us_chip_tc358743_get_audio_hz(chip_fd);
+			if (hz == US_ERROR_NO_SIGNAL) {
+				US_ONCE({ US_LOG_INFO("No audio presented from the host"); });
+				goto close_acap;
+			} else if (hz <= 0) {
+				US_ONCE({ US_LOG_PERROR("Can't get audio HZ"); });
+				goto close_acap;
+			}
+			US_ONCE({ US_LOG_INFO("Detected host audio"); });
 		}
-		if (hz == 0) {
-			US_ONCE({ US_JLOG_INFO("acap", "No audio presented from the host"); });
-			goto close_acap;
-		}
-		US_ONCE({ US_JLOG_INFO("acap", "Detected host audio"); });
+
 		if ((acap = us_acap_init(_g_config->acap_dev_name, hz)) == NULL) {
 			goto close_acap;
 		}
@@ -276,10 +252,10 @@ static void *_acap_thread(void *arg) {
 		once = 0;
 
 		while (!_STOP && _HAS_WATCHERS && _HAS_LISTENERS) {
-			if (_get_acap_hz(&hz) < 0 || acap->pcm_hz != hz) {
+			if (chip_fd >= 0 && us_chip_tc358743_get_audio_hz(chip_fd) != (int)acap->pcm_hz) {
 				goto close_acap;
 			}
-			uz size = US_RTP_DATAGRAM_SIZE - US_RTP_HEADER_SIZE;
+			uz size = US_RTP_TOTAL_SIZE - US_RTP_HEADER_SIZE;
 			u8 data[size];
 			u64 pts;
 			const int result = us_acap_get_encoded(acap, data, &size, &pts);
@@ -294,6 +270,7 @@ static void *_acap_thread(void *arg) {
 
 	close_acap:
 		US_DELETE(acap, us_acap_destroy);
+		US_CLOSE_FD(chip_fd);
 		sleep(1); // error_delay
 	}
 	return NULL;
@@ -301,10 +278,10 @@ static void *_acap_thread(void *arg) {
 
 static void *_aplay_thread(void *arg) {
 	(void)arg;
-	US_THREAD_SETTLE("us_p_ap");
+	US_THREAD_SETTLE("us_p_aplay");
 	atomic_store(&_g_aplay_tid_created, true);
 
-	assert(_g_config->aplay_dev_name != NULL);
+	US_A(us_str_is_ok(_g_config->aplay_dev_name));
 
 	int once = 0;
 
@@ -330,10 +307,10 @@ static void *_aplay_thread(void *arg) {
 					}
 				} while (skip && !_STOP);
 				us_au_pcm_mix(&mixed, &last);
-				// US_JLOG_INFO("++++++", "mixed %p", client);
+				// US_LOG_INFO("++++++ mixed %p", client);
 			});
 			_UNLOCK_APLAY;
-			// US_JLOG_INFO("++++++", "--------------");
+			// US_LOG_INFO("++++++ --------------");
 
 			if (skip) {
 				static uint skipped = 0;
@@ -345,19 +322,19 @@ static void *_aplay_thread(void *arg) {
 				}
 			}
 
-			if (!_HAS_WATCHERS || !_HAS_LISTENERS || !_HAS_SPEAKERS) {
+			if (!_HAS_WATCHERS || !_HAS_SPEAKERS) {
 				goto close_aplay;
 			}
 
 			if (dev == NULL) {
 				if (!us_au_probe(_g_config->aplay_dev_name)) {
-					US_ONCE({ US_JLOG_ERROR("aplay", "No PCM playback device"); });
+					US_ONCE({ US_LOG_ERROR("No PCM playback device"); });
 					goto close_aplay;
 				}
 
 				int err = snd_pcm_open(&dev, _g_config->aplay_dev_name, SND_PCM_STREAM_PLAYBACK, 0);
 				if (err < 0) {
-					US_ONCE({ US_JLOG_PERROR_ALSA(err, "aplay", "Can't open PCM playback"); });
+					US_ONCE({ US_LOG_PERROR_ALSA(err, "Can't open PCM playback"); });
 					goto close_aplay;
 				}
 
@@ -365,11 +342,11 @@ static void *_aplay_thread(void *arg) {
 					US_RTP_OPUS_CH, US_RTP_OPUS_HZ, 1 /* soft resample */, 50000 /* 50000 = 0.05sec */
 				);
 				if (err < 0) {
-					US_ONCE({ US_JLOG_PERROR_ALSA(err, "aplay", "Can't configure PCM playback"); });
+					US_ONCE({ US_LOG_PERROR_ALSA(err, "Can't configure PCM playback"); });
 					goto close_aplay;
 				}
 
-				US_JLOG_INFO("aplay", "Playback opened, playing ...");
+				US_LOG_INFO("Playback opened, playing ...");
 				once = 0;
 			}
 
@@ -379,20 +356,20 @@ static void *_aplay_thread(void *arg) {
 					frames = snd_pcm_recover(dev, frames, 1);
 				} else {
 					if (once != 0) {
-						US_JLOG_INFO("aplay", "Playing resumed (snd_pcm_writei) ...");
+						US_LOG_INFO("Playing resumed (snd_pcm_writei) ...");
 					}
 					once = 0;
 					skip = false;
 				}
 				if (frames < 0) {
-					US_ONCE({ US_JLOG_PERROR_ALSA(frames, "aplay", "Can't play to PCM playback"); });
+					US_ONCE({ US_LOG_PERROR_ALSA(frames, "Can't play to PCM playback"); });
 					if (frames == -ENODEV) {
 						goto close_aplay;
 					}
 					skip = true;
 				} else {
 					if (once != 0) {
-						US_JLOG_INFO("aplay", "Playing resumed (snd_pcm_recover) ...");
+						US_LOG_INFO("Playing resumed (snd_pcm_recover) ...");
 					}
 					once = 0;
 					skip = false;
@@ -403,12 +380,195 @@ static void *_aplay_thread(void *arg) {
 	close_aplay:
 		if (dev != NULL) {
 			US_DELETE(dev, snd_pcm_close);
-			US_JLOG_INFO("aplay", "Playback closed");
+			US_LOG_INFO("Playback closed");
 		}
 	}
 	return NULL;
 }
 
+static void _unsafe_camera_push_event(const char *str) {
+	json_t *const json_event_type = json_string(str);
+
+	json_t *const camera = json_object();
+	json_object_set_new(camera, "action", json_event_type);
+
+	json_t *const result = json_object();
+	json_object_set_new(result, "ustreamer", json_string("event"));
+	json_object_set_new(result, "status", json_string("camera"));
+	json_object_set_new(result, "camera", camera);
+
+	json_t *const event = json_object();
+	json_object_set_new(event, "result", result);
+
+	US_LIST_ITERATE(_g_clients, client, {
+		_g_gw->push_event(client->session, create(), NULL, event, NULL);
+	});
+
+	json_decref(event);
+}
+
+static void _camera_push_online(uint width, uint height, uint fps) {
+	bool again = false;
+	_LOCK_VPLAY;
+	if (_g_camera.active) {
+		_unsafe_camera_push_event("offline");
+		again = true;
+	}
+	const uint old_width = _g_camera.width;
+	const uint old_height = _g_camera.height;
+	const uint old_fps = _g_camera.fps;
+	_g_camera.active = true;
+	_g_camera.width = width;
+	_g_camera.height = height;
+	_g_camera.fps = fps;
+	_unsafe_camera_push_event("online");
+	_UNLOCK_VPLAY;
+	if (again) {
+		US_LOG_INFO("Camera changed: %ux%u@%u -> %ux%u@%u",
+			old_width, old_height, old_fps, width, height, fps);
+	} else {
+		US_LOG_INFO("Camera online: %ux%u@%u", width, height, fps);
+	}
+}
+
+static void _camera_push_offline() {
+	_LOCK_VPLAY;
+	if (_g_camera.active) {
+		_g_camera.active = false;
+		_unsafe_camera_push_event("offline");
+		US_LOG_INFO("Camera offline");
+	}
+	_UNLOCK_VPLAY;
+}
+
+static void _camera_push_lost() {
+	_LOCK_VPLAY;
+	if (_g_camera.active) {
+		_unsafe_camera_push_event("lost");
+		US_LOG_INFO("Camera's source client is lost");
+	}
+	_UNLOCK_VPLAY;
+}
+
+static void *_vplay_thread(void *arg) {
+	(void)arg;
+	US_THREAD_SETTLE("us_p_vplay");
+	atomic_store(&_g_vplay_tid_created, true);
+
+	US_A(us_str_is_ok(_g_config->vplay_sink_name));
+
+	us_frame_s *frame = us_frame_init();
+
+	while (!_STOP) {
+		us_memsink_s* sink = us_memsink_init_opened(
+			"H264-CAM",
+			_g_config->vplay_sink_name,
+			true, // Server
+			_g_config->vplay_sink_mode,
+			false, // Don't remote sink on destroy
+			1, // Client TTL
+			1); // Timeout
+		if (sink == NULL) {
+			goto close_memsink;
+		}
+
+		const ldf memsink_check_interval = 1;
+		ldf last_memsink_check_time = 0;
+
+		int once = 0;
+		while (!_STOP) {
+			_LOCK_VPLAY;
+			const bool c_active = _g_camera.active;
+			const uint c_width = _g_camera.width;
+			const uint c_height = _g_camera.height;
+			_UNLOCK_VPLAY;
+
+			if (!c_active) {
+				frame->used = 0;
+			}
+
+			if (frame->used > 0 && (c_width != frame->width || c_height != frame->height)) {
+				US_ONCE({ US_LOG_INFO("Got WebRTC frame with wrong resolution"); });
+				frame->used = 0;
+			}
+
+			const ldf now_ts = us_get_now_monotonic();
+			if (frame->used > 0 || last_memsink_check_time + memsink_check_interval < now_ts) {
+				last_memsink_check_time = now_ts;
+
+				us_memsink_wants_s w_get = {0};
+				switch (us_memsink_server_x_lock(sink, &w_get)) {
+					case US_MSS_SUCCESS:
+						if (w_get.format == V4L2_PIX_FMT_H264) {
+							if (w_get.width == c_width && w_get.height == c_height) {
+								if (us_memsink_server_x_is_consumed(sink)) {
+									if (!c_active) {
+										_camera_push_online(w_get.width, w_get.height, w_get.fps);
+									} else if (frame->used > 0) {
+										US_ONCE({ US_LOG_INFO("Streaming to the camera ..."); });
+										us_memsink_server_x_put(sink, frame);
+										frame->used = 0;
+									}
+								} else {
+									if (!c_active) {
+										us_memsink_server_x_set_consumed(sink);
+									}
+								}
+							} else { // Notify to changed resolution
+								_camera_push_online(w_get.width, w_get.height, w_get.fps);
+							}
+						} else {
+							US_ONCE({ US_LOG_ERROR("Got invalid format from the camera: %u", w_get.format); });
+						}
+						if (us_memsink_server_x_unlock(sink) < 0) {
+							goto close_memsink;
+						}
+						break;
+
+					case US_MSS_NO_CLIENT:
+						_camera_push_offline();
+						break;
+
+					case US_MSS_BUSY: // Busy by some client
+						break;
+
+					case US_MSS_ERROR:
+					default:
+						goto close_memsink;
+				}
+			}
+
+			if (frame->used == 0) { // Frame sent (or discarded), get a new one from a client
+				const int in_ri = us_ring_consumer_acquire(_g_vplay_ring, 0.1);
+				if (in_ri >= 0) {
+					us_frame_s *tmp_frame = frame;
+					frame = _g_vplay_ring->items[in_ri];
+					_g_vplay_ring->items[in_ri] = tmp_frame;
+					us_ring_consumer_release(_g_vplay_ring, in_ri);
+				} else {
+					US_ONCE({
+						_LOCK_VPLAY;
+						if (_g_camera.active) {
+							US_LOG_INFO("No frames have got from WebRTC");
+						}
+						_UNLOCK_VPLAY;
+					});
+				}
+			} else {
+				usleep(1000); // Don't iterate too frequently
+			}
+		}
+
+	close_memsink:
+		US_DELETE(sink, us_memsink_destroy);
+		US_LOG_INFO("Memsink closed");
+		sleep(1);
+	}
+
+	us_frame_destroy(frame);
+	return NULL;
+}
+
 static void _relay_rtp_clients(const us_rtp_s *rtp) {
 	US_LIST_ITERATE(_g_clients, client, {
 		us_janus_client_send(client, rtp);
@@ -430,8 +590,13 @@ static int _plugin_init(janus_callbacks *gw, const char *config_dir_path) {
 	// sysctl -w net.core.rmem_max=1000000 
 	// sysctl -w net.core.wmem_max=1000000
 
-	US_JLOG_INFO("main", "Initializing PiKVM uStreamer plugin %s ...", US_VERSION);
-	if (gw == NULL || config_dir_path == NULL || ((_g_config = us_config_init(config_dir_path)) == NULL)) {
+	US_LOGGING_INIT;
+
+	US_LOG_INFO("Initializing PiKVM uStreamer plugin %s ...", US_VERSION);
+	if (gw == NULL || !us_str_is_ok(config_dir_path)) {
+		return -1;
+	}
+	if ((_g_config = us_config_init(config_dir_path)) == NULL) {
 		return -1;
 	}
 	_g_gw = gw;
@@ -440,13 +605,21 @@ static int _plugin_init(janus_callbacks *gw, const char *config_dir_path) {
 
 	US_RING_INIT_WITH_ITEMS(_g_video_ring, 64, us_frame_init);
 	_g_rtpv = us_rtpv_init(_relay_rtp_clients);
-	if (_g_config->acap_dev_name != NULL) {
+
+	if (us_str_is_ok(_g_config->vplay_sink_name)) {
+		US_RING_INIT_WITH_ITEMS(_g_vplay_ring, 15, us_frame_init);
+		US_THREAD_CREATE(_g_vplay_tid, _vplay_thread, NULL);
+	}
+
+	if (us_str_is_ok(_g_config->acap_dev_name)) {
 		_g_rtpa = us_rtpa_init(_relay_rtp_clients);
 		US_THREAD_CREATE(_g_acap_tid, _acap_thread, NULL);
-		if (_g_config->aplay_dev_name != NULL) {
-			US_THREAD_CREATE(_g_aplay_tid, _aplay_thread, NULL);
-		}
 	}
+
+	if (us_str_is_ok(_g_config->aplay_dev_name)) {
+		US_THREAD_CREATE(_g_aplay_tid, _aplay_thread, NULL);
+	}
+
 	US_THREAD_CREATE(_g_video_rtp_tid, _video_rtp_thread, NULL);
 	US_THREAD_CREATE(_g_video_sink_tid, _video_sink_thread, NULL);
 
@@ -455,32 +628,39 @@ static int _plugin_init(janus_callbacks *gw, const char *config_dir_path) {
 }
 
 static void _plugin_destroy(void) {
-	US_JLOG_INFO("main", "Destroying plugin ...");
+	US_LOG_INFO("Destroying plugin ...");
 
 	atomic_store(&_g_stop, true);
 #	define JOIN(_tid) { if (atomic_load(&_tid##_created)) { US_THREAD_JOIN(_tid); } }
+	JOIN(_g_aplay_tid);
+	JOIN(_g_acap_tid);
+	JOIN(_g_vplay_tid);
 	JOIN(_g_video_sink_tid);
 	JOIN(_g_video_rtp_tid);
-	JOIN(_g_acap_tid);
-	JOIN(_g_aplay_tid);
 #	undef JOIN
 
 	US_LIST_ITERATE(_g_clients, client, {
+		if (_g_camera.client == client) {
+			_g_camera.client = NULL;
+		}
 		US_LIST_REMOVE(_g_clients, client);
 		us_janus_client_destroy(client);
 	});
 
+	US_RING_DELETE_WITH_ITEMS(_g_vplay_ring, us_frame_destroy);
 	US_RING_DELETE_WITH_ITEMS(_g_video_ring, us_frame_destroy);
 
 	US_DELETE(_g_rtpa, us_rtpa_destroy);
 	US_DELETE(_g_rtpv, us_rtpv_destroy);
 	US_DELETE(_g_config, us_config_destroy);
+
+	US_LOGGING_DESTROY;
 }
 
 static void _plugin_create_session(janus_plugin_session *session, int *err) {
 	_IF_DISABLED({ *err = -1; return; });
 	_LOCK_ALL;
-	US_JLOG_INFO("main", "Creating session %p ...", session);
+	US_LOG_INFO("Creating session %p ...", session);
 	us_janus_client_s *const client = us_janus_client_init(_g_gw, session);
 	US_LIST_APPEND(_g_clients, client);
 	atomic_store(&_g_has_watchers, true);
@@ -494,9 +674,14 @@ static void _plugin_destroy_session(janus_plugin_session* session, int *err) {
 	bool has_watchers = false;
 	bool has_listeners = false;
 	bool has_speakers = false;
+	bool camera_lost = false;
 	US_LIST_ITERATE(_g_clients, client, {
 		if (client->session == session) {
-			US_JLOG_INFO("main", "Removing session %p ...", session);
+			US_LOG_INFO("Removing session %p ...", session);
+			if (_g_camera.client == client) {
+				_g_camera.client = NULL;
+				camera_lost = true;
+			}
 			US_LIST_REMOVE(_g_clients, client);
 			us_janus_client_destroy(client);
 			found = true;
@@ -507,13 +692,16 @@ static void _plugin_destroy_session(janus_plugin_session* session, int *err) {
 		}
 	});
 	if (!found) {
-		US_JLOG_WARN("main", "No session %p", session);
+		US_LOG_ERROR("No session %p", session);
 		*err = -2;
 	}
 	atomic_store(&_g_has_watchers, has_watchers);
 	atomic_store(&_g_has_listeners, has_listeners);
 	atomic_store(&_g_has_speakers, has_speakers);
 	_UNLOCK_ALL;
+	if (camera_lost) {
+		_camera_push_lost();
+	}
 }
 
 static json_t *_plugin_query_session(janus_plugin_session *session) {
@@ -539,13 +727,13 @@ static void _set_transmit(janus_plugin_session *session, const char *msg, bool t
 	US_LIST_ITERATE(_g_clients, client, {
 		if (client->session == session) {
 			atomic_store(&client->transmit, transmit);
-			// US_JLOG_INFO("main", "%s session %p", msg, session);
+			// US_LOG_INFO("%s session %p", msg, session);
 			found = true;
 		}
 		has_watchers = (has_watchers || atomic_load(&client->transmit));
 	});
 	if (!found) {
-		US_JLOG_WARN("main", "No session %p", session);
+		US_LOG_ERROR("No session %p", session);
 	}
 	atomic_store(&_g_has_watchers, has_watchers);
 	_UNLOCK_ALL;
@@ -554,6 +742,16 @@ static void _set_transmit(janus_plugin_session *session, const char *msg, bool t
 static void _plugin_setup_media(janus_plugin_session *session) { _set_transmit(session, "Unmuted", true); }
 static void _plugin_hangup_media(janus_plugin_session *session) { _set_transmit(session, "Muted", false); }
 
+static bool _is_camera_enabled(void) {
+	bool enabled = us_str_is_ok(_g_config->vplay_sink_name);
+	if (enabled && us_str_is_ok(_g_config->vplay_dev_path)) {
+		if (access(_g_config->vplay_dev_path, F_OK) != 0) {
+			enabled = false;
+		}
+	}
+	return enabled;
+}
+
 static struct janus_plugin_result *_plugin_handle_message(
 	janus_plugin_session *session, char *transaction, json_t *msg, json_t *jsep) {
 
@@ -567,7 +765,7 @@ static struct janus_plugin_result *_plugin_handle_message(
 	}
 
 #	define PUSH_ERROR(x_error, x_reason) { \
-			/*US_JLOG_ERROR("main", "Message error in session %p: %s", session, x_reason);*/ \
+			/*US_LOG_ERROR("Message error in session %p: %s", session, x_reason);*/ \
 			json_t *m_event = json_object(); \
 			json_object_set_new(m_event, "ustreamer", json_string("event")); \
 			json_object_set_new(m_event, "error_code", json_integer(x_error)); \
@@ -587,7 +785,7 @@ static struct janus_plugin_result *_plugin_handle_message(
 		PUSH_ERROR(400, "Request not a string");
 		goto done;
 	}
-	// US_JLOG_INFO("main", "Message: %s", request_str);
+	// US_LOG_INFO("Message: %s", request_str);
 
 #	define PUSH_STATUS(x_status, x_payload, x_jsep) { \
 			json_t *const m_event = json_object(); \
@@ -612,21 +810,20 @@ static struct janus_plugin_result *_plugin_handle_message(
 		uint video_orient = 0;
 		bool with_acap = false;
 		bool with_aplay = false;
+		bool with_vplay = false;
 		{
 			json_t *const params = json_object_get(msg, "params");
 			if (params != NULL) {
-				{
-					json_t *const obj = json_object_get(params, "audio");
-					if (obj != NULL && json_is_boolean(obj)) {
-						with_acap = (us_au_probe(_g_config->acap_dev_name) && json_boolean_value(obj));
+#				define READ_BOOL(x_target, x_key, x_cond) { \
+						json_t *const m_obj = json_object_get(params, x_key); \
+						if (m_obj != NULL && json_is_boolean(m_obj)) { \
+							x_target = (json_boolean_value(m_obj) && (x_cond)); \
+						} \
 					}
-				}
-				{
-					json_t *const obj = json_object_get(params, "mic");
-					if (obj != NULL && json_is_boolean(obj)) {
-						with_aplay = (us_au_probe(_g_config->aplay_dev_name) && json_boolean_value(obj));
-					}
-				}
+				READ_BOOL(with_acap, "audio", us_au_probe(_g_config->acap_dev_name));
+				READ_BOOL(with_aplay, "mic", us_au_probe(_g_config->aplay_dev_name));
+				READ_BOOL(with_vplay, "camera", _is_camera_enabled());
+#				undef READ_BOOL
 				{
 					json_t *const obj = json_object_get(params, "orientation");
 					if (obj != NULL && json_is_integer(obj)) {
@@ -640,44 +837,32 @@ static struct janus_plugin_result *_plugin_handle_message(
 			}
 		}
 
-		{
-			char *sdp;
-			char *const video_sdp = us_rtpv_make_sdp(_g_rtpv);
-			char *const audio_sdp = (with_acap ? us_rtpa_make_sdp(_g_rtpa, with_aplay) : us_strdup(""));
-			US_ASPRINTF(sdp,
-				"v=0" RN
-				"o=- %" PRIu64 " 1 IN IP4 0.0.0.0" RN
-				"s=PiKVM uStreamer" RN
-				"t=0 0" RN
-				"%s%s",
-				us_get_now_id() >> 1,
-#				if JANUS_PLUGIN_API_VERSION >= 100
-				// Place video SDP before audio SDP so that the video and audio streams
-				// have predictable indices, even if audio is not available.
-				// See also client.c.
-				video_sdp, audio_sdp
-#				else
-				// For versions of Janus prior to 1.x, place the audio SDP first.
-				audio_sdp, video_sdp
-#				endif
-			);
-			json_t *const offer_jsep = json_pack("{ssss}", "type", "offer", "sdp", sdp);
-			PUSH_STATUS("started", NULL, offer_jsep);
-			json_decref(offer_jsep);
-			free(audio_sdp);
-			free(video_sdp);
-			free(sdp);
-		}
-
 		{
 			_LOCK_ALL;
+			with_vplay = (with_vplay && _g_camera.active && !_g_camera.client);
+
 			bool has_listeners = false;
 			bool has_speakers = false;
 			US_LIST_ITERATE(_g_clients, client, {
 				if (client->session == session) {
+					char *const sdp = us_sdp_create(
+						client->video_ssrc,
+						client->audio_ssrc,
+						with_acap,
+						with_aplay,
+						with_vplay);
+					json_t *const offer_jsep = json_pack("{ssss}", "type", "offer", "sdp", sdp);
+					PUSH_STATUS("started", NULL, offer_jsep);
+					json_decref(offer_jsep);
+					free(sdp);
+
 					atomic_store(&client->transmit_acap, with_acap);
 					atomic_store(&client->transmit_aplay, with_aplay);
 					atomic_store(&client->video_orient, video_orient);
+					if (with_vplay) {
+						us_janus_client_start_vplay(client, _g_vplay_ring);
+						_g_camera.client = client;
+					}
 				}
 				has_listeners = (has_listeners || atomic_load(&client->transmit_acap));
 				has_speakers = (has_speakers || atomic_load(&client->transmit_aplay));
@@ -689,18 +874,36 @@ static struct janus_plugin_result *_plugin_handle_message(
 
 	} else if (!strcmp(request_str, "features")) {
 		const char *const ice_url = getenv("JANUS_USTREAMER_WEB_ICE_URL");
-		const bool acap_avail = us_au_probe(_g_config->acap_dev_name);
+
+		_LOCK_VPLAY
+		json_t *camera_req = NULL;
+		if (_g_camera.active && _g_camera.client == NULL) {
+			json_t *resolution = json_object();
+			json_object_set_new(resolution, "width", json_integer(_g_camera.width));
+			json_object_set_new(resolution, "height", json_integer(_g_camera.height));
+
+			camera_req = json_object();
+			json_object_set_new(camera_req, "resolution", resolution);
+			json_object_set_new(camera_req, "fps", json_integer(_g_camera.fps));
+		}
+		_UNLOCK_VPLAY
+
 		json_t *const features = json_pack(
-			"{s:b, s:b, s:{s:s?}}",
-			"audio", acap_avail,
-			"mic", (acap_avail && us_au_probe(_g_config->aplay_dev_name)),
-			"ice", "url", (ice_url != NULL ? ice_url : default_ice_url)
+			"{s:b, s:b, s:{s:b, s:o*}, s:{s:s?}}",
+			"audio", us_au_probe(_g_config->acap_dev_name),
+			"mic", us_au_probe(_g_config->aplay_dev_name),
+			"camera",
+				"enabled", _is_camera_enabled(),
+				"request", camera_req,
+			"ice",
+				"url", (ice_url != NULL ? ice_url : _g_default_ice_url)
 		);
+
 		PUSH_STATUS("features", features, NULL);
 		json_decref(features);
 
 	} else if (!strcmp(request_str, "key_required")) {
-		// US_JLOG_INFO("main", "Got key_required message");
+		// US_LOG_INFO("Got key_required message");
 		atomic_store(&_g_key_required, true);
 
 	} else {
@@ -722,28 +925,60 @@ done:
 
 static void _plugin_incoming_rtp(janus_plugin_session *session, janus_plugin_rtp *packet) {
 	_IF_DISABLED({ return; });
-	if (session == NULL || packet == NULL || packet->video) {
-		return; // Accept only valid audio
+	if (session == NULL || packet == NULL) {
+		return; // Accept only valid packets
 	}
-	_LOCK_APLAY;
-	US_LIST_ITERATE(_g_clients, client, {
-		if (client->session == session) {
-			us_janus_client_recv(client, packet);
-			break;
+	if (packet->video) {
+		_LOCK_VPLAY;
+		if (_g_camera.client != NULL && _g_camera.client->session == session) {
+			us_janus_client_recv(_g_camera.client, packet);
 		}
-	});
-	_UNLOCK_APLAY;
+		_UNLOCK_VPLAY;
+	} else {
+		_LOCK_APLAY;
+		US_LIST_ITERATE(_g_clients, client, {
+			if (client->session == session) {
+				us_janus_client_recv(client, packet);
+				break;
+			}
+		});
+		_UNLOCK_APLAY;
+	}
 }
 
 static void _plugin_incoming_rtcp(janus_plugin_session *session, janus_plugin_rtcp *packet) {
 	_IF_DISABLED({ return; });
-	if (session == NULL || packet == NULL || !packet->video) {
-		return; // Accept only valid video
+	if (
+		session == NULL || packet == NULL || !packet->video
+		|| packet->length < sizeof(janus_rtcp_header)
+	) {
+		return;
 	}
-	if (janus_rtcp_has_pli(packet->buffer, packet->length)) {
-		// US_JLOG_INFO("main", "Got video PLI");
+
+	if (
+		janus_rtcp_has_pli(packet->buffer, packet->length)
+		|| janus_rtcp_has_fir(packet->buffer, packet->length)
+	) {
 		atomic_store(&_g_key_required, true);
 	}
+
+	janus_rtcp_header *header = (janus_rtcp_header *)packet->buffer;
+
+	if (header->type == RTCP_SR && janus_rtcp_check_sr(header, packet->length)) {
+		const janus_rtcp_sr *const sr = (janus_rtcp_sr *)header;
+
+		if (sr->si.ntp_ts_msw && sr->si.ntp_ts_lsw) {
+			_LOCK_VPLAY;
+			US_LIST_ITERATE(_g_clients, client, {
+				if (client->session == session) {
+					const ldf ntp_ts = ntohl(sr->si.ntp_ts_msw) + ntohl(sr->si.ntp_ts_lsw) / 4294967296.L;
+					us_rtpc_sync_timestamp(client->rtpc, ntp_ts, ntohl(sr->si.rtp_ts));
+					break;
+				}
+			});
+			_UNLOCK_VPLAY;
+		}
+	}
 }
 
 

janus/src/rtp.c

@@ -44,22 +44,27 @@ void us_rtp_destroy(us_rtp_s *rtp) {
 void us_rtp_assign(us_rtp_s *rtp, uint payload, bool video) {
 	rtp->payload = payload;
 	rtp->video = video;
-	rtp->ssrc = us_triple_u32(us_get_now_monotonic_u64());
 }
 
-void us_rtp_write_header(us_rtp_s *rtp, u32 pts, bool marked) {
+#define WRITE_BE_U32(x_offset, x_value) \
+	*((u32*)(rtp->datagram + x_offset)) = __builtin_bswap32(x_value)
+
+void us_rtp_write_header(us_rtp_s *rtp, u32 pts, bool last_header) {
 	u32 word0 = 0x80000000;
-	if (marked) {
+	if (last_header) {
 		word0 |= 1 << 23;
 	}
 	word0 |= (rtp->payload & 0x7F) << 16;
 	word0 |= rtp->seq;
 	++rtp->seq;
 
-#	define WRITE_BE_U32(x_offset, x_value) \
-		*((u32*)(rtp->datagram + x_offset)) = __builtin_bswap32(x_value)
 	WRITE_BE_U32(0, word0);
 	WRITE_BE_U32(4, pts);
-	WRITE_BE_U32(8, rtp->ssrc);
-#	undef WRITE_BE_U32
+	WRITE_BE_U32(8, 0); // SSRC should be written separately for each client
 }
+
+void us_rtp_write_ssrc(us_rtp_s *rtp, u32 ssrc) {
+	WRITE_BE_U32(8, ssrc);
+}
+
+#undef WRITE_BE_U32

janus/src/rtp.h

@@ -23,12 +23,20 @@
 #pragma once
 
 #include "uslibs/types.h"
+#include "uslibs/tools.h"
 
 
-// https://stackoverflow.com/questions/47635545/why-webrtc-chose-rtp-max-packet-size-to-1200-bytes
-#define US_RTP_DATAGRAM_SIZE	1200
+// Max RTP size for WebRTC is 1200 bytes:
+//   - https://stackoverflow.com/questions/47635545/why-webrtc-chose-rtp-max-packet-size-to-1200-bytes
+// But(!) Tailscale has 1280 MTU. So to fit it required to subtract:
+// 1. possible RTP extensions (see sdp.c)
+// 2. additional SRTP fields (>= 10 bytes)
+// 3. UDP header (8 bytes)
+// 4. IPv6 header (40 bytes)
+// Finally it looks like below value should be good enought for all cases
+#define US_RTP_TOTAL_SIZE		1150
 #define US_RTP_HEADER_SIZE		12
-#define US_RTP_PAYLOAD_SIZE		(US_RTP_DATAGRAM_SIZE - US_RTP_HEADER_SIZE)
+#define US_RTP_PAYLOAD_SIZE		(US_RTP_TOTAL_SIZE - US_RTP_HEADER_SIZE)
 
 #define US_RTP_H264_PAYLOAD		96
 #define US_RTP_OPUS_PAYLOAD		111
@@ -40,10 +48,9 @@
 typedef struct {
 	uint	payload;
 	bool	video;
-	u32		ssrc;
 
 	u16		seq;
-	u8		datagram[US_RTP_DATAGRAM_SIZE];
+	u8		datagram[US_RTP_TOTAL_SIZE];
 	uz		used;
 
 	bool	first_of_frame;
@@ -55,8 +62,13 @@ typedef struct {
 typedef void (*us_rtp_callback_f)(const us_rtp_s *rtp);
 
 
+static inline u32 us_get_ssrc(void) {
+	return us_triple_u32(us_get_now_monotonic_u64());
+}
+
 us_rtp_s *us_rtp_init(void);
 void us_rtp_destroy(us_rtp_s *rtp);
 
 void us_rtp_assign(us_rtp_s *rtp, uint payload, bool video);
-void us_rtp_write_header(us_rtp_s *rtp, u32 pts, bool marked);
+void us_rtp_write_header(us_rtp_s *rtp, u32 pts, bool last_header);
+void us_rtp_write_ssrc(us_rtp_s *rtp, u32 ssrc);

janus/src/rtpa.c

@@ -23,11 +23,12 @@
 #include "rtpa.h"
 
 #include <stdlib.h>
-#include <inttypes.h>
 
 #include "uslibs/types.h"
 #include "uslibs/tools.h"
 
+#include "rtp.h"
+
 
 us_rtpa_s *us_rtpa_init(us_rtp_callback_f callback) {
 	us_rtpa_s *rtpa;
@@ -43,32 +44,8 @@ void us_rtpa_destroy(us_rtpa_s *rtpa) {
 	free(rtpa);
 }
 
-char *us_rtpa_make_sdp(us_rtpa_s *rtpa, bool mic) {
-	const uint pl = rtpa->rtp->payload;
-	char *sdp;
-	US_ASPRINTF(sdp,
-		"m=audio 1 RTP/SAVPF %u" RN
-		"c=IN IP4 0.0.0.0" RN
-		"a=rtpmap:%u OPUS/%u/%u" RN
-		"a=fmtp:%u sprop-stereo=1" RN // useinbandfec=1
-		"a=rtcp-fb:%u nack" RN
-		"a=rtcp-fb:%u nack pli" RN
-		"a=rtcp-fb:%u goog-remb" RN
-		"a=mid:a" RN
-		"a=msid:audio a" RN
-		"a=ssrc:%" PRIu32 " cname:ustreamer" RN
-		"a=%s" RN,
-		pl, pl,
-		US_RTP_OPUS_HZ, US_RTP_OPUS_CH,
-		pl, pl, pl, pl,
-		rtpa->rtp->ssrc,
-		(mic ? "sendrecv" : "sendonly")
-	);
-	return sdp;
-}
-
 void us_rtpa_wrap(us_rtpa_s *rtpa, const u8 *data, uz size, u32 pts) {
-    if (size + US_RTP_HEADER_SIZE <= US_RTP_DATAGRAM_SIZE) {
+    if (size + US_RTP_HEADER_SIZE <= US_RTP_TOTAL_SIZE) {
         us_rtp_write_header(rtpa->rtp, pts, false);
         memcpy(rtpa->rtp->datagram + US_RTP_HEADER_SIZE, data, size);
 		rtpa->rtp->used = size + US_RTP_HEADER_SIZE;

janus/src/rtpa.h

@@ -36,5 +36,4 @@ typedef struct {
 us_rtpa_s *us_rtpa_init(us_rtp_callback_f callback);
 void us_rtpa_destroy(us_rtpa_s *rtpa);
 
-char *us_rtpa_make_sdp(us_rtpa_s *rtpa, bool mic);
 void us_rtpa_wrap(us_rtpa_s *rtpa, const u8 *data, uz size, u32 pts);

janus/src/rtpc.c

@@ -0,0 +1,315 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#include "rtpc.h"
+
+#include <linux/videodev2.h>
+
+#include "uslibs/logging.h"
+
+#include "h264.h"
+
+
+// H.264 fragment types. Commented types are not used in WebRTC.
+enum {
+	_FR_REGULAR_NAL_MIN = 1,
+	_FR_REGULAR_NAL_MAX = 23,
+	_FR_STAP_A = 24,		// Single Time Aggregation Packet, Type A (STAP-A)
+	// _FR_STAP_B = 25,		// Single Time Aggregation Packet, Type B (STAP-B)
+	// _FR_MTAP_16 = 26,	// Multi-Time Aggregation Packet, Type A (MTAP-16)
+	// _FR_MTAP_24 = 27,	// Multi-Time Aggregation Packet, Type B (MTAP-24)
+	_FR_FU_A = 28,			// Fragmentation unit (FU-A)
+	// _FR_FU_B = 29,		// Fragmentation unit (FU-B)
+};
+
+enum {
+	_NALU_SPS = 7
+};
+
+
+static int _update_sequence_for_new_packet(us_rtpc_s *rtpc, u16 seq);
+static ldf _rtp_to_local_ts(us_rtpc_s *rtpc, u32 rts);
+static void _update_timestamps_for_new_packet(us_rtpc_s *rtpc, u32 rts);
+
+static void _unwrapping_begin(us_rtpc_s *rtpc);
+static void _unwrapping_end(us_rtpc_s *rtpc);
+static void _unwrap_whole(us_rtpc_s *rtpc, const u8 *data, u16 size);
+
+
+us_rtpc_s *us_rtpc_init(us_frame_callback_f callback, void *user_data) {
+	us_rtpc_s *rtpc;
+	US_CALLOC(rtpc, 1);
+	rtpc->user_data = user_data;
+	rtpc->frame = us_frame_init();
+	rtpc->callback = callback;
+	return rtpc;
+}
+
+void us_rtpc_destroy(us_rtpc_s *rtpc) {
+	us_frame_destroy(rtpc->frame);
+	free(rtpc);
+}
+
+us_rtpc_unwrap_result_e us_rtpc_unwrap(
+	us_rtpc_s *rtpc,
+	const u8 *payload,
+	int payload_size,
+	u16 seq,
+	u32 rts
+) {
+	US_A(payload != NULL);
+	US_A(payload_size >= 1);
+
+	if (_update_sequence_for_new_packet(rtpc, seq) < 0) {
+		return US_RUR_INVALID_SEQ;
+	}
+	_update_timestamps_for_new_packet(rtpc, rts);
+
+	rtpc->had_packets = true;
+
+	const u8 fragment_type = payload[0] & 0x1F;
+
+	if (rtpc->fu && (rtpc->fu_rts != rts || fragment_type != _FR_FU_A)) {
+		rtpc->fu = false;
+		if (!rtpc->fu_is_bad) { // Not reported yet
+			return US_RUR_DEPACKETIZATION_FAILED_RETRY;
+		}
+	}
+
+	if (fragment_type >= _FR_REGULAR_NAL_MIN && fragment_type <= _FR_REGULAR_NAL_MAX) {
+		_unwrap_whole(rtpc, payload, payload_size);
+
+	} else if (fragment_type == _FR_STAP_A) { // Single Time Aggregation Packet, Type A (STAP-A).
+		const u8 *aggregated_unit = payload + 1; // skip aggregation header (1 byte)
+		payload_size -= 1;
+
+		while (payload_size > 0) {
+			if (payload_size < 2) { // Aggregated NAL unit size (2 bytes) is missing
+				US_LOG_ERROR("Aggregated NALU size is missing: seq=%u, rts=%u", seq, rts);
+				return US_RUR_BAD_PACKET;
+			}
+
+			const u16 nalu_size = ntohs(*((const u16 *)aggregated_unit));
+			aggregated_unit	+= 2;
+			payload_size -= 2;
+
+			if (nalu_size == 0) {
+				US_LOG_ERROR("Aggregated NALU size is zero: seq=%u, rts=%u", seq, rts);
+				return US_RUR_BAD_PACKET;
+			}
+
+			if (payload_size < nalu_size) { // aggregated unit size declared as bigger than available data size
+				US_LOG_ERROR("Aggregated NALU size is too big: seq=%u, rts=%u", seq, rts);
+				return US_RUR_BAD_PACKET;
+			}
+
+			_unwrap_whole(rtpc, aggregated_unit, nalu_size);
+
+			aggregated_unit	+= nalu_size;
+			payload_size -= nalu_size;
+		}
+
+	} else if (fragment_type == _FR_FU_A) { // Fragmentation unit (FU-A).
+		if (payload_size < 2) { // FU header is missing (2 bytes)
+			US_LOG_ERROR("FU header is missing: seq=%u, rts=%u", seq, rts);
+			return US_RUR_BAD_PACKET;
+		}
+
+		const u8 fu_indicator = payload[0];
+		const u8 fu_header = payload[1];
+		const bool first_fragment = !!(fu_header & 0x80);
+		const bool last_fragment = !!(fu_header & 0x40);
+
+		if (first_fragment && last_fragment) { // should never happen according to RFC
+			return US_RUR_BAD_PACKET;
+		}
+
+		if (rtpc->fu && rtpc->fu_is_bad) {
+			return US_RUR_DEPACKETIZATION_SKIPPED;
+		}
+
+		if (!rtpc->fu) {
+			rtpc->fu = true;
+			rtpc->fu_seq = seq;
+			rtpc->fu_rts = rts;
+
+			if (!first_fragment) {
+				// US_LOG_INFO("Ignoring FU without a first fragment: seq=%u, rts=%u", seq, rts);
+				rtpc->fu_is_bad = true;
+				return US_RUR_DEPACKETIZATION_FAILED;
+			}
+			rtpc->fu_is_bad = false;
+
+			const u8 header = (fu_indicator & 0xE0) | (fu_header & 0x1F);
+			_unwrapping_begin(rtpc);
+			us_frame_append_data(rtpc->frame, &header, sizeof(header));
+			// - FU indicator - FU header
+			us_frame_append_data(rtpc->frame, (const u8 *)payload + 2, payload_size - 2);
+
+		} else if (!first_fragment && (seq == rtpc->fu_seq + 1) && (rts == rtpc->fu_rts)) {
+			//       ^^^ Already had first fragment
+			rtpc->fu_seq = seq;
+
+			// - FU indicator - FU header
+			us_frame_append_data(rtpc->frame, (const u8 *)payload + 2, payload_size - 2);
+
+			if (last_fragment) {
+				_unwrapping_end(rtpc);
+				rtpc->fu = false;
+			}
+
+		} else {
+			// US_LOG_ERROR("Something went wrong with FU processing: seq=%u, rts=%u", seq, rts);
+			rtpc->fu_is_bad = true;
+			return US_RUR_DEPACKETIZATION_FAILED;
+		}
+
+	} else {
+		US_LOG_ERROR("Unsupported video unit type=%x: seq=%u, rts=%u", fragment_type, seq, rts);
+		return US_RUR_UNSUPPORTED_UNIT_TYPE;
+	}
+	return US_RUR_SUCCESS;
+}
+
+void us_rtpc_sync_timestamp(us_rtpc_s *rtpc, ldf ntp_ts, u32 packet_rts) {
+	if (!rtpc->had_packets) {
+		return;
+	}
+	if (rtpc->reference_ntp_ts > ntp_ts) {
+		return;
+	}
+
+	if (rtpc->reference_ntp_ts == 0.L) {
+		if (rtpc->reference_packet_rts < packet_rts) {
+			const ldf diff = (packet_rts - rtpc->reference_packet_rts) / 90000.L;
+			rtpc->reference_local_ts += diff;
+			rtpc->reference_packet_rts = packet_rts;
+			rtpc->reference_ntp_ts = ntp_ts;
+		}
+	} else {
+		const ldf diff = ntp_ts - rtpc->reference_ntp_ts;
+		rtpc->reference_local_ts += diff;
+		rtpc->reference_packet_rts = packet_rts;
+		rtpc->reference_ntp_ts = ntp_ts;
+	}
+}
+
+static int _update_sequence_for_new_packet(us_rtpc_s *rtpc, u16 seq) {
+	if (
+		rtpc->had_packets
+		&& (
+			rtpc->last_packet_seq == seq
+			|| (rtpc->last_packet_seq < seq && seq - rtpc->last_packet_seq > (UINT16_MAX >> 1))
+			|| (seq < rtpc->last_packet_seq && rtpc->last_packet_seq - seq < (UINT16_MAX >> 1))
+		)
+	) {
+		return -1;
+	}
+	rtpc->last_packet_seq = seq;
+	return 0;
+}
+
+static ldf _rtp_to_local_ts(us_rtpc_s *rtpc, u32 rts) {
+	if (rtpc->reference_packet_rts <= rts) {
+		return rtpc->reference_local_ts + (rts - rtpc->reference_packet_rts) / 90000.L;
+	} else {
+		return rtpc->reference_local_ts - (rtpc->reference_packet_rts - rts) / 90000.L;
+	}
+}
+
+static void _update_timestamps_for_new_packet(us_rtpc_s *rtpc, u32 rts) {
+	const ldf now_ts = us_get_now_monotonic();
+	ldf ts; // Fill for any goto
+
+	if (!rtpc->had_packets) {
+		ts = _rtp_to_local_ts(rtpc, rts); // Used in packet_update
+		goto full_update;
+	}
+	if (rtpc->last_packet_rts != rts) { // If this is a new frame
+		ts = _rtp_to_local_ts(rtpc, rts);
+		if (rtpc->last_packet_rts > rts || fabsl(now_ts - ts) > .1L) {
+			// ^ RTP timestamp wrapped |OR| ^ Too significant mistake
+			goto full_update;
+		}
+		goto packet_update;
+	}
+
+	return;
+
+full_update:
+	rtpc->reference_local_ts = now_ts;
+	rtpc->reference_packet_rts = rts;
+	rtpc->reference_ntp_ts = 0.L; // Force resync
+	// US_LOG_INFO("Reference timestamps reseted");
+packet_update:
+	rtpc->last_packet_rts = rts;
+	rtpc->frame_ts = ts;
+}
+
+static void _unwrapping_begin(us_rtpc_s *rtpc) {
+	us_frame_s *const frame = rtpc->frame;
+	u8 *const data = frame->data;
+	const uz allocated = frame->allocated;
+
+	// Reset the content, start from scratch
+	memset(frame, 0, sizeof(us_frame_s));
+	frame->data = data;
+	frame->allocated = allocated;
+
+	static const u8 nalu_start_code[] = {0, 0, 0, 1};
+	us_frame_set_data(rtpc->frame, nalu_start_code, sizeof(nalu_start_code));
+}
+
+static void _unwrapping_end(us_rtpc_s *rtpc) {
+	us_frame_s *const frame = rtpc->frame;
+	frame->format = V4L2_PIX_FMT_H264;
+	frame->width = rtpc->frame_width;
+	frame->height = rtpc->frame_height;
+	frame->grab_begin_ts = rtpc->frame_ts;
+	frame->grab_end_ts = rtpc->frame_ts;
+
+	// Exchange frames
+	rtpc->frame = rtpc->callback(frame, rtpc->user_data);
+	US_A(rtpc->frame != NULL);
+}
+
+static void _unwrap_whole(us_rtpc_s *rtpc, const u8 *data, u16 size) {
+	US_A(size >= 1);
+
+	const u8 nalu_type = (data[0] & 0x1F);
+	if (nalu_type == _NALU_SPS) {
+		uint width = 0;
+		uint height = 0;
+		if (
+			!us_h264_parse_sps_nalu(data, size, &width, &height)
+			&& (rtpc->frame_width != width || rtpc->frame_height != height)
+		) {
+			US_LOG_INFO("Frame size updated: width=%u, height=%u", width, height);
+			rtpc->frame_width = width;
+			rtpc->frame_height = height;
+		}
+	}
+
+	_unwrapping_begin(rtpc);
+	us_frame_append_data(rtpc->frame, data, size);
+	_unwrapping_end(rtpc);
+}

janus/src/rtpc.h

@@ -0,0 +1,82 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#pragma once
+
+#include <janus/rtp.h>
+
+#include "uslibs/types.h"
+#include "uslibs/frame.h"
+
+
+// Should return:
+// * input frame
+// * replacement frame - to use it in following parsing
+typedef us_frame_s *(*us_frame_callback_f)(us_frame_s *frame, void *user_data);
+
+typedef struct { // *_rts == [R]TP [T]ime[S]tamp
+	us_frame_callback_f		callback;
+	void					*user_data;
+
+	bool		had_packets; // Были ли вообще какие-то пакеты когда-то, взводится один раз
+
+	ldf			reference_local_ts; // Локальное время
+	u32			reference_packet_rts;
+	ldf			reference_ntp_ts;
+
+	u16			last_packet_seq;
+	u32			last_packet_rts; // Время из последнего полученного пакета
+
+	bool		fu;
+	u16			fu_seq;
+	u32			fu_rts;
+	bool		fu_is_bad;
+
+	us_frame_s	*frame;
+	uint		frame_width;
+	uint		frame_height;
+	ldf			frame_ts; // in case of fu, time of very first packet
+} us_rtpc_s;
+
+typedef enum {
+	US_RUR_SUCCESS = 0,
+	US_RUR_BAD_PACKET = -1,
+	US_RUR_INVALID_SEQ = -2,
+	US_RUR_DEPACKETIZATION_FAILED = -3,
+	// Incomplete FU found. Current packet not handled.
+	// It's required to call us_rtpc_unwrap one else time with the same packet.
+	US_RUR_DEPACKETIZATION_FAILED_RETRY = -4,
+	US_RUR_DEPACKETIZATION_SKIPPED = -5,
+	US_RUR_UNSUPPORTED_UNIT_TYPE = -6,
+} us_rtpc_unwrap_result_e;
+
+us_rtpc_s *us_rtpc_init(us_frame_callback_f callback, void *user_data);
+void us_rtpc_destroy(us_rtpc_s *rtpc);
+
+us_rtpc_unwrap_result_e us_rtpc_unwrap(
+	us_rtpc_s *rtpc,
+	const u8 *payload,
+	int payload_size,
+	u16 seq,
+	u32 rts);
+
+void us_rtpc_sync_timestamp(us_rtpc_s *rtpc, ldf ntp_ts, u32 packet_rts);

janus/src/rtpv.c

@@ -26,8 +26,6 @@
 #include "rtpv.h"
 
 #include <stdlib.h>
-#include <inttypes.h>
-#include <assert.h>
 
 #include <linux/videodev2.h>
 
@@ -35,10 +33,15 @@
 #include "uslibs/tools.h"
 #include "uslibs/frame.h"
 
+#include "rtp.h"
 
-void _rtpv_process_nalu(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool marked);
 
 static sz _find_annexb(const u8 *data, uz size);
+static bool _is_nalu_small(uz size);
+
+static void _rtpv_process_nalu(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool first_nalu, bool last_nalu);
+static void _rtpv_process_nalu_small(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool first_nalu, bool last_nalu);
+static void _rtpv_process_nalu_big(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool first_nalu, bool last_nalu);
 
 
 us_rtpv_s *us_rtpv_init(us_rtp_callback_f callback) {
@@ -55,89 +58,112 @@ void us_rtpv_destroy(us_rtpv_s *rtpv) {
 	free(rtpv);
 }
 
-char *us_rtpv_make_sdp(us_rtpv_s *rtpv) {
-	// https://tools.ietf.org/html/rfc6184
-	// https://github.com/meetecho/janus-gateway/issues/2443
-	const uint pl = rtpv->rtp->payload;
-	char *sdp;
-	US_ASPRINTF(sdp,
-		"m=video 1 RTP/SAVPF %u" RN
-		"c=IN IP4 0.0.0.0" RN
-		"a=rtpmap:%u H264/90000" RN
-		"a=fmtp:%u profile-level-id=42E01F;packetization-mode=1" RN
-		"a=rtcp-fb:%u nack" RN
-		"a=rtcp-fb:%u nack pli" RN
-		"a=rtcp-fb:%u goog-remb" RN
-		"a=mid:v" RN
-		"a=msid:video v" RN
-		"a=ssrc:%" PRIu32 " cname:ustreamer" RN
-		"a=extmap:1/sendonly urn:3gpp:video-orientation" RN
-		"a=extmap:2/sendonly http://www.webrtc.org/experiments/rtp-hdrext/playout-delay" RN
-		"a=extmap:3/sendonly http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time" RN
-		"a=sendonly" RN,
-		pl, pl, pl, pl, pl, pl,
-		rtpv->rtp->ssrc
-	);
-	return sdp;
-}
-
 #define _PRE 3 // Annex B prefix length
 
 void us_rtpv_wrap(us_rtpv_s *rtpv, const us_frame_s *frame, bool zero_playout_delay) {
-	// There is a complicated logic here but everything works as it should:
-	//   - https://github.com/pikvm/ustreamer/issues/115#issuecomment-893071775
+	US_A(frame->format == V4L2_PIX_FMT_H264);
 
-	assert(frame->format == V4L2_PIX_FMT_H264);
+	if (frame->used <= _PRE) {
+		return;
+	}
 
-	rtpv->rtp->first_of_frame = true;
-	rtpv->rtp->last_of_frame = false;
 	rtpv->rtp->zero_playout_delay = zero_playout_delay;
 	rtpv->rtp->grab_ntp_ts = us_get_now_ntp() - us_ld_to_ntp(us_get_now_monotonic() - frame->grab_begin_ts);
 
 	const u32 pts = us_get_now_monotonic_u64() * 9 / 100; // PTS units are in 90 kHz
-	sz last_offset = -_PRE;
 
-	while (true) { // Find and iterate by nalus
-		const uz next_start = last_offset + _PRE;
-		sz offset = _find_annexb(frame->data + next_start, frame->used - next_start);
-		if (offset < 0) {
-			break;
-		}
-		offset += next_start;
+	// Bytestream can be like:
+	//   [00-]00-00-01 NALU [00-]00-00-01 NALU
+	// ... with optional 00 prefix for _PRE.
+	// We need to iterate by NALUs only and throw away separators.
 
-		if (last_offset >= 0) {
-			const u8 *const data = frame->data + last_offset + _PRE;
-			uz size = offset - last_offset - _PRE;
-			if (data[size - 1] == 0) { // Check for extra 00
+	const sz offset_to_first = _find_annexb(frame->data, frame->used);
+	if (offset_to_first < 0) {
+		return;
+	}
+	uz begin = offset_to_first + _PRE;
+
+	bool first_nalu = true;
+	while (begin < frame->used) { // Find and iterate by NALUs
+		const u8 *const data = frame->data + begin;
+		const sz offset_to_next = _find_annexb(data, frame->used - begin);
+
+		if (offset_to_next >= 0) { // Process NALUs between prefixes
+			uz size = offset_to_next;
+			if (size > 1 && data[size - 1] == 0) { // Check for extra trailing zero
 				--size;
 			}
-			_rtpv_process_nalu(rtpv, data, size, pts, false);
+			if (size > 1) { // Skip too short NALUs
+				_rtpv_process_nalu(rtpv, data, size, pts, first_nalu, false);
+				first_nalu = false;
+			}
+			begin += offset_to_next + _PRE;
+
+		} else { // Process the tail
+			const uz size = frame->used - begin;
+			if (size > 1) {
+				_rtpv_process_nalu(rtpv, data, size, pts, first_nalu, true);
+				first_nalu = false;
+			}
+			break;
 		}
-
-		last_offset = offset;
-	}
-
-	if (last_offset >= 0) {
-		const u8 *const data = frame->data + last_offset + _PRE;
-		uz size = frame->used - last_offset - _PRE;
-		_rtpv_process_nalu(rtpv, data, size, pts, true);
 	}
 }
 
-void _rtpv_process_nalu(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool marked) {
+static sz _find_annexb(const u8 *data, uz size) {
+	// Parses buffer for 00 00 01 start codes
+	if (size >= _PRE) {
+		for (uz i = 0; i <= size - _PRE; ++i) {
+			if (
+				data[i] == 0
+				&& data[i + 1] == 0
+				&& data[i + 2] == 1
+			) {
+				return i;
+			}
+		}
+	}
+	return -1;
+}
+
+#undef _PRE
+
+static bool _is_nalu_small(uz size) {
+	return (size + US_RTP_HEADER_SIZE <= US_RTP_TOTAL_SIZE);
+}
+
+static void _rtpv_process_nalu(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool first_nalu, bool last_nalu) {
+	US_A(size > 1);
+	if (_is_nalu_small(size)) {
+		_rtpv_process_nalu_small(rtpv, data, size, pts, first_nalu, last_nalu);
+	} else {
+		_rtpv_process_nalu_big(rtpv, data, size, pts, first_nalu, last_nalu);
+	}
+}
+
+static void _rtpv_process_nalu_small(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool first_nalu, bool last_nalu) {
+	US_A(size > 1);
+	US_A(_is_nalu_small(size));
+
+	us_rtp_s *rtp = rtpv->rtp;
+	u8 *dg = rtp->datagram;
+
+	us_rtp_write_header(rtp, pts, last_nalu);
+	memcpy(dg + US_RTP_HEADER_SIZE, data, size);
+	rtp->used = size + US_RTP_HEADER_SIZE;
+	rtp->first_of_frame = first_nalu;
+	rtp->last_of_frame = last_nalu;
+	rtpv->callback(rtp);
+}
+
+static void _rtpv_process_nalu_big(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool first_nalu, bool last_nalu) {
+	US_A(size > 1);
+	US_A(!_is_nalu_small(size));
+
 	const uint ref_idc = (data[0] >> 5) & 3;
 	const uint type = data[0] & 0x1F;
-	u8 *dg = rtpv->rtp->datagram;
-
-	if (size + US_RTP_HEADER_SIZE <= US_RTP_DATAGRAM_SIZE) {
-		us_rtp_write_header(rtpv->rtp, pts, marked);
-		memcpy(dg + US_RTP_HEADER_SIZE, data, size);
-		rtpv->rtp->used = size + US_RTP_HEADER_SIZE;
-		rtpv->rtp->last_of_frame = true;
-		rtpv->callback(rtpv->rtp);
-		rtpv->rtp->first_of_frame = false;
-		return;
-	}
+	us_rtp_s *rtp = rtpv->rtp;
+	u8 *dg = rtp->datagram;
 
 	const uz fu_overhead = US_RTP_HEADER_SIZE + 2; // FU-A overhead
 
@@ -146,13 +172,13 @@ void _rtpv_process_nalu(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool
 
 	bool first = true;
 	while (remaining > 0) {
-		sz frag_size = US_RTP_DATAGRAM_SIZE - fu_overhead;
+		sz frag_size = US_RTP_TOTAL_SIZE - fu_overhead;
 		const bool last = (remaining <= frag_size);
 		if (last) {
 			frag_size = remaining;
 		}
 
-		us_rtp_write_header(rtpv->rtp, pts, (marked && last));
+		us_rtp_write_header(rtp, pts, (last_nalu && last));
 
 		dg[US_RTP_HEADER_SIZE] = 28 | (ref_idc << 5);
 
@@ -166,27 +192,13 @@ void _rtpv_process_nalu(us_rtpv_s *rtpv, const u8 *data, uz size, u32 pts, bool
 		dg[US_RTP_HEADER_SIZE + 1] = fu;
 
 		memcpy(dg + fu_overhead, src, frag_size);
-		rtpv->rtp->used = fu_overhead + frag_size;
-		rtpv->rtp->last_of_frame = last;
-		rtpv->callback(rtpv->rtp);
-		rtpv->rtp->first_of_frame = false;
+		rtp->used = fu_overhead + frag_size;
+		rtp->first_of_frame = (first_nalu && first);
+		rtp->last_of_frame = (last_nalu && last);
+		rtpv->callback(rtp);
 
 		src += frag_size;
 		remaining -= frag_size;
 		first = false;
 	}
 }
-
-static sz _find_annexb(const u8 *data, uz size) {
-	// Parses buffer for 00 00 01 start codes
-	if (size >= _PRE) {
-		for (uz index = 0; index <= size - _PRE; ++index) {
-			if (data[index] == 0 && data[index + 1] == 0 && data[index + 2] == 1) {
-				return index;
-			}
-		}
-	}
-	return -1;
-}
-
-#undef _PRE

janus/src/rtpv.h

@@ -37,5 +37,4 @@ typedef struct {
 us_rtpv_s *us_rtpv_init(us_rtp_callback_f callback);
 void us_rtpv_destroy(us_rtpv_s *rtpv);
 
-char *us_rtpv_make_sdp(us_rtpv_s *rtpv);
 void us_rtpv_wrap(us_rtpv_s *rtpv, const us_frame_s *frame, bool zero_playout_delay);

janus/src/sdp.c

@@ -0,0 +1,119 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#include "sdp.h"
+
+#include <inttypes.h>
+
+#include <janus/plugins/plugin.h>
+
+#include "uslibs/types.h"
+#include "uslibs/tools.h"
+
+#include "rtp.h"
+#include "rtpv.h"
+#include "rtpa.h"
+
+
+char *us_sdp_create(u32 video_ssrc, u32 audio_ssrc, bool acap, bool aplay, bool vplay) {
+	char *video_sdp;
+	{
+		// https://tools.ietf.org/html/rfc6184
+		// https://github.com/meetecho/janus-gateway/issues/2443
+		const uint pl = US_RTP_H264_PAYLOAD;
+		US_ASPRINTF(
+			video_sdp,
+			"m=video 1 RTP/SAVPF %u" RN
+			"c=IN IP4 0.0.0.0" RN
+			"a=rtpmap:%u H264/90000" RN
+			"a=fmtp:%u profile-level-id=42E01F;packetization-mode=1" RN
+			"a=rtcp-fb:%u ccm fir" RN
+			"a=rtcp-fb:%u nack" RN
+			"a=rtcp-fb:%u nack pli" RN
+			"a=rtcp-fb:%u goog-remb" RN
+			"a=mid:v" RN
+			"a=msid:video v" RN
+			"a=ssrc:%" PRIu32 " cname:ustreamer" RN
+			"a=extmap:1/sendonly urn:3gpp:video-orientation" RN
+			"a=extmap:2/sendonly http://www.webrtc.org/experiments/rtp-hdrext/playout-delay" RN
+			"a=extmap:3/sendonly http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time" RN
+			"a=%s" RN,
+			pl, pl, pl, pl, pl, pl, pl,
+			video_ssrc,
+			(vplay ? "sendrecv" : "sendonly"));
+	}
+
+	char *audio_sdp;
+	if (acap || aplay) {
+		const uint pl = US_RTP_OPUS_PAYLOAD;
+
+		char *dir = "";
+		if (acap && aplay) {
+			dir = "sendrecv";
+		} else if (acap && !aplay) {
+			dir = "sendonly";
+		} else if (!acap && aplay) {
+			dir = "recvonly";
+		}
+
+		US_ASPRINTF(
+			audio_sdp,
+			"m=audio 1 RTP/SAVPF %u" RN
+			"c=IN IP4 0.0.0.0" RN
+			"a=rtpmap:%u OPUS/%u/%u" RN
+			"a=fmtp:%u sprop-stereo=1" RN // useinbandfec=1
+			"a=mid:a" RN
+			"a=msid:audio a" RN
+			"a=ssrc:%" PRIu32 " cname:ustreamer" RN
+			"a=%s" RN,
+			pl, pl,
+			US_RTP_OPUS_HZ, US_RTP_OPUS_CH,
+			pl,
+			audio_ssrc,
+			dir);
+	} else {
+		audio_sdp = us_strdup("");
+	}
+
+	char *sdp;
+	US_ASPRINTF(sdp,
+		"v=0" RN
+		"o=- %" PRIu64 " 1 IN IP4 0.0.0.0" RN
+		"s=PiKVM uStreamer" RN
+		"t=0 0" RN
+		"%s%s",
+		us_get_now_id() >> 1,
+#		if JANUS_PLUGIN_API_VERSION >= 100
+		// Place video SDP before audio SDP so that the video and audio streams
+		// have predictable indices, even if audio is not available.
+		// See also client.c.
+		video_sdp, audio_sdp
+#		else
+		// For versions of Janus prior to 1.x, place the audio SDP first.
+		audio_sdp, video_sdp
+#		endif
+	);
+
+	free(audio_sdp);
+	free(video_sdp);
+	return sdp;
+}

janus/src/sdp.h

@@ -0,0 +1,28 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#pragma once
+
+#include "uslibs/types.h"
+
+
+char *us_sdp_create(u32 video_ssrc, u32 audio_ssrc, bool acap, bool aplay, bool vplay);

janus/src/uslibs/chip.c

@@ -0,0 +1 @@
+../../../src/libs/chip.c

janus/src/uslibs/chip.h

@@ -0,0 +1 @@
+../../../src/libs/chip.h

janus/src/uslibs/logging.c

@@ -0,0 +1 @@
+../../../src/libs/logging.c

janus/src/uslibs/logging.h

@@ -0,0 +1 @@
+../../../src/libs/logging.h

janus/src/uslibs/logging_base.h

@@ -20,35 +20,13 @@
 *****************************************************************************/
 
 
-#include "tc358743.h"
+#include <janus/plugins/plugin.h>
 
-#include <unistd.h>
-#include <fcntl.h>
-
-#include <linux/videodev2.h>
-#include <linux/v4l2-controls.h>
-
-#include "types.h"
-#include "tools.h"
-#include "xioctl.h"
+#include "threading.h"
 
 
-int us_tc358743_xioctl_get_audio_hz(int fd, uint *audio_hz) {
-	*audio_hz = 0;
-
-	struct v4l2_control ctl = {.id = TC358743_CID_AUDIO_PRESENT};
-	if (us_xioctl(fd, VIDIOC_G_CTRL, &ctl) < 0) {
-		return -1;
+#define US_LOG_PRINTF_NOLOCK(x_label_color, x_label, x_msg_color, x_msg, ...) { \
+		char m_tname_buf[US_THREAD_NAME_SIZE] = {0}; \
+		us_thread_get_name(m_tname_buf); \
+		JANUS_LOG(LOG_INFO, "== ustreamer/%-15s " x_label " -- " x_msg "\n", m_tname_buf, ##__VA_ARGS__); \
 	}
-	if (!ctl.value) {
-		return 0; // No audio
-	}
-
-	US_MEMSET_ZERO(ctl);
-	ctl.id = TC358743_CID_AUDIO_SAMPLING_RATE;
-	if (us_xioctl(fd, VIDIOC_G_CTRL, &ctl) < 0) {
-		return -1;
-	}
-	*audio_hz = ctl.value;
-	return 0;
-}

janus/src/uslibs/memsink.c

@@ -0,0 +1 @@
+../../../src/libs/memsink.c

janus/src/uslibs/memsink.h

@@ -0,0 +1 @@
+../../../src/libs/memsink.h

janus/src/uslibs/tc358743.c

@@ -1 +0,0 @@
-../../../src/libs/tc358743.c

janus/src/uslibs/tc358743.h

@@ -1 +0,0 @@
-../../../src/libs/tc358743.h

linters/Dockerfile

@@ -9,7 +9,7 @@ RUN pacman -Syu --noconfirm \
 	&& pacman -S --needed --noconfirm \
 		vim \
 		git \
-		libjpeg \
+		libjpeg-turbo \
 		libevent \
 		libutil-linux \
 		libbsd \

linters/tox.ini

@@ -37,7 +37,7 @@ deps =
 
 [testenv:mypy]
 allowlist_externals = bash
-commands = bash -c 'mypy --config-file=linters/mypy.ini tools/*.py python/*.py'
+commands = bash -c 'mypy --config-file=linters/mypy.ini --cache-dir=/dev/null tools/*.py python/*.py'
 deps =
 	mypy
 

man/ustreamer-dump.1

@@ -1,6 +1,6 @@
 .\" Manpage for ustreamer-dump.
 .\" Open an issue or pull request to https://github.com/pikvm/ustreamer to correct errors or typos
-.TH USTREAMER-DUMP 1 "version 6.49" "January 2021"
+.TH USTREAMER-DUMP 1 "version 6.63" "January 2021"
 
 .SH NAME
 ustreamer-dump \- Dump uStreamer's memory sink to file

man/ustreamer.1

@@ -1,6 +1,6 @@
 .\" Manpage for ustreamer.
 .\" Open an issue or pull request to https://github.com/pikvm/ustreamer to correct errors or typos
-.TH USTREAMER 1 "version 6.49" "November 2020"
+.TH USTREAMER 1 "version 6.63" "November 2020"
 
 .SH NAME
 ustreamer \- stream MJPEG video from any V4L2 device to the network
@@ -23,7 +23,7 @@ For example, the recommended way of running µStreamer with TC358743-based captu
 .RS
 \fB\-\-format=uyvy \e\fR # Device input format
 .nf
-\fB\-\-encoder=m2m-image \e\fR # Hardware encoding with V4L2 M2M intraface
+\fB\-\-encoder=m2m-image \e\fR # Hardware encoding with V4L2 M2M interface
 .nf
 \fB\-\-workers=3 \e\fR # Maximum workers for V4L2 encoder
 .nf
@@ -66,7 +66,7 @@ Available: MMAP, USERPTR; default: MMAP.
 Desired FPS. Default: maximum possible.
 .TP
 .BR \-z\ \fIN ", " \-\-min\-frame\-size\ \fIN
-Drop frames smaller then this limit. Useful if the device produces small\-sized garbage frames. Default: 128 bytes.
+Drop frames smaller than this limit. Useful if the device produces small\-sized garbage frames. Default: 128 bytes.
 .TP
 .BR \-T ", " \-\-allow\-truncated\-frames
 Allows to handle truncated frames. Useful if the device produces incorrect but still acceptable frames. Default: disabled.
@@ -78,7 +78,7 @@ Suppress repetitive signal source errors. Default: disabled.
 Enable DV-timings querying and events processing to automatic resolution change. Default: disabled.
 .TP
 .BR \-b\ \fIN ", " \-\-buffers\ \fIN
-The number of buffers to receive data from the device. Each buffer may processed using an independent thread.
+The number of buffers to receive data from the device. Each buffer may be processed using an independent thread.
 Default: 2 (the number of CPU cores (but not more than 4) + 1).
 .TP
 .BR \-w\ \fIN ", " \-\-workers\ \fIN
@@ -120,6 +120,12 @@ Delay before trying to connect to the device again after an error (timeout for e
 .TP
 .BR \-\-m2m\-device\ \fI/dev/path
 Path to V4L2 mem-to-mem encoder device. Default: auto-select.
+.TP
+.BR \-\-media\-device \fI/dev/path
+Path to V4L2 /dev/media* device for setting subdevices (currently necessary for Raspberry Pi 5). Default: unset.
+.TP
+.BR \-\-media\-entity\-name \fIname
+Name of the V4L2 entity to grab video from, will be router to \-\-device, for example "tc358743 10-000f". Default: unset.
 
 .SS "Image control options"
 .TP

pkg/arch/PKGBUILD

@@ -3,13 +3,13 @@
 
 
 pkgname=ustreamer
-pkgver=6.49
+pkgver=6.63
 pkgrel=1
 pkgdesc="Lightweight and fast MJPEG-HTTP streamer"
 url="https://github.com/pikvm/ustreamer"
 license=(GPL)
 arch=(i686 x86_64 armv6h armv7h aarch64)
-depends=(libjpeg libevent libbsd libgpiod systemd)
+depends=(libjpeg-turbo libevent libbsd libgpiod systemd)
 makedepends=(gcc make pkgconf systemd)
 source=(${pkgname}::"git+https://github.com/pikvm/ustreamer#commit=v${pkgver}")
 md5sums=(SKIP)
@@ -19,7 +19,7 @@ _options="WITH_GPIO=1 WITH_SYSTEMD=1"
 if [ -e /usr/bin/python3 ]; then
 	_options="$_options WITH_PYTHON=1"
 	depends+=("python>=3.14" "python<3.15")
-	makedepends+=(python-setuptools python-pip python-build python-wheel)
+	makedepends+=(python-setuptools python-pip python-build python-wheel python-installer)
 fi
 if [ -e /usr/include/janus/plugins/plugin.h ];then
 	depends+=(janus-gateway alsa-lib opus)

pkg/openwrt/Makefile

@@ -2,11 +2,14 @@
 # This is free software, licensed under the GNU General Public License v2.
 # See /LICENSE for more information.
 #
+# This package is just an example. For OpenWRT it is recommended to use upstream package:
+#   - https://github.com/openwrt/packages/tree/master/multimedia/ustreamer
+#
 
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ustreamer
-PKG_VERSION:=6.49
+PKG_VERSION:=6.63
 PKG_RELEASE:=1
 PKG_MAINTAINER:=Maxim Devaev <mdevaev@gmail.com>
 

python/Makefile

@@ -9,13 +9,12 @@ all: root
 root: $(shell find src -type f,l) setup.py
 	$(info == PY_BUILD ustreamer-*.so)
 	rm -rf root
-	$(ECHO) $(PY) -m build --skip-dependency-check --no-isolation
-	$(ECHO) $(PY) -m pip install dist/*.whl --ignore-installed --root=./root
+	$(ECHO) $(PY) -m build --wheel --skip-dependency-check --no-isolation
 
 
 install:
-	$(PY) -m pip install dist/*.whl --ignore-installed --prefix=$(PREFIX) --root=$(if $(R_DESTDIR),$(R_DESTDIR),/)
+	$(PY) -m installer --prefix=$(PREFIX) $(if $(R_DESTDIR),--destdir=$(R_DESTDIR),) dist/*.whl
 
 
 clean:
-	rm -rf root dist ustreamer.egg-info
+	rm -rf root dist build ustreamer.egg-info

python/setup.py

@@ -34,7 +34,7 @@ def main() -> None:
     flags = _find_flags()
     setup(
         name="ustreamer",
-        version="6.49",
+        version="6.63",
         description="uStreamer tools",
         author="Maxim Devaev",
         author_email="mdevaev@gmail.com",

python/src/ustreamer.c

@@ -63,7 +63,7 @@ typedef struct {
 
 static void _MemsinkObject_destroy_internals(_MemsinkObject *self) {
 	if (self->mem != NULL) {
-		us_memsink_shared_unmap(self->mem, self->data_size);
+		us_memsinksh_unmap(self->mem, self->data_size);
 		self->mem = NULL;
 	}
 	US_CLOSE_FD(self->fd);
@@ -94,7 +94,7 @@ static int _MemsinkObject_init(_MemsinkObject *self, PyObject *args, PyObject *k
 	SET_DOUBLE(drop_same_frames, >= 0);
 #	undef SET_DOUBLE
 
-	if ((self->data_size = us_memsink_calculate_size(self->obj)) == 0) {
+	if ((self->data_size = us_memsinksh_calculate_size(self->obj)) == 0) {
 		PyErr_SetString(PyExc_ValueError, "Invalid memsink object suffix");
 		return -1;
 	}
@@ -105,7 +105,7 @@ static int _MemsinkObject_init(_MemsinkObject *self, PyObject *args, PyObject *k
 		PyErr_SetFromErrno(PyExc_OSError);
 		goto error;
 	}
-	if ((self->mem = us_memsink_shared_map(self->fd, self->data_size)) == NULL) {
+	if ((self->mem = us_memsinksh_map(self->fd, self->data_size)) == NULL) {
 		PyErr_SetFromErrno(PyExc_OSError);
 		goto error;
 	}
@@ -165,6 +165,7 @@ static int _wait_frame(_MemsinkObject *self) {
 
 		// Let the sink know that the client is alive
 		mem->last_client_ts = now_ts;
+		mem->client_magic = US_MEMSINK_MAGIC;
 
 		if (mem->id == self->frame_id) {
 			goto retry;
@@ -174,7 +175,7 @@ static int _wait_frame(_MemsinkObject *self) {
 			if (
 				US_FRAME_COMPARE_GEOMETRY(self->mem, self->frame)
 				&& (self->frame_ts + self->drop_same_frames > now_ts)
-				&& !memcmp(self->frame->data, us_memsink_get_data(mem), mem->used)
+				&& !memcmp(self->frame->data, us_memsinksh_get_data(mem), mem->used)
 			) {
 				self->frame_id = mem->id;
 				goto retry;
@@ -225,12 +226,12 @@ static PyObject *_MemsinkObject_wait_frame(_MemsinkObject *self, PyObject *args,
 	}
 
 	us_memsink_shared_s *mem = self->mem;
-	us_frame_set_data(self->frame, us_memsink_get_data(mem), mem->used);
+	us_frame_set_data(self->frame, us_memsinksh_get_data(mem), mem->used);
 	US_FRAME_COPY_META(self->mem, self->frame);
 	self->frame_id = mem->id;
 	self->frame_ts = us_get_now_monotonic();
 	if (key_required) {
-		mem->key_requested = true;
+		mem->wants.key = true;
 	}
 
 	if (flock(self->fd, LOCK_UN) < 0) {

src/Makefile

@@ -46,7 +46,7 @@ _OBJS = $(_USTR_SRCS:%.c=$(_BUILD)/%.o) $(_DUMP_SRCS:%.c=$(_BUILD)/%.o)
 
 
 # =====
-ifneq ($(shell sh -c 'uname 2>/dev/null || echo Unknown'),FreeBSD)
+ifeq ($(findstring bsd,$(shell $(CC) -dumpmachine)),)
 override _USTR_LDFLAGS += -latomic
 override _DUMP_LDFLAGS += -latomic
 override _V4P_LDFLAGS += -latomic
@@ -78,7 +78,7 @@ endif
 
 ifneq ($(MK_WITH_SETPROCTITLE),)
 override _CFLAGS += -DMK_WITH_SETPROCTITLE -DWITH_SETPROCTITLE
-ifeq ($(shell uname -s | tr A-Z a-z),linux)
+ifneq ($(findstring linux,$(shell $(CC) -dumpmachine)),)
 override _USTR_LDFLAGS += -lbsd
 endif
 endif

src/dump/main.c

@@ -177,14 +177,14 @@ int main(int argc, char *argv[]) {
 #	undef OPT_NUMBER
 #	undef OPT_SET
 
-	if (sink_name == NULL || sink_name[0] == '\0') {
+	if (!us_str_is_ok(sink_name)) {
 		puts("Missing option --sink. See --help for details.");
 		return 1;
 	}
 
 	_output_context_s ctx = {0};
 
-	if (out_path && out_path[0] != '\0') {
+	if (us_str_is_ok(out_path)) {
 		if ((ctx.v_out = (void*)us_output_file_init(out_path, out_json)) == NULL) {
 			return 1;
 		}
@@ -236,8 +236,9 @@ static int _dump_sink(
 	long double last_ts = 0;
 
 	while (!_g_stop) {
-		bool key_requested;
-		const int got = us_memsink_client_get(sink, frame, &key_requested, key_required);
+		us_memsink_wants_s w_get = {0};
+		us_memsink_wants_s w_put = {.key = key_required};
+		const int got = us_memsink_client_get(sink, frame, &w_get, (key_required ? &w_put : NULL));
 		if (got == 0) {
 			key_required = false;
 
@@ -252,7 +253,7 @@ static int _dump_sink(
 				frame->height,
 				frame->gop,
 				frame->key,
-				key_requested,
+				w_get.key,
 				frame->grab_end_ts - frame->grab_begin_ts,
 				frame->encode_begin_ts - frame->grab_end_ts,
 				frame->encode_end_ts - frame->encode_begin_ts,

src/libs/capture.c

@@ -34,10 +34,12 @@
 #include <sys/select.h>
 #include <sys/mman.h>
 #include <sys/time.h>
+#include <sys/stat.h>
 
 #include <pthread.h>
 #include <linux/videodev2.h>
-#include <linux/v4l2-controls.h>
+#include <linux/media.h>
+#include <linux/v4l2-subdev.h>
 
 #include "types.h"
 #include "errors.h"
@@ -47,7 +49,8 @@
 #include "threading.h"
 #include "frame.h"
 #include "xioctl.h"
-#include "tc358743.h"
+#include "media.h"
+#include "chip.h"
 
 
 static const struct {
@@ -63,18 +66,19 @@ static const struct {
 static const struct {
 	const char *name; // cppcheck-suppress unusedStructMember
 	const uint format; // cppcheck-suppress unusedStructMember
+	const uint media_format; // cppcheck-suppress unusedStructMember
 } _FORMATS[] = {
-	{"YUYV",	V4L2_PIX_FMT_YUYV},
-	{"YVYU",	V4L2_PIX_FMT_YVYU},
-	{"UYVY",	V4L2_PIX_FMT_UYVY},
-	{"YUV420",	V4L2_PIX_FMT_YUV420},
-	{"YVU420",	V4L2_PIX_FMT_YVU420},
-	{"GREY",	V4L2_PIX_FMT_GREY},
-	{"RGB565",	V4L2_PIX_FMT_RGB565},
-	{"RGB24",	V4L2_PIX_FMT_RGB24},
-	{"BGR24",	V4L2_PIX_FMT_BGR24},
-	{"MJPEG",	V4L2_PIX_FMT_MJPEG},
-	{"JPEG",	V4L2_PIX_FMT_JPEG},
+	{"YUYV",	V4L2_PIX_FMT_YUYV,		MEDIA_BUS_FMT_YUYV8_1X16},
+	{"YVYU",	V4L2_PIX_FMT_YVYU,		MEDIA_BUS_FMT_YVYU8_1X16},
+	{"UYVY",	V4L2_PIX_FMT_UYVY,		MEDIA_BUS_FMT_UYVY8_1X16},
+	{"YUV420",	V4L2_PIX_FMT_YUV420,	MEDIA_BUS_FMT_YUV8_1X24},
+	{"YVU420",	V4L2_PIX_FMT_YVU420,	0},
+	{"GREY",	V4L2_PIX_FMT_GREY,		MEDIA_BUS_FMT_Y8_1X8},
+	{"RGB565",	V4L2_PIX_FMT_RGB565,	MEDIA_BUS_FMT_RGB565_1X16},
+	{"RGB24",	V4L2_PIX_FMT_RGB24,		MEDIA_BUS_FMT_RGB888_1X24},
+	{"BGR24",	V4L2_PIX_FMT_BGR24,		MEDIA_BUS_FMT_BGR888_1X24},
+	{"MJPEG",	V4L2_PIX_FMT_MJPEG,		0},
+	{"JPEG",	V4L2_PIX_FMT_JPEG,		MEDIA_BUS_FMT_JPEG_1X8},
 };
 
 static const struct {
@@ -86,10 +90,12 @@ static const struct {
 };
 
 static int _capture_wait_buffer(us_capture_s *cap);
-static int _capture_consume_event(const us_capture_s *cap);
+static int _v4l2_consume_event(int fd);
 static void _v4l2_buffer_copy(const struct v4l2_buffer *src, struct v4l2_buffer *dest);
 static bool _capture_is_buffer_valid(const us_capture_s *cap, const struct v4l2_buffer *buf, const u8 *data);
 static int _capture_open_check_cap(us_capture_s *cap);
+static int _capture_open_media_pads(const us_capture_s *cap);
+static int _capture_open_media_format(const us_capture_s *cap, const us_media_pad_s *pad);
 static int _capture_open_dv_timings(us_capture_s *cap, bool apply);
 static int _capture_open_format(us_capture_s *cap, bool first);
 static void _capture_open_hw_fps(us_capture_s *cap);
@@ -101,27 +107,11 @@ static int _capture_open_queue_buffers(us_capture_s *cap);
 static int _capture_open_export_to_dma(us_capture_s *cap);
 static int _capture_apply_resolution(us_capture_s *cap, uint width, uint height, float hz);
 
-static void _capture_apply_controls(const us_capture_s *cap);
-
-static int _capture_query_control(
-	const us_capture_s *cap,
-	struct v4l2_queryctrl *query,
-	const char *name,
-	uint cid,
-	bool quiet);
-
-static void _capture_set_control(
-	const us_capture_s *cap,
-	const struct v4l2_queryctrl *query,
-	const char *name,
-	uint cid,
-	int value,
-	bool quiet);
-
 static const char *_format_to_string_nullable(uint format);
 static const char *_format_to_string_supported(uint format);
 static const char *_standard_to_string(v4l2_std_id standard);
 static const char *_io_method_to_string_supported(enum v4l2_memory io_method);
+static uint _format_to_media(uint format);
 
 
 #define _LOG_ERROR(x_msg, ...)	US_LOG_ERROR("CAP: " x_msg, ##__VA_ARGS__)
@@ -135,10 +125,14 @@ us_capture_s *us_capture_init(void) {
 	us_capture_runtime_s *run;
 	US_CALLOC(run, 1);
 	run->fd = -1;
+	run->dv_timings_fd = -1;
+	US_ARRAY_ITERATE(run->media_pads, 0, item, { item->fd = -1; });
 
 	us_capture_s *cap;
 	US_CALLOC(cap, 1);
 	cap->path = "/dev/video0";
+	cap->media_path = "";
+	cap->media_entity_name = "tc358743 10-000f"; // Will not be enabled without media_path
 	cap->width = 640;
 	cap->height = 480;
 	cap->format = V4L2_PIX_FMT_YUYV;
@@ -148,11 +142,13 @@ us_capture_s *us_capture_init(void) {
 	cap->n_bufs = us_get_cores_available() + 1;
 	cap->min_frame_size = 128;
 	cap->timeout = 1;
+	cap->ctl = us_controls_init();
 	cap->run = run;
 	return cap;
 }
 
 void us_capture_destroy(us_capture_s *cap) {
+	us_controls_destroy(cap->ctl);
 	free(cap->run);
 	free(cap);
 }
@@ -201,12 +197,20 @@ int us_capture_open(us_capture_s *cap) {
 	}
 	_LOG_DEBUG("Capture device fd=%d opened", run->fd);
 
+	if (cap->dv_timings && us_str_is_ok(cap->media_path)) {
+		if (_capture_open_media_pads(cap) < 0) {
+			goto error_no_device;
+		}
+		run->dv_timings_fd = run->media_pads[0].fd;
+	} else {
+		run->dv_timings_fd = run->fd;
+	}
+
 	if (cap->dv_timings && cap->persistent) {
-		struct v4l2_control ctl = {.id = V4L2_CID_DV_RX_POWER_PRESENT};
-		if (!us_xioctl(run->fd, VIDIOC_G_CTRL, &ctl)) {
-			if (!ctl.value) {
-				goto error_no_cable;
-			}
+		_LOG_DEBUG("Probing the cable ...")
+		switch (us_chip_check_cable(run->fd)) {
+			case 0: break;
+			case US_ERROR_NO_CABLE: goto error_no_cable;
 		}
 		_LOG_DEBUG("Probing DV-timings or QuerySTD ...");
 		switch (_capture_open_dv_timings(cap, false)) {
@@ -228,16 +232,20 @@ int us_capture_open(us_capture_s *cap) {
 	if (cap->dv_timings && _capture_open_dv_timings(cap, true) < 0) {
 		goto error;
 	}
+	if (cap->dv_timings && us_str_is_ok(cap->media_path)) {
+		US_ARRAY_ITERATE(run->media_pads, 0, pad, {
+			if (pad->fd >= -1 && _capture_open_media_format(cap, pad) < 0) {
+				goto error;
+			}
+		});
+	}
 	if (_capture_open_format(cap, true) < 0) {
 		goto error;
 	}
 	if (cap->dv_timings && cap->persistent) {
-		struct v4l2_control ctl = {.id = TC358743_CID_LANES_ENOUGH};
-		if (!us_xioctl(run->fd, VIDIOC_G_CTRL, &ctl)) {
-			if (!ctl.value) {
-				_LOG_ERROR("Not enough lanes, hardware can't handle this signal");
-				goto error_no_lanes;
-			}
+		switch (us_chip_tc358743_check_lanes(run->fd)) {
+			case 0: break;
+			case US_ERROR_NO_LANES: goto error_no_lanes;
 		}
 	}
 	_capture_open_hw_fps(cap);
@@ -255,7 +263,7 @@ int us_capture_open(us_capture_s *cap) {
 			goto error;
 		}
 	}
-	_capture_apply_controls(cap);
+	us_controls_apply(cap->ctl, cap->run->fd);
 
 	enum v4l2_buf_type type = run->capture_type;
 	if (us_xioctl(run->fd, VIDIOC_STREAMON, &type) < 0) {
@@ -287,6 +295,7 @@ error_no_sync:
 	return US_ERROR_NO_SYNC;
 
 error_no_lanes:
+	US_ONCE_FOR(run->open_error_once, __LINE__, { _LOG_ERROR("Not enough lanes, the hardware can't handle this signal"); });
 	us_capture_close(cap);
 	return US_ERROR_NO_LANES;
 
@@ -337,7 +346,9 @@ void us_capture_close(us_capture_s *cap) {
 		run->n_bufs = 0;
 	}
 
+	US_ARRAY_ITERATE(run->media_pads, 0, pad, { US_CLOSE_FD(pad->fd); });
 	US_CLOSE_FD(run->fd);
+	run->dv_timings_fd = -1; // It's just a copy of the media or device descriptor
 
 	if (say) {
 		_LOG_INFO("Capturing stopped");
@@ -498,6 +509,7 @@ int _capture_wait_buffer(us_capture_s *cap) {
 	INIT_FD_SET(read_fds);
 	INIT_FD_SET(error_fds);
 #	undef INIT_FD_SET
+	FD_SET(run->dv_timings_fd, &error_fds);
 
 	// Раньше мы проверяли и has_write, но потом выяснилось, что libcamerify зачем-то
 	// генерирует эвенты на запись, вероятно ошибочно. Судя по всему, игнорирование
@@ -511,10 +523,16 @@ int _capture_wait_buffer(us_capture_s *cap) {
 
 	bool has_read = false;
 	bool has_error = false;
-	const int selected = select(run->fd + 1, &read_fds, NULL, &error_fds, &timeout);
+	bool has_dv_error = false;
+	const int max_fd = US_MAX(run->fd, run->dv_timings_fd);
+	const int selected = select(max_fd + 1, &read_fds, NULL, &error_fds, &timeout);
+
 	if (selected > 0) {
 		has_read = FD_ISSET(run->fd, &read_fds);
 		has_error = FD_ISSET(run->fd, &error_fds);
+		if (run->fd != run->dv_timings_fd) {
+			has_dv_error = FD_ISSET(run->dv_timings_fd, &error_fds);
+		}
 	}
 	_LOG_DEBUG("Device select() --> %d; has_read=%d, has_error=%d", selected, has_read, has_error);
 
@@ -527,16 +545,19 @@ int _capture_wait_buffer(us_capture_s *cap) {
 		_LOG_ERROR("Device select() timeout");
 		return -1;
 	} else {
-		if (has_error && _capture_consume_event(cap) < 0) {
+		if (has_error && _v4l2_consume_event(run->fd) < 0) {
+			return -1; // Restart required
+		}
+		if (has_dv_error && _v4l2_consume_event(run->dv_timings_fd) < 0) {
 			return -1; // Restart required
 		}
 	}
 	return 0;
 }
 
-static int _capture_consume_event(const us_capture_s *cap) {
+static int _v4l2_consume_event(int fd) {
 	struct v4l2_event event;
-	if (us_xioctl(cap->run->fd, VIDIOC_DQEVENT, &event) < 0) {
+	if (us_xioctl(fd, VIDIOC_DQEVENT, &event) < 0) {
 		_LOG_PERROR("Can't consume V4L2 event");
 		return -1;
 	}
@@ -661,6 +682,192 @@ static int _capture_open_check_cap(us_capture_s *cap) {
 	return 0;
 }
 
+static int _open_subdev_for_entity(
+	struct media_v2_topology *topology,
+	int entity_id
+) {
+	struct media_v2_interface *interfaces = (struct media_v2_interface *)topology->ptr_interfaces;
+	int subdev_intf = us_media_find_link(topology, MEDIA_LNK_FL_INTERFACE_LINK, -1, entity_id, NULL);
+	int dev_major = -1;
+	int dev_minor = -1;
+	char *dev_path = NULL;
+	int fd;
+
+	if (subdev_intf < 0) {
+		_LOG_ERROR("Failed to find interface for entity %d", entity_id);
+		return -1;
+	}
+
+	for (uint i = 0; i < topology->num_interfaces; ++i) {
+		if (
+			interfaces[i].intf_type == MEDIA_INTF_T_V4L_SUBDEV
+			&& interfaces[i].id == (u32)subdev_intf
+		) {
+			dev_major = interfaces[i].devnode.major;
+			dev_minor = interfaces[i].devnode.minor;
+		}
+	}
+
+	if (dev_major < 0 || dev_minor < 0) {
+		_LOG_ERROR("Cannot find interface %d", subdev_intf);
+		return -1;
+	}
+
+	US_ASPRINTF(dev_path, "/dev/char/%d:%d", dev_major, dev_minor);
+	if ((fd = open(dev_path, O_RDWR | O_NONBLOCK)) < 0) {
+		_LOG_PERROR("Failed to open %s", dev_path);
+		free(dev_path);
+		return -1;
+	}
+	free(dev_path);
+
+	return fd;
+}
+
+static int _capture_open_media_pads(const us_capture_s *cap) {
+	// This function needs to:
+	// 1. Find entity related to actual input signal
+	// 2. Find entity related to the selected v4l2 device (cap->path)
+	// 3. Figure out links to connect them
+	// The whole path is:
+	// - source_intf_id (points at relevant /dev/v4l-subdev*)
+	// - source_entity_id
+	// - source_pad
+	// - mux_source_pad_id, mux_source_pad
+	// - mux_entity_id (usually named "csi2" here)
+	// - mux_sink_pad_id, mux_sink_pad
+	// - sink_pad
+	// - sink_entity_id
+	// - sink_intf_id (points at cap->path / run->fd)
+	// Pads need saving both ID (for links lookup) and index (for MEDIA_IOC_SETUP_LINK ioctl),
+	// so lookup relevant media_v2_pad structure.
+	// Walk the relevant paths from both ends in parallel to reduce numbers of loops.
+
+	us_capture_runtime_s *const run = cap->run;
+	struct media_v2_topology *topology = NULL;
+	int media_fd = -1;
+	int mux_subdev_fd = -1;
+	int source_subdev_fd = -1;
+
+	if ((media_fd = open(cap->media_path, O_RDWR | O_NONBLOCK)) < 0) {
+		_LOG_PERROR("Can't open media device");
+		goto error;
+	}
+
+	if ((topology = us_media_topology_init(media_fd)) == NULL) {
+		goto error;
+	}
+
+	const int source_entity_id = us_media_find_entity_by_name(topology, cap->media_entity_name);
+	if (source_entity_id == -1) {
+		_LOG_ERROR("Can't find source media entity '%s'", cap->media_entity_name);
+		goto error;
+	}
+	const struct media_v2_pad *source_pad = us_media_find_pad_by_entity(topology, MEDIA_PAD_FL_SOURCE, source_entity_id);
+	if (source_pad == NULL) {
+		_LOG_ERROR("Failed to find pad for entity %d ('%s')", source_entity_id, cap->media_entity_name);
+		goto error;
+	}
+
+	struct stat st = {0};
+	if (fstat(run->fd, &st) < 0) {
+		_LOG_PERROR("Can't stat() video device");
+		goto error;
+	}
+	const int sink_entity_id = us_media_find_entity_by_devnode(topology, st.st_rdev);
+	if (sink_entity_id == -1) {
+		_LOG_ERROR("Can't find sink media entity '%s'", cap->path);
+		goto error;
+	}
+	const struct media_v2_pad *sink_pad = us_media_find_pad_by_entity(topology, MEDIA_PAD_FL_SINK, sink_entity_id);
+	if (sink_pad == NULL) {
+		_LOG_ERROR("Failed to find pad for entity %d ('%s')", sink_entity_id, cap->path);
+		goto error;
+	}
+
+	u32 source_link_flags = 0;
+	u32 sink_link_flags = 0;
+	int mux_source_pad_id = us_media_find_link(topology, MEDIA_LNK_FL_DATA_LINK, source_pad->id, -1, &source_link_flags);
+	int mux_sink_pad_id = us_media_find_link(topology, MEDIA_LNK_FL_DATA_LINK, -1, sink_pad->id, &sink_link_flags);
+
+	const struct media_v2_pad *mux_source_pad = us_media_find_pad(topology, MEDIA_PAD_FL_SINK, mux_source_pad_id);
+	const struct media_v2_pad *mux_sink_pad = us_media_find_pad(topology, MEDIA_PAD_FL_SOURCE, mux_sink_pad_id);
+
+	// Those were referenced by relevant links, not finding them means kernel gave us incomplete info
+	US_A(mux_source_pad);
+	US_A(mux_sink_pad);
+
+	if (mux_source_pad->entity_id != mux_sink_pad->entity_id) {
+		_LOG_ERROR(
+			"Media source entity %d '%s' connected to a different entity (%d) "
+			"than the sink entity %d '%s' (connected to %d). Select a different device.",
+			source_entity_id, cap->media_entity_name, mux_source_pad->entity_id,
+			sink_entity_id, cap->path, mux_sink_pad->entity_id);
+		goto error;
+	}
+
+	const int mux_entity_id = mux_source_pad->entity_id;
+
+	if (mux_entity_id == -1) {
+		_LOG_ERROR("Failed to find media mux entity");
+		goto error;
+	}
+
+	// this one is usually MEDIA_LNK_FL_IMMUTABLE, so it's always enabled,
+	// but check anyway
+	if (!(source_link_flags & MEDIA_LNK_FL_ENABLED)) {
+		if (us_media_xioctl_setup_link(
+			media_fd,
+			source_pad, mux_source_pad,
+			source_link_flags | MEDIA_LNK_FL_ENABLED
+		) < 0) {
+			_LOG_PERROR("Failed to enable source link");
+			goto error;
+		}
+	}
+
+	if (!(sink_link_flags & MEDIA_LNK_FL_ENABLED)) {
+		if (us_media_xioctl_setup_link(
+			media_fd,
+			mux_sink_pad, sink_pad,
+			sink_link_flags | MEDIA_LNK_FL_ENABLED
+		) < 0) {
+			_LOG_PERROR("Failed to enable sink link");
+			goto error;
+		}
+	}
+
+	if ((source_subdev_fd = _open_subdev_for_entity(topology, source_entity_id)) < 0) {
+		_LOG_PERROR("Failed to open source v4l2 subdevice");
+		goto error;
+	}
+
+	if ((mux_subdev_fd = _open_subdev_for_entity(topology, mux_entity_id)) < 0) {
+		_LOG_PERROR("Failed to open mux v4l2 subdevice");
+		goto error;
+	}
+
+	// Save PAD indices for VIDIOC_SUBDEV_S_FMT
+	run->media_pads[0].fd = source_subdev_fd;
+	run->media_pads[0].pad = source_pad->index;
+	run->media_pads[1].fd = mux_subdev_fd;
+	run->media_pads[1].pad = mux_source_pad->index;
+	run->media_pads[2].fd = mux_subdev_fd;
+	run->media_pads[2].pad = mux_sink_pad->index;
+
+	goto ok;
+
+error:
+	US_CLOSE_FD(source_subdev_fd);
+	US_CLOSE_FD(mux_subdev_fd); // cppcheck-suppress unreadVariable
+	source_subdev_fd = -1; // cppcheck-suppress redundantAssignment
+
+ok:
+	US_CLOSE_FD(media_fd); // cppcheck-suppress unreadVariable
+	US_DELETE(topology, us_media_topology_destroy);
+	return (source_subdev_fd >= 0 ? 0 : -1);
+}
+
 static int _capture_open_dv_timings(us_capture_s *cap, bool apply) {
 	// Just probe only if @apply is false
 
@@ -670,7 +877,7 @@ static int _capture_open_dv_timings(us_capture_s *cap, bool apply) {
 
 	struct v4l2_dv_timings dv = {0};
 	_LOG_DEBUG("Querying DV-timings (apply=%u) ...", apply);
-	if (us_xioctl(run->fd, VIDIOC_QUERY_DV_TIMINGS, &dv) < 0) {
+	if (us_xioctl(run->dv_timings_fd, VIDIOC_QUERY_DV_TIMINGS, &dv) < 0) {
 		// TC358743 errors here (see in the kernel: drivers/media/i2c/tc358743.c):
 		//   - ENOLINK: No valid signal (SYS_STATUS & MASK_S_TMDS)
 		//   - ENOLCK:  No sync on signal (SYS_STATUS & MASK_S_SYNC)
@@ -701,7 +908,7 @@ static int _capture_open_dv_timings(us_capture_s *cap, bool apply) {
 	}
 
 	_LOG_DEBUG("Applying DV-timings ...");
-	if (us_xioctl(run->fd, VIDIOC_S_DV_TIMINGS, &dv) < 0) {
+	if (us_xioctl(run->dv_timings_fd, VIDIOC_S_DV_TIMINGS, &dv) < 0) {
 		_LOG_PERROR("Failed to apply DV-timings");
 		return -1;
 	}
@@ -712,7 +919,7 @@ static int _capture_open_dv_timings(us_capture_s *cap, bool apply) {
 
 querystd:
 	_LOG_DEBUG("Failed to query DV-timings, trying QuerySTD ...");
-	if (us_xioctl(run->fd, VIDIOC_QUERYSTD, &cap->standard) < 0) {
+	if (us_xioctl(run->dv_timings_fd, VIDIOC_QUERYSTD, &cap->standard) < 0) {
 		if (apply) {
 			char *std_error = us_errno_to_string(errno); // Read the errno first
 			char *dv_error = us_errno_to_string(dv_errno);
@@ -724,7 +931,7 @@ querystd:
 	} else if (!apply) {
 		goto probe_only;
 	}
-	if (us_xioctl(run->fd, VIDIOC_S_STD, &cap->standard) < 0) {
+	if (us_xioctl(run->dv_timings_fd, VIDIOC_S_STD, &cap->standard) < 0) {
 		_LOG_PERROR("Can't set apply standard: %s", _standard_to_string(cap->standard));
 		return -1;
 	}
@@ -734,7 +941,7 @@ subscribe:
 	; // Empty statement for the goto label above
 	struct v4l2_event_subscription sub = {.type = V4L2_EVENT_SOURCE_CHANGE};
 	_LOG_DEBUG("Subscribing to V4L2_EVENT_SOURCE_CHANGE ...")
-	if (us_xioctl(cap->run->fd, VIDIOC_SUBSCRIBE_EVENT, &sub) < 0) {
+	if (us_xioctl(cap->run->dv_timings_fd, VIDIOC_SUBSCRIBE_EVENT, &sub) < 0) {
 		_LOG_PERROR("Can't subscribe to V4L2_EVENT_SOURCE_CHANGE");
 		return -1;
 	}
@@ -743,6 +950,47 @@ probe_only:
 	return 0;
 }
 
+static int _capture_open_media_format(const us_capture_s *cap, const us_media_pad_s *pad) {
+	const us_capture_runtime_s *const run = cap->run;
+
+	uint media_format = _format_to_media(cap->format);
+	if (media_format == 0) {
+		_LOG_ERROR("Format not supported for V4L2-subdev: %#x", cap->format);
+		return -1;
+	}
+
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = pad->pad,
+		.format.width = run->width,
+		.format.height = run->height,
+		.format.code = media_format,
+		.format.field = V4L2_FIELD_NONE,
+	};
+	// Thankfully fmt.stream seems to be irrelevant on RPi5:
+	// # v4l2-ctl -d /dev/v4l-subdev0 --get-routing
+	// Streams API not supported.
+	if (us_xioctl(pad->fd, VIDIOC_SUBDEV_S_FMT, &fmt) < 0) {
+		_LOG_PERROR("Can't set device format");
+		return -1;
+	}
+
+	// This could use negotiation similar to _capture_open_format
+	if (
+		fmt.format.width != run->width ||
+		fmt.format.height != run->height ||
+		fmt.format.code != media_format
+	) {
+		_LOG_ERROR("Format not accepted by pad %u: attempted %ux%u format %#x, got %ux%u format %#x",
+			pad->pad,
+			run->width, run->height, media_format,
+			fmt.format.width, fmt.format.height, fmt.format.code);
+		return -1;
+	}
+
+	return 0;
+}
+
 static int _capture_open_format(us_capture_s *cap, bool first) {
 	us_capture_runtime_s *const run = cap->run;
 
@@ -1094,111 +1342,6 @@ static int _capture_apply_resolution(us_capture_s *cap, uint width, uint height,
 	return 0;
 }
 
-static void _capture_apply_controls(const us_capture_s *cap) {
-#	define SET_CID_VALUE(x_cid, x_field, x_value, x_quiet) { \
-			struct v4l2_queryctrl m_query; \
-			if (_capture_query_control(cap, &m_query, #x_field, x_cid, x_quiet) == 0) { \
-				_capture_set_control(cap, &m_query, #x_field, x_cid, x_value, x_quiet); \
-			} \
-		}
-
-#	define SET_CID_DEFAULT(x_cid, x_field, x_quiet) { \
-			struct v4l2_queryctrl m_query; \
-			if (_capture_query_control(cap, &m_query, #x_field, x_cid, x_quiet) == 0) { \
-				_capture_set_control(cap, &m_query, #x_field, x_cid, m_query.default_value, x_quiet); \
-			} \
-		}
-
-#	define CONTROL_MANUAL_CID(x_cid, x_field) { \
-			if (cap->ctl.x_field.mode == CTL_MODE_VALUE) { \
-				SET_CID_VALUE(x_cid, x_field, cap->ctl.x_field.value, false); \
-			} else if (cap->ctl.x_field.mode == CTL_MODE_DEFAULT) { \
-				SET_CID_DEFAULT(x_cid, x_field, false); \
-			} \
-		}
-
-#	define CONTROL_AUTO_CID(x_cid_auto, x_cid_manual, x_field) { \
-			if (cap->ctl.x_field.mode == CTL_MODE_VALUE) { \
-				SET_CID_VALUE(x_cid_auto, x_field##_auto, 0, true); \
-				SET_CID_VALUE(x_cid_manual, x_field, cap->ctl.x_field.value, false); \
-			} else if (cap->ctl.x_field.mode == CTL_MODE_AUTO) { \
-				SET_CID_VALUE(x_cid_auto, x_field##_auto, 1, false); \
-			} else if (cap->ctl.x_field.mode == CTL_MODE_DEFAULT) { \
-				SET_CID_VALUE(x_cid_auto, x_field##_auto, 0, true); /* Reset inactive flag */ \
-				SET_CID_DEFAULT(x_cid_manual, x_field, false); \
-				SET_CID_DEFAULT(x_cid_auto, x_field##_auto, false); \
-			} \
-		}
-
-	CONTROL_AUTO_CID	(V4L2_CID_AUTOBRIGHTNESS,		V4L2_CID_BRIGHTNESS,				brightness);
-	CONTROL_MANUAL_CID	(								V4L2_CID_CONTRAST,					contrast);
-	CONTROL_MANUAL_CID	(								V4L2_CID_SATURATION,				saturation);
-	CONTROL_AUTO_CID	(V4L2_CID_HUE_AUTO,				V4L2_CID_HUE,						hue);
-	CONTROL_MANUAL_CID	(								V4L2_CID_GAMMA,						gamma);
-	CONTROL_MANUAL_CID	(								V4L2_CID_SHARPNESS,					sharpness);
-	CONTROL_MANUAL_CID	(								V4L2_CID_BACKLIGHT_COMPENSATION,	backlight_compensation);
-	CONTROL_AUTO_CID	(V4L2_CID_AUTO_WHITE_BALANCE,	V4L2_CID_WHITE_BALANCE_TEMPERATURE,	white_balance);
-	CONTROL_AUTO_CID	(V4L2_CID_AUTOGAIN,				V4L2_CID_GAIN,						gain);
-	CONTROL_MANUAL_CID	(								V4L2_CID_COLORFX,					color_effect);
-	CONTROL_MANUAL_CID	(								V4L2_CID_ROTATE,	   				rotate);
-	CONTROL_MANUAL_CID	(								V4L2_CID_VFLIP,						flip_vertical);
-	CONTROL_MANUAL_CID	(								V4L2_CID_HFLIP,						flip_horizontal);
-
-#	undef CONTROL_AUTO_CID
-#	undef CONTROL_MANUAL_CID
-#	undef SET_CID_DEFAULT
-#	undef SET_CID_VALUE
-}
-
-static int _capture_query_control(
-	const us_capture_s *cap,
-	struct v4l2_queryctrl *query,
-	const char *name,
-	uint cid,
-	bool quiet
-) {
-	// cppcheck-suppress redundantPointerOp
-	US_MEMSET_ZERO(*query);
-	query->id = cid;
-
-	if (us_xioctl(cap->run->fd, VIDIOC_QUERYCTRL, query) < 0 || query->flags & V4L2_CTRL_FLAG_DISABLED) {
-		if (!quiet) {
-			_LOG_ERROR("Changing control %s is unsupported", name);
-		}
-		return -1;
-	}
-	return 0;
-}
-
-static void _capture_set_control(
-	const us_capture_s *cap,
-	const struct v4l2_queryctrl *query,
-	const char *name,
-	uint cid,
-	int value,
-	bool quiet
-) {
-	if (value < query->minimum || value > query->maximum || value % query->step != 0) {
-		if (!quiet) {
-			_LOG_ERROR("Invalid value %d of control %s: min=%d, max=%d, default=%d, step=%u",
-				value, name, query->minimum, query->maximum, query->default_value, query->step);
-		}
-		return;
-	}
-
-	struct v4l2_control ctl = {
-		.id = cid,
-		.value = value,
-	};
-	if (us_xioctl(cap->run->fd, VIDIOC_S_CTRL, &ctl) < 0) {
-		if (!quiet) {
-			_LOG_PERROR("Can't set control %s", name);
-		}
-	} else if (!quiet) {
-		_LOG_INFO("Applying control %s: %d", name, ctl.value);
-	}
-}
-
 static const char *_format_to_string_nullable(uint format) {
 	US_ARRAY_ITERATE(_FORMATS, 0, item, {
 		if (item->format == format) {
@@ -1230,3 +1373,12 @@ static const char *_io_method_to_string_supported(enum v4l2_memory io_method) {
 	});
 	return "unsupported";
 }
+
+static uint _format_to_media(uint format) {
+	US_ARRAY_ITERATE(_FORMATS, 0, item, {
+		if (item->format == format) {
+			return item->media_format;
+		}
+	});
+	return 0;
+}

src/libs/capture.h

@@ -28,6 +28,8 @@
 
 #include "types.h"
 #include "frame.h"
+#include "media.h"
+#include "controls.h"
 
 
 #define US_VIDEO_MIN_WIDTH		((uint)160)
@@ -53,6 +55,8 @@ typedef struct {
 
 typedef struct {
 	int					fd;
+	int					dv_timings_fd;
+	us_media_pad_s		media_pads[3]; // for VIDIOC_SUBDEV_S_FMT
 	uint				width;
 	uint				height;
 	uint				format;
@@ -69,40 +73,14 @@ typedef struct {
 	int					open_error_once;
 } us_capture_runtime_s;
 
-typedef enum {
-	CTL_MODE_NONE = 0,
-	CTL_MODE_VALUE,
-	CTL_MODE_AUTO,
-	CTL_MODE_DEFAULT,
-} us_control_mode_e;
-
-typedef struct {
-	us_control_mode_e	mode;
-	int					value;
-} us_control_s;
-
-typedef struct {
-	us_control_s	brightness;
-	us_control_s	contrast;
-	us_control_s	saturation;
-	us_control_s	hue;
-	us_control_s	gamma;
-	us_control_s	sharpness;
-	us_control_s	backlight_compensation;
-	us_control_s	white_balance;
-	us_control_s	gain;
-	us_control_s	color_effect;
-	us_control_s	rotate;
-	us_control_s	flip_vertical;
-	us_control_s	flip_horizontal;
-} us_controls_s;
-
 typedef struct {
 	char				*path;
 	uint				input;
 	uint				width;
 	uint				height;
 	uint				format;
+	char				*media_path;
+	char				*media_entity_name;
 
 	bool				format_swap_rgb;
 	uint				jpeg_quality;
@@ -116,7 +94,7 @@ typedef struct {
 	bool				allow_truncated_frames;
 	bool				persistent;
 	uint				timeout;
-	us_controls_s 		ctl;
+	us_controls_s 		*ctl;
 	us_capture_runtime_s *run;
 } us_capture_s;
 

src/libs/chip.c

@@ -20,61 +20,66 @@
 *****************************************************************************/
 
 
-#include "memsinkfd.h"
+#include "chip.h"
 
 #include <unistd.h>
+#include <fcntl.h>
 
 #include <linux/videodev2.h>
+#include <linux/v4l2-controls.h>
 
-#include "uslibs/types.h"
-#include "uslibs/errors.h"
-#include "uslibs/tools.h"
-#include "uslibs/frame.h"
-#include "uslibs/memsinksh.h"
-
-#include "logging.h"
+#include "types.h"
+#include "errors.h"
+#include "tools.h"
+#include "xioctl.h"
 
 
-int us_memsink_fd_wait_frame(int fd, us_memsink_shared_s *mem, u64 last_id) {
-	const ldf deadline_ts = us_get_now_monotonic() + 1; // wait_timeout
-	ldf now_ts;
-	do {
-		const int result = us_flock_timedwait_monotonic(fd, 1); // lock_timeout
-		now_ts = us_get_now_monotonic();
-		if (result < 0 && errno != EWOULDBLOCK) {
-			US_JLOG_PERROR("video", "Can't lock memsink");
-			return -1;
-		} else if (result == 0) {
-			if (mem->magic == US_MEMSINK_MAGIC && mem->version == US_MEMSINK_VERSION && mem->id != last_id) {
-				return 0;
-			}
-			if (flock(fd, LOCK_UN) < 0) {
-				US_JLOG_PERROR("video", "Can't unlock memsink");
-				return -1;
-			}
-		}
-		usleep(1000); // lock_polling
-	} while (now_ts < deadline_ts);
-	return US_ERROR_NO_DATA;
+#ifndef V4L2_CID_USER_TC358743_BASE
+#	define V4L2_CID_USER_TC358743_BASE (V4L2_CID_USER_BASE + 0x1080)
+#endif
+
+#ifndef TC358743_CID_AUDIO_SAMPLING_RATE
+#	define TC358743_CID_AUDIO_SAMPLING_RATE (V4L2_CID_USER_TC358743_BASE + 0)
+#endif
+
+#ifndef TC358743_CID_AUDIO_PRESENT
+#	define TC358743_CID_AUDIO_PRESENT (V4L2_CID_USER_TC358743_BASE + 1)
+#endif
+
+#ifndef TC358743_CID_LANES_ENOUGH
+#	define TC358743_CID_LANES_ENOUGH (V4L2_CID_USER_TC358743_BASE + 2)
+#endif
+
+
+int us_chip_check_cable(int fd) {
+	struct v4l2_control ctl = {.id = V4L2_CID_DV_RX_POWER_PRESENT};
+	if (us_xioctl(fd, VIDIOC_G_CTRL, &ctl) < 0) {
+		return US_ERROR_COMMON;
+	}
+	return (ctl.value ? 0 : US_ERROR_NO_CABLE);
 }
 
-int us_memsink_fd_get_frame(int fd, us_memsink_shared_s *mem, us_frame_s *frame, u64 *frame_id, bool key_required) {
-	us_frame_set_data(frame, us_memsink_get_data(mem), mem->used);
-	US_FRAME_COPY_META(mem, frame);
-	*frame_id = mem->id;
-	mem->last_client_ts = us_get_now_monotonic();
-	if (key_required) {
-		mem->key_requested = true;
+int us_chip_tc358743_check_lanes(int fd) {
+	struct v4l2_control ctl = {.id = TC358743_CID_LANES_ENOUGH};
+	if (us_xioctl(fd, VIDIOC_G_CTRL, &ctl) < 0) {
+		return US_ERROR_COMMON;
+	}
+	return (ctl.value ? 0 : US_ERROR_NO_LANES);
+}
+
+int us_chip_tc358743_get_audio_hz(int fd) {
+	struct v4l2_control ctl = {.id = TC358743_CID_AUDIO_PRESENT};
+	if (us_xioctl(fd, VIDIOC_G_CTRL, &ctl) < 0) {
+		return US_ERROR_COMMON;
+	}
+	if (!ctl.value) {
+		return US_ERROR_NO_SIGNAL; // No audio
 	}
 
-	bool retval = 0;
-	if (frame->format != V4L2_PIX_FMT_H264) {
-		US_JLOG_ERROR("video", "Got non-H264 frame from memsink");
-		retval = -1;
+	US_MEMSET_ZERO(ctl);
+	ctl.id = TC358743_CID_AUDIO_SAMPLING_RATE;
+	if (us_xioctl(fd, VIDIOC_G_CTRL, &ctl) < 0) {
+		return US_ERROR_COMMON;
 	}
-	if (flock(fd, LOCK_UN) < 0) {
-		US_JLOG_PERROR("video", "Can't unlock memsink");
-		retval = -1;
-	}
-	return retval;
+	return (ctl.value ? ctl.value : US_ERROR_NO_SIGNAL);
 }

src/libs/chip.h

@@ -22,26 +22,10 @@
 
 #pragma once
 
-#include <linux/v4l2-controls.h>
-
 #include "types.h"
 
 
-#ifndef V4L2_CID_USER_TC358743_BASE
-#	define V4L2_CID_USER_TC358743_BASE (V4L2_CID_USER_BASE + 0x1080)
-#endif
+int us_chip_check_cable(int fd);
 
-#ifndef TC358743_CID_AUDIO_SAMPLING_RATE
-#	define TC358743_CID_AUDIO_SAMPLING_RATE (V4L2_CID_USER_TC358743_BASE + 0)
-#endif
-
-#ifndef TC358743_CID_AUDIO_PRESENT
-#	define TC358743_CID_AUDIO_PRESENT (V4L2_CID_USER_TC358743_BASE + 1)
-#endif
-
-#ifndef TC358743_CID_LANES_ENOUGH
-#	define TC358743_CID_LANES_ENOUGH (V4L2_CID_USER_TC358743_BASE + 2)
-#endif
-
-
-int us_tc358743_xioctl_get_audio_hz(int fd, uint *audio_hz);
+int us_chip_tc358743_check_lanes(int fd);
+int us_chip_tc358743_get_audio_hz(int fd);

src/libs/const.h

@@ -26,7 +26,7 @@
 
 
 #define US_VERSION_MAJOR 6
-#define US_VERSION_MINOR 49
+#define US_VERSION_MINOR 63
 
 #define US_MAKE_VERSION2(_major, _minor) #_major "." #_minor
 #define US_MAKE_VERSION1(_major, _minor) US_MAKE_VERSION2(_major, _minor)

src/libs/controls.c

@@ -0,0 +1,173 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#include "controls.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include <linux/videodev2.h>
+#include <linux/v4l2-controls.h>
+
+#include "types.h"
+#include "tools.h"
+#include "logging.h"
+#include "xioctl.h"
+
+
+static int _query_control(
+	int fd,
+	struct v4l2_queryctrl *query,
+	const char *name,
+	uint cid,
+	bool quiet);
+
+static void _set_control(
+	int fd,
+	const struct v4l2_queryctrl *query,
+	const char *name,
+	uint cid,
+	int value,
+	bool quiet);
+
+
+#define _LOG_ERROR(x_msg, ...)	US_LOG_ERROR("CAP: " x_msg, ##__VA_ARGS__)
+#define _LOG_PERROR(x_msg, ...)	US_LOG_PERROR("CAP: " x_msg, ##__VA_ARGS__)
+#define _LOG_INFO(x_msg, ...)	US_LOG_INFO("CAP: " x_msg, ##__VA_ARGS__)
+
+
+us_controls_s *us_controls_init(void) {
+	us_controls_s *ctl;
+	US_CALLOC(ctl, 1);
+	return ctl;
+}
+
+void us_controls_destroy(us_controls_s *ctl) {
+	free(ctl);
+}
+
+void us_controls_apply(const us_controls_s *ctl, int fd) {
+#	define SET_CID_VALUE(x_cid, x_field, x_value, x_quiet) { \
+			struct v4l2_queryctrl m_query; \
+			if (_query_control(fd, &m_query, #x_field, x_cid, x_quiet) == 0) { \
+				_set_control(fd, &m_query, #x_field, x_cid, x_value, x_quiet); \
+			} \
+		}
+
+#	define SET_CID_DEFAULT(x_cid, x_field, x_quiet) { \
+			struct v4l2_queryctrl m_query; \
+			if (_query_control(fd, &m_query, #x_field, x_cid, x_quiet) == 0) { \
+				_set_control(fd, &m_query, #x_field, x_cid, m_query.default_value, x_quiet); \
+			} \
+		}
+
+#	define CONTROL_MANUAL_CID(x_cid, x_field) { \
+			if (ctl->x_field.mode == US_CTL_MODE_VALUE) { \
+				SET_CID_VALUE(x_cid, x_field, ctl->x_field.value, false); \
+			} else if (ctl->x_field.mode == US_CTL_MODE_DEFAULT) { \
+				SET_CID_DEFAULT(x_cid, x_field, false); \
+			} \
+		}
+
+#	define CONTROL_AUTO_CID(x_cid_auto, x_cid_manual, x_field) { \
+			if (ctl->x_field.mode == US_CTL_MODE_VALUE) { \
+				SET_CID_VALUE(x_cid_auto, x_field##_auto, 0, true); \
+				SET_CID_VALUE(x_cid_manual, x_field, ctl->x_field.value, false); \
+			} else if (ctl->x_field.mode == US_CTL_MODE_AUTO) { \
+				SET_CID_VALUE(x_cid_auto, x_field##_auto, 1, false); \
+			} else if (ctl->x_field.mode == US_CTL_MODE_DEFAULT) { \
+				SET_CID_VALUE(x_cid_auto, x_field##_auto, 0, true); /* Reset inactive flag */ \
+				SET_CID_DEFAULT(x_cid_manual, x_field, false); \
+				SET_CID_DEFAULT(x_cid_auto, x_field##_auto, false); \
+			} \
+		}
+
+	CONTROL_AUTO_CID	(V4L2_CID_AUTOBRIGHTNESS,		V4L2_CID_BRIGHTNESS,				brightness);
+	CONTROL_MANUAL_CID	(								V4L2_CID_CONTRAST,					contrast);
+	CONTROL_MANUAL_CID	(								V4L2_CID_SATURATION,				saturation);
+	CONTROL_AUTO_CID	(V4L2_CID_HUE_AUTO,				V4L2_CID_HUE,						hue);
+	CONTROL_MANUAL_CID	(								V4L2_CID_GAMMA,						gamma);
+	CONTROL_MANUAL_CID	(								V4L2_CID_SHARPNESS,					sharpness);
+	CONTROL_MANUAL_CID	(								V4L2_CID_BACKLIGHT_COMPENSATION,	backlight_compensation);
+	CONTROL_AUTO_CID	(V4L2_CID_AUTO_WHITE_BALANCE,	V4L2_CID_WHITE_BALANCE_TEMPERATURE,	white_balance);
+	CONTROL_AUTO_CID	(V4L2_CID_AUTOGAIN,				V4L2_CID_GAIN,						gain);
+	CONTROL_MANUAL_CID	(								V4L2_CID_COLORFX,					color_effect);
+	CONTROL_MANUAL_CID	(								V4L2_CID_ROTATE,	   				rotate);
+	CONTROL_MANUAL_CID	(								V4L2_CID_VFLIP,						flip_vertical);
+	CONTROL_MANUAL_CID	(								V4L2_CID_HFLIP,						flip_horizontal);
+
+#	undef CONTROL_AUTO_CID
+#	undef CONTROL_MANUAL_CID
+#	undef SET_CID_DEFAULT
+#	undef SET_CID_VALUE
+}
+
+static int _query_control(
+	int fd,
+	struct v4l2_queryctrl *query,
+	const char *name,
+	uint cid,
+	bool quiet
+) {
+	// cppcheck-suppress redundantPointerOp
+	US_MEMSET_ZERO(*query);
+	query->id = cid;
+
+	if (us_xioctl(fd, VIDIOC_QUERYCTRL, query) < 0 || query->flags & V4L2_CTRL_FLAG_DISABLED) {
+		if (!quiet) {
+			_LOG_ERROR("Changing control %s is unsupported", name);
+		}
+		return -1;
+	}
+	return 0;
+}
+
+static void _set_control(
+	int fd,
+	const struct v4l2_queryctrl *query,
+	const char *name,
+	uint cid,
+	int value,
+	bool quiet
+) {
+	if (value < query->minimum || value > query->maximum || value % query->step != 0) {
+		if (!quiet) {
+			_LOG_ERROR("Invalid value %d of control %s: min=%d, max=%d, default=%d, step=%u",
+				value, name, query->minimum, query->maximum, query->default_value, query->step);
+		}
+		return;
+	}
+
+	struct v4l2_control ctl = {
+		.id = cid,
+		.value = value,
+	};
+	if (us_xioctl(fd, VIDIOC_S_CTRL, &ctl) < 0) {
+		if (!quiet) {
+			_LOG_PERROR("Can't set control %s", name);
+		}
+	} else if (!quiet) {
+		_LOG_INFO("Applying control %s: %d", name, ctl.value);
+	}
+}

src/libs/controls.h

@@ -0,0 +1,60 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#pragma once
+
+#include "types.h"
+
+
+typedef enum {
+	US_CTL_MODE_NONE = 0,
+	US_CTL_MODE_VALUE,
+	US_CTL_MODE_AUTO,
+	US_CTL_MODE_DEFAULT,
+} us_control_mode_e;
+
+typedef struct {
+	us_control_mode_e	mode;
+	int					value;
+} us_control_s;
+
+typedef struct {
+	us_control_s	brightness;
+	us_control_s	contrast;
+	us_control_s	saturation;
+	us_control_s	hue;
+	us_control_s	gamma;
+	us_control_s	sharpness;
+	us_control_s	backlight_compensation;
+	us_control_s	white_balance;
+	us_control_s	gain;
+	us_control_s	color_effect;
+	us_control_s	rotate;
+	us_control_s	flip_vertical;
+	us_control_s	flip_horizontal;
+} us_controls_s;
+
+
+us_controls_s *us_controls_init(void);
+void us_controls_destroy(us_controls_s *ctl);
+
+void us_controls_apply(const us_controls_s *ctl, int fd);

src/libs/drm/drm.c

@@ -56,7 +56,7 @@ static int _drm_find_sink(us_drm_s *drm, uint width, uint height, float hz);
 
 static drmModeModeInfo *_find_best_mode(drmModeConnector *conn, uint width, uint height, float hz);
 static u32 _find_dpms(int fd, drmModeConnector *conn);
-static u32 _find_crtc(int fd, drmModeRes *res, drmModeConnector *conn, u32 *taken_crtcs);
+static u32 _find_crtc(int fd, const drmModeRes *res, drmModeConnector *conn, u32 *taken_crtcs);
 static const char *_connector_type_to_string(u32 type);
 static float _get_refresh_rate(const drmModeModeInfo *mode);
 
@@ -726,7 +726,7 @@ static u32 _find_dpms(int fd, drmModeConnector *conn) {
 	return 0;
 }
 
-static u32 _find_crtc(int fd, drmModeRes *res, drmModeConnector *conn, u32 *taken_crtcs) {
+static u32 _find_crtc(int fd, const drmModeRes *res, drmModeConnector *conn, u32 *taken_crtcs) {
 	for (int ei = 0; ei < conn->count_encoders; ++ei) {
 		drmModeEncoder *enc = drmModeGetEncoder(fd, conn->encoders[ei]);
 		if (enc == NULL) {

src/libs/frametext.c

@@ -112,7 +112,7 @@ void us_frametext_draw(us_frametext_s *ft, const char *text, uint width, uint he
 	}
 
 	char *str = us_strdup(text);
-	char *line;
+	const char *line;
 	char *rest;
 
 	uint block_width = 0;

src/libs/logging.h

@@ -34,6 +34,7 @@
 #include "types.h"
 #include "tools.h"
 #include "threading.h"
+#include "logging_base.h"
 
 
 enum us_log_level_t {
@@ -72,37 +73,7 @@ extern pthread_mutex_t us_g_log_mutex;
 #define US_COLOR_RESET		"\x1b[0m"
 
 
-#define US_SEP_INFO(x_ch) { \
-		US_LOGGING_LOCK; \
-		for (int m_i = 0; m_i < 80; ++m_i) { \
-			fputc((x_ch), stderr); \
-		} \
-		fputc('\n', stderr); \
-		fflush(stderr); \
-		US_LOGGING_UNLOCK; \
-	}
-
-#define US_SEP_DEBUG(x_ch) { \
-		if (us_g_log_level >= US_LOG_LEVEL_DEBUG) { \
-			US_SEP_INFO(x_ch); \
-		} \
-	}
-
-
-#define US_LOG_PRINTF_NOLOCK(x_label_color, x_label, x_msg_color, x_msg, ...) { \
-		char m_tname_buf[US_THREAD_NAME_SIZE] = {0}; \
-		us_thread_get_name(m_tname_buf); \
-		if (us_g_log_colored) { \
-			fprintf(stderr, US_COLOR_GRAY "-- " x_label_color x_label US_COLOR_GRAY \
-				" [%.03Lf %9s]" " -- " US_COLOR_RESET x_msg_color x_msg US_COLOR_RESET, \
-				us_get_now_monotonic(), m_tname_buf, ##__VA_ARGS__); \
-		} else { \
-			fprintf(stderr, "-- " x_label " [%.03Lf %9s] -- " x_msg, \
-				us_get_now_monotonic(), m_tname_buf, ##__VA_ARGS__); \
-		} \
-		fputc('\n', stderr); \
-		fflush(stderr); \
-	}
+// XXX: See the definition of US_LOG_PRINTF_NOLOCK() in logging_base.h
 
 #define US_LOG_PRINTF(x_label_color, x_label, x_msg_color, x_msg, ...) { \
 		US_LOGGING_LOCK; \
@@ -110,6 +81,10 @@ extern pthread_mutex_t us_g_log_mutex;
 		US_LOGGING_UNLOCK; \
 	}
 
+#define US_LOG_SEP_INFO { \
+		US_LOG_PRINTF(US_COLOR_GREEN, "INFO ", US_COLOR_GREEN, "============================================="); \
+	}
+
 #define US_LOG_ERROR(x_msg, ...) { \
 		US_LOG_PRINTF(US_COLOR_RED, "ERROR", US_COLOR_RED, x_msg, ##__VA_ARGS__); \
 	}
@@ -120,6 +95,11 @@ extern pthread_mutex_t us_g_log_mutex;
 		free(m_perror_str); \
 	}
 
+// We don't include alsa, speex and opus headers here
+#define US_LOG_PERROR_ALSA(x_err, x_msg, ...)	US_LOG_ERROR(x_msg ": %s", ##__VA_ARGS__, snd_strerror(x_err))
+#define US_LOG_PERROR_RES(x_err, x_msg, ...)	US_LOG_ERROR(x_msg ": %s", ##__VA_ARGS__, speex_resampler_strerror(x_err))
+#define US_LOG_PERROR_OPUS(x_err, x_msg, ...)	US_LOG_ERROR(x_msg ": %s", ##__VA_ARGS__, opus_strerror(x_err))
+
 #define US_LOG_INFO(x_msg, ...) { \
 		US_LOG_PRINTF(US_COLOR_GREEN, "INFO ", "", x_msg, ##__VA_ARGS__); \
 	}

src/libs/logging_base.h

@@ -20,24 +20,29 @@
 *****************************************************************************/
 
 
-#include <janus/plugins/plugin.h>
+#pragma once
 
-#include "uslibs/tools.h"
+#include <stdio.h>
+#include <stdlib.h>
 
-#include "const.h"
+#include <pthread.h>
+
+#include "types.h"
+#include "tools.h"
+#include "threading.h"
 
 
-#define US_JLOG_INFO(x_prefix, x_msg, ...)	JANUS_LOG(LOG_INFO, "== %s/%-9s -- " x_msg "\n", US_PLUGIN_NAME, x_prefix, ##__VA_ARGS__)
-#define US_JLOG_WARN(x_prefix, x_msg, ...)	JANUS_LOG(LOG_WARN, "== %s/%-9s -- " x_msg "\n", US_PLUGIN_NAME, x_prefix, ##__VA_ARGS__)
-#define US_JLOG_ERROR(x_prefix, x_msg, ...)	JANUS_LOG(LOG_ERR, "== %s/%-9s -- " x_msg "\n", US_PLUGIN_NAME, x_prefix, ##__VA_ARGS__)
-
-#define US_JLOG_PERROR(x_prefix, x_msg, ...) { \
-		char *const m_perror_str = us_errno_to_string(errno); \
-		JANUS_LOG(LOG_ERR, "[%s/%-9s] " x_msg ": %s\n", US_PLUGIN_NAME, x_prefix, ##__VA_ARGS__, m_perror_str); \
-		free(m_perror_str); \
+#define US_LOG_PRINTF_NOLOCK(x_label_color, x_label, x_msg_color, x_msg, ...) { \
+		char m_tname_buf[US_THREAD_NAME_SIZE] = {0}; \
+		us_thread_get_name(m_tname_buf); \
+		if (us_g_log_colored) { \
+			fprintf(stderr, US_COLOR_GRAY "-- " x_label_color x_label US_COLOR_GRAY \
+				" [%.03Lf %9s]" " -- " US_COLOR_RESET x_msg_color x_msg US_COLOR_RESET, \
+				us_get_now_monotonic(), m_tname_buf, ##__VA_ARGS__); \
+		} else { \
+			fprintf(stderr, "-- " x_label " [%.03Lf %9s] -- " x_msg, \
+				us_get_now_monotonic(), m_tname_buf, ##__VA_ARGS__); \
+		} \
+		fputc('\n', stderr); \
+		fflush(stderr); \
 	}
-
-// We don't include alsa, speex and opus headers here
-#define US_JLOG_PERROR_ALSA(_err, _prefix, _msg, ...)	US_JLOG_ERROR(_prefix, _msg ": %s", ##__VA_ARGS__, snd_strerror(_err))
-#define US_JLOG_PERROR_RES(_err, _prefix, _msg, ...)	US_JLOG_ERROR(_prefix, _msg ": %s", ##__VA_ARGS__, speex_resampler_strerror(_err))
-#define US_JLOG_PERROR_OPUS(_err, _prefix, _msg, ...)	US_JLOG_ERROR(_prefix, _msg ": %s", ##__VA_ARGS__, opus_strerror(_err))

src/libs/media.c

@@ -0,0 +1,215 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#include "media.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <sys/sysmacros.h>
+
+#include <linux/videodev2.h>
+#include <linux/media.h>
+#include <linux/v4l2-subdev.h>
+
+#include "types.h"
+#include "errors.h"
+#include "tools.h"
+#include "logging.h"
+#include "xioctl.h"
+
+
+#define _LOG_ERROR(x_msg, ...)	US_LOG_ERROR("CAP: " x_msg, ##__VA_ARGS__)
+#define _LOG_PERROR(x_msg, ...)	US_LOG_PERROR("CAP: " x_msg, ##__VA_ARGS__)
+
+
+int us_media_find_link(
+	struct media_v2_topology *topology,
+	u32 link_type,
+	int source_id,
+	int sink_id,
+	u32 *link_flags
+) {
+	struct media_v2_link *links = (struct media_v2_link *)topology->ptr_links;
+
+	for (uint i = 0; i < topology->num_links; ++i) {
+		if ((links[i].flags & MEDIA_LNK_FL_LINK_TYPE) != link_type) {
+			continue;
+		}
+		if (source_id >= 0 && (u32)source_id == links[i].source_id) {
+			if (link_flags) {
+				*link_flags = links[i].flags;
+			}
+			return links[i].sink_id;
+		}
+		if (sink_id >= 0 && (u32)sink_id == links[i].sink_id) {
+			if (link_flags) {
+				*link_flags = links[i].flags;
+			}
+			return links[i].source_id;
+		}
+	}
+	return -1;
+}
+
+int us_media_find_entity_by_name(
+	const struct media_v2_topology *topology,
+	const char *name
+) {
+	const struct media_v2_entity *entities = (struct media_v2_entity *)topology->ptr_entities;
+
+	for (uint i = 0; i < topology->num_entities; ++i) {
+		if (!strcmp(entities[i].name, name)) {
+			return entities[i].id;
+		}
+	}
+	return -1;
+}
+
+int us_media_find_entity_by_devnode(
+	struct media_v2_topology *topology,
+	const dev_t devnode
+) {
+	const struct media_v2_interface *interfaces = (struct media_v2_interface *)topology->ptr_interfaces;
+
+	for (uint i = 0; i < topology->num_interfaces; ++i) {
+		if (
+			interfaces[i].intf_type == MEDIA_INTF_T_V4L_VIDEO
+			&& interfaces[i].devnode.major == major(devnode)
+			&& interfaces[i].devnode.minor == minor(devnode)
+		) {
+			return us_media_find_link(
+				topology, MEDIA_LNK_FL_INTERFACE_LINK,
+				interfaces[i].id, -1, NULL);
+		}
+	}
+	return -1;
+}
+
+const struct media_v2_pad *us_media_find_pad_by_entity(
+	const struct media_v2_topology *topology,
+	u32 pad_type,
+	u32 entity_id
+) {
+	const struct media_v2_pad *pads = (const struct media_v2_pad *)topology->ptr_pads;
+
+	for (uint i = 0; i < topology->num_pads; ++i) {
+		if ((pads[i].flags & pad_type) && pads[i].entity_id == entity_id) {
+			return &pads[i];
+		}
+	}
+	return NULL;
+}
+
+const struct media_v2_pad *us_media_find_pad(
+	const struct media_v2_topology *topology,
+	u32 pad_type,
+	u32 pad_id
+) {
+	const struct media_v2_pad *pads = (const struct media_v2_pad *)topology->ptr_pads;
+
+	for (uint i = 0; i < topology->num_pads; ++i) {
+		if ((pads[i].flags & pad_type) && pads[i].id == pad_id) {
+			return &pads[i];
+		}
+	}
+	return NULL;
+}
+
+int us_media_xioctl_setup_link(
+	int fd,
+	const struct media_v2_pad *source_pad,
+	const struct media_v2_pad *sink_pad,
+	u32 link_flags
+) {
+	struct media_link_desc link_desc = {
+		.source = {
+			.entity = source_pad->entity_id,
+			.index = source_pad->index,
+		},
+		.sink = {
+			.entity = sink_pad->entity_id,
+			.index = sink_pad->index,
+		},
+		.flags = link_flags,
+	};
+	return us_xioctl(fd, MEDIA_IOC_SETUP_LINK, &link_desc);
+}
+
+struct media_v2_topology *us_media_topology_init(int fd) {
+	struct media_v2_topology *topology = NULL;
+	struct media_device_info dev_info = {0};
+
+	if (us_xioctl(fd, MEDIA_IOC_DEVICE_INFO, &dev_info) < 0) {
+		_LOG_PERROR("Can't to query media device info");
+		goto error;
+	}
+
+	if (!MEDIA_V2_PAD_HAS_INDEX(dev_info.media_version)) {
+		_LOG_ERROR("Media topology doesn't have pad indices, too old kernel?");
+		goto error;
+	}
+
+	US_CALLOC(topology, 1);
+
+	if (us_xioctl(fd, MEDIA_IOC_G_TOPOLOGY, topology) < 0) {
+		_LOG_PERROR("Can't query media topology info");
+		goto error;
+	}
+
+#	define INIT_FIELD(x_field, x_type) { \
+			void *m_ptr = calloc(topology->num_##x_field, sizeof(x_type)); \
+			US_A(m_ptr); \
+			topology->ptr_##x_field = (uintptr_t)m_ptr; \
+		}
+	INIT_FIELD(entities, struct media_v2_entity);
+	INIT_FIELD(links, struct media_v2_link);
+	INIT_FIELD(interfaces, struct media_v2_interface);
+	INIT_FIELD(pads, struct media_v2_pad);
+#	undef INIT_FIELD
+
+	if (us_xioctl(fd, MEDIA_IOC_G_TOPOLOGY, topology) < 0) {
+		_LOG_PERROR("Failed to retrieve topology info");
+		goto error;
+	}
+
+	return topology;
+
+error:
+	US_DELETE(topology, us_media_topology_destroy);
+	return NULL;
+}
+
+void us_media_topology_destroy(struct media_v2_topology *topology) {
+#	define DELETE_FIELD(x_field) { \
+			if (topology->ptr_##x_field) { \
+				free((void *)topology->ptr_##x_field); \
+			} \
+		}
+	DELETE_FIELD(pads);
+	DELETE_FIELD(interfaces);
+	DELETE_FIELD(links);
+	DELETE_FIELD(entities);
+#	undef DELETE_FIELD
+	free(topology);
+}

src/libs/media.h

@@ -0,0 +1,70 @@
+/*****************************************************************************
+#                                                                            #
+#    uStreamer - Lightweight and fast MJPEG-HTTP streamer.                   #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <mdevaev@gmail.com>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+#pragma once
+
+#include <linux/videodev2.h>
+#include <linux/media.h>
+#include <linux/v4l2-subdev.h>
+
+#include "types.h"
+
+
+typedef struct {
+	int		fd;
+	uint	pad;
+} us_media_pad_s;
+
+
+int us_media_find_link(
+	struct media_v2_topology *topology,
+	u32 link_type,
+	int source_id,
+	int sink_id,
+	u32 *link_flags);
+
+int us_media_find_entity_by_name(
+	const struct media_v2_topology *topology,
+	const char *name);
+
+int us_media_find_entity_by_devnode(
+	struct media_v2_topology *topology,
+	const dev_t devnode);
+
+const struct media_v2_pad *us_media_find_pad_by_entity(
+	const struct media_v2_topology *topology,
+	u32 pad_type,
+	u32 entity_id);
+
+const struct media_v2_pad *us_media_find_pad(
+	const struct media_v2_topology *topology,
+	u32 pad_type,
+	u32 pad_id);
+
+int us_media_xioctl_setup_link(
+	int fd,
+	const struct media_v2_pad *source_pad,
+	const struct media_v2_pad *sink_pad,
+	u32 link_flags);
+
+struct media_v2_topology *us_media_topology_init(int fd);
+void us_media_topology_destroy(struct media_v2_topology *topology);

src/libs/memsink.c

@@ -41,8 +41,8 @@
 
 us_memsink_s *us_memsink_init_opened(
 	const char *name, const char *obj, bool server,
-	mode_t mode, bool rm, uint client_ttl, uint timeout) {
-
+	mode_t mode, bool rm, uint client_ttl, uint timeout
+) {
 	us_memsink_s *sink;
 	US_CALLOC(sink, 1);
 	sink->name = name;
@@ -56,7 +56,7 @@ us_memsink_s *us_memsink_init_opened(
 
 	US_LOG_INFO("Using %s-sink: %s", name, obj);
 
-	if ((sink->data_size = us_memsink_calculate_size(obj)) == 0) {
+	if ((sink->data_size = us_memsinksh_calculate_size(obj)) == 0) {
 		US_LOG_ERROR("%s-sink: Invalid object suffix", name);
 		goto error;
 	}
@@ -75,7 +75,7 @@ us_memsink_s *us_memsink_init_opened(
 		goto error;
 	}
 
-	if ((sink->mem = us_memsink_shared_map(sink->fd, sink->data_size)) == NULL) {
+	if ((sink->mem = us_memsinksh_map(sink->fd, sink->data_size)) == NULL) {
 		US_LOG_PERROR("%s-sink: Can't mmap shared memory", name);
 		goto error;
 	}
@@ -88,7 +88,7 @@ error:
 
 void us_memsink_destroy(us_memsink_s *sink) {
 	if (sink->mem != NULL) {
-		if (us_memsink_shared_unmap(sink->mem, sink->data_size) < 0) {
+		if (us_memsinksh_unmap(sink->mem, sink->data_size) < 0) {
 			US_LOG_PERROR("%s-sink: Can't unmap shared memory", sink->name);
 		}
 	}
@@ -143,11 +143,10 @@ bool us_memsink_server_check(us_memsink_s *sink, const us_frame_s *frame) {
 	}
 
 	// Проверяем, есть ли у нас живой клиент по таймауту
-	const bool has_clients = (sink->mem->last_client_ts + sink->client_ttl > us_get_now_monotonic());
+	const bool has_clients = us_memsinksh_has_clients(sink->mem, sink->client_ttl);
 	atomic_store(&sink->has_clients, has_clients);
 
-	if (flock(sink->fd, LOCK_UN) < 0) {
-		US_LOG_PERROR("%s-sink: Can't unlock memory", sink->name);
+	if (us_memsink_server_x_unlock(sink) < 0) {
 		return false;
 	}
 	if (has_clients) {
@@ -160,43 +159,37 @@ bool us_memsink_server_check(us_memsink_s *sink, const us_frame_s *frame) {
 	return false;
 }
 
-int us_memsink_server_put(us_memsink_s *sink, const us_frame_s *frame, bool *key_requested) {
+int us_memsink_server_put(
+	us_memsink_s *sink,
+	const us_frame_s *frame,
+	us_memsink_wants_s *w_get
+) {
 	US_A(sink->server);
 
 	const ldf now = us_get_now_monotonic();
 
-	if (frame->used > sink->data_size) {
-		US_LOG_ERROR("%s-sink: Can't put frame: is too big (%zu > %zu)",
-			sink->name, frame->used, sink->data_size);
-		return 0;
-	}
-
 	if (us_flock_timedwait_monotonic(sink->fd, 1) == 0) {
+		const bool has_clients = us_memsinksh_has_clients(sink->mem, sink->client_ttl);
+		atomic_store(&sink->has_clients, has_clients);
+
 		US_LOG_VERBOSE("%s-sink: >>>>> Exposing new frame ...", sink->name);
+		const bool exposed = !us_memsink_server_x_put(sink, frame);
 
-		sink->mem->id = us_get_now_id();
-		if (sink->mem->key_requested && frame->key) {
-			sink->mem->key_requested = false;
-		}
-		if (key_requested != NULL) { // We don't need it for non-H264 sinks
-			*key_requested = sink->mem->key_requested;
+		if (w_get != NULL) {
+			if (has_clients) {
+				memcpy(w_get, &sink->mem->wants, sizeof(us_memsink_wants_s));
+			} else {
+				US_MEMSET_ZERO(*w_get);
+			}
 		}
 
-		memcpy(us_memsink_get_data(sink->mem), frame->data, frame->used);
-		sink->mem->used = frame->used;
-		US_FRAME_COPY_META(frame, sink->mem);
-
-		sink->mem->magic = US_MEMSINK_MAGIC;
-		sink->mem->version = US_MEMSINK_VERSION;
-
-		atomic_store(&sink->has_clients, (sink->mem->last_client_ts + sink->client_ttl > us_get_now_monotonic()));
-
-		if (flock(sink->fd, LOCK_UN) < 0) {
-			US_LOG_PERROR("%s-sink: Can't unlock memory", sink->name);
+		if (us_memsink_server_x_unlock(sink) < 0) {
 			return -1;
 		}
-		US_LOG_VERBOSE("%s-sink: Exposed new frame; full exposition time = %.3Lf",
-			sink->name, us_get_now_monotonic() - now);
+		if (exposed) {
+			US_LOG_VERBOSE("%s-sink: Exposed new frame; full exposition time = %.3Lf",
+				sink->name, us_get_now_monotonic() - now);
+		}
 
 	} else if (errno == EWOULDBLOCK) {
 		US_LOG_VERBOSE("%s-sink: ===== Shared memory is busy now; frame skipped", sink->name);
@@ -208,7 +201,12 @@ int us_memsink_server_put(us_memsink_s *sink, const us_frame_s *frame, bool *key
 	return 0;
 }
 
-int us_memsink_client_get(us_memsink_s *sink, us_frame_s *frame, bool *key_requested, bool key_required) {
+int us_memsink_client_get(
+	us_memsink_s *sink,
+	us_frame_s *frame,
+	us_memsink_wants_s *w_get,
+	const us_memsink_wants_s *w_put
+) {
 	US_A(!sink->server); // Client only
 
 	if (us_flock_timedwait_monotonic(sink->fd, sink->timeout) < 0) {
@@ -232,28 +230,121 @@ int us_memsink_client_get(us_memsink_s *sink, us_frame_s *frame, bool *key_reque
 		goto done;
 	}
 
+	const bool had_client_magic = (sink->mem->client_magic == US_MEMSINK_MAGIC);
 	// Let the sink know that the client is alive
 	sink->mem->last_client_ts = us_get_now_monotonic();
+	sink->mem->client_magic = US_MEMSINK_MAGIC;
 
-	if (sink->mem->id == sink->last_readed_id) {
-		retval = US_ERROR_NO_DATA; // Not updated
+	if (sink->mem->used == 0 || sink->mem->id == sink->last_readed_id) {
+		// ^ Zero frame     |OR| ^ Not updated
+		retval = US_ERROR_NO_DATA;
 		goto done;
 	}
 
 	sink->last_readed_id = sink->mem->id;
-	us_frame_set_data(frame, us_memsink_get_data(sink->mem), sink->mem->used);
+	us_frame_set_data(frame, us_memsinksh_get_data(sink->mem), sink->mem->used);
 	US_FRAME_COPY_META(sink->mem, frame);
-	if (key_requested != NULL) { // We don't need it for non-H264 sinks
-		*key_requested = sink->mem->key_requested;
+
+	if (w_get != NULL) {
+		if (had_client_magic) { // Чтобы не прочитать мусор, если не было клиентов
+			memcpy(w_get, &sink->mem->wants, sizeof(us_memsink_wants_s));
+		} else {
+			US_MEMSET_ZERO(*w_get);
+		}
 	}
-	if (key_required) {
-		sink->mem->key_requested = true;
+
+	if (w_put != NULL) {
+		const bool key = sink->mem->wants.key;
+		memcpy(&sink->mem->wants, w_put, sizeof(us_memsink_wants_s));
+		if (key) {
+			sink->mem->wants.key = key;
+		}
+	} else {
+		US_MEMSET_ZERO(sink->mem->wants);
 	}
 
 done:
-	if (flock(sink->fd, LOCK_UN) < 0) {
-		US_LOG_PERROR("%s-sink: Can't unlock memory", sink->name);
+	if (us_memsink_server_x_unlock(sink) < 0) {
 		retval = -1;
 	}
 	return retval;
 }
+
+us_mss_lock_result_e us_memsink_server_x_lock(
+	us_memsink_s *sink,
+	us_memsink_wants_s *w_get
+) {
+	US_A(sink->server);
+
+	if (us_flock_timedwait_monotonic(sink->fd, 1) < 0) {
+		if (errno == EWOULDBLOCK) {
+			return US_MSS_BUSY;
+		}
+		US_LOG_PERROR("%s-sink: Can't lock memory", sink->name);
+		return US_MSS_ERROR;
+	}
+
+	if (sink->mem->magic == US_MEMSINK_MAGIC && sink->mem->version == US_MEMSINK_VERSION) {
+		// Если инициализировано - значит клиенту что-то уже могли написать
+		const bool has_clients = us_memsinksh_has_clients(sink->mem, sink->client_ttl);
+		atomic_store(&sink->has_clients, has_clients);
+		if (has_clients) {
+			if (w_get != NULL) {
+				memcpy(w_get, &sink->mem->wants, sizeof(us_memsink_wants_s));
+			}
+			// sink->unsafe_last_client_ts = sink->mem->last_client_ts; // _x_ functions don't use it
+			return US_MSS_SUCCESS;
+		}
+	} else {
+		// А иначе скормим клиентам инициализацию с пустым фреймом,
+		// в следующий раз прочтем от них что-то путное.
+		us_memsink_server_x_put(sink, NULL);
+	}
+
+	if (us_memsink_server_x_unlock(sink) < 0) {
+		return US_MSS_ERROR;
+	}
+	return US_MSS_NO_CLIENT;
+}
+
+bool us_memsink_server_x_is_consumed(us_memsink_s *sink) {
+	return (sink->mem->used == 0);
+}
+
+void us_memsink_server_x_set_consumed(us_memsink_s *sink) {
+	sink->mem->used = 0;
+}
+
+int us_memsink_server_x_put(us_memsink_s *sink, const us_frame_s *frame) {
+	if (frame != NULL) {
+		if (frame->used > sink->data_size) {
+			US_LOG_ERROR("%s-sink: Can't put frame: it's too big (%zu > %zu)",
+				sink->name, frame->used, sink->data_size);
+			return -1;
+		}
+
+		memcpy(us_memsinksh_get_data(sink->mem), frame->data, frame->used);
+		sink->mem->used = frame->used;
+		US_FRAME_COPY_META(frame, sink->mem);
+
+		if (sink->mem->wants.key && frame->key) {
+			// Можем писать в мусор, но пофигу
+			sink->mem->wants.key = false;
+		}
+	} else {
+		sink->mem->used = 0;
+	}
+
+	sink->mem->id = us_get_now_id();
+	sink->mem->version = US_MEMSINK_VERSION;
+	sink->mem->magic = US_MEMSINK_MAGIC;
+	return 0;
+}
+
+int us_memsink_server_x_unlock(us_memsink_s *sink) {
+	if (flock(sink->fd, LOCK_UN) < 0) {
+		US_LOG_PERROR("%s-sink: Can't unlock memory", sink->name);
+		return -1;
+	}
+	return 0;
+}

src/libs/memsink.h

@@ -57,6 +57,36 @@ us_memsink_s *us_memsink_init_opened(
 void us_memsink_destroy(us_memsink_s *sink);
 
 bool us_memsink_server_check(us_memsink_s *sink, const us_frame_s *frame);
-int us_memsink_server_put(us_memsink_s *sink, const us_frame_s *frame, bool *key_requested);
 
-int us_memsink_client_get(us_memsink_s *sink, us_frame_s *frame, bool *key_requested, bool key_required);
+int us_memsink_server_put(
+	us_memsink_s *sink,
+	const us_frame_s *frame,
+	us_memsink_wants_s *w_get);
+
+int us_memsink_client_get(
+	us_memsink_s *sink,
+	us_frame_s *frame,
+	us_memsink_wants_s *w_get,
+	const us_memsink_wants_s *w_put);
+
+
+// Low-level
+
+typedef enum {
+	US_MSS_SUCCESS = 0,
+	US_MSS_ERROR = -1,
+	US_MSS_BUSY = -2,
+	US_MSS_NO_CLIENT = -3,
+} us_mss_lock_result_e;
+
+
+us_mss_lock_result_e us_memsink_server_x_lock(
+	us_memsink_s *sink,
+	us_memsink_wants_s *w_get);
+
+bool us_memsink_server_x_is_consumed(us_memsink_s *sink);
+void us_memsink_server_x_set_consumed(us_memsink_s *sink);
+
+int us_memsink_server_x_put(us_memsink_s *sink, const us_frame_s *frame);
+
+int us_memsink_server_x_unlock(us_memsink_s *sink);

src/libs/memsinksh.c

@@ -31,7 +31,7 @@
 #include "tools.h"
 
 
-us_memsink_shared_s *us_memsink_shared_map(int fd, uz data_size) {
+us_memsink_shared_s *us_memsinksh_map(int fd, uz data_size) {
 	us_memsink_shared_s *mem = mmap(
 		NULL,
 		sizeof(us_memsink_shared_s) + data_size,
@@ -44,12 +44,12 @@ us_memsink_shared_s *us_memsink_shared_map(int fd, uz data_size) {
 	return mem;
 }
 
-int us_memsink_shared_unmap(us_memsink_shared_s *mem, uz data_size) {
+int us_memsinksh_unmap(us_memsink_shared_s *mem, uz data_size) {
 	US_A(mem != NULL);
 	return munmap(mem, sizeof(us_memsink_shared_s) + data_size);
 }
 
-uz us_memsink_calculate_size(const char *obj) {
+uz us_memsinksh_calculate_size(const char *obj) {
 	const char *ptr = strrchr(obj, ':');
 	if (ptr == NULL) {
 		ptr = strrchr(obj, '.');
@@ -61,12 +61,19 @@ uz us_memsink_calculate_size(const char *obj) {
 		} else if (!strcasecmp(ptr, "h264")) {
 			return 2 * 1024 * 1024;
 		} else if (!strcasecmp(ptr, "raw")) {
-			return 1920 * 1200 * 3; // RGB
+			return 3840 * 2160 * 3; // RGB
 		}
 	}
 	return 0;
 }
 
-u8 *us_memsink_get_data(us_memsink_shared_s *mem) {
+u8 *us_memsinksh_get_data(us_memsink_shared_s *mem) {
 	return (u8*)(mem) + sizeof(us_memsink_shared_s);
 }
+
+bool us_memsinksh_has_clients(const us_memsink_shared_s *mem, u32 ttl) {
+	return (
+		mem->client_magic == US_MEMSINK_MAGIC
+		&& mem->last_client_ts + ttl > us_get_now_monotonic()
+	);
+}

src/libs/memsinksh.h

@@ -27,24 +27,34 @@
 
 
 #define US_MEMSINK_MAGIC	((u64)0xCAFEBABECAFEBABE)
-#define US_MEMSINK_VERSION	((u32)7)
+#define US_MEMSINK_VERSION	((u32)11)
 
 
+typedef struct {
+	uint	width;
+	uint	height;
+	uint	format;
+	uint	fps;
+	bool	key;
+} us_memsink_wants_s;
+
 typedef struct {
 	u64		magic;
 	u32		version;
 	u64		id;
 	uz		used;
 
+	u64		client_magic;
 	ldf		last_client_ts;
-	bool	key_requested;
+	us_memsink_wants_s	wants;
 
 	US_FRAME_META_DECLARE;
 } us_memsink_shared_s;
 
 
-us_memsink_shared_s *us_memsink_shared_map(int fd, uz data_size);
-int us_memsink_shared_unmap(us_memsink_shared_s *mem, uz data_size);
+us_memsink_shared_s *us_memsinksh_map(int fd, uz data_size);
+int us_memsinksh_unmap(us_memsink_shared_s *mem, uz data_size);
 
-uz us_memsink_calculate_size(const char *obj);
-u8 *us_memsink_get_data(us_memsink_shared_s *mem);
+uz us_memsinksh_calculate_size(const char *obj);
+u8 *us_memsinksh_get_data(us_memsink_shared_s *mem);
+bool us_memsinksh_has_clients(const us_memsink_shared_s *mem, u32 ttl);

src/libs/threading.h

@@ -94,13 +94,13 @@ INLINE void us_thread_get_name(char *name) { // Always required for logging
 #ifdef WITH_PTHREAD_NP
 	int retval = -1;
 #	if defined(__linux__) || defined (__NetBSD__)
-	retval = pthread_getname_np(pthread_self(), name, US_THREAD_NAME_SIZE - 1);
+	retval = pthread_getname_np(pthread_self(), name, US_THREAD_NAME_SIZE); // Always null-terminated
 #	elif \
 		(defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1103500) \
 		|| (defined(__OpenBSD__) && defined(OpenBSD) && OpenBSD >= 201905) \
 		|| defined(__DragonFly__)
-	pthread_get_name_np(pthread_self(), name, US_THREAD_NAME_SIZE - 1);
-	if (name[0] != '\0') {
+	pthread_get_name_np(pthread_self(), name, US_THREAD_NAME_SIZE); // Also null-terminated
+	if (us_str_is_ok(name)) {
 		retval = 0;
 	}
 #	else

src/libs/tools.h

@@ -92,6 +92,10 @@ INLINE char *us_strdup(const char *str) {
 	return new;
 }
 
+INLINE bool us_str_is_ok(const char *str) {
+	return (str != NULL && str[0] != '\0');
+}
+
 INLINE const char *us_bool_to_string(bool flag) {
 	return (flag ? "true" : "false");
 }

src/ustreamer/encoders/cpu/encoder.c

@@ -216,17 +216,18 @@ static void _jpeg_write_scanlines_yuv_planar(struct jpeg_compress_struct *jpeg,
 	US_CALLOC(line_buf, frame->width * 3);
 
 	const uint padding = us_frame_get_padding(frame);
-	const uint image_size = frame->width * frame->height;
-	const uint chroma_array_size = (frame->used - image_size) / 2;
-	const uint chroma_matrix_order = (image_size / chroma_array_size) == 16 ? 4 : 2;
+	const uint luma_array_size = (frame->width + padding) * frame->height;
+	const uint chroma_array_size = (frame->used - luma_array_size) >> 1;
 	const u8 *data = frame->data;
-	const u8 *chroma1_data = frame->data + image_size;
-	const u8 *chroma2_data = frame->data + image_size + chroma_array_size;
-
-	//US_LOG_DEBUG("Planar data: Image Size %u, Chroma Array Size %u, Chroma Matrix Order %u",
-	//	image_size, chroma_array_size, chroma_matrix_order);
+	const u8 *chroma1_data = frame->data + luma_array_size;
+	const u8 *chroma2_data = frame->data + luma_array_size + chroma_array_size;
 
 	while (jpeg->next_scanline < frame->height) {
+		if (jpeg->next_scanline > 0 && jpeg->next_scanline % 2 == 0) {
+			chroma1_data += (frame->width + padding) >> 1;
+			chroma2_data += (frame->width + padding) >> 1;
+		}
+
 		u8 *ptr = line_buf;
 
 		for (uint x = 0; x < frame->width; ++x) {
@@ -234,7 +235,7 @@ static void _jpeg_write_scanlines_yuv_planar(struct jpeg_compress_struct *jpeg,
 			u8 y = data[x];
 			u8 u;
 			u8 v;
-			uint chroma_position = x / chroma_matrix_order;
+			uint chroma_position = x >> 1;
 
 			switch (frame->format) {
 				case V4L2_PIX_FMT_YUV420:
@@ -257,11 +258,6 @@ static void _jpeg_write_scanlines_yuv_planar(struct jpeg_compress_struct *jpeg,
 
 		data += frame->width + padding;
 
-		if (jpeg->next_scanline > 0 && jpeg->next_scanline % chroma_matrix_order == 0) {
-			chroma1_data += (frame->width + padding) / chroma_matrix_order;
-			chroma2_data += (frame->width + padding) / chroma_matrix_order;
-		}
-
 		JSAMPROW scanlines[1] = {line_buf};
 		jpeg_write_scanlines(jpeg, scanlines, 1);
 	}

src/ustreamer/http/server.c

@@ -183,7 +183,7 @@ int us_server_listen(us_server_s *server) {
 	us_stream_s *const stream = server->stream;
 
 	{
-		if (server->static_path[0] != '\0') {
+		if (us_str_is_ok(server->static_path)) {
 			_LOG_INFO("Enabling the file server: %s", server->static_path);
 			evhttp_set_gencb(run->http, _http_callback_static, (void*)server);
 		} else {
@@ -202,7 +202,7 @@ int us_server_listen(us_server_s *server) {
 
 	evhttp_set_timeout(run->http, server->timeout);
 
-	if (server->user[0] != '\0') {
+	if (us_str_is_ok(server->user)) {
 		char *encoded_token = NULL;
 
 		char *raw_token;
@@ -216,7 +216,7 @@ int us_server_listen(us_server_s *server) {
 		_LOG_INFO("Using HTTP basic auth");
 	}
 
-	if (server->unix_path[0] != '\0') {
+	if (us_str_is_ok(server->unix_path)) {
 		_LOG_DEBUG("Binding server to UNIX socket '%s' ...", server->unix_path);
 		if ((run->ext_fd = us_evhttp_bind_unix(
 			run->http,
@@ -264,7 +264,7 @@ static int _http_preprocess_request(struct evhttp_request *req, us_server_s *ser
 
 	atomic_store(&server->stream->run->http->last_req_ts, us_get_now_monotonic());
 
-	if (server->allow_origin[0] != '\0') {
+	if (us_str_is_ok(server->allow_origin)) {
 		const char *const cors_headers = us_evhttp_get_header(req, "Access-Control-Request-Headers");
 		const char *const cors_method = us_evhttp_get_header(req, "Access-Control-Request-Method");
 
@@ -666,7 +666,7 @@ static void _http_callback_stream_write(struct bufferevent *buf_event, void *v_c
 	if (client->need_initial) {
 		_A_EVBUFFER_ADD_PRINTF(buf, "HTTP/1.0 200 OK" RN);
 		
-		if (client->server->allow_origin[0] != '\0') {
+		if (us_str_is_ok(client->server->allow_origin)) {
 			const char *const cors_headers = us_evhttp_get_header(client->req, "Access-Control-Request-Headers");
 			const char *const cors_method = us_evhttp_get_header(client->req, "Access-Control-Request-Method");
 
@@ -693,8 +693,8 @@ static void _http_callback_stream_write(struct bufferevent *buf_event, void *v_c
 			"Content-Type: multipart/x-mixed-replace;boundary=" BOUNDARY RN
 			RN
 			"--" BOUNDARY RN,
-			(server->instance_id[0] == '\0' ? "" : "_"),
-			server->instance_id,
+			(us_str_is_ok(server->instance_id) ? "_" : ""),
+			(us_str_is_ok(server->instance_id) ? server->instance_id : ""),
 			(client->key != NULL ? client->key : "0"),
 			client->id);
 

src/ustreamer/http/static.c

@@ -39,7 +39,7 @@ char *us_find_static_file_path(const char *root_path, const char *req_path) {
 	char *path = NULL;
 
 	char *const simplified_path = us_simplify_request_path(req_path);
-	if (simplified_path[0] == '\0') {
+	if (!us_str_is_ok(simplified_path)) {
 		US_LOG_VERBOSE("HTTP: Invalid request path %s to static", req_path);
 		goto error;
 	}

src/ustreamer/options.c

@@ -93,6 +93,8 @@ enum _US_OPT_VALUES {
 	_O_DEVICE_ERROR_DELAY,
 	_O_FORMAT_SWAP_RGB,
 	_O_M2M_DEVICE,
+	_O_MEDIA_DEVICE,
+	_O_MEDIA_ENTITY_NAME,
 
 	_O_IMAGE_DEFAULT,
 	_O_BRIGHTNESS,
@@ -188,6 +190,8 @@ static const struct option _LONG_OPTS[] = {
 	{"device-timeout",			required_argument,	NULL,	_O_DEVICE_TIMEOUT},
 	{"device-error-delay",		required_argument,	NULL,	_O_DEVICE_ERROR_DELAY},
 	{"m2m-device",				required_argument,	NULL,	_O_M2M_DEVICE},
+	{"media-device",			required_argument,	NULL,	_O_MEDIA_DEVICE},
+	{"media-entity-name",		required_argument,	NULL,	_O_MEDIA_ENTITY_NAME},
 
 	{"image-default",			no_argument,		NULL,	_O_IMAGE_DEFAULT},
 	{"brightness",				required_argument,	NULL,	_O_BRIGHTNESS},
@@ -300,10 +304,11 @@ us_options_s *us_options_init(uint argc, char *argv[]) {
 	opts->argc = argc;
 	opts->argv = argv;
 
-	US_CALLOC(opts->argv_copy, argc);
+	US_CALLOC(opts->argv_copy, argc + 1);
 	for (uint i = 0; i < argc; ++i) {
 		opts->argv_copy[i] = us_strdup(argv[i]);
 	}
+	opts->argv_copy[argc] = NULL;
 	return opts;
 }
 
@@ -374,15 +379,15 @@ int us_options_parse(
 		}
 
 #	define OPT_CTL_DEFAULT_NOBREAK(x_dest) { \
-			cap->ctl.x_dest.mode = CTL_MODE_DEFAULT; \
+			cap->ctl->x_dest.mode = US_CTL_MODE_DEFAULT; \
 		}
 
 #	define OPT_CTL_MANUAL(x_dest) { \
 			if (!strcasecmp(optarg, "default")) { \
 				OPT_CTL_DEFAULT_NOBREAK(x_dest); \
 			} else { \
-				cap->ctl.x_dest.mode = CTL_MODE_VALUE; \
-				OPT_NUMBER("--"#x_dest, cap->ctl.x_dest.value, INT_MIN, INT_MAX, 0); \
+				cap->ctl->x_dest.mode = US_CTL_MODE_VALUE; \
+				OPT_NUMBER("--"#x_dest, cap->ctl->x_dest.value, INT_MIN, INT_MAX, 0); \
 			} \
 			break; \
 		}
@@ -391,10 +396,10 @@ int us_options_parse(
 			if (!strcasecmp(optarg, "default")) { \
 				OPT_CTL_DEFAULT_NOBREAK(x_dest); \
 			} else if (!strcasecmp(optarg, "auto")) { \
-				cap->ctl.x_dest.mode = CTL_MODE_AUTO; \
+				cap->ctl->x_dest.mode = US_CTL_MODE_AUTO; \
 			} else { \
-				cap->ctl.x_dest.mode = CTL_MODE_VALUE; \
-				OPT_NUMBER("--"#x_dest, cap->ctl.x_dest.value, INT_MIN, INT_MAX, 0); \
+				cap->ctl->x_dest.mode = US_CTL_MODE_VALUE; \
+				OPT_NUMBER("--"#x_dest, cap->ctl->x_dest.value, INT_MIN, INT_MAX, 0); \
 			} \
 			break; \
 		}
@@ -445,6 +450,8 @@ int us_options_parse(
 			case _O_DEVICE_TIMEOUT:		OPT_NUMBER("--device-timeout", cap->timeout, 1, 60, 0);
 			case _O_DEVICE_ERROR_DELAY:	OPT_NUMBER("--device-error-delay", stream->error_delay, 1, 60, 0);
 			case _O_M2M_DEVICE:			OPT_SET(enc->m2m_path, optarg);
+			case _O_MEDIA_DEVICE:		OPT_SET(cap->media_path, optarg);
+			case _O_MEDIA_ENTITY_NAME:	OPT_SET(cap->media_entity_name, optarg);
 
 			case _O_IMAGE_DEFAULT:
 				OPT_CTL_DEFAULT_NOBREAK(brightness);
@@ -562,7 +569,7 @@ int us_options_parse(
 	US_LOG_INFO("Starting PiKVM uStreamer %s ...", US_VERSION);
 
 #	define ADD_SINK(x_label, x_prefix) { \
-			if (x_prefix##_name && x_prefix##_name[0] != '\0') { \
+			if (us_str_is_ok(x_prefix##_name)) { \
 				opts->x_prefix = us_memsink_init_opened( \
 					x_label, \
 					x_prefix##_name, \
@@ -704,7 +711,7 @@ static void _help(
 	SAY("                                           Changing of this parameter may increase the performance. Or not.");
 	SAY("                                           Available: %s; default: MMAP.\n", US_IO_METHODS_STR);
 	SAY("    -f|--desired-fps <N>  ──────────────── Desired FPS. Default: maximum possible.\n");
-	SAY("    -z|--min-frame-size <N>  ───────────── Drop frames smaller then this limit. Useful if the device");
+	SAY("    -z|--min-frame-size <N>  ───────────── Drop frames smaller than this limit. Useful if the device");
 	SAY("                                           produces small-sized garbage frames. Default: %zu bytes.\n", cap->min_frame_size);
 	SAY("    -T|--allow-truncated-frames  ───────── Allows to handle truncated frames. Useful if the device");
 	SAY("                                           produces incorrect but still acceptable frames. Default: disabled.\n");
@@ -712,7 +719,7 @@ static void _help(
 	SAY("    -t|--dv-timings  ───────────────────── Enable DV-timings querying and events processing");
 	SAY("                                           to automatic resolution change. Default: disabled.\n");
 	SAY("    -b|--buffers <N>  ──────────────────── The number of buffers to receive data from the device.");
-	SAY("                                           Each buffer may processed using an independent thread.");
+	SAY("                                           Each buffer may be processed using an independent thread.");
 	SAY("                                           Default: %u (the number of CPU cores (but not more than 4) + 1).\n", cap->n_bufs);
 	SAY("    -w|--workers <N>  ──────────────────── The number of worker threads but not more than buffers.");
 	SAY("                                           Default: %u (the number of CPU cores (but not more than 4)).\n", enc->n_workers);
@@ -737,6 +744,10 @@ static void _help(
 	SAY("    --device-error-delay <sec>  ────────── Delay before trying to connect to the device again");
 	SAY("                                           after an error (timeout for example). Default: %u.\n", stream->error_delay);
 	SAY("    --m2m-device </dev/path>  ──────────── Path to V4L2 M2M encoder device. Default: auto select.\n");
+	SAY("    --media-device </dev/path>  ────────── Path to V4L2 /dev/media* device for setting subdevices");
+	SAY("                                           (currently necessary for RPi5). Default: unset.\n");
+	SAY("    --media-entity-name <name>  ────────── Name of the V4L2 entity to grab video from,");
+	SAY("                                           will be routed to --device. Default: none.\n");
 	SAY("Image control options:");
 	SAY("══════════════════════");
 	SAY("    --image-default  ────────────────────── Reset all image settings below to default. Default: no change.\n");

src/ustreamer/stream.c

@@ -272,7 +272,7 @@ void us_stream_loop(us_stream_s *stream) {
 		us_capture_close(cap);
 
 		if (!atomic_load(&run->stop)) {
-			US_SEP_INFO('=');
+			US_LOG_SEP_INFO;
 		}
 	}
 
@@ -758,7 +758,9 @@ static void _stream_encode_expose_h264(us_stream_s *stream, const us_frame_s *fr
 		force_key = true;
 	}
 	if (!us_m2m_encoder_compress(run->h264_enc, frame, run->h264_dest, force_key)) {
-		meta.online = !us_memsink_server_put(stream->h264_sink, run->h264_dest, &run->h264_key_requested);
+		us_memsink_wants_s wants = {0};
+		meta.online = !us_memsink_server_put(stream->h264_sink, run->h264_dest, &wants);
+		run->h264_key_requested = wants.key;
 	}
 
 done:
@@ -770,7 +772,7 @@ static void _stream_check_suicide(us_stream_s *stream) {
 		return;
 	}
 
-	const atomic_ullong *last_req_ts = &stream->run->http->last_req_ts;
+	atomic_ullong *last_req_ts = &stream->run->http->last_req_ts;
 	const ldf now_ts = us_get_now_monotonic();
 
 	if (_stream_has_any_clients_cached(stream)) {