diff --git a/src/ustreamer/h264/encoder.c b/src/ustreamer/h264/encoder.c
index d280052..546b953 100644
--- a/src/ustreamer/h264/encoder.c
+++ b/src/ustreamer/h264/encoder.c
@@ -46,7 +46,6 @@ h264_encoder_s *h264_encoder_init(unsigned bitrate, unsigned gop, unsigned fps)
 	enc->gop = gop; // Interval between keyframes
 	enc->fps = fps;
 
-	enc->tmp = frame_init("h264_tmp");
 	enc->last_online = -1;
 
 	if (vcos_semaphore_create(&enc->handler_sem, "h264_handler_sem", 0) != VCOS_SUCCESS) {
@@ -87,7 +86,6 @@ void h264_encoder_destroy(h264_encoder_s *enc) {
 		vcos_semaphore_delete(&enc->handler_sem);
 	}
 
-	frame_destroy(enc->tmp);
 	free(enc);
 }
 
@@ -154,8 +152,6 @@ int h264_encoder_prepare(h264_encoder_s *enc, const frame_s *frame) {
 			case V4L2_PIX_FMT_YUYV:		IFMT(encoding) = MMAL_ENCODING_YUYV; break;
 			case V4L2_PIX_FMT_UYVY:		IFMT(encoding) = MMAL_ENCODING_UYVY; break;
 			case V4L2_PIX_FMT_RGB565:	IFMT(encoding) = MMAL_ENCODING_RGB16; break;
-			case V4L2_PIX_FMT_MJPEG:
-			case V4L2_PIX_FMT_JPEG:		// See unjpeg.c
 			case V4L2_PIX_FMT_RGB24:	IFMT(encoding) = MMAL_ENCODING_RGB24; break;
 			default: assert(0 && "Unsupported pixelformat");
 		}
@@ -260,32 +256,17 @@ static void _h264_encoder_cleanup(h264_encoder_s *enc) {
 
 int h264_encoder_compress(h264_encoder_s *enc, const frame_s *src, frame_s *dest) {
 	assert(enc->ready);
+	assert(src->used > 0);
+	assert(src->width == enc->width);
+	assert(src->height == enc->height);
+	assert(src->format == enc->format);
+	assert(src->stride == enc->stride);
 
 	frame_copy_meta(src, dest);
 	dest->encode_begin_ts = get_now_monotonic();
 	dest->format = V4L2_PIX_FMT_H264;
 	dest->stride = 0;
 
-	if (is_jpeg(src->format)) {
-		LOG_DEBUG("H264: Input frame format is JPEG; decoding ...");
-		if (unjpeg(src, enc->tmp, true) < 0) {
-			return -1;
-		}
-		assert(enc->tmp->format == V4L2_PIX_FMT_RGB24);
-		LOG_VERBOSE("H264: JPEG decoded; time=%Lf", get_now_monotonic() - dest->encode_begin_ts);
-	} else {
-		LOG_DEBUG("H264: Copying source to tmp buffer ...");
-		frame_copy(src, enc->tmp);
-		assert(src->format == enc->format);
-		assert(enc->tmp->stride == enc->stride);
-		LOG_VERBOSE("H264: Source copied; time=%Lf", get_now_monotonic() - dest->encode_begin_ts);
-	}
-	src = enc->tmp;
-
-	assert(src->used > 0);
-	assert(src->width == enc->width);
-	assert(src->height == enc->height);
-
 	bool force_key = (enc->last_online != src->online);
 
 	if (_h264_encoder_compress_raw(enc, src, dest, force_key) < 0) {
diff --git a/src/ustreamer/h264/encoder.h b/src/ustreamer/h264/encoder.h
index 4d83454..6348e74 100644
--- a/src/ustreamer/h264/encoder.h
+++ b/src/ustreamer/h264/encoder.h
@@ -37,7 +37,6 @@
 #include "../../libs/tools.h"
 #include "../../libs/logging.h"
 #include "../../libs/frame.h"
-#include "../../libs/unjpeg.h"
 
 
 typedef struct {
@@ -51,8 +50,7 @@ typedef struct {
 	VCOS_SEMAPHORE_T	handler_sem;
 	bool				i_handler_sem;
 
-	frame_s		*tmp;
-	int			last_online;
+	int last_online;
 
 	unsigned	width;
 	unsigned	height;
diff --git a/src/ustreamer/stream.c b/src/ustreamer/stream.c
index 69a0d86..bc51fc2 100644
--- a/src/ustreamer/stream.c
+++ b/src/ustreamer/stream.c
@@ -412,15 +412,15 @@ static bool _worker_run_job(worker_s *wr) {
 static h264_stream_s *_h264_stream_init(memsink_s *sink, unsigned bitrate, unsigned gop) {
 	h264_stream_s *h264;
 	A_CALLOC(h264, 1);
+	h264->sink = sink;
+	h264->tmp_src = frame_init("h264_tmp_src");
+	h264->dest = frame_init("h264_dest");
 
 	// FIXME: 30 or 0? https://github.com/6by9/yavta/blob/master/yavta.c#L210
 	if ((h264->enc = h264_encoder_init(bitrate, gop, 0)) == NULL) {
 		goto error;
 	}
 
-	h264->dest = frame_init("h264_dest");
-	h264->sink = sink;
-
 	return h264;
 
 	error:
@@ -432,18 +432,31 @@ static void _h264_stream_destroy(h264_stream_s *h264) {
 	if (h264->enc) {
 		h264_encoder_destroy(h264->enc);
 	}
-	if (h264->dest) {
-		frame_destroy(h264->dest);
-	}
+	frame_destroy(h264->dest);
+	frame_destroy(h264->tmp_src);
 	free(h264);
 }
 
 static void _h264_stream_process(h264_stream_s *h264, const frame_s *frame) {
-	if (!h264_encoder_is_prepared_for(h264->enc, frame)) {
-		h264_encoder_prepare(h264->enc, frame);
+	long double now = get_now_monotonic();
+	if (is_jpeg(frame->format)) {
+		LOG_DEBUG("H264: Input frame is JPEG; decoding ...");
+		if (unjpeg(frame, h264->tmp_src, true) < 0) {
+			return;
+		}
+		LOG_VERBOSE("H264: JPEG decoded; time=%Lf", get_now_monotonic() - now);
+	} else {
+		LOG_DEBUG("H264: Copying source to tmp buffer ...");
+		frame_copy(frame, h264->tmp_src);
+		LOG_VERBOSE("H264: Source copied; time=%Lf", get_now_monotonic() - now);
 	}
+
+	if (!h264_encoder_is_prepared_for(h264->enc, h264->tmp_src)) {
+		h264_encoder_prepare(h264->enc, h264->tmp_src);
+	}
+
 	if (h264->enc->ready) {
-		if (h264_encoder_compress(h264->enc, frame, h264->dest) == 0) {
+		if (h264_encoder_compress(h264->enc, h264->tmp_src, h264->dest) == 0) {
 			memsink_server_put(h264->sink, h264->dest);
 		}
 	}
diff --git a/src/ustreamer/stream.h b/src/ustreamer/stream.h
index 839bd7f..2168f92 100644
--- a/src/ustreamer/stream.h
+++ b/src/ustreamer/stream.h
@@ -43,6 +43,7 @@
 #include "encoder.h"
 #include "workers.h"
 #ifdef WITH_OMX
+#	include "../libs/unjpeg.h"
 #	include "h264/encoder.h"
 #endif
 #ifdef WITH_GPIO
@@ -65,9 +66,10 @@ typedef struct {
 
 #ifdef WITH_OMX
 typedef struct {
-	h264_encoder_s	*enc;
-	frame_s			*dest;
 	memsink_s		*sink;
+	frame_s			*tmp_src;
+	frame_s			*dest;
+	h264_encoder_s	*enc;
 } h264_stream_s;
 #endif