eessppeelllloo/vimix
1
0
Fork 0
mirror of https://github.com/brunoherbelin/vimix.git synced 2025-12-11 10:19:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improv MultiFileRecorder to use gstreamer instead of STB

Browse Source 
Allows adding images of different size and formats, faster decoding and encoding, add keyframes in recorded video.
This commit is contained in:
Bruno Herbelin
2024-04-29 23:49:41 +02:00
parent cd8f9792ab
commit ec78631691
2 changed files with 107 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
src/MultiFileRecorder.cpp
View File

@@ -4,8 +4,6 @@
#include <gst/gstformat.h>
#include <gst/video/video.h>
#include <stb_image.h>
#include "Log.h"
#include "GstToolkit.h"
#include "BaseToolkit.h"
@@ -16,7 +14,7 @@
#include "MultiFileRecorder.h"
MultiFileRecorder::MultiFileRecorder() :
fps_(0), width_(0), height_(0), bpp_(3),
fps_(0), width_(0), height_(0),
pipeline_(nullptr), src_(nullptr), frame_count_(0), timestamp_(0), frame_duration_(0),
cancel_(false), endofstream_(false), accept_buffer_(false), progress_(0.f)
{
@@ -66,55 +64,85 @@ void MultiFileRecorder::callback_enough_data (GstAppSrc *, gpointer p)
grabber->accept_buffer_ = false;
}
bool MultiFileRecorder::add_image (const std::string &image_filename)
bool MultiFileRecorder::add_image (const std::string &image_filename, GstCaps *caps)
{
if (image_filename.empty())
std::string uri = GstToolkit::filename_to_uri(image_filename);
if (uri.empty())
return false;
// read pix
int c = 0;
int w = 0;
int h = 0;
unsigned char* rgb = stbi_load(image_filename.c_str(), &w, &h, &c, bpp_);
// create playbin
GstElement *img_pipeline = gst_element_factory_make("playbin", "imgreader");
if ( rgb && w == width_ && h == height_ && c == bpp_) {
// set uri of file to open
g_object_set(G_OBJECT(img_pipeline), "uri", uri.c_str(), NULL);
// new buffer
guint size = width_ * height_ * bpp_;
GstBuffer *buffer = gst_buffer_new_and_alloc (size);
// set flag to only read VIDEO
g_object_set(G_OBJECT(img_pipeline), "flags", 0x00000001, NULL);
// map gst buffer into a memory WRITE target
GstMapInfo map;
gst_buffer_map (buffer, &map, GST_MAP_WRITE);
// instruct sink to use the required caps (without framerate)
GstCaps *sinkcaps = gst_caps_copy(caps);
GValue v = {GST_TYPE_FRACTION, {{0}, {1}}};
gst_caps_set_value(sinkcaps, "framerate", &v);
// transfer pixels from memory to buffer memory
memmove(map.data, rgb, size);
GstElement *sink = gst_element_factory_make("appsink", "imgsink");
gst_app_sink_set_caps(GST_APP_SINK(sink), sinkcaps);
// un-map
gst_buffer_unmap (buffer, &map);
// set playbin sink
g_object_set(G_OBJECT(img_pipeline), "video-sink", sink, NULL);
// free stbi memory
stbi_image_free( rgb );
/* Start the pipeline */
gst_element_set_state(img_pipeline, GST_STATE_PLAYING);
//g_print("frame_added @ timestamp = %ld\n", timestamp_);
GST_BUFFER_DTS(buffer) = GST_BUFFER_PTS(buffer) = timestamp_;
/* Wait for the pipeline to preroll, i.e., wait for the image to be loaded */
gst_element_get_state(img_pipeline, NULL, NULL, GST_CLOCK_TIME_NONE);
// set frame duration
buffer->duration = frame_duration_;
/* Get the sample from appsink */
GstSample *sample;
g_signal_emit_by_name(sink, "pull-sample", &sample, NULL);
// monotonic time increment to keep fixed FPS
timestamp_ += frame_duration_;
/* Extract the buffer */
GstBuffer *buffer_read = gst_sample_get_buffer(sample);
// push frame
if ( gst_app_src_push_buffer (src_, buffer) != GST_FLOW_OK )
return false;
bool ret = false;
// map the buffer to access the data
GstMapInfo map_read;
if ( gst_buffer_map(buffer_read, &map_read, GST_MAP_READ) && map_read.size > 0 ) {
// map a new gst buffer into memory to WRITE target
GstMapInfo map_write;
GstBuffer *buffer_write = gst_buffer_new_and_alloc(map_read.size);
if ( gst_buffer_map(buffer_write, &map_write, GST_MAP_WRITE) ) {
// transfer pixels from map_read memory to map_write memory (buffer to write to)
memmove(map_write.data, map_read.data, map_read.size);
// un-map buffer
gst_buffer_unmap(buffer_write, &map_write);
//g_print("frame_added @ timestamp = %ld\n", timestamp_);
GST_BUFFER_DTS(buffer_write) = GST_BUFFER_PTS(buffer_write) = timestamp_;
// set frame duration
buffer_write->duration = frame_duration_;
// monotonic time increment to keep fixed FPS
timestamp_ += frame_duration_;
// push buffer as new frame in appsrc
ret = gst_app_src_push_buffer(src_, buffer_write) == GST_FLOW_OK;
}
// unmap read buffer
gst_buffer_unmap(buffer_read, &map_read);
}
else
return false;
/* Clean up */
gst_caps_unref(caps);
gst_sample_unref(sample);
gst_element_set_state(img_pipeline, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(img_pipeline));
return true;
return ret;
}
@@ -132,7 +160,19 @@ bool MultiFileRecorder::start_record (const std::string &video_filename)
// create a gstreamer pipeline
std::string description = "appsrc name=src ! queue ! videoconvert ! videoscale ! ";
description += VideoRecorder::profile_description[ profile_ ];
// test for a hardware accelerated encoder
if (Settings::application.render.gpu_decoding && (int) VideoRecorder::hardware_encoder.size() > 0 &&
GstToolkit::has_feature(VideoRecorder::hardware_encoder[profile_]) ) {
description += VideoRecorder::hardware_profile_description[Settings::application.record.profile];
Log::Info("MultiFileRecorder use hardware accelerated encoder (%s)", VideoRecorder::hardware_encoder[profile_].c_str());
}
// revert to software encoder
else
description += VideoRecorder::profile_description[profile_];
// qt muxer in .mov file
description += "qtmux ! filesink name=sink";
// parse pipeline descriptor
@@ -179,7 +219,7 @@ bool MultiFileRecorder::start_record (const std::string &video_filename)
// specify recorder resolution and framerate in the source caps
GstCaps *caps = gst_caps_new_simple ("video/x-raw",
"format", G_TYPE_STRING, bpp_ < 4 ? "RGB" : "RGBA",
"format", G_TYPE_STRING, "RGB",
"width", G_TYPE_INT, width_ - width_%2,
"height", G_TYPE_INT, height_ - height_%2,
"framerate", GST_TYPE_FRACTION, fps_, 1,
@@ -204,35 +244,7 @@ bool MultiFileRecorder::start_record (const std::string &video_filename)
int max = 100;
accept_buffer_ = false;
while (!accept_buffer_ && --max > 0)
std::this_thread::sleep_for(std::chrono::milliseconds(5));
// // send request key frame upstream
// GstEvent* event = gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, TRUE, 1);
// if (!gst_element_send_event( GST_ELEMENT(sink), event) )
// Log::Warning("MultiFileRecorder: Failed to request key unit.");
// GstPad *padsrc = gst_element_get_static_pad ( GST_ELEMENT (sink), "sink");
// gst_pad_push_event(padsrc, gst_event_new_custom(GST_EVENT_CUSTOM_UPSTREAM, gst_structure_new("GstForceKeyUnit", "all-headers",
// G_TYPE_BOOLEAN, TRUE, NULL)));
// gst_object_unref (padsrc);
// // send request key frame downstream
// GstPad *padsrc = gst_element_get_static_pad ( GST_ELEMENT (src_), "src");
// GstStructure *s = gst_structure_new("GstForceKeyUnit",
// "timestamp", G_TYPE_UINT64, 0,
// "stream-time", G_TYPE_UINT64, 0,
// "running-time", G_TYPE_UINT64, 0,
// "all-headers", G_TYPE_BOOLEAN, TRUE,
// NULL);
// if ( !gst_pad_push_event(padsrc, gst_event_new_custom(GST_EVENT_CUSTOM_DOWNSTREAM, s )) )
// Log::Warning("MultiFileRecorder: Failed to force key unit.");
// gst_object_unref (padsrc);
GstEvent* event = gst_video_event_new_downstream_force_key_unit (GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, TRUE, 1);
if (!gst_element_send_event( GST_ELEMENT(src_), event) )
Log::Warning("MultiFileRecorder: Failed to force key unit.");
std::this_thread::sleep_for(std::chrono::milliseconds(4));
return true;
}
@@ -324,25 +336,31 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
rec->progress_ = 0.f;
rec->width_ = 0;
rec->height_ = 0;
rec->bpp_ = 0;
// input files
if ( rec->files_.size() < 1 ) {
Log::Warning("MultiFileRecorder: No image given.");
Log::Warning("MultiFileRecorder No image given.");
return filename;
}
// get info first file
std::string uri = GstToolkit::filename_to_uri(rec->files_.front());
MediaInfo media = MediaPlayer::UriDiscoverer(uri);
if (!media.valid || !media.isimage || media.width < 10 || media.height < 10) {
Log::Warning("MultiFileRecorder Invalid file %s.", rec->files_.front().c_str());
return filename;
}
// set recorder resolution from first image
stbi_info( rec->files_.front().c_str(), &rec->width_, &rec->height_, &rec->bpp_);
if ( rec->width_ < 10 || rec->height_ < 10 || rec->bpp_ < 3 ) {
Log::Warning("MultiFileRecorder: Invalid image %s.", rec->files_.front().c_str());
return filename;
}
rec->width_ = media.width;
rec->height_ = media.height;
// progress increment
float inc_ = 1.f / ( (float) rec->files_.size() + 2.f);
// keyframe increment
guint64 keyf_ = MAXI( 2, rec->files_.size() / 20);
// initialize
rec->frame_count_ = 0;
filename = BaseToolkit::common_prefix (rec->files_);
@@ -350,7 +368,7 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
filename += "image";
filename += "_sequence.mov";
Log::Info("MultiFileRecorder creating %s, %d x %d px.", filename.c_str(), rec->width_, rec->height_);
Log::Info("MultiFileRecorder Creating %s, %d x %d px.", filename.c_str(), rec->width_, rec->height_);
if ( rec->start_record( filename ) )
{
@@ -363,16 +381,25 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
if ( rec->cancel_ )
break;
if ( rec->add_image( *file ) )
if ( rec->add_image( *file, gst_app_src_get_caps(rec->src_) ) ) {
// validate file
rec->frame_count_++;
// add a key frame every
if ( rec->frame_count_%keyf_ < 1 ) {
GstEvent *event = gst_video_event_new_downstream_force_key_unit(
rec->timestamp_, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, FALSE, rec->frame_count_ / keyf_);
if (!gst_element_send_event(GST_ELEMENT(rec->src_), event))
Log::Info("MultiFileRecorder Failed to force key unit %l.", rec->timestamp_);
}
}
else
Log::Info("MultiFileRecorder could not add %s.", file->c_str());
Log::Info("MultiFileRecorder Could not add %s.", file->c_str());
// pause in case appsrc buffer is full
int max = 100;
while (!rec->accept_buffer_ && --max > 0)
std::this_thread::sleep_for(std::chrono::milliseconds(10));
std::this_thread::sleep_for(std::chrono::milliseconds(4));
// progressing
rec->progress_ += inc_;
@@ -380,7 +407,7 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
// Give more explanation for possible errors
if ( rec->frame_count_ < rec->files_.size())
Log::Info("MultiFileRecorder not fully successful; are all images %d x %d px?",rec->width_, rec->height_);
Log::Info("MultiFileRecorder Not fully successful; are all images %d x %d px?",rec->width_, rec->height_);
// close file properly
if ( rec->end_record() )

3
src/MultiFileRecorder.h
View File

@@ -44,7 +44,7 @@ protected:
// gstreamer functions
static std::string assemble (MultiFileRecorder *rec);
bool start_record (const std::string &video_filename);
bool add_image (const std::string &image_filename);
bool add_image (const std::string &image_filename, GstCaps *caps);
bool end_record();
// gstreamer callbacks
@@ -59,7 +59,6 @@ private:
int fps_;
int width_;
int height_;
int bpp_;
// encoder
std::list<std::string> files_;