eessppeelllloo/vimix
1
0
Fork 0
mirror of https://github.com/brunoherbelin/vimix.git synced 2025-12-13 19:29:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improv MultiFileRecorder to use gstreamer instead of STB

Browse Source 
Allows adding images of different size and formats, faster decoding and encoding, add keyframes in recorded video.
This commit is contained in:
Bruno Herbelin
2024-04-29 23:49:41 +02:00
parent cd8f9792ab
commit ec78631691
2 changed files with 107 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
src/MultiFileRecorder.cpp
View File

@@ -4,8 +4,6 @@
#include <gst/gstformat.h> #include <gst/gstformat.h>
#include <gst/video/video.h> #include <gst/video/video.h>
#include <stb_image.h>
#include "Log.h" #include "Log.h"
#include "GstToolkit.h" #include "GstToolkit.h"
#include "BaseToolkit.h" #include "BaseToolkit.h"
@@ -16,7 +14,7 @@
#include "MultiFileRecorder.h" #include "MultiFileRecorder.h"
MultiFileRecorder::MultiFileRecorder() : MultiFileRecorder::MultiFileRecorder() :
fps_(0), width_(0), height_(0), bpp_(3), fps_(0), width_(0), height_(0),
pipeline_(nullptr), src_(nullptr), frame_count_(0), timestamp_(0), frame_duration_(0), pipeline_(nullptr), src_(nullptr), frame_count_(0), timestamp_(0), frame_duration_(0),
cancel_(false), endofstream_(false), accept_buffer_(false), progress_(0.f) cancel_(false), endofstream_(false), accept_buffer_(false), progress_(0.f)
{ {
@@ -66,55 +64,85 @@ void MultiFileRecorder::callback_enough_data (GstAppSrc *, gpointer p)
grabber->accept_buffer_ = false; grabber->accept_buffer_ = false;
} }
bool MultiFileRecorder::add_image (const std::string &image_filename) bool MultiFileRecorder::add_image (const std::string &image_filename, GstCaps *caps)
{ {
if (image_filename.empty()) std::string uri = GstToolkit::filename_to_uri(image_filename);
if (uri.empty())
return false; return false;
// read pix // create playbin
int c = 0; GstElement *img_pipeline = gst_element_factory_make("playbin", "imgreader");
int w = 0;
int h = 0;
unsigned char* rgb = stbi_load(image_filename.c_str(), &w, &h, &c, bpp_);
if ( rgb && w == width_ && h == height_ && c == bpp_) { // set uri of file to open
g_object_set(G_OBJECT(img_pipeline), "uri", uri.c_str(), NULL);
// new buffer // set flag to only read VIDEO
guint size = width_ * height_ * bpp_; g_object_set(G_OBJECT(img_pipeline), "flags", 0x00000001, NULL);
GstBuffer *buffer = gst_buffer_new_and_alloc (size);
// map gst buffer into a memory WRITE target // instruct sink to use the required caps (without framerate)
GstMapInfo map; GstCaps *sinkcaps = gst_caps_copy(caps);
gst_buffer_map (buffer, &map, GST_MAP_WRITE); GValue v = {GST_TYPE_FRACTION, {{0}, {1}}};
gst_caps_set_value(sinkcaps, "framerate", &v);
// transfer pixels from memory to buffer memory GstElement *sink = gst_element_factory_make("appsink", "imgsink");
memmove(map.data, rgb, size); gst_app_sink_set_caps(GST_APP_SINK(sink), sinkcaps);
// un-map // set playbin sink
gst_buffer_unmap (buffer, &map); g_object_set(G_OBJECT(img_pipeline), "video-sink", sink, NULL);
// free stbi memory /* Start the pipeline */
stbi_image_free( rgb ); gst_element_set_state(img_pipeline, GST_STATE_PLAYING);
//g_print("frame_added @ timestamp = %ld\n", timestamp_); /* Wait for the pipeline to preroll, i.e., wait for the image to be loaded */
GST_BUFFER_DTS(buffer) = GST_BUFFER_PTS(buffer) = timestamp_; gst_element_get_state(img_pipeline, NULL, NULL, GST_CLOCK_TIME_NONE);
// set frame duration /* Get the sample from appsink */
buffer->duration = frame_duration_; GstSample *sample;
g_signal_emit_by_name(sink, "pull-sample", &sample, NULL);
// monotonic time increment to keep fixed FPS /* Extract the buffer */
timestamp_ += frame_duration_; GstBuffer *buffer_read = gst_sample_get_buffer(sample);
// push frame bool ret = false;
if ( gst_app_src_push_buffer (src_, buffer) != GST_FLOW_OK ) // map the buffer to access the data
return false; GstMapInfo map_read;
if ( gst_buffer_map(buffer_read, &map_read, GST_MAP_READ) && map_read.size > 0 ) {
// map a new gst buffer into memory to WRITE target
GstMapInfo map_write;
GstBuffer *buffer_write = gst_buffer_new_and_alloc(map_read.size);
if ( gst_buffer_map(buffer_write, &map_write, GST_MAP_WRITE) ) {
// transfer pixels from map_read memory to map_write memory (buffer to write to)
memmove(map_write.data, map_read.data, map_read.size);
// un-map buffer
gst_buffer_unmap(buffer_write, &map_write);
//g_print("frame_added @ timestamp = %ld\n", timestamp_);
GST_BUFFER_DTS(buffer_write) = GST_BUFFER_PTS(buffer_write) = timestamp_;
// set frame duration
buffer_write->duration = frame_duration_;
// monotonic time increment to keep fixed FPS
timestamp_ += frame_duration_;
// push buffer as new frame in appsrc
ret = gst_app_src_push_buffer(src_, buffer_write) == GST_FLOW_OK;
}
// unmap read buffer
gst_buffer_unmap(buffer_read, &map_read);
} }
else
return false;
/* Clean up */
gst_caps_unref(caps);
gst_sample_unref(sample);
gst_element_set_state(img_pipeline, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(img_pipeline));
return true; return ret;
} }
@@ -132,7 +160,19 @@ bool MultiFileRecorder::start_record (const std::string &video_filename)
// create a gstreamer pipeline // create a gstreamer pipeline
std::string description = "appsrc name=src ! queue ! videoconvert ! videoscale ! "; std::string description = "appsrc name=src ! queue ! videoconvert ! videoscale ! ";
description += VideoRecorder::profile_description[ profile_ ];
// test for a hardware accelerated encoder
if (Settings::application.render.gpu_decoding && (int) VideoRecorder::hardware_encoder.size() > 0 &&
GstToolkit::has_feature(VideoRecorder::hardware_encoder[profile_]) ) {
description += VideoRecorder::hardware_profile_description[Settings::application.record.profile];
Log::Info("MultiFileRecorder use hardware accelerated encoder (%s)", VideoRecorder::hardware_encoder[profile_].c_str());
}
// revert to software encoder
else
description += VideoRecorder::profile_description[profile_];
// qt muxer in .mov file
description += "qtmux ! filesink name=sink"; description += "qtmux ! filesink name=sink";
// parse pipeline descriptor // parse pipeline descriptor
@@ -179,7 +219,7 @@ bool MultiFileRecorder::start_record (const std::string &video_filename)
// specify recorder resolution and framerate in the source caps // specify recorder resolution and framerate in the source caps
GstCaps *caps = gst_caps_new_simple ("video/x-raw", GstCaps *caps = gst_caps_new_simple ("video/x-raw",
"format", G_TYPE_STRING, bpp_ < 4 ? "RGB" : "RGBA", "format", G_TYPE_STRING, "RGB",
"width", G_TYPE_INT, width_ - width_%2, "width", G_TYPE_INT, width_ - width_%2,
"height", G_TYPE_INT, height_ - height_%2, "height", G_TYPE_INT, height_ - height_%2,
"framerate", GST_TYPE_FRACTION, fps_, 1, "framerate", GST_TYPE_FRACTION, fps_, 1,
@@ -204,35 +244,7 @@ bool MultiFileRecorder::start_record (const std::string &video_filename)
int max = 100; int max = 100;
accept_buffer_ = false; accept_buffer_ = false;
while (!accept_buffer_ && --max > 0) while (!accept_buffer_ && --max > 0)
std::this_thread::sleep_for(std::chrono::milliseconds(5)); std::this_thread::sleep_for(std::chrono::milliseconds(4));
// // send request key frame upstream
// GstEvent* event = gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, TRUE, 1);
// if (!gst_element_send_event( GST_ELEMENT(sink), event) )
// Log::Warning("MultiFileRecorder: Failed to request key unit.");
// GstPad *padsrc = gst_element_get_static_pad ( GST_ELEMENT (sink), "sink");
// gst_pad_push_event(padsrc, gst_event_new_custom(GST_EVENT_CUSTOM_UPSTREAM, gst_structure_new("GstForceKeyUnit", "all-headers",
// G_TYPE_BOOLEAN, TRUE, NULL)));
// gst_object_unref (padsrc);
// // send request key frame downstream
// GstPad *padsrc = gst_element_get_static_pad ( GST_ELEMENT (src_), "src");
// GstStructure *s = gst_structure_new("GstForceKeyUnit",
// "timestamp", G_TYPE_UINT64, 0,
// "stream-time", G_TYPE_UINT64, 0,
// "running-time", G_TYPE_UINT64, 0,
// "all-headers", G_TYPE_BOOLEAN, TRUE,
// NULL);
// if ( !gst_pad_push_event(padsrc, gst_event_new_custom(GST_EVENT_CUSTOM_DOWNSTREAM, s )) )
// Log::Warning("MultiFileRecorder: Failed to force key unit.");
// gst_object_unref (padsrc);
GstEvent* event = gst_video_event_new_downstream_force_key_unit (GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, TRUE, 1);
if (!gst_element_send_event( GST_ELEMENT(src_), event) )
Log::Warning("MultiFileRecorder: Failed to force key unit.");
return true; return true;
} }
@@ -324,25 +336,31 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
rec->progress_ = 0.f; rec->progress_ = 0.f;
rec->width_ = 0; rec->width_ = 0;
rec->height_ = 0; rec->height_ = 0;
rec->bpp_ = 0;
// input files // input files
if ( rec->files_.size() < 1 ) { if ( rec->files_.size() < 1 ) {
Log::Warning("MultiFileRecorder: No image given."); Log::Warning("MultiFileRecorder No image given.");
return filename;
}
// get info first file
std::string uri = GstToolkit::filename_to_uri(rec->files_.front());
MediaInfo media = MediaPlayer::UriDiscoverer(uri);
if (!media.valid || !media.isimage || media.width < 10 || media.height < 10) {
Log::Warning("MultiFileRecorder Invalid file %s.", rec->files_.front().c_str());
return filename; return filename;
} }
// set recorder resolution from first image // set recorder resolution from first image
stbi_info( rec->files_.front().c_str(), &rec->width_, &rec->height_, &rec->bpp_); rec->width_ = media.width;
rec->height_ = media.height;
if ( rec->width_ < 10 || rec->height_ < 10 || rec->bpp_ < 3 ) {
Log::Warning("MultiFileRecorder: Invalid image %s.", rec->files_.front().c_str());
return filename;
}
// progress increment // progress increment
float inc_ = 1.f / ( (float) rec->files_.size() + 2.f); float inc_ = 1.f / ( (float) rec->files_.size() + 2.f);
// keyframe increment
guint64 keyf_ = MAXI( 2, rec->files_.size() / 20);
// initialize // initialize
rec->frame_count_ = 0; rec->frame_count_ = 0;
filename = BaseToolkit::common_prefix (rec->files_); filename = BaseToolkit::common_prefix (rec->files_);
@@ -350,7 +368,7 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
filename += "image"; filename += "image";
filename += "_sequence.mov"; filename += "_sequence.mov";
Log::Info("MultiFileRecorder creating %s, %d x %d px.", filename.c_str(), rec->width_, rec->height_); Log::Info("MultiFileRecorder Creating %s, %d x %d px.", filename.c_str(), rec->width_, rec->height_);
if ( rec->start_record( filename ) ) if ( rec->start_record( filename ) )
{ {
@@ -363,16 +381,25 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
if ( rec->cancel_ ) if ( rec->cancel_ )
break; break;
if ( rec->add_image( *file ) ) if ( rec->add_image( *file, gst_app_src_get_caps(rec->src_) ) ) {
// validate file // validate file
rec->frame_count_++; rec->frame_count_++;
// add a key frame every
if ( rec->frame_count_%keyf_ < 1 ) {
GstEvent *event = gst_video_event_new_downstream_force_key_unit(
rec->timestamp_, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, FALSE, rec->frame_count_ / keyf_);
if (!gst_element_send_event(GST_ELEMENT(rec->src_), event))
Log::Info("MultiFileRecorder Failed to force key unit %l.", rec->timestamp_);
}
}
else else
Log::Info("MultiFileRecorder could not add %s.", file->c_str()); Log::Info("MultiFileRecorder Could not add %s.", file->c_str());
// pause in case appsrc buffer is full // pause in case appsrc buffer is full
int max = 100; int max = 100;
while (!rec->accept_buffer_ && --max > 0) while (!rec->accept_buffer_ && --max > 0)
std::this_thread::sleep_for(std::chrono::milliseconds(10)); std::this_thread::sleep_for(std::chrono::milliseconds(4));
// progressing // progressing
rec->progress_ += inc_; rec->progress_ += inc_;
@@ -380,7 +407,7 @@ std::string MultiFileRecorder::assemble (MultiFileRecorder *rec)
// Give more explanation for possible errors // Give more explanation for possible errors
if ( rec->frame_count_ < rec->files_.size()) if ( rec->frame_count_ < rec->files_.size())
Log::Info("MultiFileRecorder not fully successful; are all images %d x %d px?",rec->width_, rec->height_); Log::Info("MultiFileRecorder Not fully successful; are all images %d x %d px?",rec->width_, rec->height_);
// close file properly // close file properly
if ( rec->end_record() ) if ( rec->end_record() )

3
src/MultiFileRecorder.h
View File

@@ -44,7 +44,7 @@ protected:
// gstreamer functions // gstreamer functions
static std::string assemble (MultiFileRecorder *rec); static std::string assemble (MultiFileRecorder *rec);
bool start_record (const std::string &video_filename); bool start_record (const std::string &video_filename);
bool add_image (const std::string &image_filename); bool add_image (const std::string &image_filename, GstCaps *caps);
bool end_record(); bool end_record();
// gstreamer callbacks // gstreamer callbacks
@@ -59,7 +59,6 @@ private:
int fps_; int fps_;
int width_; int width_;
int height_; int height_;
int bpp_;
// encoder // encoder
std::list<std::string> files_; std::list<std::string> files_;