eessppeelllloo/vimix
1
0
Fork 0
mirror of https://github.com/brunoherbelin/vimix.git synced 2025-12-11 18:34:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Video Recoding Buffer management

Browse Source 
Implemented methods to supervise encoding in FrameGrabber, avoid running out of buffer, and give user a selection of buffer sizes for recording.
This commit is contained in:
Bruno
2021-08-08 23:58:35 +02:00
parent ddd9bb4e99
commit 5a18dbaf37
9 changed files with 161 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
FrameGrabber.cpp
View File

@@ -214,8 +214,8 @@ void FrameGrabbing::grabFrame(FrameBuffer *frame_buffer, float dt)
FrameGrabber::FrameGrabber(): finished_(false), expecting_finished_(false), active_(false), accept_buffer_(false),
pipeline_(nullptr), src_(nullptr), caps_(nullptr), timer_(nullptr), timestamp_(0)
FrameGrabber::FrameGrabber(): finished_(false), active_(false), endofstream_(false), accept_buffer_(false), buffering_full_(false),
pipeline_(nullptr), src_(nullptr), caps_(nullptr), timer_(nullptr), timestamp_(0), frame_count_(0), buffering_size_(MIN_BUFFER_SIZE)
{
// unique id
id_ = BaseToolkit::uniqueId();
@@ -255,12 +255,11 @@ uint64_t FrameGrabber::duration() const
void FrameGrabber::stop ()
{
// send end of stream
expecting_finished_ = true;
gst_app_src_end_of_stream (src_);
// stop recording
active_ = false;
// send end of stream
gst_app_src_end_of_stream (src_);
}
std::string FrameGrabber::info() const
@@ -285,7 +284,6 @@ void FrameGrabber::callback_enough_data (GstAppSrc *, gpointer p)
FrameGrabber *grabber = static_cast<FrameGrabber *>(p);
if (grabber)
grabber->accept_buffer_ = false;
}
GstPadProbeReturn FrameGrabber::callback_event_probe(GstPad *, GstPadProbeInfo * info, gpointer p)
@@ -295,7 +293,7 @@ GstPadProbeReturn FrameGrabber::callback_event_probe(GstPad *, GstPadProbeInfo *
{
FrameGrabber *grabber = static_cast<FrameGrabber *>(p);
if (grabber)
grabber->finished_ = true;
grabber->endofstream_ = true;
}
return GST_PAD_PROBE_OK;
@@ -316,54 +314,97 @@ void FrameGrabber::addFrame (GstBuffer *buffer, GstCaps *caps, float dt)
gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, FrameGrabber::callback_event_probe, this, NULL);
gst_object_unref (pad);
}
// terminate properly if finished
if (finished_)
terminate();
// stop if an incompatilble frame buffer given
else if ( !gst_caps_is_equal( caps_, caps ))
{
stop();
Log::Warning("FrameGrabber interrupted because the resolution changed.");
Log::Warning("Frame capture interrupted because the resolution changed.");
}
// store a frame if recording is active
// and if the encoder accepts data
else if (active_ && accept_buffer_)
// store a frame if recording is active and if the encoder accepts data
if (active_)
{
GstClockTime t = 0;
if (accept_buffer_) {
GstClockTime t = 0;
// initialize timer on first occurence
if (timer_ == nullptr) {
timer_ = gst_pipeline_get_clock ( GST_PIPELINE(pipeline_) );
timer_firstframe_ = gst_clock_get_time(timer_);
}
else
// time since timer starts (first frame registered)
t = gst_clock_get_time(timer_) - timer_firstframe_;
// if time is zero (first frame) or if delta time is passed one frame duration (with a margin)
if ( t == 0 || (t - timestamp_) > (frame_duration_ - 3000) ) {
// round time to a multiples of frame duration
t = ( t / frame_duration_) * frame_duration_;
// set frame presentation time stamp
buffer->pts = t;
// if time since last timestamp is more than 1 frame
if (t - timestamp_ > frame_duration_) {
// compute duration
buffer->duration = t - timestamp_;
// keep timestamp for next addFrame to one frame later
timestamp_ = t + frame_duration_;
}
// normal case (not delayed)
else {
// normal frame duration
buffer->duration = frame_duration_;
// keep timestamp for next addFrame
timestamp_ = t;
}
// when buffering is full, refuse buffer every frame
if (buffering_full_)
accept_buffer_ = false;
else
{
// enter buffering_full_ mode if the space left in buffering is for only few frames
// (this prevents filling the buffer entirely)
// if ( (double) gst_app_src_get_current_level_bytes(src_) / (double) buffering_size_ > 0.8) // 80% test
if ( buffering_size_ - gst_app_src_get_current_level_bytes(src_) < 4 * gst_buffer_get_size(buffer))
buffering_full_ = true;
}
// increment ref counter to make sure the frame remains available
gst_buffer_ref(buffer);
// push frame
gst_app_src_push_buffer (src_, buffer);
// NB: buffer will be unrefed by the appsrc
// count frames
frame_count_++;
}
// initialize timer on first occurence
if (timer_ == nullptr) {
timer_ = gst_pipeline_get_clock ( GST_PIPELINE(pipeline_) );
timer_firstframe_ = gst_clock_get_time(timer_);
}
}
// if we received and end of stream (from callback_event_probe)
if (endofstream_)
{
// try to stop properly when interrupted
if (active_) {
// de-activate and re-send EOS
stop();
// inform
Log::Warning("Frame capture : interrupted after %s.", GstToolkit::time_to_string(timestamp_).c_str());
Log::Info("Frame capture: not space left on drive / encoding buffer full.");
}
// terminate properly if finished
else
// time since timer starts (first frame registered)
t = gst_clock_get_time(timer_) - timer_firstframe_;
// if time is zero (first frame)
// of if delta time is passed one frame duration (with a margin)
if ( t == 0 || (t - timestamp_) > (frame_duration_ - 3000) ) {
// round t to multiple of frame duration
t = ( t / frame_duration_) * frame_duration_;
// set timing of buffer
buffer->pts = t;
buffer->duration = t - timestamp_;
// increment ref counter to make sure the frame remains available
gst_buffer_ref(buffer);
// push frame
gst_app_src_push_buffer (src_, buffer);
// NB: buffer will be unrefed by the appsrc
// keep timestamp for next addFrame
timestamp_ = t;
{
finished_ = true;
terminate();
}
}
}

7
FrameGrabber.h
View File

@@ -14,6 +14,8 @@
// https://stackoverflow.com/questions/38140527/glreadpixels-vs-glgetteximage
#define USE_GLREADPIXEL
#define MIN_BUFFER_SIZE 33177600 // 33177600 bytes = 1 frames 4K, 9 frames 720p
class FrameBuffer;
@@ -53,9 +55,9 @@ protected:
virtual void terminate() = 0;
// thread-safe testing termination
std::atomic<bool> expecting_finished_;
std::atomic<bool> finished_;
std::atomic<bool> active_;
std::atomic<bool> endofstream_;
std::atomic<bool> accept_buffer_;
// gstreamer pipeline
@@ -65,6 +67,9 @@ protected:
GstClockTime timestamp_;
GstClockTime frame_duration_;
guint64 frame_count_;
guint64 buffering_size_;
std::atomic<bool> buffering_full_;
GstClockTime timer_firstframe_;
GstClock *timer_;

17
Loopback.cpp
View File

@@ -152,7 +152,6 @@ bool Loopback::systemLoopbackInitialized()
Loopback::Loopback() : FrameGrabber()
{
frame_duration_ = gst_util_uint64_scale_int (1, GST_SECOND, 60);
}
void Loopback::init(GstCaps *caps)
@@ -190,25 +189,26 @@ void Loopback::init(GstCaps *caps)
if (src_) {
g_object_set (G_OBJECT (src_),
"stream-type", GST_APP_STREAM_TYPE_STREAM,
"is-live", TRUE,
"format", GST_FORMAT_TIME,
// "do-timestamp", TRUE,
NULL);
// Direct encoding (no buffering)
gst_app_src_set_max_bytes( src_, 0 );
// configure stream
gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_latency( src_, -1, 0);
// Set buffer size
gst_app_src_set_max_bytes( src_, buffering_size_ );
// instruct src to use the required caps
caps_ = gst_caps_copy( caps );
gst_app_src_set_caps (src_, caps_);
gst_app_src_set_caps( src_, caps_);
// setup callbacks
GstAppSrcCallbacks callbacks;
callbacks.need_data = FrameGrabber::callback_need_data;
callbacks.enough_data = FrameGrabber::callback_enough_data;
callbacks.seek_data = NULL; // stream type is not seekable
gst_app_src_set_callbacks (src_, &callbacks, this, NULL);
gst_app_src_set_callbacks( src_, &callbacks, this, NULL);
}
else {
@@ -237,6 +237,5 @@ void Loopback::init(GstCaps *caps)
void Loopback::terminate()
{
active_ = false;
Log::Notify("Loopback to %s terminated.", Loopback::system_loopback_name.c_str());
}

49
Recorder.cpp
View File

@@ -60,12 +60,13 @@ void PNGRecorder::init(GstCaps *caps)
if (src_) {
g_object_set (G_OBJECT (src_),
"stream-type", GST_APP_STREAM_TYPE_STREAM,
"is-live", TRUE,
"format", GST_FORMAT_TIME,
// "do-timestamp", TRUE,
NULL);
// configure stream
gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_latency( src_, -1, 0);
// Direct encoding (no buffering)
gst_app_src_set_max_bytes( src_, 0 );
@@ -104,7 +105,6 @@ void PNGRecorder::init(GstCaps *caps)
void PNGRecorder::terminate()
{
active_ = false;
Log::Notify("PNG Capture %s is ready.", filename_.c_str());
}
@@ -178,7 +178,7 @@ const std::vector<std::string> VideoRecorder::profile_description {
// 2 ‘standard’
// 3 ‘hq’
// 4 ‘4444’
"avenc_prores_ks pass=2 profile=2 quantizer=26 ! ",
"video/x-raw, format=I422_10LE ! avenc_prores_ks pass=2 profile=2 quantizer=26 ! ",
"video/x-raw, format=Y444_10LE ! avenc_prores_ks pass=2 profile=4 quantizer=12 ! ",
// VP8 WebM encoding
"vp8enc end-usage=vbr cpu-used=8 max-quantizer=35 deadline=100000 target-bitrate=200000 keyframe-max-dist=360 token-partitions=2 static-threshold=100 ! ",
@@ -198,8 +198,13 @@ const std::vector<std::string> VideoRecorder::profile_description {
// "qtmux ! filesink name=sink";
VideoRecorder::VideoRecorder() : FrameGrabber()
const char* VideoRecorder::buffering_preset_name[VIDEO_RECORDER_BUFFERING_NUM_PRESET] = { "30 MB", "100 MB", "200 MB", "500 MB", "1 GB", "2 GB" };
const guint64 VideoRecorder::buffering_preset_value[VIDEO_RECORDER_BUFFERING_NUM_PRESET] = { MIN_BUFFER_SIZE, 104857600, 209715200, 524288000, 1073741824, 2147483648};
VideoRecorder::VideoRecorder(guint64 buffersize) : FrameGrabber()
{
buffering_size_ = MAX( MIN_BUFFER_SIZE, buffersize);
}
void VideoRecorder::init(GstCaps *caps)
@@ -239,7 +244,7 @@ void VideoRecorder::init(GstCaps *caps)
GError *error = NULL;
pipeline_ = gst_parse_launch (description.c_str(), &error);
if (error != NULL) {
Log::Warning("VideoRecorder Could not construct pipeline %s:\n%s", description.c_str(), error->message);
Log::Warning("Video Recording : Could not construct pipeline %s:\n%s", description.c_str(), error->message);
g_clear_error (&error);
finished_ = true;
return;
@@ -256,14 +261,17 @@ void VideoRecorder::init(GstCaps *caps)
if (src_) {
g_object_set (G_OBJECT (src_),
"stream-type", GST_APP_STREAM_TYPE_STREAM,
"is-live", TRUE,
"format", GST_FORMAT_TIME,
// "do-timestamp", TRUE,
// "do-timestamp", TRUE,
NULL);
// Direct encoding (no buffering)
gst_app_src_set_max_bytes( src_, 0 );
// configure stream
gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_latency( src_, -1, 0);
// Set buffer size
gst_app_src_set_max_bytes( src_, buffering_size_);
// instruct src to use the required caps
caps_ = gst_caps_copy( caps );
@@ -278,7 +286,7 @@ void VideoRecorder::init(GstCaps *caps)
}
else {
Log::Warning("VideoRecorder Could not configure source");
Log::Warning("Video Recording : Could not configure source");
finished_ = true;
return;
}
@@ -286,7 +294,7 @@ void VideoRecorder::init(GstCaps *caps)
// start recording
GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE) {
Log::Warning("VideoRecorder Could not record %s", filename_.c_str());
Log::Warning("Video Recording : Could not record %s", filename_.c_str());
finished_ = true;
return;
}
@@ -300,10 +308,15 @@ void VideoRecorder::init(GstCaps *caps)
void VideoRecorder::terminate()
{
active_ = false;
// stop the pipeline
gst_element_set_state (pipeline_, GST_STATE_NULL);
if (!expecting_finished_)
Log::Warning("Video Recording interrupted (no more disk space?).");
guint64 N = MAX( (guint64) timestamp_ / (guint64) frame_duration_, frame_count_);
float loss = 100.f * ((float) (N - frame_count_) ) / (float) N;
Log::Info("Video Recording : %ld frames in %s (aming for %ld, %.0f%% lost)", frame_count_, GstToolkit::time_to_string(timestamp_).c_str(), N, loss);
Log::Info("Video Recording : try with a lower resolution / a larger buffer size / a faster codec.");
if (loss > 20.f)
Log::Warning("Video Recording lost %.0f%% of frames.", loss);
Log::Notify("Video Recording %s is ready.", filename_.c_str());
}
@@ -312,6 +325,8 @@ std::string VideoRecorder::info() const
{
if (active_)
return GstToolkit::time_to_string(timestamp_);
else
else if (!endofstream_)
return "Saving file...";
else
return "...";
}

9
Recorder.h
View File

@@ -25,6 +25,8 @@ protected:
};
#define VIDEO_RECORDER_BUFFERING_NUM_PRESET 6
class VideoRecorder : public FrameGrabber
{
std::string filename_;
@@ -45,10 +47,13 @@ public:
JPEG_MULTI,
DEFAULT
} Profile;
static const char* profile_name[DEFAULT];
static const char* profile_name[DEFAULT];
static const std::vector<std::string> profile_description;
VideoRecorder();
static const char* buffering_preset_name[VIDEO_RECORDER_BUFFERING_NUM_PRESET];
static const guint64 buffering_preset_value[VIDEO_RECORDER_BUFFERING_NUM_PRESET];
VideoRecorder(guint64 buffersize = 0);
std::string info() const override;
};

2
Settings.cpp
View File

@@ -102,6 +102,7 @@ void Settings::Save()
RecordNode->SetAttribute("profile", application.record.profile);
RecordNode->SetAttribute("timeout", application.record.timeout);
RecordNode->SetAttribute("delay", application.record.delay);
RecordNode->SetAttribute("buffering_mode", application.record.buffering_mode);
pRoot->InsertEndChild(RecordNode);
// Transition
@@ -314,6 +315,7 @@ void Settings::Load()
recordnode->QueryIntAttribute("profile", &application.record.profile);
recordnode->QueryUnsignedAttribute("timeout", &application.record.timeout);
recordnode->QueryIntAttribute("delay", &application.record.delay);
recordnode->QueryIntAttribute("buffering_mode", &application.record.buffering_mode);
const char *path_ = recordnode->Attribute("path");
if (path_)

2
Settings.h
View File

@@ -77,11 +77,13 @@ struct RecordConfig
int profile;
uint timeout;
int delay;
int buffering_mode;
RecordConfig() : path("") {
profile = 0;
timeout = RECORD_MAX_TIMEOUT;
delay = 0;
buffering_mode = 0;
}
};

11
Streamer.cpp
View File

@@ -345,14 +345,17 @@ void VideoStreamer::init(GstCaps *caps)
if (src_) {
g_object_set (G_OBJECT (src_),
"stream-type", GST_APP_STREAM_TYPE_STREAM,
"is-live", TRUE,
"format", GST_FORMAT_TIME,
// "do-timestamp", TRUE,
NULL);
// Direct encoding (no buffering)
gst_app_src_set_max_bytes( src_, 0 );
// configure stream
gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_latency( src_, -1, 0);
// Set buffer size
gst_app_src_set_max_bytes( src_, buffering_size_ );
// instruct src to use the required caps
caps_ = gst_caps_copy( caps );
@@ -389,8 +392,6 @@ void VideoStreamer::init(GstCaps *caps)
void VideoStreamer::terminate()
{
active_ = false;
// send EOS
gst_app_src_end_of_stream (src_);

24
UserInterfaceManager.cpp
View File

@@ -275,7 +275,9 @@ void UserInterface::handleKeyboard()
// video_recorder_ = nullptr;
}
else {
_video_recorders.emplace_back( std::async(std::launch::async, delayTrigger, new VideoRecorder, std::chrono::seconds(Settings::application.record.delay)) );
_video_recorders.emplace_back( std::async(std::launch::async, delayTrigger,
new VideoRecorder(VideoRecorder::buffering_preset_value[Settings::application.record.buffering_mode]),
std::chrono::seconds(Settings::application.record.delay)) );
}
}
}
@@ -1100,7 +1102,9 @@ void UserInterface::RenderPreview()
else {
ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(IMGUI_COLOR_RECORD, 0.9f));
if ( ImGui::MenuItem( ICON_FA_CIRCLE " Record", CTRL_MOD "R") ) {
_video_recorders.emplace_back( std::async(std::launch::async, delayTrigger, new VideoRecorder, std::chrono::seconds(Settings::application.record.delay)) );
_video_recorders.emplace_back( std::async(std::launch::async, delayTrigger,
new VideoRecorder(VideoRecorder::buffering_preset_value[Settings::application.record.buffering_mode]),
std::chrono::seconds(Settings::application.record.delay)) );
}
ImGui::PopStyleColor(1);
// select profile
@@ -1138,6 +1142,11 @@ void UserInterface::RenderPreview()
ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN);
ImGui::SliderInt("Trigger", &Settings::application.record.delay, 0, 5,
Settings::application.record.delay < 1 ? "Immediate" : "After %d s");
ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN);
ImGui::SliderInt("Buffer", &Settings::application.record.buffering_mode, 0, VIDEO_RECORDER_BUFFERING_NUM_PRESET-1,
VideoRecorder::buffering_preset_name[Settings::application.record.buffering_mode]);
}
ImGui::EndMenu();
}
@@ -4548,8 +4557,6 @@ void Navigator::RenderMainPannelSettings()
ImGui::SetCursorPosY(width_);
// Appearance
// ImGuiToolkit::Icon(3, 2);
// ImGui::SameLine(0, 10);
ImGui::Text("Appearance");
ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN);
if ( ImGui::DragFloat("Scale", &Settings::application.scale, 0.01, 0.5f, 2.0f, "%.1f"))
@@ -4562,8 +4569,6 @@ void Navigator::RenderMainPannelSettings()
// Options
ImGui::Spacing();
// ImGuiToolkit::Icon(2, 2);
// ImGui::SameLine(0, 10);
ImGui::Text("Options");
ImGuiToolkit::ButtonSwitch( ICON_FA_MOUSE_POINTER " Smooth cursor", &Settings::application.smooth_cursor);
ImGuiToolkit::ButtonSwitch( ICON_FA_TACHOMETER_ALT " Metrics", &Settings::application.widget.stats);
@@ -4578,13 +4583,8 @@ void Navigator::RenderMainPannelSettings()
// system preferences
ImGui::Spacing();
//#ifdef LINUX
// ImGuiToolkit::Icon(12, 6);
//#else
// ImGuiToolkit::Icon(6, 0);
//#endif
// ImGui::SameLine(0, 10);
ImGui::Text("System");
static bool need_restart = false;
static bool vsync = (Settings::application.render.vsync > 0);
static bool blit = Settings::application.render.blit;