diff --git a/FrameGrabber.cpp b/FrameGrabber.cpp index 7fba468..2ec1e38 100644 --- a/FrameGrabber.cpp +++ b/FrameGrabber.cpp @@ -214,8 +214,8 @@ void FrameGrabbing::grabFrame(FrameBuffer *frame_buffer, float dt) -FrameGrabber::FrameGrabber(): finished_(false), expecting_finished_(false), active_(false), accept_buffer_(false), - pipeline_(nullptr), src_(nullptr), caps_(nullptr), timer_(nullptr), timestamp_(0) +FrameGrabber::FrameGrabber(): finished_(false), active_(false), endofstream_(false), accept_buffer_(false), buffering_full_(false), + pipeline_(nullptr), src_(nullptr), caps_(nullptr), timer_(nullptr), timestamp_(0), frame_count_(0), buffering_size_(MIN_BUFFER_SIZE) { // unique id id_ = BaseToolkit::uniqueId(); @@ -255,12 +255,11 @@ uint64_t FrameGrabber::duration() const void FrameGrabber::stop () { - // send end of stream - expecting_finished_ = true; - gst_app_src_end_of_stream (src_); - // stop recording active_ = false; + + // send end of stream + gst_app_src_end_of_stream (src_); } std::string FrameGrabber::info() const @@ -285,7 +284,6 @@ void FrameGrabber::callback_enough_data (GstAppSrc *, gpointer p) FrameGrabber *grabber = static_cast(p); if (grabber) grabber->accept_buffer_ = false; - } GstPadProbeReturn FrameGrabber::callback_event_probe(GstPad *, GstPadProbeInfo * info, gpointer p) @@ -295,7 +293,7 @@ GstPadProbeReturn FrameGrabber::callback_event_probe(GstPad *, GstPadProbeInfo * { FrameGrabber *grabber = static_cast(p); if (grabber) - grabber->finished_ = true; + grabber->endofstream_ = true; } return GST_PAD_PROBE_OK; @@ -316,54 +314,97 @@ void FrameGrabber::addFrame (GstBuffer *buffer, GstCaps *caps, float dt) gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, FrameGrabber::callback_event_probe, this, NULL); gst_object_unref (pad); } - - // terminate properly if finished - if (finished_) - terminate(); - // stop if an incompatilble frame buffer given else if ( !gst_caps_is_equal( caps_, caps )) { stop(); - Log::Warning("FrameGrabber interrupted because the resolution changed."); + Log::Warning("Frame capture interrupted because the resolution changed."); } - // store a frame if recording is active - // and if the encoder accepts data - else if (active_ && accept_buffer_) + // store a frame if recording is active and if the encoder accepts data + if (active_) { - GstClockTime t = 0; + if (accept_buffer_) { + GstClockTime t = 0; + + // initialize timer on first occurence + if (timer_ == nullptr) { + timer_ = gst_pipeline_get_clock ( GST_PIPELINE(pipeline_) ); + timer_firstframe_ = gst_clock_get_time(timer_); + } + else + // time since timer starts (first frame registered) + t = gst_clock_get_time(timer_) - timer_firstframe_; + + // if time is zero (first frame) or if delta time is passed one frame duration (with a margin) + if ( t == 0 || (t - timestamp_) > (frame_duration_ - 3000) ) { + + // round time to a multiples of frame duration + t = ( t / frame_duration_) * frame_duration_; + + // set frame presentation time stamp + buffer->pts = t; + + // if time since last timestamp is more than 1 frame + if (t - timestamp_ > frame_duration_) { + // compute duration + buffer->duration = t - timestamp_; + // keep timestamp for next addFrame to one frame later + timestamp_ = t + frame_duration_; + } + // normal case (not delayed) + else { + // normal frame duration + buffer->duration = frame_duration_; + // keep timestamp for next addFrame + timestamp_ = t; + } + + // when buffering is full, refuse buffer every frame + if (buffering_full_) + accept_buffer_ = false; + else + { + // enter buffering_full_ mode if the space left in buffering is for only few frames + // (this prevents filling the buffer entirely) +// if ( (double) gst_app_src_get_current_level_bytes(src_) / (double) buffering_size_ > 0.8) // 80% test + if ( buffering_size_ - gst_app_src_get_current_level_bytes(src_) < 4 * gst_buffer_get_size(buffer)) + buffering_full_ = true; + } + + // increment ref counter to make sure the frame remains available + gst_buffer_ref(buffer); + + // push frame + gst_app_src_push_buffer (src_, buffer); + // NB: buffer will be unrefed by the appsrc + + // count frames + frame_count_++; + + } - // initialize timer on first occurence - if (timer_ == nullptr) { - timer_ = gst_pipeline_get_clock ( GST_PIPELINE(pipeline_) ); - timer_firstframe_ = gst_clock_get_time(timer_); } + } + + // if we received and end of stream (from callback_event_probe) + if (endofstream_) + { + // try to stop properly when interrupted + if (active_) { + // de-activate and re-send EOS + stop(); + // inform + Log::Warning("Frame capture : interrupted after %s.", GstToolkit::time_to_string(timestamp_).c_str()); + Log::Info("Frame capture: not space left on drive / encoding buffer full."); + } + // terminate properly if finished else - // time since timer starts (first frame registered) - t = gst_clock_get_time(timer_) - timer_firstframe_; - - // if time is zero (first frame) - // of if delta time is passed one frame duration (with a margin) - if ( t == 0 || (t - timestamp_) > (frame_duration_ - 3000) ) { - - // round t to multiple of frame duration - t = ( t / frame_duration_) * frame_duration_; - - // set timing of buffer - buffer->pts = t; - buffer->duration = t - timestamp_; - - // increment ref counter to make sure the frame remains available - gst_buffer_ref(buffer); - - // push frame - gst_app_src_push_buffer (src_, buffer); - // NB: buffer will be unrefed by the appsrc - - // keep timestamp for next addFrame - timestamp_ = t; + { + finished_ = true; + terminate(); } + } } diff --git a/FrameGrabber.h b/FrameGrabber.h index 502ed11..aee3e4e 100644 --- a/FrameGrabber.h +++ b/FrameGrabber.h @@ -14,6 +14,8 @@ // https://stackoverflow.com/questions/38140527/glreadpixels-vs-glgetteximage #define USE_GLREADPIXEL +#define MIN_BUFFER_SIZE 33177600 // 33177600 bytes = 1 frames 4K, 9 frames 720p + class FrameBuffer; @@ -53,9 +55,9 @@ protected: virtual void terminate() = 0; // thread-safe testing termination - std::atomic expecting_finished_; std::atomic finished_; std::atomic active_; + std::atomic endofstream_; std::atomic accept_buffer_; // gstreamer pipeline @@ -65,6 +67,9 @@ protected: GstClockTime timestamp_; GstClockTime frame_duration_; + guint64 frame_count_; + guint64 buffering_size_; + std::atomic buffering_full_; GstClockTime timer_firstframe_; GstClock *timer_; diff --git a/Loopback.cpp b/Loopback.cpp index d462272..7e0230a 100644 --- a/Loopback.cpp +++ b/Loopback.cpp @@ -152,7 +152,6 @@ bool Loopback::systemLoopbackInitialized() Loopback::Loopback() : FrameGrabber() { frame_duration_ = gst_util_uint64_scale_int (1, GST_SECOND, 60); - } void Loopback::init(GstCaps *caps) @@ -190,25 +189,26 @@ void Loopback::init(GstCaps *caps) if (src_) { g_object_set (G_OBJECT (src_), - "stream-type", GST_APP_STREAM_TYPE_STREAM, "is-live", TRUE, - "format", GST_FORMAT_TIME, - // "do-timestamp", TRUE, NULL); - // Direct encoding (no buffering) - gst_app_src_set_max_bytes( src_, 0 ); + // configure stream + gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM); + gst_app_src_set_latency( src_, -1, 0); + + // Set buffer size + gst_app_src_set_max_bytes( src_, buffering_size_ ); // instruct src to use the required caps caps_ = gst_caps_copy( caps ); - gst_app_src_set_caps (src_, caps_); + gst_app_src_set_caps( src_, caps_); // setup callbacks GstAppSrcCallbacks callbacks; callbacks.need_data = FrameGrabber::callback_need_data; callbacks.enough_data = FrameGrabber::callback_enough_data; callbacks.seek_data = NULL; // stream type is not seekable - gst_app_src_set_callbacks (src_, &callbacks, this, NULL); + gst_app_src_set_callbacks( src_, &callbacks, this, NULL); } else { @@ -237,6 +237,5 @@ void Loopback::init(GstCaps *caps) void Loopback::terminate() { - active_ = false; Log::Notify("Loopback to %s terminated.", Loopback::system_loopback_name.c_str()); } diff --git a/Recorder.cpp b/Recorder.cpp index 5c02eb0..fa7f838 100644 --- a/Recorder.cpp +++ b/Recorder.cpp @@ -60,12 +60,13 @@ void PNGRecorder::init(GstCaps *caps) if (src_) { g_object_set (G_OBJECT (src_), - "stream-type", GST_APP_STREAM_TYPE_STREAM, "is-live", TRUE, - "format", GST_FORMAT_TIME, - // "do-timestamp", TRUE, NULL); + // configure stream + gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM); + gst_app_src_set_latency( src_, -1, 0); + // Direct encoding (no buffering) gst_app_src_set_max_bytes( src_, 0 ); @@ -104,7 +105,6 @@ void PNGRecorder::init(GstCaps *caps) void PNGRecorder::terminate() { - active_ = false; Log::Notify("PNG Capture %s is ready.", filename_.c_str()); } @@ -178,7 +178,7 @@ const std::vector VideoRecorder::profile_description { // 2 ‘standard’ // 3 ‘hq’ // 4 ‘4444’ - "avenc_prores_ks pass=2 profile=2 quantizer=26 ! ", + "video/x-raw, format=I422_10LE ! avenc_prores_ks pass=2 profile=2 quantizer=26 ! ", "video/x-raw, format=Y444_10LE ! avenc_prores_ks pass=2 profile=4 quantizer=12 ! ", // VP8 WebM encoding "vp8enc end-usage=vbr cpu-used=8 max-quantizer=35 deadline=100000 target-bitrate=200000 keyframe-max-dist=360 token-partitions=2 static-threshold=100 ! ", @@ -198,8 +198,13 @@ const std::vector VideoRecorder::profile_description { // "qtmux ! filesink name=sink"; -VideoRecorder::VideoRecorder() : FrameGrabber() +const char* VideoRecorder::buffering_preset_name[VIDEO_RECORDER_BUFFERING_NUM_PRESET] = { "30 MB", "100 MB", "200 MB", "500 MB", "1 GB", "2 GB" }; +const guint64 VideoRecorder::buffering_preset_value[VIDEO_RECORDER_BUFFERING_NUM_PRESET] = { MIN_BUFFER_SIZE, 104857600, 209715200, 524288000, 1073741824, 2147483648}; + + +VideoRecorder::VideoRecorder(guint64 buffersize) : FrameGrabber() { + buffering_size_ = MAX( MIN_BUFFER_SIZE, buffersize); } void VideoRecorder::init(GstCaps *caps) @@ -239,7 +244,7 @@ void VideoRecorder::init(GstCaps *caps) GError *error = NULL; pipeline_ = gst_parse_launch (description.c_str(), &error); if (error != NULL) { - Log::Warning("VideoRecorder Could not construct pipeline %s:\n%s", description.c_str(), error->message); + Log::Warning("Video Recording : Could not construct pipeline %s:\n%s", description.c_str(), error->message); g_clear_error (&error); finished_ = true; return; @@ -256,14 +261,17 @@ void VideoRecorder::init(GstCaps *caps) if (src_) { g_object_set (G_OBJECT (src_), - "stream-type", GST_APP_STREAM_TYPE_STREAM, "is-live", TRUE, "format", GST_FORMAT_TIME, - // "do-timestamp", TRUE, +// "do-timestamp", TRUE, NULL); - // Direct encoding (no buffering) - gst_app_src_set_max_bytes( src_, 0 ); + // configure stream + gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM); + gst_app_src_set_latency( src_, -1, 0); + + // Set buffer size + gst_app_src_set_max_bytes( src_, buffering_size_); // instruct src to use the required caps caps_ = gst_caps_copy( caps ); @@ -278,7 +286,7 @@ void VideoRecorder::init(GstCaps *caps) } else { - Log::Warning("VideoRecorder Could not configure source"); + Log::Warning("Video Recording : Could not configure source"); finished_ = true; return; } @@ -286,7 +294,7 @@ void VideoRecorder::init(GstCaps *caps) // start recording GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_PLAYING); if (ret == GST_STATE_CHANGE_FAILURE) { - Log::Warning("VideoRecorder Could not record %s", filename_.c_str()); + Log::Warning("Video Recording : Could not record %s", filename_.c_str()); finished_ = true; return; } @@ -300,10 +308,15 @@ void VideoRecorder::init(GstCaps *caps) void VideoRecorder::terminate() { - active_ = false; + // stop the pipeline + gst_element_set_state (pipeline_, GST_STATE_NULL); - if (!expecting_finished_) - Log::Warning("Video Recording interrupted (no more disk space?)."); + guint64 N = MAX( (guint64) timestamp_ / (guint64) frame_duration_, frame_count_); + float loss = 100.f * ((float) (N - frame_count_) ) / (float) N; + Log::Info("Video Recording : %ld frames in %s (aming for %ld, %.0f%% lost)", frame_count_, GstToolkit::time_to_string(timestamp_).c_str(), N, loss); + Log::Info("Video Recording : try with a lower resolution / a larger buffer size / a faster codec."); + if (loss > 20.f) + Log::Warning("Video Recording lost %.0f%% of frames.", loss); Log::Notify("Video Recording %s is ready.", filename_.c_str()); } @@ -312,6 +325,8 @@ std::string VideoRecorder::info() const { if (active_) return GstToolkit::time_to_string(timestamp_); - else + else if (!endofstream_) return "Saving file..."; + else + return "..."; } diff --git a/Recorder.h b/Recorder.h index fdda4e1..df4b334 100644 --- a/Recorder.h +++ b/Recorder.h @@ -25,6 +25,8 @@ protected: }; +#define VIDEO_RECORDER_BUFFERING_NUM_PRESET 6 + class VideoRecorder : public FrameGrabber { std::string filename_; @@ -45,10 +47,13 @@ public: JPEG_MULTI, DEFAULT } Profile; - static const char* profile_name[DEFAULT]; + static const char* profile_name[DEFAULT]; static const std::vector profile_description; - VideoRecorder(); + static const char* buffering_preset_name[VIDEO_RECORDER_BUFFERING_NUM_PRESET]; + static const guint64 buffering_preset_value[VIDEO_RECORDER_BUFFERING_NUM_PRESET]; + + VideoRecorder(guint64 buffersize = 0); std::string info() const override; }; diff --git a/Settings.cpp b/Settings.cpp index 141adfb..83e3995 100644 --- a/Settings.cpp +++ b/Settings.cpp @@ -102,6 +102,7 @@ void Settings::Save() RecordNode->SetAttribute("profile", application.record.profile); RecordNode->SetAttribute("timeout", application.record.timeout); RecordNode->SetAttribute("delay", application.record.delay); + RecordNode->SetAttribute("buffering_mode", application.record.buffering_mode); pRoot->InsertEndChild(RecordNode); // Transition @@ -314,6 +315,7 @@ void Settings::Load() recordnode->QueryIntAttribute("profile", &application.record.profile); recordnode->QueryUnsignedAttribute("timeout", &application.record.timeout); recordnode->QueryIntAttribute("delay", &application.record.delay); + recordnode->QueryIntAttribute("buffering_mode", &application.record.buffering_mode); const char *path_ = recordnode->Attribute("path"); if (path_) diff --git a/Settings.h b/Settings.h index dcc133e..35c242d 100644 --- a/Settings.h +++ b/Settings.h @@ -77,11 +77,13 @@ struct RecordConfig int profile; uint timeout; int delay; + int buffering_mode; RecordConfig() : path("") { profile = 0; timeout = RECORD_MAX_TIMEOUT; delay = 0; + buffering_mode = 0; } }; diff --git a/Streamer.cpp b/Streamer.cpp index b65afe2..cbb6951 100644 --- a/Streamer.cpp +++ b/Streamer.cpp @@ -345,14 +345,17 @@ void VideoStreamer::init(GstCaps *caps) if (src_) { g_object_set (G_OBJECT (src_), - "stream-type", GST_APP_STREAM_TYPE_STREAM, "is-live", TRUE, "format", GST_FORMAT_TIME, // "do-timestamp", TRUE, NULL); - // Direct encoding (no buffering) - gst_app_src_set_max_bytes( src_, 0 ); + // configure stream + gst_app_src_set_stream_type( src_, GST_APP_STREAM_TYPE_STREAM); + gst_app_src_set_latency( src_, -1, 0); + + // Set buffer size + gst_app_src_set_max_bytes( src_, buffering_size_ ); // instruct src to use the required caps caps_ = gst_caps_copy( caps ); @@ -389,8 +392,6 @@ void VideoStreamer::init(GstCaps *caps) void VideoStreamer::terminate() { - active_ = false; - // send EOS gst_app_src_end_of_stream (src_); diff --git a/UserInterfaceManager.cpp b/UserInterfaceManager.cpp index fa1db56..fa732ef 100644 --- a/UserInterfaceManager.cpp +++ b/UserInterfaceManager.cpp @@ -275,7 +275,9 @@ void UserInterface::handleKeyboard() // video_recorder_ = nullptr; } else { - _video_recorders.emplace_back( std::async(std::launch::async, delayTrigger, new VideoRecorder, std::chrono::seconds(Settings::application.record.delay)) ); + _video_recorders.emplace_back( std::async(std::launch::async, delayTrigger, + new VideoRecorder(VideoRecorder::buffering_preset_value[Settings::application.record.buffering_mode]), + std::chrono::seconds(Settings::application.record.delay)) ); } } } @@ -1100,7 +1102,9 @@ void UserInterface::RenderPreview() else { ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(IMGUI_COLOR_RECORD, 0.9f)); if ( ImGui::MenuItem( ICON_FA_CIRCLE " Record", CTRL_MOD "R") ) { - _video_recorders.emplace_back( std::async(std::launch::async, delayTrigger, new VideoRecorder, std::chrono::seconds(Settings::application.record.delay)) ); + _video_recorders.emplace_back( std::async(std::launch::async, delayTrigger, + new VideoRecorder(VideoRecorder::buffering_preset_value[Settings::application.record.buffering_mode]), + std::chrono::seconds(Settings::application.record.delay)) ); } ImGui::PopStyleColor(1); // select profile @@ -1138,6 +1142,11 @@ void UserInterface::RenderPreview() ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN); ImGui::SliderInt("Trigger", &Settings::application.record.delay, 0, 5, Settings::application.record.delay < 1 ? "Immediate" : "After %d s"); + + ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN); + ImGui::SliderInt("Buffer", &Settings::application.record.buffering_mode, 0, VIDEO_RECORDER_BUFFERING_NUM_PRESET-1, + VideoRecorder::buffering_preset_name[Settings::application.record.buffering_mode]); + } ImGui::EndMenu(); } @@ -4548,8 +4557,6 @@ void Navigator::RenderMainPannelSettings() ImGui::SetCursorPosY(width_); // Appearance -// ImGuiToolkit::Icon(3, 2); -// ImGui::SameLine(0, 10); ImGui::Text("Appearance"); ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN); if ( ImGui::DragFloat("Scale", &Settings::application.scale, 0.01, 0.5f, 2.0f, "%.1f")) @@ -4562,8 +4569,6 @@ void Navigator::RenderMainPannelSettings() // Options ImGui::Spacing(); -// ImGuiToolkit::Icon(2, 2); -// ImGui::SameLine(0, 10); ImGui::Text("Options"); ImGuiToolkit::ButtonSwitch( ICON_FA_MOUSE_POINTER " Smooth cursor", &Settings::application.smooth_cursor); ImGuiToolkit::ButtonSwitch( ICON_FA_TACHOMETER_ALT " Metrics", &Settings::application.widget.stats); @@ -4578,13 +4583,8 @@ void Navigator::RenderMainPannelSettings() // system preferences ImGui::Spacing(); -//#ifdef LINUX -// ImGuiToolkit::Icon(12, 6); -//#else -// ImGuiToolkit::Icon(6, 0); -//#endif -// ImGui::SameLine(0, 10); ImGui::Text("System"); + static bool need_restart = false; static bool vsync = (Settings::application.render.vsync > 0); static bool blit = Settings::application.render.blit;