From 68b2c5e0c1f3eea5beb02567517796afba755e97 Mon Sep 17 00:00:00 2001 From: Bruno Herbelin Date: Thu, 2 Dec 2021 11:45:22 +0100 Subject: [PATCH] Frame grabber threaded initialization Start gstreamer init of frame grabber in a thread and wait future return from initializer before switching to active recording mode. --- FrameGrabber.cpp | 52 +++++++++++++++++++++++++++++++++++++--------- FrameGrabber.h | 10 +++++++-- Loopback.cpp | 27 +++++++++--------------- Loopback.h | 2 +- Recorder.cpp | 54 +++++++++++++++++++++--------------------------- Recorder.h | 4 ++-- Streamer.cpp | 28 ++++++++++++------------- Streamer.h | 2 +- 8 files changed, 100 insertions(+), 79 deletions(-) diff --git a/FrameGrabber.cpp b/FrameGrabber.cpp index 5be1878..2ea2bdf 100644 --- a/FrameGrabber.cpp +++ b/FrameGrabber.cpp @@ -271,8 +271,8 @@ void FrameGrabbing::grabFrame(FrameBuffer *frame_buffer) -FrameGrabber::FrameGrabber(): finished_(false), active_(false), endofstream_(false), accept_buffer_(false), buffering_full_(false), - pipeline_(nullptr), src_(nullptr), caps_(nullptr), timer_(nullptr), +FrameGrabber::FrameGrabber(): finished_(false), initialized_(false), active_(false), endofstream_(false), accept_buffer_(false), buffering_full_(false), + pipeline_(nullptr), src_(nullptr), caps_(nullptr), timer_(nullptr), timer_firstframe_(0), timestamp_(0), duration_(0), frame_count_(0), buffering_size_(MIN_BUFFER_SIZE), timestamp_on_clock_(false) { // unique id @@ -315,6 +315,8 @@ uint64_t FrameGrabber::duration() const void FrameGrabber::stop () { + // TODO if not initialized wait for initializer + // stop recording active_ = false; @@ -324,6 +326,8 @@ void FrameGrabber::stop () std::string FrameGrabber::info() const { + if (!initialized_) + return "Initializing"; if (active_) return GstToolkit::time_to_string(duration_); else @@ -363,6 +367,12 @@ GstPadProbeReturn FrameGrabber::callback_event_probe(GstPad *, GstPadProbeInfo * return GST_PAD_PROBE_OK; } + +std::string FrameGrabber::initialize(FrameGrabber *rec, GstCaps *caps) +{ + return rec->init(caps); +} + void FrameGrabber::addFrame (GstBuffer *buffer, GstCaps *caps) { // ignore @@ -371,15 +381,37 @@ void FrameGrabber::addFrame (GstBuffer *buffer, GstCaps *caps) // first time initialization if (pipeline_ == nullptr) { - // type specific initialisation - init(caps); - // attach EOS detector - GstPad *pad = gst_element_get_static_pad (gst_bin_get_by_name (GST_BIN (pipeline_), "sink"), "sink"); - gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, FrameGrabber::callback_event_probe, this, NULL); - gst_object_unref (pad); + initializer_ = std::async( FrameGrabber::initialize, this, caps); } - // stop if an incompatilble frame buffer given - else if ( !gst_caps_is_subset( caps_, caps )) + + // initializer ongoing in separate thread + if (initializer_.valid()) { + // try to get info from initializer + if (initializer_.wait_for( std::chrono::milliseconds(4) ) == std::future_status::ready ) + { + // done initialization + std::string msg = initializer_.get(); + + // if initialization succeeded + if (initialized_) { + // attach EOS detector + GstPad *pad = gst_element_get_static_pad (gst_bin_get_by_name (GST_BIN (pipeline_), "sink"), "sink"); + gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, FrameGrabber::callback_event_probe, this, NULL); + gst_object_unref (pad); + // start recording + active_ = true; + // inform + Log::Info("%s", msg.c_str()); + } + // else show warning + else { + Log::Warning("%s", msg.c_str()); + } + } + } + + // stop if an incompatilble frame buffer given after initialization + if (initialized_ && !gst_caps_is_subset( caps_, caps )) { stop(); Log::Warning("Frame capture interrupted because the resolution changed."); diff --git a/FrameGrabber.h b/FrameGrabber.h index 44dc397..f9ebba6 100644 --- a/FrameGrabber.h +++ b/FrameGrabber.h @@ -2,6 +2,7 @@ #define FRAMEGRABBER_H #include +#include #include #include #include @@ -52,11 +53,12 @@ protected: virtual void addFrame(GstBuffer *buffer, GstCaps *caps); // only addFrame method shall call those - virtual void init(GstCaps *caps) = 0; + virtual std::string init(GstCaps *caps) = 0; virtual void terminate() = 0; // thread-safe testing termination std::atomic finished_; + std::atomic initialized_; std::atomic active_; std::atomic endofstream_; std::atomic accept_buffer_; @@ -68,6 +70,7 @@ protected: GstCaps *caps_; GstClock *timer_; + GstClockTime timer_firstframe_; GstClockTime timestamp_; GstClockTime duration_; GstClockTime frame_duration_; @@ -75,7 +78,10 @@ protected: guint64 buffering_size_; bool timestamp_on_clock_; - GstClockTime timer_firstframe_; + + // async threaded initializer + std::future initializer_; + static std::string initialize(FrameGrabber *rec, GstCaps *caps); // gstreamer callbacks static void callback_need_data (GstAppSrc *, guint, gpointer user_data); diff --git a/Loopback.cpp b/Loopback.cpp index 3c22565..5098ab9 100644 --- a/Loopback.cpp +++ b/Loopback.cpp @@ -171,16 +171,15 @@ Loopback::Loopback() : FrameGrabber() frame_duration_ = gst_util_uint64_scale_int (1, GST_SECOND, 30); // fixed 30 FPS } -void Loopback::init(GstCaps *caps) +std::string Loopback::init(GstCaps *caps) { // ignore if (caps == nullptr) - return; + return std::string("Invalid caps"); if (!Loopback::systemLoopbackInitialized()){ - Log::Warning("Loopback system shall be initialized first."); finished_ = true; - return; + return std::string("Loopback system shall be initialized first."); } // create a gstreamer pipeline @@ -190,10 +189,10 @@ void Loopback::init(GstCaps *caps) GError *error = NULL; pipeline_ = gst_parse_launch (description.c_str(), &error); if (error != NULL) { - Log::Warning("Loopback Could not construct pipeline %s:\n%s", description.c_str(), error->message); + std::string msg = std::string("Loopback : Could not construct pipeline ") + description + "\n" + std::string(error->message); g_clear_error (&error); finished_ = true; - return; + return msg; } // setup device sink @@ -238,27 +237,21 @@ void Loopback::init(GstCaps *caps) } else { - Log::Warning("Loopback Could not configure source"); finished_ = true; - return; + return std::string("Loopback : Could not configure source."); } // start recording GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_PLAYING); if (ret == GST_STATE_CHANGE_FAILURE) { - Log::Warning("Loopback Could not open %s", Loopback::system_loopback_name.c_str()); finished_ = true; - return; + return std::string("Loopback : Could not open ") + Loopback::system_loopback_name; } // all good -#if defined(LINUX) - Log::Notify("Loopback started (v4l2loopback on %s)", Loopback::system_loopback_name.c_str()); -#else - Log::Notify("Loopback started (%s)", Loopback::system_loopback_name.c_str()); -#endif - // start - active_ = true; + initialized_ = true; + + return std::string("Loopback started on ") + Loopback::system_loopback_name; } void Loopback::terminate() diff --git a/Loopback.h b/Loopback.h index b6bd406..7f047c3 100644 --- a/Loopback.h +++ b/Loopback.h @@ -15,7 +15,7 @@ class Loopback : public FrameGrabber static std::string system_loopback_name; static bool system_loopback_initialized; - void init(GstCaps *caps) override; + std::string init(GstCaps *caps) override; void terminate() override; public: diff --git a/Recorder.cpp b/Recorder.cpp index f9f3ac1..277be01 100644 --- a/Recorder.cpp +++ b/Recorder.cpp @@ -43,11 +43,11 @@ PNGRecorder::PNGRecorder() : FrameGrabber() { } -void PNGRecorder::init(GstCaps *caps) +std::string PNGRecorder::init(GstCaps *caps) { // ignore if (caps == nullptr) - return; + return std::string("Invalid caps"); // create a gstreamer pipeline std::string description = "appsrc name=src ! videoconvert ! pngenc ! filesink name=sink"; @@ -56,10 +56,10 @@ void PNGRecorder::init(GstCaps *caps) GError *error = NULL; pipeline_ = gst_parse_launch (description.c_str(), &error); if (error != NULL) { - Log::Warning("PNG Capture Could not construct pipeline %s:\n%s", description.c_str(), error->message); + std::string msg = std::string("PNG Capture Could not construct pipeline ") + description + "\n" + std::string(error->message); g_clear_error (&error); finished_ = true; - return; + return msg; } // verify location path (path is always terminated by the OS dependent separator) @@ -102,24 +102,21 @@ void PNGRecorder::init(GstCaps *caps) } else { - Log::Warning("PNG Capture Could not configure source"); finished_ = true; - return; + return std::string("PNG Capture : Failed to configure frame grabber."); } // start pipeline GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_PLAYING); if (ret == GST_STATE_CHANGE_FAILURE) { - Log::Warning("PNG Capture Could not record %s", filename_.c_str()); finished_ = true; - return; + return std::string("PNG Capture : Failed to start frame grabber."); } // all good - Log::Info("PNG Capture started."); + initialized_ = true; - // start recording !! - active_ = true; + return std::string("PNG Capture started "); } void PNGRecorder::terminate() @@ -289,11 +286,11 @@ VideoRecorder::VideoRecorder() : FrameGrabber() } -void VideoRecorder::init(GstCaps *caps) +std::string VideoRecorder::init(GstCaps *caps) { // ignore if (caps == nullptr) - return; + return std::string("Invalid caps"); // apply settings buffering_size_ = MAX( MIN_BUFFER_SIZE, buffering_preset_value[Settings::application.record.buffering_mode]); @@ -307,10 +304,10 @@ void VideoRecorder::init(GstCaps *caps) // test for a hardware accelerated encoder if (Settings::application.render.gpu_decoding && -#if GST_GL_HAVE_PLATFORM_GLX +//#if GST_GL_HAVE_PLATFORM_GLX - glGetString(GL_VENDOR)[0] == 'N' && glGetString(GL_VENDOR)[1] == 'V' && // TODO; hack to test for NVIDIA GPU support -#endif +// glGetString(GL_VENDOR)[0] == 'N' && glGetString(GL_VENDOR)[1] == 'V' && // TODO; hack to test for NVIDIA GPU support +//#endif GstToolkit::has_feature(hardware_encoder[Settings::application.record.profile]) ) { description += hardware_profile_description[Settings::application.record.profile]; @@ -345,11 +342,10 @@ void VideoRecorder::init(GstCaps *caps) GError *error = NULL; pipeline_ = gst_parse_launch (description.c_str(), &error); if (error != NULL) { - Log::Info("Video Recording : Could not construct pipeline %s\n%s", description.c_str(), error->message); - Log::Warning("Video Recording : Failed to initiate GStreamer."); + std::string msg = std::string("Video Recording : Could not construct pipeline ") + description + "\n" + std::string(error->message); g_clear_error (&error); finished_ = true; - return; + return msg; } // setup file sink @@ -399,24 +395,22 @@ void VideoRecorder::init(GstCaps *caps) } else { - Log::Warning("Video Recording : Failed to configure frame grabber."); finished_ = true; - return; + return std::string("Video Recording : Failed to configure frame grabber."); } // start recording GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_PLAYING); if (ret == GST_STATE_CHANGE_FAILURE) { - Log::Warning("Video Recording : Failed to start frame grabber."); finished_ = true; - return; + return std::string("Video Recording : Failed to start frame grabber."); } // all good - Log::Info("Video Recording started (%s)", profile_name[Settings::application.record.profile]); + initialized_ = true; + + return std::string("Video Recording started ") + profile_name[Settings::application.record.profile]; - // start recording !! - active_ = true; } void VideoRecorder::terminate() @@ -445,10 +439,8 @@ void VideoRecorder::terminate() std::string VideoRecorder::info() const { - if (active_) - return FrameGrabber::info(); - else if (!endofstream_) + if (initialized_ && !active_ && !endofstream_) return "Saving file..."; - else - return "..."; + + return FrameGrabber::info(); } diff --git a/Recorder.h b/Recorder.h index caf7766..b084929 100644 --- a/Recorder.h +++ b/Recorder.h @@ -18,7 +18,7 @@ public: protected: - void init(GstCaps *caps) override; + std::string init(GstCaps *caps) override; void terminate() override; void addFrame(GstBuffer *buffer, GstCaps *caps) override; @@ -28,7 +28,7 @@ class VideoRecorder : public FrameGrabber { std::string filename_; - void init(GstCaps *caps) override; + std::string init(GstCaps *caps) override; void terminate() override; public: diff --git a/Streamer.cpp b/Streamer.cpp index 3576ff5..54ce675 100644 --- a/Streamer.cpp +++ b/Streamer.cpp @@ -308,11 +308,11 @@ VideoStreamer::VideoStreamer(const NetworkToolkit::StreamConfig &conf): FrameGra frame_duration_ = gst_util_uint64_scale_int (1, GST_SECOND, STREAMING_FPS); // fixed 30 FPS } -void VideoStreamer::init(GstCaps *caps) +std::string VideoStreamer::init(GstCaps *caps) { // ignore if (caps == nullptr) - return; + return std::string("Invalid caps"); // check that config matches the given buffer properties gint w = 0, h = 0; @@ -322,10 +322,9 @@ void VideoStreamer::init(GstCaps *caps) if ( gst_structure_has_field (capstruct, "height")) gst_structure_get_int (capstruct, "height", &h); if ( config_.width != w || config_.height != h) { - Log::Warning("Streaming cannot start: given frames (%d x %d) incompatible with stream (%d x %d)", - w, w, config_.width, config_.height); finished_ = true; - return; + return std::string("Video Streamer cannot start: given frames (") + std::to_string(w) + " x " + std::to_string(h) + + ") are incompatible with stream (" + std::to_string(config_.width) + " x " + std::to_string(config_.height) + ")"; } // prevent eroneous protocol values @@ -340,10 +339,10 @@ void VideoStreamer::init(GstCaps *caps) GError *error = NULL; pipeline_ = gst_parse_launch (description.c_str(), &error); if (error != NULL) { - Log::Warning("VideoStreamer Could not construct pipeline %s:\n%s", description.c_str(), error->message); + std::string msg = std::string("Video Streamer : Could not construct pipeline ") + description + "\n" + std::string(error->message); g_clear_error (&error); finished_ = true; - return; + return msg; } // setup streaming sink @@ -398,24 +397,21 @@ void VideoStreamer::init(GstCaps *caps) } else { - Log::Warning("VideoStreamer Could not configure capture source"); finished_ = true; - return; + return std::string("Video Streamer : Failed to configure frame grabber."); } // start recording GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_PLAYING); if (ret == GST_STATE_CHANGE_FAILURE) { - Log::Warning("VideoStreamer failed"); finished_ = true; - return; + return std::string("Video Streamer : Failed to start frame grabber."); } // all good - Log::Notify("Streaming to %s.", config_.client_name.c_str()); + initialized_ = true; - // start streaming !! - active_ = true; + return std::string("Streaming to ") + config_.client_name + "started."; } void VideoStreamer::terminate() @@ -451,7 +447,9 @@ void VideoStreamer::stop () std::string VideoStreamer::info() const { std::ostringstream ret; - if (active_) { + if (!initialized_) + ret << "Connecting"; + else if (active_) { ret << NetworkToolkit::protocol_name[config_.protocol]; ret << " to "; ret << config_.client_name; diff --git a/Streamer.h b/Streamer.h index 88c7b8c..f57dae7 100644 --- a/Streamer.h +++ b/Streamer.h @@ -70,7 +70,7 @@ class VideoStreamer : public FrameGrabber { friend class Streaming; - void init(GstCaps *caps) override; + std::string init(GstCaps *caps) override; void terminate() override; void stop() override;