Implementation of Recorder with dual PBO mechanism for best efficiency

and compatibility. Fixed user interface and avoid user creating multiple
recorders.

FrameBuffer.cpp

@@ -116,17 +116,12 @@ glm::vec3 FrameBuffer::resolution() const
     return glm::vec3(attrib_.viewport.x, attrib_.viewport.y, 0.f);
 }
 
-void FrameBuffer::bind()
+void FrameBuffer::begin()
 {
     if (!framebufferid_)
         init();
 
     glBindFramebuffer(GL_FRAMEBUFFER, framebufferid_);
-}
-
-void FrameBuffer::begin()
-{
-    bind();
 
     Rendering::manager().pushAttrib(attrib_);
 
@@ -155,6 +150,21 @@ void FrameBuffer::release()
     glBindFramebuffer(GL_FRAMEBUFFER, 0);
 }
 
+void FrameBuffer::readPixels()
+{
+    if (!framebufferid_)
+        return;
+
+    if (use_multi_sampling_)
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, intermediate_framebufferid_);
+    else
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, framebufferid_);
+
+    glPixelStorei(GL_PACK_ALIGNMENT, 1);
+    glReadPixels(0, 0, attrib_.viewport.x, attrib_.viewport.y, (use_alpha_? GL_RGBA : GL_RGB), GL_UNSIGNED_BYTE, 0);   
+    glBindFramebuffer(GL_FRAMEBUFFER, 0);
+}
+
 bool FrameBuffer::blit(FrameBuffer *other)
 {
     if (!framebufferid_ || !other || !other->framebufferid_)

FrameBuffer.h

@@ -18,8 +18,6 @@ public:
     FrameBuffer(uint width, uint height, bool useAlpha = false, bool multiSampling = false);
     ~FrameBuffer();
 
-    // bind the FrameBuffer as current to draw into
-    void bind();
     // unbind any framebuffer object
     static void release();
     // Bind & push attribs to prepare draw
@@ -29,6 +27,9 @@ public:
 
     // blit copy to another, returns true on success
     bool blit(FrameBuffer *other);
+    // bind the FrameBuffer in READ and perform glReadPixels
+    // return the size of the buffer
+    void readPixels();
 
     // clear color
     inline void setClearColor(glm::vec4 color) { attrib_.clear_color = color; }

Recorder.cpp

@@ -20,11 +20,16 @@
 
 #include "Recorder.h"
 
+// use glReadPixel or glGetTextImage ?
+// read pixels & pbo should be the fastest
+// https://stackoverflow.com/questions/38140527/glreadpixels-vs-glgetteximage
+#define USE_GLREADPIXEL
+
 using namespace std;
 
-Recorder::Recorder() : finished_(false)
+Recorder::Recorder() : finished_(false), pbo_index_(0), pbo_next_index_(0), size_(0)
 {
-
+    pbo_[0] = pbo_[1] = 0;
 }
 
 PNGRecorder::PNGRecorder() : Recorder()
@@ -34,6 +39,7 @@ PNGRecorder::PNGRecorder() : Recorder()
         path = SystemToolkit::home_path();
 
     filename_ = path + SystemToolkit::date_time_string() + "_vimix.png";
+
 }
 
 // Thread to perform slow operation of saving to file
@@ -44,7 +50,7 @@ void save_png(std::string filename, unsigned char *data, uint w, uint h, uint c)
         // save file
         stbi_write_png(filename.c_str(), w, h, c, data, w * c);
         // notify
-        Log::Notify("Capture %s saved.", filename.c_str());
+        Log::Notify("Capture %s ready (%d x %d %d)", filename.c_str(), w, h, c);
         // done
         free(data);
     }
@@ -52,21 +58,68 @@ void save_png(std::string filename, unsigned char *data, uint w, uint h, uint c)
 
 void PNGRecorder::addFrame(FrameBuffer *frame_buffer, float)
 {
+    // ignore
+    if (frame_buffer == nullptr)
+        return;
 
+    // get what is needed from frame buffer
     uint w = frame_buffer->width();
     uint h = frame_buffer->height();
     uint c = frame_buffer->use_alpha() ? 4 : 3;
-    GLenum format = frame_buffer->use_alpha() ? GL_RGBA : GL_RGB;
-    uint size = w * h * c;
-    unsigned char * data = (unsigned char*) malloc(size);
 
-    glGetTextureSubImage( frame_buffer->texture(), 0, 0, 0, 0, w, h, 1, format, GL_UNSIGNED_BYTE, size, data);
+    // first iteration: initialize and get frame
+    if (size_ < 1)
+    {
+        // init size
+        size_ = w * h * c;
 
-    // save in separate thread
+        // create PBO
-    std::thread(save_png, filename_, data, w, h, c).detach();
+        glGenBuffers(2, pbo_);
+
+        // set writing PBO
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[0]);
+        glBufferData(GL_PIXEL_PACK_BUFFER, size_, NULL, GL_STREAM_READ);
+
+#ifdef USE_GLREADPIXEL
+        // get frame
+        frame_buffer->readPixels();
+#else
+        glBindTexture(GL_TEXTURE_2D, frame_buffer->texture());
+        glGetTexImage(GL_TEXTURE_2D, 0, GL_RGB, GL_UNSIGNED_BYTE, 0);
+#endif
+    }
+    // second iteration; get frame and save file
+    else {
+
+        // set reading PBO
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[0]);
+
+        // get pixels
+        unsigned char* ptr = (unsigned char*) glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);
+        if (NULL != ptr) {
+            // prepare memory buffer0
+            unsigned char * data = (unsigned char*) malloc(size_);
+            // transfer frame to data
+            memmove(data, ptr, size_);
+            // save in separate thread
+            std::thread(save_png, filename_, data, w, h, c).detach();
+        }
+        // unmap
+        glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
+
+        // ok done
+        glDeleteBuffers(2, pbo_);
+
+        // recorded one frame
+        finished_ = true;
+    }
+
+    glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+
+//    unsigned char * data = (unsigned char*) malloc(size);
+//    GLenum format = frame_buffer->use_alpha() ? GL_RGBA : GL_RGB;
+//    glGetTextureSubImage( frame_buffer->texture(), 0, 0, 0, 0, w, h, 1, format, GL_UNSIGNED_BYTE, size, data);
 
-    // record one frame only
-    finished_ = true;
 }
 
 const char* VideoRecorder::profile_name[4] = { "H264 (low)", "H264 (high)", "Apple ProRes 4444", "WebM VP9" };
@@ -104,7 +157,7 @@ const std::vector<std::string> VideoRecorder::profile_description {
 //               "qtmux ! filesink name=sink";
 
 
-VideoRecorder::VideoRecorder() : Recorder(), frame_buffer_(nullptr), width_(0), height_(0), buf_size_(0),
+VideoRecorder::VideoRecorder() : Recorder(), frame_buffer_(nullptr), width_(0), height_(0),
     recording_(false), pipeline_(nullptr), src_(nullptr), timestamp_(0), timeframe_(0), accept_buffer_(false)
 {
     // auto filename
@@ -121,12 +174,14 @@ VideoRecorder::VideoRecorder() : Recorder(), frame_buffer_(nullptr), width_(0),
 
 VideoRecorder::~VideoRecorder()
 {
+    if (src_ != nullptr)
+        gst_object_unref (src_);
     if (pipeline_ != nullptr) {
         gst_element_set_state (pipeline_, GST_STATE_NULL);
         gst_object_unref (pipeline_);
     }
-    if (src_ != nullptr)
+
-        gst_object_unref (src_);
+    glDeleteBuffers(2, pbo_);
 }
 
 void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
@@ -146,7 +201,14 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
        // define stream properties
        width_ = frame_buffer_->width();
        height_ = frame_buffer_->height();
-       buf_size_ = width_ * height_ * (frame_buffer_->use_alpha() ? 4 : 3);
+       size_ = width_ * height_ * (frame_buffer_->use_alpha() ? 4 : 3);
+
+       // create PBOs
+       glGenBuffers(2, pbo_);
+       glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[1]);
+       glBufferData(GL_PIXEL_PACK_BUFFER, size_, NULL, GL_STREAM_READ);
+       glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[0]);
+       glBufferData(GL_PIXEL_PACK_BUFFER, size_, NULL, GL_STREAM_READ);
 
        // create a gstreamer pipeline
        string description = "appsrc name=src ! videoconvert ! ";
@@ -164,18 +226,10 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
        }
 
        // setup file sink
-       sink_ = GST_BASE_SINK( gst_bin_get_by_name (GST_BIN (pipeline_), "sink") );
+       g_object_set (G_OBJECT (gst_bin_get_by_name (GST_BIN (pipeline_), "sink")),
-       if (sink_) {
+                     "location", filename_.c_str(),
-           g_object_set (G_OBJECT (sink_),
+                     "sync", FALSE,
-                         "location", filename_.c_str(),
+                     NULL);
-                         "sync", FALSE,
-                         NULL);
-       }
-       else {
-           Log::Warning("VideoRecorder Could not configure file");
-           finished_ = true;
-           return;
-       }
 
        // setup custom app source
        src_ = GST_APP_SRC( gst_bin_get_by_name (GST_BIN (pipeline_), "src") );
@@ -239,7 +293,7 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
             frame_buffer->use_alpha() != frame_buffer_->use_alpha()) {
 
            stop();
-           Log::Info("Recording interrupted: new session (%d x %d) incompatible with recording (%d x %d)", frame_buffer->width(), frame_buffer->height(), width_, height_);
+           Log::Warning("Recording interrupted: new session (%d x %d) incompatible with recording (%d x %d)", frame_buffer->width(), frame_buffer->height(), width_, height_);
        }
        else {
            // accepting a new frame buffer as input
@@ -247,10 +301,8 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
        }
    }
 
-   static int count = 0;
-
    // store a frame if recording is active
-   if (recording_ && buf_size_ > 0)
+   if (recording_ && size_ > 0)
    {
        // calculate dt in ns
        timeframe_ +=  gst_gdouble_to_guint64( dt * 1000000.f);
@@ -259,28 +311,62 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
        // and if the encoder accepts data
        if ( timeframe_ > frame_duration_ - 3000000 && accept_buffer_) {
 
-           GstBuffer *buffer = gst_buffer_new_and_alloc (buf_size_);
+           // set buffer target for writing in a new frame
-           GLenum format = frame_buffer_->use_alpha() ? GL_RGBA : GL_RGB;
+           glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[pbo_index_]);
 
-           // set timing of buffer
+#ifdef USE_GLREADPIXEL
-           buffer->pts = timestamp_;
+           // get frame
-           buffer->duration = frame_duration_;
+           frame_buffer->readPixels();
+#else
+           glBindTexture(GL_TEXTURE_2D, frame_buffer->texture());
+           glGetTexImage(GL_TEXTURE_2D, 0, GL_RGB, GL_UNSIGNED_BYTE, 0);
+#endif
 
-           // OpenGL capture
+           // update case ; alternating indices
-           GstMapInfo map;
+           if ( pbo_next_index_ != pbo_index_ ) {
-           gst_buffer_map (buffer, &map, GST_MAP_WRITE);
-           glGetTextureSubImage( frame_buffer_->texture(), 0, 0, 0, 0, width_, height_, 1, format, GL_UNSIGNED_BYTE, buf_size_, map.data);
-           gst_buffer_unmap (buffer, &map);
 
-           // push
+               // set buffer target for saving the frame
-//           Log::Info("VideoRecorder push data %ld", buffer->pts);
+               glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[pbo_next_index_]);
-           gst_app_src_push_buffer (src_, buffer);
-           // NB: buffer will be unrefed by the appsrc
 
-           // restart counter
+               // new buffer
+               GstBuffer *buffer = gst_buffer_new_and_alloc (size_);
+
+               // set timing of buffer
+               buffer->pts = timestamp_;
+               buffer->duration = frame_duration_;
+
+               // map gst buffer into a memory  WRITE target
+               GstMapInfo map;
+               gst_buffer_map (buffer, &map, GST_MAP_WRITE);
+
+               // map PBO pixels into a memory READ pointer
+               unsigned char* ptr = (unsigned char*) glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);
+
+               // transfer pixels from PBO memory to buffer memory
+               if (NULL != ptr)
+                   memmove(map.data, ptr, size_);
+
+               // un-map
+               glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
+               gst_buffer_unmap (buffer, &map);
+
+               // push
+    //           Log::Info("VideoRecorder push data %ld", buffer->pts);
+               gst_app_src_push_buffer (src_, buffer);
+               // NB: buffer will be unrefed by the appsrc
+
+               // next timestamp
+               timestamp_ += frame_duration_;
+           }
+
+           glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+
+           // alternate indices
+           pbo_next_index_ = pbo_index_;
+           pbo_index_ = (pbo_index_ + 1) % 2;
+
+           // restart frame counter
            timeframe_ = 0;
-           // next timestamp
-           timestamp_ += frame_duration_;
        }
 
    }
@@ -293,7 +379,6 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
 
        if (msg) {
 //           Log::Info("received EOS");
-
            // stop the pipeline
            GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_NULL);
            if (ret == GST_STATE_CHANGE_FAILURE)
@@ -301,7 +386,6 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
            else
                Log::Notify("Recording %s ready.", filename_.c_str());
 
-           count = 0;
            finished_ = true;
        }
    }

Recorder.h

@@ -29,7 +29,13 @@ public:
     inline bool finished() const { return finished_; }
 
 protected:
+    // thread-safe testing termination
     std::atomic<bool> finished_;
+
+    // PBO
+    guint pbo_[2];
+    guint pbo_index_, pbo_next_index_;
+    guint size_;
 };
 
 class PNGRecorder : public Recorder
@@ -52,7 +58,6 @@ class VideoRecorder : public Recorder
     FrameBuffer  *frame_buffer_;
     uint width_;
     uint height_;
-    uint buf_size_;
 
     // operation
     std::atomic<bool> recording_;
@@ -61,7 +66,6 @@ class VideoRecorder : public Recorder
     // gstreamer pipeline
     GstElement   *pipeline_;
     GstAppSrc    *src_;
-    GstBaseSink  *sink_;
     GstClockTime timeframe_;
     GstClockTime timestamp_;
     GstClockTime frame_duration_;

Screenshot.cpp

@@ -24,6 +24,7 @@ Screenshot::Screenshot()
 
 Screenshot::~Screenshot()
 {
+    glDeleteBuffers(1, &Pbo);
     if (Data)  free(Data);
 }
 

UserInterfaceManager.cpp

@@ -57,6 +57,7 @@ using namespace std;
 
 #include "TextEditor.h"
 static TextEditor editor;
+static Recorder *main_video_recorder = nullptr;
 
 // utility functions
 void ShowAboutGStreamer(bool* p_open);
@@ -292,7 +293,14 @@ void UserInterface::handleKeyboard()
         }
         else if (ImGui::IsKeyPressed( GLFW_KEY_R )) {
             // toggle recording
-            Mixer::manager().session()->addRecorder(new VideoRecorder);
+            if (main_video_recorder){
+                main_video_recorder->stop();
+                main_video_recorder = nullptr;
+            }
+            else {
+                main_video_recorder = new VideoRecorder;
+                Mixer::manager().session()->addRecorder(main_video_recorder);
+            }
         }
 
     }
@@ -871,9 +879,6 @@ void UserInterface::RenderPreview()
             return;
 
         }
-        // adapt rendering if there is a recording ongoing
-        Recorder *rec = Mixer::manager().session()->frontRecorder();
-
         // menu (no title bar)
         if (ImGui::BeginMenuBar())
         {
@@ -895,15 +900,19 @@ void UserInterface::RenderPreview()
                 ImGui::Separator();
 
                 // Stop recording menu if recording exists
-                if (rec) {
+                if (main_video_recorder) {
 
-                    if ( ImGui::MenuItem( ICON_FA_SQUARE "  Stop Record") )
+                    if ( ImGui::MenuItem( ICON_FA_SQUARE "  Stop Record", CTRL_MOD "R") ) {
-                        rec->stop();
+                        main_video_recorder->stop();
+                        main_video_recorder = nullptr;
+                    }
                 }
                 // start recording menu
                 else {
-                    if ( ImGui::MenuItem( ICON_FA_CIRCLE "  Record") )
+                    if ( ImGui::MenuItem( ICON_FA_CIRCLE "  Record", CTRL_MOD "R") ) {
-                        Mixer::manager().session()->addRecorder(new VideoRecorder);
+                        main_video_recorder = new VideoRecorder;
+                        Mixer::manager().session()->addRecorder(main_video_recorder);
+                    }
 
                     ImGui::SetNextItemWidth(300);
                     ImGui::Combo("##RecProfile", &Settings::application.record.profile, VideoRecorder::profile_name, IM_ARRAYSIZE(VideoRecorder::profile_name) );
@@ -948,13 +957,13 @@ void UserInterface::RenderPreview()
         // preview image
         ImGui::Image((void*)(intptr_t)output->texture(), imagesize);
         // recording indicator overlay
-        if (rec)
+        if (main_video_recorder)
         {
             float r = ImGui::GetTextLineHeightWithSpacing();
             ImGui::SetCursorScreenPos(ImVec2(draw_pos.x + r, draw_pos.y + r));
             ImGuiToolkit::PushFont(ImGuiToolkit::FONT_LARGE);
             ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(1.0, 0.05, 0.05, 0.8f));
-            ImGui::Text(ICON_FA_CIRCLE " %s", rec->info().c_str() );
+            ImGui::Text(ICON_FA_CIRCLE " %s", main_video_recorder->info().c_str() );
             ImGui::PopStyleColor(1);
             ImGui::PopFont();
         }