eessppeelllloo/vimix
1
0
Fork 0
mirror of https://github.com/brunoherbelin/vimix.git synced 2025-12-05 15:30:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

Implementation of Recorder with dual PBO mechanism for best efficiency

Browse Source 
and compatibility. Fixed user interface and avoid user creating multiple
recorders.
This commit is contained in:
brunoherbelin
2020-07-27 15:56:24 +02:00
parent 3f782736ac
commit 3bb3e66f55
6 changed files with 180 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
FrameBuffer.cpp
View File

@@ -116,17 +116,12 @@ glm::vec3 FrameBuffer::resolution() const
return glm::vec3(attrib_.viewport.x, attrib_.viewport.y, 0.f);
}
void FrameBuffer::bind()
void FrameBuffer::begin()
{
if (!framebufferid_)
init();
glBindFramebuffer(GL_FRAMEBUFFER, framebufferid_);
}
void FrameBuffer::begin()
{
bind();
Rendering::manager().pushAttrib(attrib_);
@@ -155,6 +150,21 @@ void FrameBuffer::release()
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
void FrameBuffer::readPixels()
{
if (!framebufferid_)
return;
if (use_multi_sampling_)
glBindFramebuffer(GL_READ_FRAMEBUFFER, intermediate_framebufferid_);
else
glBindFramebuffer(GL_READ_FRAMEBUFFER, framebufferid_);
glPixelStorei(GL_PACK_ALIGNMENT, 1);
glReadPixels(0, 0, attrib_.viewport.x, attrib_.viewport.y, (use_alpha_? GL_RGBA : GL_RGB), GL_UNSIGNED_BYTE, 0);
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
bool FrameBuffer::blit(FrameBuffer *other)
{
if (!framebufferid_ || !other || !other->framebufferid_)

5
FrameBuffer.h
View File

@@ -18,8 +18,6 @@ public:
FrameBuffer(uint width, uint height, bool useAlpha = false, bool multiSampling = false);
~FrameBuffer();
// bind the FrameBuffer as current to draw into
void bind();
// unbind any framebuffer object
static void release();
// Bind & push attribs to prepare draw
@@ -29,6 +27,9 @@ public:
// blit copy to another, returns true on success
bool blit(FrameBuffer *other);
// bind the FrameBuffer in READ and perform glReadPixels
// return the size of the buffer
void readPixels();
// clear color
inline void setClearColor(glm::vec4 color) { attrib_.clear_color = color; }

184
Recorder.cpp
View File

@@ -20,11 +20,16 @@
#include "Recorder.h"
// use glReadPixel or glGetTextImage ?
// read pixels & pbo should be the fastest
// https://stackoverflow.com/questions/38140527/glreadpixels-vs-glgetteximage
#define USE_GLREADPIXEL
using namespace std;
Recorder::Recorder() : finished_(false)
Recorder::Recorder() : finished_(false), pbo_index_(0), pbo_next_index_(0), size_(0)
{
pbo_[0] = pbo_[1] = 0;
}
PNGRecorder::PNGRecorder() : Recorder()
@@ -34,6 +39,7 @@ PNGRecorder::PNGRecorder() : Recorder()
path = SystemToolkit::home_path();
filename_ = path + SystemToolkit::date_time_string() + "_vimix.png";
}
// Thread to perform slow operation of saving to file
@@ -44,7 +50,7 @@ void save_png(std::string filename, unsigned char *data, uint w, uint h, uint c)
// save file
stbi_write_png(filename.c_str(), w, h, c, data, w * c);
// notify
Log::Notify("Capture %s saved.", filename.c_str());
Log::Notify("Capture %s ready (%d x %d %d)", filename.c_str(), w, h, c);
// done
free(data);
}
@@ -52,21 +58,68 @@ void save_png(std::string filename, unsigned char *data, uint w, uint h, uint c)
void PNGRecorder::addFrame(FrameBuffer *frame_buffer, float)
{
// ignore
if (frame_buffer == nullptr)
return;
// get what is needed from frame buffer
uint w = frame_buffer->width();
uint h = frame_buffer->height();
uint c = frame_buffer->use_alpha() ? 4 : 3;
GLenum format = frame_buffer->use_alpha() ? GL_RGBA : GL_RGB;
uint size = w * h * c;
unsigned char * data = (unsigned char*) malloc(size);
glGetTextureSubImage( frame_buffer->texture(), 0, 0, 0, 0, w, h, 1, format, GL_UNSIGNED_BYTE, size, data);
// first iteration: initialize and get frame
if (size_ < 1)
{
// init size
size_ = w * h * c;
// save in separate thread
std::thread(save_png, filename_, data, w, h, c).detach();
// create PBO
glGenBuffers(2, pbo_);
// set writing PBO
glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[0]);
glBufferData(GL_PIXEL_PACK_BUFFER, size_, NULL, GL_STREAM_READ);
#ifdef USE_GLREADPIXEL
// get frame
frame_buffer->readPixels();
#else
glBindTexture(GL_TEXTURE_2D, frame_buffer->texture());
glGetTexImage(GL_TEXTURE_2D, 0, GL_RGB, GL_UNSIGNED_BYTE, 0);
#endif
}
// second iteration; get frame and save file
else {
// set reading PBO
glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[0]);
// get pixels
unsigned char* ptr = (unsigned char*) glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);
if (NULL != ptr) {
// prepare memory buffer0
unsigned char * data = (unsigned char*) malloc(size_);
// transfer frame to data
memmove(data, ptr, size_);
// save in separate thread
std::thread(save_png, filename_, data, w, h, c).detach();
}
// unmap
glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
// ok done
glDeleteBuffers(2, pbo_);
// recorded one frame
finished_ = true;
}
glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
// unsigned char * data = (unsigned char*) malloc(size);
// GLenum format = frame_buffer->use_alpha() ? GL_RGBA : GL_RGB;
// glGetTextureSubImage( frame_buffer->texture(), 0, 0, 0, 0, w, h, 1, format, GL_UNSIGNED_BYTE, size, data);
// record one frame only
finished_ = true;
}
const char* VideoRecorder::profile_name[4] = { "H264 (low)", "H264 (high)", "Apple ProRes 4444", "WebM VP9" };
@@ -104,7 +157,7 @@ const std::vector<std::string> VideoRecorder::profile_description {
// "qtmux ! filesink name=sink";
VideoRecorder::VideoRecorder() : Recorder(), frame_buffer_(nullptr), width_(0), height_(0), buf_size_(0),
VideoRecorder::VideoRecorder() : Recorder(), frame_buffer_(nullptr), width_(0), height_(0),
recording_(false), pipeline_(nullptr), src_(nullptr), timestamp_(0), timeframe_(0), accept_buffer_(false)
{
// auto filename
@@ -121,12 +174,14 @@ VideoRecorder::VideoRecorder() : Recorder(), frame_buffer_(nullptr), width_(0),
VideoRecorder::~VideoRecorder()
{
if (src_ != nullptr)
gst_object_unref (src_);
if (pipeline_ != nullptr) {
gst_element_set_state (pipeline_, GST_STATE_NULL);
gst_object_unref (pipeline_);
}
if (src_ != nullptr)
gst_object_unref (src_);
glDeleteBuffers(2, pbo_);
}
void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
@@ -146,7 +201,14 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
// define stream properties
width_ = frame_buffer_->width();
height_ = frame_buffer_->height();
buf_size_ = width_ * height_ * (frame_buffer_->use_alpha() ? 4 : 3);
size_ = width_ * height_ * (frame_buffer_->use_alpha() ? 4 : 3);
// create PBOs
glGenBuffers(2, pbo_);
glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[1]);
glBufferData(GL_PIXEL_PACK_BUFFER, size_, NULL, GL_STREAM_READ);
glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[0]);
glBufferData(GL_PIXEL_PACK_BUFFER, size_, NULL, GL_STREAM_READ);
// create a gstreamer pipeline
string description = "appsrc name=src ! videoconvert ! ";
@@ -164,18 +226,10 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
}
// setup file sink
sink_ = GST_BASE_SINK( gst_bin_get_by_name (GST_BIN (pipeline_), "sink") );
if (sink_) {
g_object_set (G_OBJECT (sink_),
"location", filename_.c_str(),
"sync", FALSE,
NULL);
}
else {
Log::Warning("VideoRecorder Could not configure file");
finished_ = true;
return;
}
g_object_set (G_OBJECT (gst_bin_get_by_name (GST_BIN (pipeline_), "sink")),
"location", filename_.c_str(),
"sync", FALSE,
NULL);
// setup custom app source
src_ = GST_APP_SRC( gst_bin_get_by_name (GST_BIN (pipeline_), "src") );
@@ -239,7 +293,7 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
frame_buffer->use_alpha() != frame_buffer_->use_alpha()) {
stop();
Log::Info("Recording interrupted: new session (%d x %d) incompatible with recording (%d x %d)", frame_buffer->width(), frame_buffer->height(), width_, height_);
Log::Warning("Recording interrupted: new session (%d x %d) incompatible with recording (%d x %d)", frame_buffer->width(), frame_buffer->height(), width_, height_);
}
else {
// accepting a new frame buffer as input
@@ -247,10 +301,8 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
}
}
static int count = 0;
// store a frame if recording is active
if (recording_ && buf_size_ > 0)
if (recording_ && size_ > 0)
{
// calculate dt in ns
timeframe_ += gst_gdouble_to_guint64( dt * 1000000.f);
@@ -259,28 +311,62 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
// and if the encoder accepts data
if ( timeframe_ > frame_duration_ - 3000000 && accept_buffer_) {
GstBuffer *buffer = gst_buffer_new_and_alloc (buf_size_);
GLenum format = frame_buffer_->use_alpha() ? GL_RGBA : GL_RGB;
// set buffer target for writing in a new frame
glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[pbo_index_]);
// set timing of buffer
buffer->pts = timestamp_;
buffer->duration = frame_duration_;
#ifdef USE_GLREADPIXEL
// get frame
frame_buffer->readPixels();
#else
glBindTexture(GL_TEXTURE_2D, frame_buffer->texture());
glGetTexImage(GL_TEXTURE_2D, 0, GL_RGB, GL_UNSIGNED_BYTE, 0);
#endif
// OpenGL capture
GstMapInfo map;
gst_buffer_map (buffer, &map, GST_MAP_WRITE);
glGetTextureSubImage( frame_buffer_->texture(), 0, 0, 0, 0, width_, height_, 1, format, GL_UNSIGNED_BYTE, buf_size_, map.data);
gst_buffer_unmap (buffer, &map);
// update case ; alternating indices
if ( pbo_next_index_ != pbo_index_ ) {
// push
// Log::Info("VideoRecorder push data %ld", buffer->pts);
gst_app_src_push_buffer (src_, buffer);
// NB: buffer will be unrefed by the appsrc
// set buffer target for saving the frame
glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_[pbo_next_index_]);
// restart counter
// new buffer
GstBuffer *buffer = gst_buffer_new_and_alloc (size_);
// set timing of buffer
buffer->pts = timestamp_;
buffer->duration = frame_duration_;
// map gst buffer into a memory WRITE target
GstMapInfo map;
gst_buffer_map (buffer, &map, GST_MAP_WRITE);
// map PBO pixels into a memory READ pointer
unsigned char* ptr = (unsigned char*) glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);
// transfer pixels from PBO memory to buffer memory
if (NULL != ptr)
memmove(map.data, ptr, size_);
// un-map
glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
gst_buffer_unmap (buffer, &map);
// push
// Log::Info("VideoRecorder push data %ld", buffer->pts);
gst_app_src_push_buffer (src_, buffer);
// NB: buffer will be unrefed by the appsrc
// next timestamp
timestamp_ += frame_duration_;
}
glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
// alternate indices
pbo_next_index_ = pbo_index_;
pbo_index_ = (pbo_index_ + 1) % 2;
// restart frame counter
timeframe_ = 0;
// next timestamp
timestamp_ += frame_duration_;
}
}
@@ -293,7 +379,6 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
if (msg) {
// Log::Info("received EOS");
// stop the pipeline
GstStateChangeReturn ret = gst_element_set_state (pipeline_, GST_STATE_NULL);
if (ret == GST_STATE_CHANGE_FAILURE)
@@ -301,7 +386,6 @@ void VideoRecorder::addFrame (FrameBuffer *frame_buffer, float dt)
else
Log::Notify("Recording %s ready.", filename_.c_str());
count = 0;
finished_ = true;
}
}

8
Recorder.h
View File

@@ -29,7 +29,13 @@ public:
inline bool finished() const { return finished_; }
protected:
// thread-safe testing termination
std::atomic<bool> finished_;
// PBO
guint pbo_[2];
guint pbo_index_, pbo_next_index_;
guint size_;
};
class PNGRecorder : public Recorder
@@ -52,7 +58,6 @@ class VideoRecorder : public Recorder
FrameBuffer *frame_buffer_;
uint width_;
uint height_;
uint buf_size_;
// operation
std::atomic<bool> recording_;
@@ -61,7 +66,6 @@ class VideoRecorder : public Recorder
// gstreamer pipeline
GstElement *pipeline_;
GstAppSrc *src_;
GstBaseSink *sink_;
GstClockTime timeframe_;
GstClockTime timestamp_;
GstClockTime frame_duration_;

1
Screenshot.cpp
View File

@@ -24,6 +24,7 @@ Screenshot::Screenshot()
Screenshot::~Screenshot()
{
glDeleteBuffers(1, &Pbo);
if (Data) free(Data);
}

31
UserInterfaceManager.cpp
View File

@@ -57,6 +57,7 @@ using namespace std;
#include "TextEditor.h"
static TextEditor editor;
static Recorder *main_video_recorder = nullptr;
// utility functions
void ShowAboutGStreamer(bool* p_open);
@@ -292,7 +293,14 @@ void UserInterface::handleKeyboard()
}
else if (ImGui::IsKeyPressed( GLFW_KEY_R )) {
// toggle recording
Mixer::manager().session()->addRecorder(new VideoRecorder);
if (main_video_recorder){
main_video_recorder->stop();
main_video_recorder = nullptr;
}
else {
main_video_recorder = new VideoRecorder;
Mixer::manager().session()->addRecorder(main_video_recorder);
}
}
}
@@ -871,9 +879,6 @@ void UserInterface::RenderPreview()
return;
}
// adapt rendering if there is a recording ongoing
Recorder *rec = Mixer::manager().session()->frontRecorder();
// menu (no title bar)
if (ImGui::BeginMenuBar())
{
@@ -895,15 +900,19 @@ void UserInterface::RenderPreview()
ImGui::Separator();
// Stop recording menu if recording exists
if (rec) {
if (main_video_recorder) {
if ( ImGui::MenuItem( ICON_FA_SQUARE " Stop Record") )
rec->stop();
if ( ImGui::MenuItem( ICON_FA_SQUARE " Stop Record", CTRL_MOD "R") ) {
main_video_recorder->stop();
main_video_recorder = nullptr;
}
}
// start recording menu
else {
if ( ImGui::MenuItem( ICON_FA_CIRCLE " Record") )
Mixer::manager().session()->addRecorder(new VideoRecorder);
if ( ImGui::MenuItem( ICON_FA_CIRCLE " Record", CTRL_MOD "R") ) {
main_video_recorder = new VideoRecorder;
Mixer::manager().session()->addRecorder(main_video_recorder);
}
ImGui::SetNextItemWidth(300);
ImGui::Combo("##RecProfile", &Settings::application.record.profile, VideoRecorder::profile_name, IM_ARRAYSIZE(VideoRecorder::profile_name) );
@@ -948,13 +957,13 @@ void UserInterface::RenderPreview()
// preview image
ImGui::Image((void*)(intptr_t)output->texture(), imagesize);
// recording indicator overlay
if (rec)
if (main_video_recorder)
{
float r = ImGui::GetTextLineHeightWithSpacing();
ImGui::SetCursorScreenPos(ImVec2(draw_pos.x + r, draw_pos.y + r));
ImGuiToolkit::PushFont(ImGuiToolkit::FONT_LARGE);
ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(1.0, 0.05, 0.05, 0.8f));
ImGui::Text(ICON_FA_CIRCLE " %s", rec->info().c_str() );
ImGui::Text(ICON_FA_CIRCLE " %s", main_video_recorder->info().c_str() );
ImGui::PopStyleColor(1);
ImGui::PopFont();
}