From 9251aff19f84acef23e4e719c9b643fce0dc62ba Mon Sep 17 00:00:00 2001 From: brunoherbelin Date: Mon, 21 Sep 2020 22:41:20 +0200 Subject: [PATCH] Create Device Source and integration of Stream --- CMakeLists.txt | 1 + DeviceSource.cpp | 170 +++++++++++++++++++++++++++++++++++++++ DeviceSource.h | 48 +++++++++++ Mixer.cpp | 14 ++++ Mixer.h | 1 + PatternSource.cpp | 7 +- Stream.cpp | 58 +++++++++++-- Stream.h | 5 ++ UserInterfaceManager.cpp | 37 +++++++-- UserInterfaceManager.h | 3 +- 10 files changed, 323 insertions(+), 21 deletions(-) create mode 100644 DeviceSource.cpp create mode 100644 DeviceSource.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6353991..bc67a43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,6 +239,7 @@ set(VMIX_SRCS MediaPlayer.cpp MediaSource.cpp PatternSource.cpp + DeviceSource.cpp FrameBuffer.cpp RenderingManager.cpp UserInterfaceManager.cpp diff --git a/DeviceSource.cpp b/DeviceSource.cpp new file mode 100644 index 0000000..68c1293 --- /dev/null +++ b/DeviceSource.cpp @@ -0,0 +1,170 @@ +#include +#include + +#include "DeviceSource.h" + +#include "defines.h" +#include "ImageShader.h" +#include "ImageProcessingShader.h" +#include "Resource.h" +#include "Primitives.h" +#include "Stream.h" +#include "Visitor.h" +#include "Log.h" + +Device::Device() : Stream() +{ + +} + +glm::ivec2 Device::resolution() +{ + return glm::ivec2( width_, height_); +} + + +void Device::open( uint device ) +{ + device_ = CLAMP(device, 0, 2); + + single_frame_ = false; + live_ = true; + +// std::string desc = "v4l2src ! video/x-raw,width=320,height=240,framerate=30/1 ! videoconvert"; +// std::string desc = "v4l2src ! jpegdec ! videoconvert"; +// std::string desc = "v4l2src ! image/jpeg,width=640,height=480,framerate=30/1 ! jpegdec ! videoconvert"; + +// std::string desc = "videotestsrc pattern=snow is-live=true "; + std::string desc = "ximagesrc endx=640 endy=480 ! video/x-raw,framerate=5/1 ! videoconvert ! queue"; + + // (private) open stream + open(desc); +} + +void Device::open(std::string gstreamer_description) +{ + // set gstreamer pipeline source + description_ = gstreamer_description; + + // close before re-openning + if (isOpen()) + close(); + + execute_open(); +} + + +DeviceSource::DeviceSource() : Source() +{ + // create stream + stream_ = new Device(); + + // create surface + devicesurface_ = new Surface(renderingshader_); +} + +DeviceSource::~DeviceSource() +{ + // delete media surface & stream + delete devicesurface_; + delete stream_; +} + +bool DeviceSource::failed() const +{ + return stream_->failed(); +} + +uint DeviceSource::texture() const +{ + return stream_->texture(); +} + +void DeviceSource::replaceRenderingShader() +{ + devicesurface_->replaceShader(renderingshader_); +} + +void DeviceSource::setDevice(int id) +{ + Log::Notify("Openning device %d", id); + + stream_->open(id); + stream_->play(true); +} + + +void DeviceSource::init() +{ + if ( stream_->isOpen() ) { + + // update video + stream_->update(); + + // once the texture of media player is created + if (stream_->texture() != Resource::getTextureBlack()) { + + // get the texture index from media player, apply it to the media surface + devicesurface_->setTextureIndex( stream_->texture() ); + + // create Frame buffer matching size of media player + float height = float(stream_->width()) / stream_->aspectRatio(); + FrameBuffer *renderbuffer = new FrameBuffer(stream_->width(), (uint)height, true); + + // set the renderbuffer of the source and attach rendering nodes + attach(renderbuffer); + + // icon in mixing view + overlays_[View::MIXING]->attach( new Symbol(Symbol::EMPTY, glm::vec3(0.8f, 0.8f, 0.01f)) ); + overlays_[View::LAYER]->attach( new Symbol(Symbol::EMPTY, glm::vec3(0.8f, 0.8f, 0.01f)) ); + + // done init + initialized_ = true; + Log::Info("Source Device linked to Stream %d.", stream_->description().c_str()); + + // force update of activation mode + active_ = true; + touch(); + } + } + +} + +void DeviceSource::setActive (bool on) +{ + bool was_active = active_; + + Source::setActive(on); + + // change status of media player (only if status changed) + if ( active_ != was_active ) { + stream_->enable(active_); + } +} + +void DeviceSource::update(float dt) +{ + Source::update(dt); + + // update stream + stream_->update(); +} + +void DeviceSource::render() +{ + if (!initialized_) + init(); + else { + // render the media player into frame buffer + static glm::mat4 projection = glm::ortho(-1.f, 1.f, 1.f, -1.f, -1.f, 1.f); + renderbuffer_->begin(); + devicesurface_->draw(glm::identity(), projection); + renderbuffer_->end(); + } +} + +void DeviceSource::accept(Visitor& v) +{ + Source::accept(v); + v.visit(*this); +} diff --git a/DeviceSource.h b/DeviceSource.h new file mode 100644 index 0000000..279d247 --- /dev/null +++ b/DeviceSource.h @@ -0,0 +1,48 @@ +#ifndef DEVICESOURCE_H +#define DEVICESOURCE_H + +#include "Stream.h" +#include "Source.h" + +class Device : public Stream +{ +public: + + Device(); + void open( uint deviceid ); + + glm::ivec2 resolution(); + +private: + void open( std::string description ) override; + uint device_; +}; + +class DeviceSource : public Source +{ +public: + DeviceSource(); + ~DeviceSource(); + + // implementation of source API + void update (float dt) override; + void setActive (bool on) override; + void render() override; + bool failed() const override; + uint texture() const override; + void accept (Visitor& v) override; + + // Pattern specific interface + inline Device *device() const { return stream_; } + void setDevice(int id); + +protected: + + void init() override; + void replaceRenderingShader() override; + + Surface *devicesurface_; + Device *stream_; +}; + +#endif // DEVICESOURCE_H diff --git a/Mixer.cpp b/Mixer.cpp index 43ef6cb..9b95176 100644 --- a/Mixer.cpp +++ b/Mixer.cpp @@ -23,6 +23,7 @@ using namespace tinyxml2; #include "SessionSource.h" #include "MediaSource.h" #include "PatternSource.h" +#include "DeviceSource.h" #include "Mixer.h" @@ -277,6 +278,19 @@ Source * Mixer::createSourcePattern(int pattern, glm::ivec2 res) return s; } +Source * Mixer::createSourceDevice(int id) +{ + // ready to create a source + DeviceSource *s = new DeviceSource(); + s->setDevice(id); + + // propose a new name based on pattern name + renameSource(s, "Device"); + + return s; +} + + Source * Mixer::createSourceClone(const std::string &namesource) { // ready to create a source diff --git a/Mixer.h b/Mixer.h index 0615a73..437b817 100644 --- a/Mixer.h +++ b/Mixer.h @@ -40,6 +40,7 @@ public: Source * createSourceClone (const std::string &namesource = ""); Source * createSourceRender (); Source * createSourcePattern(int pattern, glm::ivec2 res); + Source * createSourceDevice (int id); // operations on sources void addSource (Source *s); diff --git a/PatternSource.cpp b/PatternSource.cpp index 0fd2930..553950a 100644 --- a/PatternSource.cpp +++ b/PatternSource.cpp @@ -109,7 +109,7 @@ void Pattern::open( uint pattern ) type_ = CLAMP(pattern, 0, 25); std::string gstreamer_pattern = pattern_internal_[type_]; - // always some special cases... + // there is always a special case... switch(type_) { case 18: @@ -178,10 +178,10 @@ void PatternSource::replaceRenderingShader() void PatternSource::setPattern(int id) { + Log::Notify("Creating pattern %s", Pattern::pattern_types[id].c_str()); + stream_->open(id); stream_->play(true); - - Log::Notify("Creating pattern %s", Pattern::pattern_types[id].c_str()); } @@ -238,7 +238,6 @@ void PatternSource::update(float dt) Source::update(dt); // update stream - // TODO : update only if animated pattern stream_->update(); } diff --git a/Stream.cpp b/Stream.cpp index 5bdf7b1..3084bff 100644 --- a/Stream.cpp +++ b/Stream.cpp @@ -123,12 +123,18 @@ void Stream::execute_open() #ifdef USE_GST_APPSINK_CALLBACKS_ // set the callbacks GstAppSinkCallbacks callbacks; - callbacks.new_preroll = callback_new_preroll; if (single_frame_) { + callbacks.new_preroll = callback_new_preroll; callbacks.eos = NULL; callbacks.new_sample = NULL; } +// else if (live_) { +// callbacks.new_preroll = NULL; +// callbacks.eos = NULL; +// callbacks.new_sample = callback_new_sample; +// } else { + callbacks.new_preroll = callback_new_preroll; callbacks.eos = callback_end_of_stream; callbacks.new_sample = callback_new_sample; } @@ -136,9 +142,11 @@ void Stream::execute_open() gst_app_sink_set_emit_signals (GST_APP_SINK(sink), false); #else // connect signals callbacks - g_signal_connect(G_OBJECT(sink), "new-sample", G_CALLBACK (callback_new_sample), this); g_signal_connect(G_OBJECT(sink), "new-preroll", G_CALLBACK (callback_new_preroll), this); - g_signal_connect(G_OBJECT(sink), "eos", G_CALLBACK (callback_end_of_stream), this); + if (!single_frame_) { + g_signal_connect(G_OBJECT(sink), "new-sample", G_CALLBACK (callback_new_sample), this); + g_signal_connect(G_OBJECT(sink), "eos", G_CALLBACK (callback_end_of_stream), this); + } gst_app_sink_set_emit_signals (GST_APP_SINK(sink), true); #endif // done with ref to sink @@ -157,10 +165,17 @@ void Stream::execute_open() Log::Warning("Stream %s Could not open '%s'", std::to_string(id_).c_str(), description_.c_str()); failed_ = true; return; + } else if (ret == GST_STATE_CHANGE_NO_PREROLL) { + Log::Info("Stream %s is a live stream", std::to_string(id_).c_str()); + live_ = true; } // all good - Log::Info("Stream %d Opened '%s' (%d x %d)", id_, description_.c_str(), width_, height_); + Log::Info("Stream %d Opened '%s' (%d x %d)", id_, description.c_str(), width_, height_); +// if (desired_state_ == GST_STATE_PLAYING) +// Log::Info("Stream %d Playing", id_); +// else +// Log::Info("Stream %d Paused", id_); ready_ = true; } @@ -272,6 +287,11 @@ bool Stream::singleFrame() const return single_frame_; } +bool Stream::live() const +{ + return live_; +} + void Stream::play(bool on) { // ignore if disabled, and cannot play an image @@ -305,6 +325,18 @@ void Stream::play(bool on) Log::Info("Stream %s Stop", std::to_string(id_).c_str()); #endif + // DEBUG : LIVE SOURCE ?? + if (live_) { + GstState state; + gst_element_get_state (pipeline_, &state, NULL, GST_CLOCK_TIME_NONE); + + while (state != desired_state_) { + Log::Info("Stream %s Live stream did not change state", std::to_string(id_).c_str()); + gst_element_set_state (pipeline_, desired_state_); + gst_element_get_state (pipeline_, &state, NULL, GST_CLOCK_TIME_NONE); + } + } + // reset time counter timecount_.reset(); @@ -441,6 +473,9 @@ void Stream::update() // if (!enabled_) // return; +// // DEBUG : LIVE SOURCE ?? +// GstState state; +// gst_element_get_state (pipeline_, &state, NULL, GST_CLOCK_TIME_NONE); // local variables before trying to update guint read_index = 0; @@ -451,7 +486,7 @@ void Stream::update() // get the last frame filled from fill_frame() read_index = last_index_; - // Do NOT miss and jump directly (after seek) to a pre-roll + // Do NOT miss and jump directly to a pre-roll for (guint i = 0; i < N_FRAME; ++i) { if (frame_[i].status == PREROLL) { read_index = i; @@ -506,7 +541,7 @@ double Stream::updateFrameRate() const bool Stream::fill_frame(GstBuffer *buf, FrameStatus status) { -// Log::Info("Stream fill frame"); + Log::Info("Stream fill frame"); // Do NOT overwrite an unread EOS if ( frame_[write_index_].status == EOS ) @@ -548,12 +583,17 @@ bool Stream::fill_frame(GstBuffer *buf, FrameStatus status) } // full but invalid frame : will be deleted next iteration // (should never happen) - else - frame_[write_index_].status = INVALID; + else { + Log::Info("Stream %s Invalid frame", std::to_string(id_).c_str()); + frame_[write_index_].status = INVALID; + frame_[write_index_].access.unlock(); + return false; + } } // else; null buffer for EOS: give a position else { frame_[write_index_].status = EOS; + Log::Info("Stream EOS"); } // unlock access to frame @@ -617,6 +657,8 @@ GstFlowReturn Stream::callback_new_sample (GstAppSink *sink, gpointer p) { GstFlowReturn ret = GST_FLOW_OK; + Log::Info("callback_new_sample"); + // non-blocking read new sample GstSample *sample = gst_app_sink_pull_sample(sink); diff --git a/Stream.h b/Stream.h index 2c22f35..0a4f355 100644 --- a/Stream.h +++ b/Stream.h @@ -65,6 +65,10 @@ public: * True if its an image * */ bool singleFrame() const; + /** + * True if its a live stream + * */ + bool live() const; /** * Pause / Play * Can play backward if play speed is negative @@ -115,6 +119,7 @@ protected: guint width_; guint height_; bool single_frame_; + bool live_; // GST & Play status GstState desired_state_; diff --git a/UserInterfaceManager.cpp b/UserInterfaceManager.cpp index 8bdd4f0..d6483a7 100644 --- a/UserInterfaceManager.cpp +++ b/UserInterfaceManager.cpp @@ -1835,6 +1835,11 @@ void SourcePreview::Render(float width, bool controlbutton) } } +bool SourcePreview::ready() const +{ + return source_ != nullptr && source_->ready(); +} + void Navigator::RenderNewPannel() { // Next window is a side pannel @@ -1852,9 +1857,11 @@ void Navigator::RenderNewPannel() ImGui::SetCursorPosY(width_); ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN); - static const char* origin_names[3] = { ICON_FA_FILE " File", + static const char* origin_names[4] = { ICON_FA_FILE " File", ICON_FA_SITEMAP " Internal", - ICON_FA_COG " Generated" }; + ICON_FA_COG " Generated", + ICON_FA_CUBES " External" + }; // TODO IMPLEMENT EXTERNAL SOURCES static const char* origin_names[3] = { ICON_FA_FILE " File", ICON_FA_SITEMAP " Internal", ICON_FA_PLUG " External" }; if (ImGui::Combo("Origin", &Settings::application.source.new_type, origin_names, IM_ARRAYSIZE(origin_names)) ) new_source_preview_.setSource(); @@ -1988,21 +1995,35 @@ void Navigator::RenderNewPannel() } } // Hardware - else { - // helper - ImGui::SetCursorPosX(pannel_width_ - 30 + IMGUI_RIGHT_ALIGN); - ImGuiToolkit::HelpMarker("Create a source capturing images\nfrom external devices or network."); + else if (Settings::application.source.new_type == 3){ + + ImGui::SetCursorPosY(2.f * width_); + + ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN); + if (ImGui::BeginCombo("##Hardware", "Select device")) + { + if (ImGui::Selectable( "device 0" )) { + + new_source_preview_.setSource( Mixer::manager().createSourceDevice(0), "device"); + } + ImGui::EndCombo(); + } + + // Indication + ImGui::SameLine(); + ImGuiToolkit::HelpMarker("Create a source images\nfrom external devices or network."); + } ImGui::NewLine(); // if a new source was added - if (new_source_preview_.ready()) { + if (new_source_preview_.filled()) { // show preview new_source_preview_.Render(ImGui::GetContentRegionAvail().x IMGUI_RIGHT_ALIGN, Settings::application.source.new_type != 1); // ask to import the source in the mixer ImGui::NewLine(); - if ( ImGui::Button( ICON_FA_CHECK " Create", ImVec2(pannel_width_ - padding_width_, 0)) ) { + if (new_source_preview_.ready() && ImGui::Button( ICON_FA_CHECK " Create", ImVec2(pannel_width_ - padding_width_, 0)) ) { Mixer::manager().addSource(new_source_preview_.getSource()); selected_button[NAV_NEW] = false; } diff --git a/UserInterfaceManager.h b/UserInterfaceManager.h index ee41011..e5f7eb9 100644 --- a/UserInterfaceManager.h +++ b/UserInterfaceManager.h @@ -26,7 +26,8 @@ public: Source *getSource(); void Render(float width, bool controlbutton = false); - inline bool ready() const { return source_ != nullptr; } + bool ready() const; + inline bool filled() const { return source_ != nullptr; } }; class Navigator