Create Device Source and integration of Stream

CMakeLists.txt

@@ -239,6 +239,7 @@ set(VMIX_SRCS
     MediaPlayer.cpp
     MediaSource.cpp
     PatternSource.cpp
+    DeviceSource.cpp
     FrameBuffer.cpp
     RenderingManager.cpp
     UserInterfaceManager.cpp

DeviceSource.cpp

@@ -0,0 +1,170 @@
+#include <sstream>
+#include <glm/gtc/matrix_transform.hpp>
+
+#include "DeviceSource.h"
+
+#include "defines.h"
+#include "ImageShader.h"
+#include "ImageProcessingShader.h"
+#include "Resource.h"
+#include "Primitives.h"
+#include "Stream.h"
+#include "Visitor.h"
+#include "Log.h"
+
+Device::Device() : Stream()
+{
+
+}
+
+glm::ivec2 Device::resolution()
+{
+    return glm::ivec2( width_, height_);
+}
+
+
+void Device::open( uint device )
+{
+    device_ = CLAMP(device, 0, 2);
+
+    single_frame_ = false;
+    live_ = true;
+
+//    std::string desc = "v4l2src ! video/x-raw,width=320,height=240,framerate=30/1 ! videoconvert";
+//    std::string desc = "v4l2src ! jpegdec ! videoconvert";
+//    std::string desc = "v4l2src ! image/jpeg,width=640,height=480,framerate=30/1 ! jpegdec ! videoconvert";
+
+//    std::string desc = "videotestsrc pattern=snow is-live=true ";
+    std::string desc = "ximagesrc endx=640 endy=480 ! video/x-raw,framerate=5/1 ! videoconvert ! queue";
+
+    // (private) open stream
+    open(desc);
+}
+
+void Device::open(std::string gstreamer_description)
+{
+    // set gstreamer pipeline source
+    description_ = gstreamer_description;
+
+    // close before re-openning
+    if (isOpen())
+        close();
+
+    execute_open();
+}
+
+
+DeviceSource::DeviceSource() : Source()
+{
+    // create stream
+    stream_ = new Device();
+
+    // create surface
+    devicesurface_ = new Surface(renderingshader_);
+}
+
+DeviceSource::~DeviceSource()
+{
+    // delete media surface & stream
+    delete devicesurface_;
+    delete stream_;
+}
+
+bool DeviceSource::failed() const
+{
+    return stream_->failed();
+}
+
+uint DeviceSource::texture() const
+{
+    return stream_->texture();
+}
+
+void DeviceSource::replaceRenderingShader()
+{
+    devicesurface_->replaceShader(renderingshader_);
+}
+
+void DeviceSource::setDevice(int id)
+{
+    Log::Notify("Openning device %d", id);
+
+    stream_->open(id);
+    stream_->play(true);
+}
+
+
+void DeviceSource::init()
+{
+    if ( stream_->isOpen() ) {
+
+        // update video
+        stream_->update();
+
+        // once the texture of media player is created
+        if (stream_->texture() != Resource::getTextureBlack()) {
+
+            // get the texture index from media player, apply it to the media surface
+            devicesurface_->setTextureIndex( stream_->texture() );
+
+            // create Frame buffer matching size of media player
+            float height = float(stream_->width()) / stream_->aspectRatio();
+            FrameBuffer *renderbuffer = new FrameBuffer(stream_->width(), (uint)height, true);
+
+            // set the renderbuffer of the source and attach rendering nodes
+            attach(renderbuffer);
+
+            // icon in mixing view
+            overlays_[View::MIXING]->attach( new Symbol(Symbol::EMPTY, glm::vec3(0.8f, 0.8f, 0.01f)) );
+            overlays_[View::LAYER]->attach( new Symbol(Symbol::EMPTY, glm::vec3(0.8f, 0.8f, 0.01f)) );
+
+            // done init
+            initialized_ = true;
+            Log::Info("Source Device linked to Stream %d.", stream_->description().c_str());
+
+            // force update of activation mode
+            active_ = true;
+            touch();
+        }
+    }
+
+}
+
+void DeviceSource::setActive (bool on)
+{
+    bool was_active = active_;
+
+    Source::setActive(on);
+
+    // change status of media player (only if status changed)
+    if ( active_ != was_active ) {
+        stream_->enable(active_);
+    }
+}
+
+void DeviceSource::update(float dt)
+{
+    Source::update(dt);
+
+    // update stream
+    stream_->update();
+}
+
+void DeviceSource::render()
+{
+    if (!initialized_)
+        init();
+    else {
+        // render the media player into frame buffer
+        static glm::mat4 projection = glm::ortho(-1.f, 1.f, 1.f, -1.f, -1.f, 1.f);
+        renderbuffer_->begin();
+        devicesurface_->draw(glm::identity<glm::mat4>(), projection);
+        renderbuffer_->end();
+    }
+}
+
+void DeviceSource::accept(Visitor& v)
+{
+    Source::accept(v);
+    v.visit(*this);
+}

DeviceSource.h

@@ -0,0 +1,48 @@
+#ifndef DEVICESOURCE_H
+#define DEVICESOURCE_H
+
+#include "Stream.h"
+#include "Source.h"
+
+class Device : public Stream
+{
+public:
+
+    Device();
+    void open( uint deviceid );
+
+    glm::ivec2 resolution();
+
+private:
+    void open( std::string description ) override;
+    uint device_;
+};
+
+class DeviceSource : public Source
+{
+public:
+    DeviceSource();
+    ~DeviceSource();
+
+    // implementation of source API
+    void update (float dt) override;
+    void setActive (bool on) override;
+    void render() override;
+    bool failed() const override;
+    uint texture() const override;
+    void accept (Visitor& v) override;
+
+    // Pattern specific interface
+    inline Device *device() const { return stream_; }
+    void setDevice(int id);
+
+protected:
+
+    void init() override;
+    void replaceRenderingShader() override;
+
+    Surface *devicesurface_;
+    Device  *stream_;
+};
+
+#endif // DEVICESOURCE_H

Mixer.cpp

@@ -23,6 +23,7 @@ using namespace tinyxml2;
 #include "SessionSource.h"
 #include "MediaSource.h"
 #include "PatternSource.h"
+#include "DeviceSource.h"
 
 #include "Mixer.h"
 
@@ -277,6 +278,19 @@ Source * Mixer::createSourcePattern(int pattern, glm::ivec2 res)
     return s;
 }
 
+Source * Mixer::createSourceDevice(int id)
+{
+    // ready to create a source
+    DeviceSource *s = new DeviceSource();
+    s->setDevice(id);
+
+    // propose a new name based on pattern name
+    renameSource(s, "Device");
+
+    return s;
+}
+
+
 Source * Mixer::createSourceClone(const std::string &namesource)
 {
     // ready to create a source

Mixer.h

@@ -40,6 +40,7 @@ public:
     Source * createSourceClone  (const std::string &namesource = "");
     Source * createSourceRender ();
     Source * createSourcePattern(int pattern, glm::ivec2 res);
+    Source * createSourceDevice (int id);
 
     // operations on sources
     void addSource    (Source *s);

PatternSource.cpp

@@ -109,7 +109,7 @@ void Pattern::open( uint pattern )
     type_ = CLAMP(pattern, 0, 25);
     std::string gstreamer_pattern = pattern_internal_[type_];
 
-    // always some special cases...
+    // there is always a special case...
     switch(type_)
     {
     case 18:
@@ -178,10 +178,10 @@ void PatternSource::replaceRenderingShader()
 
 void PatternSource::setPattern(int id)
 {
+    Log::Notify("Creating pattern %s", Pattern::pattern_types[id].c_str());
+
     stream_->open(id);
     stream_->play(true);
-
-    Log::Notify("Creating pattern %s", Pattern::pattern_types[id].c_str());
 }
 
 
@@ -238,7 +238,6 @@ void PatternSource::update(float dt)
     Source::update(dt);
 
     // update stream
-    // TODO : update only if animated pattern
     stream_->update();
 }
 

Stream.cpp

@@ -123,12 +123,18 @@ void Stream::execute_open()
 #ifdef USE_GST_APPSINK_CALLBACKS_
         // set the callbacks
         GstAppSinkCallbacks callbacks;
-        callbacks.new_preroll = callback_new_preroll;
         if (single_frame_) {
+            callbacks.new_preroll = callback_new_preroll;
             callbacks.eos = NULL;
             callbacks.new_sample = NULL;
         }
+//        else if (live_) {
+//            callbacks.new_preroll = NULL;
+//            callbacks.eos = NULL;
+//            callbacks.new_sample = callback_new_sample;
+//        }
         else {
+            callbacks.new_preroll = callback_new_preroll;
             callbacks.eos = callback_end_of_stream;
             callbacks.new_sample = callback_new_sample;
         }
@@ -136,9 +142,11 @@ void Stream::execute_open()
         gst_app_sink_set_emit_signals (GST_APP_SINK(sink), false);
 #else
         // connect signals callbacks
-        g_signal_connect(G_OBJECT(sink), "new-sample", G_CALLBACK (callback_new_sample), this);
         g_signal_connect(G_OBJECT(sink), "new-preroll", G_CALLBACK (callback_new_preroll), this);
-        g_signal_connect(G_OBJECT(sink), "eos", G_CALLBACK (callback_end_of_stream), this);
+        if (!single_frame_) {
+            g_signal_connect(G_OBJECT(sink), "new-sample", G_CALLBACK (callback_new_sample), this);
+            g_signal_connect(G_OBJECT(sink), "eos", G_CALLBACK (callback_end_of_stream), this);
+        }
         gst_app_sink_set_emit_signals (GST_APP_SINK(sink), true);
 #endif
         // done with ref to sink
@@ -157,10 +165,17 @@ void Stream::execute_open()
         Log::Warning("Stream %s Could not open '%s'", std::to_string(id_).c_str(), description_.c_str());
         failed_ = true;
         return;
+    } else if (ret == GST_STATE_CHANGE_NO_PREROLL) {
+        Log::Info("Stream %s is a live stream", std::to_string(id_).c_str());
+        live_ = true;
     }
 
     // all good
-    Log::Info("Stream %d Opened '%s' (%d x %d)", id_, description_.c_str(), width_, height_);
+    Log::Info("Stream %d Opened '%s' (%d x %d)", id_, description.c_str(), width_, height_);
+//    if (desired_state_ == GST_STATE_PLAYING)
+//        Log::Info("Stream %d Playing", id_);
+//    else
+//        Log::Info("Stream %d Paused", id_);
 
     ready_ = true;
 }
@@ -272,6 +287,11 @@ bool Stream::singleFrame() const
     return single_frame_;
 }
 
+bool Stream::live() const
+{
+    return live_;
+}
+
 void Stream::play(bool on)
 {
     // ignore if disabled, and cannot play an image
@@ -305,6 +325,18 @@ void Stream::play(bool on)
         Log::Info("Stream %s Stop", std::to_string(id_).c_str());
 #endif
 
+    // DEBUG  : LIVE SOURCE ??
+    if (live_) {
+        GstState state;
+        gst_element_get_state (pipeline_, &state, NULL, GST_CLOCK_TIME_NONE);
+
+        while (state != desired_state_) {
+            Log::Info("Stream %s Live stream did not change state", std::to_string(id_).c_str());
+            gst_element_set_state (pipeline_, desired_state_);
+            gst_element_get_state (pipeline_, &state, NULL, GST_CLOCK_TIME_NONE);
+        }
+    }
+
     // reset time counter
     timecount_.reset();
 
@@ -441,6 +473,9 @@ void Stream::update()
 //    if (!enabled_)
 //        return;
 
+//    // DEBUG  : LIVE SOURCE ??
+//    GstState state;
+//    gst_element_get_state (pipeline_, &state, NULL, GST_CLOCK_TIME_NONE);
 
     // local variables before trying to update
     guint read_index = 0;
@@ -451,7 +486,7 @@ void Stream::update()
     // get the last frame filled from fill_frame()
     read_index = last_index_;
 
-    // Do NOT miss and jump directly (after seek) to a pre-roll
+    // Do NOT miss and jump directly to a pre-roll
     for (guint i = 0; i < N_FRAME; ++i) {
         if (frame_[i].status == PREROLL) {
             read_index = i;
@@ -506,7 +541,7 @@ double Stream::updateFrameRate() const
 
 bool Stream::fill_frame(GstBuffer *buf, FrameStatus status)
 {
-//    Log::Info("Stream fill frame");
+    Log::Info("Stream fill frame");
 
     // Do NOT overwrite an unread EOS
     if ( frame_[write_index_].status == EOS )
@@ -548,12 +583,17 @@ bool Stream::fill_frame(GstBuffer *buf, FrameStatus status)
         }
         // full but invalid frame : will be deleted next iteration
         // (should never happen)
-        else
+        else {
-            frame_[write_index_].status = INVALID;
+            Log::Info("Stream %s Invalid frame", std::to_string(id_).c_str());
+            frame_[write_index_].status = INVALID;            
+            frame_[write_index_].access.unlock();
+            return false;
+        }
     }
     // else; null buffer for EOS: give a position
     else {
         frame_[write_index_].status = EOS;
+        Log::Info("Stream EOS");
     }
 
     // unlock access to frame
@@ -617,6 +657,8 @@ GstFlowReturn Stream::callback_new_sample (GstAppSink *sink, gpointer p)
 {
     GstFlowReturn ret = GST_FLOW_OK;
 
+    Log::Info("callback_new_sample");
+
     // non-blocking read new sample
     GstSample *sample = gst_app_sink_pull_sample(sink);
 

Stream.h

@@ -65,6 +65,10 @@ public:
      * True if its an image
      * */
     bool singleFrame() const;
+    /**
+     * True if its a live stream
+     * */
+    bool live() const;
     /**
      * Pause / Play
      * Can play backward if play speed is negative
@@ -115,6 +119,7 @@ protected:
     guint width_;
     guint height_;
     bool single_frame_;
+    bool live_;
 
     // GST & Play status
     GstState desired_state_;

UserInterfaceManager.cpp

@@ -1835,6 +1835,11 @@ void SourcePreview::Render(float width, bool controlbutton)
     }
 }
 
+bool SourcePreview::ready() const
+{
+    return source_ != nullptr && source_->ready();
+}
+
 void Navigator::RenderNewPannel()
 {
     // Next window is a side pannel
@@ -1852,9 +1857,11 @@ void Navigator::RenderNewPannel()
         ImGui::SetCursorPosY(width_);
         ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN);
 
-        static const char* origin_names[3] = { ICON_FA_FILE "    File",
+        static const char* origin_names[4] = { ICON_FA_FILE "    File",
                                                ICON_FA_SITEMAP "  Internal",
-                                               ICON_FA_COG "   Generated" };
+                                               ICON_FA_COG "   Generated",
+                                               ICON_FA_CUBES "  External"
+                                             };
         // TODO IMPLEMENT EXTERNAL SOURCES static const char* origin_names[3] = { ICON_FA_FILE " File", ICON_FA_SITEMAP " Internal", ICON_FA_PLUG " External" };
         if (ImGui::Combo("Origin", &Settings::application.source.new_type, origin_names, IM_ARRAYSIZE(origin_names)) )
             new_source_preview_.setSource();
@@ -1988,21 +1995,35 @@ void Navigator::RenderNewPannel()
             }
         }
         // Hardware
-        else {
+        else if (Settings::application.source.new_type == 3){
-            // helper
+
-            ImGui::SetCursorPosX(pannel_width_ - 30 + IMGUI_RIGHT_ALIGN);
+            ImGui::SetCursorPosY(2.f * width_);
-            ImGuiToolkit::HelpMarker("Create a source capturing images\nfrom external devices or network.");
+
+            ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN);
+            if (ImGui::BeginCombo("##Hardware", "Select device"))
+            {
+                if (ImGui::Selectable( "device 0" )) {
+
+                    new_source_preview_.setSource( Mixer::manager().createSourceDevice(0), "device");
+                }
+                ImGui::EndCombo();
+            }
+
+            // Indication
+            ImGui::SameLine();
+            ImGuiToolkit::HelpMarker("Create a source images\nfrom external devices or network.");
+
         }
 
         ImGui::NewLine();
 
         // if a new source was added
-        if (new_source_preview_.ready()) {
+        if (new_source_preview_.filled()) {
             // show preview
             new_source_preview_.Render(ImGui::GetContentRegionAvail().x IMGUI_RIGHT_ALIGN, Settings::application.source.new_type != 1);
             // ask to import the source in the mixer
             ImGui::NewLine();
-            if ( ImGui::Button( ICON_FA_CHECK "  Create", ImVec2(pannel_width_ - padding_width_, 0)) ) {
+            if (new_source_preview_.ready() && ImGui::Button( ICON_FA_CHECK "  Create", ImVec2(pannel_width_ - padding_width_, 0)) ) {
                 Mixer::manager().addSource(new_source_preview_.getSource());
                 selected_button[NAV_NEW] = false;
             }

UserInterfaceManager.h

@@ -26,7 +26,8 @@ public:
     Source *getSource();
 
     void Render(float width, bool controlbutton = false);
-    inline bool ready() const { return source_ != nullptr; }
+    bool ready() const;
+    inline bool filled() const { return source_ != nullptr; }
 };
 
 class Navigator