Implementation of source frame buffer; rendering of source and scene

integration

CMakeLists.txt

@@ -241,12 +241,14 @@ set(VMIX_RSC_FILES
     ./rsc/images/icons.dds
     ./rsc/images/seed_512.jpg
     ./rsc/images/transparencygrid.png
+    ./rsc/images/shadow.png
+    ./rsc/images/shadow_dark.png
     ./rsc/mesh/point.ply
     ./rsc/mesh/disk.ply
     ./rsc/mesh/shadow.ply
-    ./rsc/mesh/shadow.png
     ./rsc/mesh/target.ply
     ./rsc/mesh/border.ply
+    ./rsc/mesh/circle.ply
     ./rsc/mesh/icon_video.ply
     ./rsc/mesh/icon_image.ply
 )

ImGuiVisitor.cpp

@@ -96,10 +96,10 @@ void ImGuiVisitor::visit(FrameBufferSurface &n)
 
 void ImGuiVisitor::visit(MediaSurface &n)
 {
-    ImGui::Text("%s", n.getUri().c_str());
+    ImGui::Text("%s", n.uri().c_str());
 
-    if (n.getMediaPlayer())
+    if (n.mediaPlayer())
-        n.getMediaPlayer()->accept(*this);
+        n.mediaPlayer()->accept(*this);
 }
 
 void ImGuiVisitor::visit(MediaPlayer &n)

ImageProcessingShader.cpp

@@ -0,0 +1,40 @@
+#include "ImageProcessingShader.h"
+
+
+ShadingProgram imageProcessingShadingProgram("shaders/processing-shader.vs", "shaders/texture-shader.fs");
+
+ImageProcessingShader::ImageProcessingShader()
+{
+    program_ = &imageProcessingShadingProgram;
+    reset();
+}
+
+void ImageProcessingShader::use()
+{
+    Shader::use();
+
+    program_->setUniform("brightness", brightness);
+    program_->setUniform("contrast", contrast);
+    program_->setUniform("stipple", stipple);
+}
+
+
+void ImageProcessingShader::reset()
+{
+    Shader::reset();
+
+    brightness = 0.f;
+    contrast = 0.f;
+    stipple = 0.f;
+}
+
+void ImageProcessingShader::accept(Visitor& v) {
+    Shader::accept(v);
+    v.visit(*this);
+}
+
+
+::ImageProcessingShader()
+{
+
+}

ImageProcessingShader.h

@@ -0,0 +1,24 @@
+#ifndef IMAGEPROCESSINGSHADER_H
+#define IMAGEPROCESSINGSHADER_H
+
+#include "Shader.h"
+
+class ImageProcessingShader : public Shader
+{
+public:
+
+    ImageProcessingShader();
+    virtual ~ImageShader() {}
+
+    virtual void use();
+    virtual void reset();
+    virtual void accept(Visitor& v);
+
+    float brightness;
+    float contrast;
+    float stipple;
+};
+
+
+
+#endif // IMAGEPROCESSINGSHADER_H

Mesh.cpp

@@ -379,42 +379,67 @@ void Mesh::accept(Visitor& v)
     v.visit(*this);
 }
 
-Frame::Frame() : Node()
+Frame::Frame(Style style) : Node()
 {
-    icon_   = new Mesh("mesh/icon_video.ply");
+    switch (style) {
-    border_ = new Mesh("mesh/border.ply");
+    case MIXING_OVERLAY:
-    shadow_ = new Mesh("mesh/shadow.ply", "mesh/shadow.png");
+        overlay_ = new Mesh("mesh/icon_video.ply");
+        border_  = nullptr;
+        shadow_  = new Mesh("mesh/shadow.ply", "images/shadow.png");
+        break;
+    default:
+    case MIXING:
+        overlay_ = nullptr;
+        border_  = new Mesh("mesh/border.ply");
+        shadow_  = new Mesh("mesh/shadow.ply", "images/shadow.png");
+        break;
+    }
     color   = glm::vec4( 0.8f, 0.8f, 0.f, 1.f);
 }
 
+Frame::~Frame()
+{
+    if(overlay_) delete overlay_;
+    if(border_) delete border_;
+    delete shadow_;
+}
+
 void Frame::draw(glm::mat4 modelview, glm::mat4 projection)
 {
     if ( !initialized() ) {
-        icon_->init();
+        if(overlay_) overlay_->init();
-        border_->init();
+        if(border_) border_->init();
         shadow_->init();
         init();
     }
 
-    // shadow
+    if ( visible_ ) { // not absolutely necessary but saves some CPU time..
-    shadow_->draw( modelview * transform_, projection);
 
-    // right side
+        // shadow
-    float ar = scale_.x / scale_.y;
+        shadow_->draw( modelview * transform_, projection);
-    glm::vec3 s(scale_.y, scale_.y, 1.0);
-    glm::vec3 t(translation_.x - 1.0 +ar, translation_.y, translation_.z);
-    glm::mat4 ctm = modelview * transform(t, rotation_, s);
 
-    border_->shader()->color = color;
+        // right side
-    border_->draw( ctm, projection );
+        float ar = scale_.x / scale_.y;
-    icon_->shader()->color = color;
+        glm::vec3 s(scale_.y, scale_.y, 1.0);
-    icon_->draw( ctm, projection );
+        glm::vec3 t(translation_.x - 1.0 +ar, translation_.y, translation_.z);
+        glm::mat4 ctm = modelview * transform(t, rotation_, s);
 
-    // left side
+        if(overlay_) {
-    t.x = -t.x;
+            overlay_->shader()->color = color;
-    s.x = -s.x;
+            overlay_->draw( ctm, projection );
-    ctm = modelview * transform(t, rotation_, s);
+        }
-    border_->draw( ctm, projection );
 
+        if(border_) {
+            // right side
+            border_->shader()->color = color;
+            border_->draw( ctm, projection );
+            // left side
+            t.x = -t.x;
+            s.x = -s.x;
+            ctm = modelview * transform(t, rotation_, s);
+            border_->draw( ctm, projection );
+        }
+
+    }
 }
 

Mesh.h

@@ -37,11 +37,14 @@ protected:
 class Frame : public Node
 {
     Mesh *border_;
-    Mesh *icon_;
+    Mesh *overlay_;
     Mesh *shadow_;
 
 public:
-    Frame();
+
+    typedef enum { MIXING = 0, MIXING_OVERLAY } Style;
+    Frame(Style style);
+    ~Frame();
 
     void draw (glm::mat4 modelview, glm::mat4 projection) override;
 

Mixer.cpp

@@ -26,7 +26,7 @@ void Mixer::update()
 
     // render of all sources
     for( SourceList::iterator it = Source::begin(); it != Source::end(); it++){
-        (*it)->render();
+        (*it)->render( it == current_source_ );
     }
 
     // recursive update of ALL views

Primitives.cpp

@@ -26,7 +26,7 @@ static const std::vector<glm::vec3> square_points {
 };
 
 
-Surface::Surface(Shader *s) : Primitive(s)
+Surface::Surface(Shader *s) : Primitive(s), textureindex_(0)
 {
     // geometry
     points_ = std::vector<glm::vec3> { glm::vec3( -1.f, -1.f, 0.f ), glm::vec3( -1.f, 1.f, 0.f ),
@@ -68,7 +68,22 @@ void Surface::accept(Visitor& v)
     v.visit(*this);
 }
 
-ImageSurface::ImageSurface(const std::string& path, Shader *s) : Surface(s), resource_(path), textureindex_(0)
+void Surface::draw(glm::mat4 modelview, glm::mat4 projection)
+{
+    if ( !initialized() )
+        init();
+
+    if ( textureindex_ )
+        glBindTexture(GL_TEXTURE_2D, textureindex_);
+    else
+        glBindTexture(GL_TEXTURE_2D, Resource::getTextureBlack());
+
+    Primitive::draw(modelview, projection);
+
+    glBindTexture(GL_TEXTURE_2D, 0);
+}
+
+ImageSurface::ImageSurface(const std::string& path, Shader *s) : Surface(s), resource_(path)
 {
 
 }
@@ -78,28 +93,12 @@ void ImageSurface::init()
     Surface::init();
 
     // load image if specified (should always be the case)
-    if ( resource_.empty())
+    if ( !resource_.empty()) {
-        textureindex_ = Resource::getTextureBlack();
-    else {
         float ar = 1.0;
         textureindex_ = Resource::getTextureImage(resource_, &ar);
         scale_.x = ar;
     }
 
-    // a new shader for a new image
-    shader_ = new ImageShader();
-}
-
-void ImageSurface::draw(glm::mat4 modelview, glm::mat4 projection)
-{
-    if ( !initialized() )
-        init();
-
-    glBindTexture(GL_TEXTURE_2D, textureindex_);
-
-    Primitive::draw(modelview, projection);
-
-    glBindTexture(GL_TEXTURE_2D, 0);
 }
 
 void ImageSurface::accept(Visitor& v)
@@ -126,7 +125,6 @@ void MediaSurface::init()
     mediaplayer_->open(uri_);
     mediaplayer_->play(true);
 
-
 }
 
 void MediaSurface::draw(glm::mat4 modelview, glm::mat4 projection)
@@ -134,14 +132,13 @@ void MediaSurface::draw(glm::mat4 modelview, glm::mat4 projection)
     if ( !initialized() )
         init();
 
+    // set the texture to the media player once openned
+    // TODO: avoid to repeat with a static flag?
     if ( mediaplayer_->isOpen() )
-        glBindTexture(GL_TEXTURE_2D, mediaplayer_->texture());
+        textureindex_ = mediaplayer_->texture();
-    else
-        glBindTexture(GL_TEXTURE_2D, Resource::getTextureBlack());
 
-    Primitive::draw(modelview, projection);
+    Surface::draw(modelview, projection);
 
-    glBindTexture(GL_TEXTURE_2D, 0);
 }
 
 void MediaSurface::update( float dt )
@@ -173,8 +170,6 @@ void FrameBufferSurface::init()
     // set aspect ratio
     scale_.x = frame_buffer_->aspectRatio();
 
-    // a new shader for a new image
-    shader_ = new ImageShader();
 }
 
 void FrameBufferSurface::draw(glm::mat4 modelview, glm::mat4 projection)

Primitives.h

@@ -25,7 +25,14 @@ public:
     Surface(Shader *s = new ImageShader);
 
     void init () override;
+    void draw (glm::mat4 modelview, glm::mat4 projection) override;
     void accept (Visitor& v) override;
+
+    inline void setTextureIndex(uint t) { textureindex_ = t; }
+    inline uint textureIndex() const { return textureindex_; }
+
+protected:
+    uint textureindex_;
 };
 
 
@@ -43,14 +50,12 @@ public:
     ImageSurface(const std::string& path, Shader *s = new ImageShader);
 
     void init () override;
-    void draw (glm::mat4 modelview, glm::mat4 projection) override;
     void accept (Visitor& v) override;
 
-    inline std::string getResource() const { return resource_; }
+    inline std::string resource() const { return resource_; }
 
 protected:
     std::string resource_;
-    uint textureindex_;
 };
 
 
@@ -71,8 +76,8 @@ public:
     void accept (Visitor& v) override;
     void update (float dt) override;
 
-    inline std::string getUri() const { return uri_; }
+    inline std::string uri() const { return uri_; }
-    MediaPlayer *getMediaPlayer() { return mediaplayer_; }
+    inline MediaPlayer *mediaPlayer() const { return mediaplayer_; }
 
 protected:
     std::string uri_;

RenderingManager.cpp

@@ -136,6 +136,7 @@ bool Rendering::Init()
     // Antialiasing
     glEnable(GL_LINE_SMOOTH);
     glHint(GL_LINE_SMOOTH_HINT, GL_NICEST);
+    glHint(GL_POLYGON_SMOOTH_HINT, GL_NICEST);
     // This hint can improve the speed of texturing when perspective-correct texture coordinate interpolation isn't needed
     glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_FASTEST);
     // This hint can improve the speed of shading when dFdx dFdy aren't needed in GLSL

Scene.cpp

@@ -289,6 +289,11 @@ void Switch::removeChild(Node *child)
 }
 
 
+void Switch::unsetActiveChild ()
+{
+    active_ == children_.end();
+}
+
 void Switch::setActiveChild(Node *child)
 {
     setActiveChild( std::find_if(children_.begin(), children_.end(), hasId(child->id())) );

Scene.h

@@ -187,6 +187,7 @@ public:
     void addChild (Node *child) override;
     void removeChild (Node *child) override;
 
+    void unsetActiveChild ();
     void setActiveChild (Node *child);
     void setActiveChild (NodeSet::iterator n);
     NodeSet::iterator activeChild () const;

SessionVisitor.cpp

@@ -108,7 +108,7 @@ void SessionVisitor::visit(ImageSurface &n)
     // Node of a different type
     xmlCurrent_->SetAttribute("type", "ImageSurface");
 
-    XMLText *filename = xmlDoc_->NewText( n.getResource().c_str() );
+    XMLText *filename = xmlDoc_->NewText( n.resource().c_str() );
     XMLElement *image = xmlDoc_->NewElement("resource");
     image->InsertEndChild(filename);
     xmlCurrent_->InsertEndChild(image);
@@ -125,7 +125,7 @@ void SessionVisitor::visit(MediaSurface &n)
     // Node of a different type
     xmlCurrent_->SetAttribute("type", "MediaSurface");
 
-    n.getMediaPlayer()->accept(*this);
+    n.mediaPlayer()->accept(*this);
 }
 
 void SessionVisitor::visit(MediaPlayer &n)

Shader.cpp

@@ -210,8 +210,8 @@ void Shader::use()
 
 void Shader::reset()
 {
-    projection =  glm::identity<glm::mat4>();
+    projection = glm::identity<glm::mat4>();
-    modelview =  glm::identity<glm::mat4>();
+    modelview  = glm::identity<glm::mat4>();
     resolution = glm::vec3(1280.f, 720.f, 0.f);
     color = glm::vec4(1.f, 1.f, 1.f, 1.f);
 }

Source.cpp

@@ -1,11 +1,13 @@
 
 #include <algorithm>
+#include <glm/gtc/matrix_transform.hpp>
 
 #include "Source.h"
 
 #include "defines.h"
 #include "FrameBuffer.h"
 #include "ImageShader.h"
+#include "Resource.h"
 #include "Primitives.h"
 #include "Mesh.h"
 #include "MediaPlayer.h"
@@ -28,7 +30,7 @@ Source::Source(std::string name) : name_(""), initialized_(false)
 
     // default mixing nodes
     groups_[View::MIXING] = new Group;
-    Frame *frame = new Frame;
+    Frame *frame = new Frame(Frame::MIXING);
     frame->translation_.z = 0.1;
     groups_[View::MIXING]->addChild(frame);
     groups_[View::MIXING]->scale_ = glm::vec3(0.2f, 0.2f, 1.f);
@@ -118,61 +120,117 @@ uint Source::numSource()
     return sources_.size();
 }
 
-MediaSource::MediaSource(std::string name, std::string uri) : Source(name)
+MediaSource::MediaSource(std::string name, std::string uri) : Source(name), uri_(uri)
 {
-    surface_ = new MediaSurface(uri);
+    // create media player
+    mediaplayer_ = new MediaPlayer;
+    mediaplayer_->open(uri_);
+    mediaplayer_->play(true);
 
-    // add the surface to draw in the views
+    // create media surface:
-    groups_[View::RENDERING]->addChild(surface_);
+    // - textured with original texture from media player
-    groups_[View::MIXING]->addChild(surface_);
+    // - crop & repeat UV can be managed here
+    // - additional custom shader can be associated
+    mediasurface_ = new Surface;
+
+    // extra overlay for mixing view
+    mixingoverlay_ = new Frame(Frame::MIXING_OVERLAY);
+    groups_[View::MIXING]->addChild(mixingoverlay_);
+    mixingoverlay_->translation_.z = 0.1;
+    mixingoverlay_->visible_ = false;
 
 }
 
 MediaSource::~MediaSource()
 {
-    // TODO verify that surface_ node is deleted in Source destructor
+    delete mediasurface_;
+    delete mediaplayer_;
+    // TODO verify that all surfaces and node is deleted in Source destructor
 }
 
-Shader *MediaSource::shader() const
+ImageShader *MediaSource::shader() const
 {
-    return surface_->shader();
+    if (!rendersurface_)
+        return nullptr;
+
+    return static_cast<ImageShader *>(rendersurface_->shader());
 }
 
 std::string MediaSource::uri() const
 {
-    return surface_->getUri();
+    return uri_;
 }
 
 MediaPlayer *MediaSource::mediaplayer() const
 {
-    return surface_->getMediaPlayer();
+    return mediaplayer_;
 }
 
 void MediaSource::init()
 {
-    ImageShader *is = static_cast<ImageShader *>(surface_->shader());
+    if ( mediaplayer_->isOpen() ) {
-    if (is)
+
-        is->stipple = 1.0;
+        // update video
+        mediaplayer_->update();
+
+        // once the texture of media player is created
+        if (mediaplayer_->texture() != Resource::getTextureBlack()) {
+
+            // get the texture index from media player, apply it to the media surface
+            mediasurface_->setTextureIndex( mediaplayer_->texture() );
+
+            // create Frame buffer matching size of media player
+            renderbuffer_ = new FrameBuffer(mediaplayer()->width(), mediaplayer()->height());
+
+            // create the surfaces to draw the frame buffer in the views
+            // TODO Provide the source specific effect shader
+            rendersurface_ = new FrameBufferSurface(renderbuffer_);
+            groups_[View::RENDERING]->addChild(rendersurface_);
+            groups_[View::MIXING]->addChild(rendersurface_);
+
+            // for mixing view, add another surface to overlay (for stippled view in transparency)
+            Surface *surfacemix = new Surface();
+            surfacemix->setTextureIndex( mediaplayer_->texture() );
+            ImageShader *is = static_cast<ImageShader *>(surfacemix->shader());
+            if (is)  is->stipple = 1.0;
+            groups_[View::MIXING]->addChild(surfacemix);
+
+            // scale all mixing nodes to match aspect ratio of the media
+            for (NodeSet::iterator node = groups_[View::MIXING]->begin();
+                 node != groups_[View::MIXING]->end(); node++) {
+                (*node)->scale_.x = mediaplayer_->aspectRatio();
+            }
+
+            // done init once and for all
+            initialized_ = true;
+        }
+    }
 
-    initialized_ = true;
 }
 
-void MediaSource::render()
+void MediaSource::render(bool current)
 {
     if (!initialized_)
         init();
-//    surface_->shader()
+    else {
+        // update video
+        mediaplayer_->update();
 
-    // scalle all mixing nodes to match scale of surface
+        // render the media player into frame buffer
-    for (NodeSet::iterator node = groups_[View::MIXING]->begin();
+        static glm::mat4 projection = glm::ortho(-1.f, 1.f, 1.f, -1.f, -1.f, 1.f);
-         node != groups_[View::MIXING]->end(); node++) {
+        renderbuffer_->begin();
-        (*node)->scale_ = surface_->scale_;
+        mediasurface_->draw(glm::identity<glm::mat4>(), projection);
+        renderbuffer_->end();
+
+
+        // read position of the mixing node and interpret this as transparency of render output
+        float alpha = 1.0 - CLAMP( SQUARE( glm::length(groups_[View::MIXING]->translation_) ), 0.f, 1.f );
+        rendersurface_->shader()->color.a = alpha;
+
+
+        // make Mixing Overlay visible if it is current source
+        mixingoverlay_->visible_ = current;
     }
-
-    // read position of the mixing node and interpret this as transparency change
-    float alpha = 1.0 - SQUARE( glm::length(groups_[View::MIXING]->translation_) );
-    surface_->shader()->color.a = alpha;
-
-
-
 }
+
+

Source.h

@@ -11,7 +11,8 @@ class ImageShader;
 class Surface;
 class FrameBuffer;
 class MediaPlayer;
-class MediaSurface;
+class FrameBufferSurface;
+class Frame;
 
 class Source;
 // TODO : source set sorted by shader
@@ -34,10 +35,10 @@ public:
     inline Group *group(View::Mode m) const { return groups_.at(m); }
 
     // every Source have a shader to control visual effects
-    virtual Shader *shader() const = 0;
+    virtual ImageShader *shader() const = 0;
 
     // every Source shall be rendered before draw
-    virtual void render() = 0;
+    virtual void render(bool current) = 0;
 
     // global management of list of sources
     static SourceList::iterator begin();
@@ -58,6 +59,15 @@ protected:
     // nodes
     std::map<View::Mode, Group*> groups_;
 
+    // render() fills in the renderbuffer at every frame
+    // NB: additional shader (custom) are applied inside render()
+    FrameBuffer *renderbuffer_;
+
+    // the rendersurface draws the renderbuffer in the scene
+    // It is associated to the sourceshader for mixing effects
+    // (aka visual effect applied in scene, not in render() )
+    FrameBufferSurface *rendersurface_;
+
     // static global list of sources
     static SourceList sources_;
 };
@@ -89,10 +99,10 @@ public:
     MediaSource(std::string name, std::string uri);
     ~MediaSource();
 
-    void render();
+    void render(bool current);
 
     // Source interface
-    Shader *shader() const;
+    ImageShader *shader() const;
 
     // Media specific interface
     std::string uri() const;
@@ -102,7 +112,10 @@ protected:
 
     virtual void init();
 
-    MediaSurface *surface_;
+    Frame *mixingoverlay_;
+    Surface *mediasurface_;
+    std::string uri_;
+    MediaPlayer *mediaplayer_;
 };
 
 

View.cpp

@@ -39,7 +39,9 @@ MixingView::MixingView() : View()
     backgound_.addChild(disk);
 
     glm::vec4 pink( 0.8f, 0.f, 0.8f, 1.f );
-    LineCircle *circle = new LineCircle(pink, 5);
+//    LineCircle *circle = new LineCircle(pink, 5);
+    Mesh *circle = new Mesh("mesh/circle.ply");
+    circle->shader()->color = pink;
     backgound_.addChild(circle);
 
     scene.root()->addChild(&backgound_);
@@ -102,14 +104,10 @@ void MixingView::grab (glm::vec2 from, glm::vec2 to, Source *s)
     }
 
     // unproject
-//    glm::vec3 gl_Position_from = Rendering::manager().unProject(from, sourceNode->transform_);
-//    glm::vec3 gl_Position_to = Rendering::manager().unProject(to, sourceNode->transform_);
     glm::vec3 gl_Position_from = Rendering::manager().unProject(from, sourceNode->parent_->transform_);
     glm::vec3 gl_Position_to = Rendering::manager().unProject(to, sourceNode->parent_->transform_);
 
     // compute delta translation
- //   node->translation_ = start_translation + gl_Position_to - gl_Position_from;
-
     sourceNode->translation_ = start_translation + gl_Position_to - gl_Position_from;
 
 }

rsc/shaders/texture-shader.fs

@@ -13,12 +13,16 @@ uniform float stipple;
 uniform vec3  resolution;           // viewport resolution (in pixels)
 
 void main()
-{		
+{
+    // color is a mix of texture (manipulated with brightness & contrast), vertex and uniform colors
     vec4 textureColor = texture(sourceTexture, vertexUV);
-    vec3 transformedRGB = mix(vec3(0.62), textureColor.rgb, contrast + 1.0) + brightness;
+    vec3 RGB = mix(vec3(0.62), textureColor.rgb, contrast + 1.0) + brightness;
-    transformedRGB *= vertexColor.rgb * color.rgb;
+    RGB *= vertexColor.rgb * color.rgb;
 
-    float a = int(gl_FragCoord.x + gl_FragCoord.y)%2 > 1 - int(stipple) ? 1.0 : color.a;
+    // alpha is a mix of texture alpha, vertex alpha, and uniform alpha affected by stippling
+    float A = textureColor.a * vertexColor.a * color.a;
+    A *= int(gl_FragCoord.x + gl_FragCoord.y) % 2 > (1 - int(stipple)) ? 0.0 : 1.0;
 
-    FragColor = vec4(transformedRGB, textureColor.a * vertexColor.a * a);
+    // output RGBA
+    FragColor = vec4(RGB, A);
 }