From d41a85f4a1c5e32538d496e8de3e31a6a3e87f42 Mon Sep 17 00:00:00 2001 From: Bruno Herbelin Date: Tue, 19 Apr 2022 01:23:50 +0200 Subject: [PATCH] New ImageFilter applied to Clone Sources Preliminary implementation, effective but without consequence on the rendering. --- CMakeLists.txt | 1 + CloneSource.cpp | 40 ++++++-- CloneSource.h | 4 + ImageFilter.cpp | 238 ++++++++++++++++++++++++++++++++++++++++++++++++ ImageFilter.h | 69 ++++++++++++++ 5 files changed, 346 insertions(+), 6 deletions(-) create mode 100644 ImageFilter.cpp create mode 100644 ImageFilter.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c28815d..e50e1f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -382,6 +382,7 @@ set(VMIX_SRCS Shader.cpp ImageShader.cpp ImageProcessingShader.cpp + ImageFilter.cpp UpdateCallback.cpp Scene.cpp Primitives.cpp diff --git a/CloneSource.cpp b/CloneSource.cpp index 18d1b1e..6f4f943 100644 --- a/CloneSource.cpp +++ b/CloneSource.cpp @@ -28,12 +28,13 @@ #include "Visitor.h" #include "FrameBuffer.h" #include "Decorations.h" +#include "ImageFilter.h" #include "CloneSource.h" CloneSource::CloneSource(Source *origin, uint64_t id) : Source(id), origin_(origin), cloningsurface_(nullptr), - garbage_image_(nullptr), timer_reset_(false), delay_(0.0), paused_(false) + garbage_image_(nullptr), timer_reset_(false), delay_(0.0), paused_(false), filter_(nullptr) { // initial name copies the origin name: diplucates are namanged in session name_ = origin->name(); @@ -44,7 +45,7 @@ CloneSource::CloneSource(Source *origin, uint64_t id) : Source(id), origin_(orig // init connecting line connection_ = new DotLine; - connection_->color = glm::vec4(COLOR_DEFAULT_SOURCE, 0.5f); + connection_->color = glm::vec4(COLOR_HIGHLIGHT_SOURCE, 0.6f); connection_->target = origin->groups_[View::MIXING]->translation_; groups_[View::MIXING]->attach(connection_); @@ -64,6 +65,9 @@ CloneSource::~CloneSource() images_.pop(); } + if (filter_) + delete filter_; + if (cloningsurface_) delete cloningsurface_; @@ -77,15 +81,20 @@ void CloneSource::init() // frame buffers where to draw frames from the origin source glm::vec3 res = origin_->frame()->resolution(); images_.push( new FrameBuffer( res, origin_->frame()->use_alpha() ) ); + origin_->frame()->blit( images_.front() ); timestamps_.push( origin_->playtime() ); elapsed_.push( 0. ); // ask to reset elapsed-timer timer_reset_ = true; + // create filter + filter_ = new ImageFilter( res, true ); + filter_->setInputTexture( images_.front()->texture() ); + // set initial texture surface - origin_->frame()->blit( images_.front() ); - texturesurface_->setTextureIndex( images_.front()->texture() ); + texturesurface_->setTextureIndex( filter_->getOutputTexture() ); +// texturesurface_->setTextureIndex( images_.front()->texture() ); // create render Frame buffer matching size of images FrameBuffer *renderbuffer = new FrameBuffer( res, true ); @@ -104,6 +113,23 @@ void CloneSource::init() } } +void CloneSource::render() +{ + if ( renderbuffer_ == nullptr ) + init(); + else { + // render filtered image + filter_->draw(); + texturesurface_->setTextureIndex( filter_->getOutputTexture() ); + + // render textured surface into frame buffer + renderbuffer_->begin(); + texturesurface_->draw(glm::identity(), renderbuffer_->projection()); + renderbuffer_->end(); + ready_ = true; + } +} + void CloneSource::setActive (bool on) { @@ -183,8 +209,10 @@ void CloneSource::update(float dt) // blit rendered framebuffer in the newest image (back) origin_->frame()->blit( images_.back() ); - // update the source surface to be rendered with the oldest image (front) - texturesurface_->setTextureIndex( images_.front()->texture() ); +// update the source surface to be rendered with the oldest image (front) +// texturesurface_->setTextureIndex( images_.front()->texture() ); + filter_->setInputTexture( images_.front()->texture() ); + } } diff --git a/CloneSource.h b/CloneSource.h index 2a86f94..3b00ddf 100644 --- a/CloneSource.h +++ b/CloneSource.h @@ -23,6 +23,7 @@ public: uint texture() const override; bool failed() const override { return origin_ == nullptr; } void accept (Visitor& v) override; + void render() override; // implementation of cloning mechanism inline void detach() { origin_ = nullptr; } @@ -57,6 +58,9 @@ protected: // control bool paused_; + // filter + class ImageFilter *filter_; + // connecting line class DotLine *connection_; }; diff --git a/ImageFilter.cpp b/ImageFilter.cpp new file mode 100644 index 0000000..178b1ef --- /dev/null +++ b/ImageFilter.cpp @@ -0,0 +1,238 @@ +/* + * This file is part of vimix - video live mixer + * + * **Copyright** (C) 2019-2022 Bruno Herbelin + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +**/ + +#include +#include +#include + +#include "defines.h" +#include "Visitor.h" +#include "FrameBuffer.h" +#include "Primitives.h" +#include "Log.h" + +#include "ImageFilter.h" + +std::string fragmentDeclaration = "#version 330 core\n" + "out vec4 FragColor;\n" + "in vec4 vertexColor;\n" + "in vec2 vertexUV;\n" + "vec3 iChannelResolution[2];\n" + "uniform mat4 iTransform;\n" + "uniform vec4 color;\n" + "uniform float stipple;\n"; + +std::string filterHeader = "uniform vec3 iResolution; // viewport resolution (in pixels)\n" + "uniform sampler2D iChannel0; // input channel (texture)\n" + "uniform sampler2D iChannel1; // input channel (mask)\n" + "uniform float iTime; // shader playback time (in seconds)\n" + "uniform float iTimeDelta; // render time (in seconds)\n" + "uniform int iFrame; // shader playback frame\n" + "uniform vec4 iDate; // (year, month, day, time in seconds)\n" ; + +std::string filterDefaultCode = "void mainImage( out vec4 fragColor, in vec2 fragCoord )\n" + "{\n" + " vec2 uv = fragCoord.xy / iResolution.xy;\n" + " fragColor = texture(iChannel0, uv);\n" + "}\n"; + +std::string fragmentMainCode = "\nvoid main() {\n" + " iChannelResolution[0] = vec3(textureSize(iChannel0, 0), 0.f);\n" + " iChannelResolution[1] = vec3(textureSize(iChannel1, 0), 0.f);\n" + " vec4 texcoord = iTransform * vec4(vertexUV.x, vertexUV.y, 0.0, 1.0);\n" + " mainImage( FragColor, texcoord.xy * iChannelResolution[0].xy );\n" + "}\n"; + + +std::string filterBloomCode = "float Threshold = 0.1;" + "float Intensity = 1.0;" + "float BlurSize = 6.0;" + "vec4 BlurColor (in vec2 Coord, in sampler2D Tex, in float MipBias)" + "{" + " vec2 TexelSize = MipBias/iChannelResolution[0].xy;" + " vec4 Color = texture(Tex, Coord, MipBias);" + " Color += texture(Tex, Coord + vec2(TexelSize.x,0.0), MipBias);" + " Color += texture(Tex, Coord + vec2(-TexelSize.x,0.0), MipBias);" + " Color += texture(Tex, Coord + vec2(0.0,TexelSize.y), MipBias);" + " Color += texture(Tex, Coord + vec2(0.0,-TexelSize.y), MipBias);" + " Color += texture(Tex, Coord + vec2(TexelSize.x,TexelSize.y), MipBias);" + " Color += texture(Tex, Coord + vec2(-TexelSize.x,TexelSize.y), MipBias);" + " Color += texture(Tex, Coord + vec2(TexelSize.x,-TexelSize.y), MipBias);" + " Color += texture(Tex, Coord + vec2(-TexelSize.x,-TexelSize.y), MipBias);" + " return Color/9.0;" + "}" + "void mainImage( out vec4 fragColor, in vec2 fragCoord )" + "{" + " vec2 uv = (fragCoord.xy/iResolution.xy);" + " vec4 Color = texture(iChannel0, uv);" + " vec4 Highlight = clamp(BlurColor(uv, iChannel0, BlurSize)-Threshold,0.0,1.0)*1.0/(1.0-Threshold);" + " fragColor = 1.0-(1.0-Color)*(1.0-Highlight*Intensity);" + "}"; + + +std::string montecarloCode = "#version 330 core\n" + "#define ITER 32\n" + "#define SIZE 100.0\n" + "out vec4 FragColor;\n" + "in vec4 vertexColor;\n" + "in vec2 vertexUV;\n" + "uniform mat4 iTransform;\n" + "uniform vec4 color;\n" + "uniform sampler2D iChannel0;\n" + "uniform sampler2D iChannel1;\n" + "uniform float size;\n" + "uniform float factor;\n" + "void srand(vec2 a, out float r) {r=sin(dot(a,vec2(1233.224,1743.335)));}\n" + "float rand(inout float r) { r=fract(3712.65*r+0.61432); return (r-0.5)*2.0;}\n" + "void main() {" + "vec4 texcoord = iTransform * vec4(vertexUV.x, vertexUV.y, 0.0, 1.0);\n" + "float p = (SIZE * size + 1.0)/textureSize(iChannel0, 0).y * factor;" + "vec4 c=vec4(0.0);" + "float r;" + "srand(vec2(texcoord), r);" + "vec2 rv;" + "for(int i=0;i montecarloParam = { { "factor", 0.9 }, {"size", 0.1} }; + + +ImageFilteringShader::ImageFilteringShader(): Shader() +{ + program_ = &custom_shading_; + custom_shading_.setShaders("shaders/image.vs", "shaders/image.fs"); + + Shader::reset(); +} + +ImageFilteringShader::~ImageFilteringShader() +{ + custom_shading_.reset(); +} + +void ImageFilteringShader::use() +{ + Shader::use(); + + + program_->setUniform("iTime", 0.f ); + program_->setUniform("iTimeDelta", 0.f ); + + static int f = 0; + program_->setUniform("iFrame", ++f); + program_->setUniform("iDate", glm::vec4(0.f, 0.f, 0.f, 0.f) ); + + // loop over all uniforms + for (auto u = uniforms_.begin(); u != uniforms_.end(); ++u) + // set uniform to current value + program_->setUniform( u->first, u->second.x ); +} + +void ImageFilteringShader::reset() +{ + Shader::reset(); + + // loop over all uniforms + for (auto u = uniforms_.begin(); u != uniforms_.end(); ++u) + // reset current value to detault value + u->second.x = u->second.y; +} + +void ImageFilteringShader::setFragmentCode(const std::string &code, std::map parameters) +{ + // change the shading code for fragment + code_ = code; + custom_shading_.setShaders("shaders/image.vs", code_); + + // set the list of uniforms + uniforms_.clear(); + for (auto u = parameters.begin(); u != parameters.end(); ++u) + // set uniform with key name to have default and current values + uniforms_[u->first] = glm::vec2(u->second, u->second); +} + +void ImageFilteringShader::copy(ImageFilteringShader const& S) +{ + // change the shading code for fragment + code_ = S.code_; + custom_shading_.setShaders("shaders/image.vs", code_); + + // set the list of uniforms + uniforms_.clear(); + for (auto u = S.uniforms_.begin(); u != S.uniforms_.end(); ++u) + // set uniform with key name to have default and current values + uniforms_[u->first] = u->second; +} + +void ImageFilteringShader::accept(Visitor& v) +{ +// Shader::accept(v); + v.visit(*this); +} + + +ImageFilter::ImageFilter(glm::vec3 resolution, bool useAlpha) +{ + shader_ = new ImageFilteringShader; + surface_ = new Surface(shader_); + buffer_ = new FrameBuffer( resolution, useAlpha ); + + std::string codefilter = fragmentDeclaration + filterHeader + filterDefaultCode + fragmentMainCode; +// std::string codefilter = fragmentDeclaration + filterHeader + filterBloomCode + fragmentMainCode; + + g_print("Filter code:\n%s", codefilter.c_str()); + + std::map< std::string, float > paramfilter; + + shader_->setFragmentCode(codefilter, paramfilter); +} + +ImageFilter::~ImageFilter() +{ + if (buffer_) + delete buffer_; + // NB: shader_ is removed with surface + if (surface_) + delete surface_; +} + +void ImageFilter::setInputTexture(uint t) +{ + surface_->setTextureIndex( t ); +} + +uint ImageFilter::getOutputTexture() const +{ + return buffer_->texture(); +} + +void ImageFilter::draw() +{ + // render filtered surface into frame buffer + buffer_->begin(); + surface_->draw(glm::identity(), buffer_->projection()); + buffer_->end(); +} + + diff --git a/ImageFilter.h b/ImageFilter.h new file mode 100644 index 0000000..0a553ad --- /dev/null +++ b/ImageFilter.h @@ -0,0 +1,69 @@ +#ifndef IMAGEFILTER_H +#define IMAGEFILTER_H + +#include + +class Surface; +class FrameBuffer; + +#include "Shader.h" + +class ImageFilteringShader : public Shader +{ + ShadingProgram custom_shading_; + + // fragment shader GLSL code + std::string code_; + + // list of uniform vars in GLSL + // with associated pair (current_value, default_values) in range [0.f 1.f] + std::map uniforms_; + +public: + + ImageFilteringShader(); + ~ImageFilteringShader(); + + void use() override; + void reset() override; + void accept(Visitor& v) override; + void copy(ImageFilteringShader const& S); + + // set fragment shader code and uniforms (with default values) + void setFragmentCode(const std::string &code, std::map parameters); +}; + + +class ImageFilter +{ + Surface *surface_; + FrameBuffer *buffer_; + ImageFilteringShader *shader_; + uint type_; + +public: + + // instanciate an image filter at given resolution, with alpha channel + ImageFilter(glm::vec3 resolution, bool useAlpha = false); + ~ImageFilter(); + + // set the texture to draw into the framebuffer + void setInputTexture(uint t); + +// typedef enum { + +// } ; + + + void setFilter(uint filter); + + // draw the input texture with filter on the framebuffer + void draw(); + + // get the texture id of the rendered framebuffer + uint getOutputTexture() const; + +}; + + +#endif // IMAGEFILTER_H