Original implementation of Resampling Image filters

This involves also resizing the renderbuffer of the clone source. Upsampling is cubic (faster approximation) and Downsampling is bilinear.

CMakeLists.txt

@@ -595,6 +595,9 @@ set(VMIX_RSC_FILES
     ./rsc/shaders/filters/dilation.glsl
     ./rsc/shaders/filters/tophat.glsl
     ./rsc/shaders/filters/blackhat.glsl
+    ./rsc/shaders/filters/resample_double.glsl
+    ./rsc/shaders/filters/resample_half.glsl
+    ./rsc/shaders/filters/resample_quarter.glsl
 )
 
 # Include the CMake RC module

CloneSource.cpp

@@ -87,14 +87,21 @@ void CloneSource::render()
     if ( renderbuffer_ == nullptr )
         init();
     else {
-
         // render filter image
         filter_->draw( origin_->frame() );
 
         // ensure correct output texture is displayed (could have changed if filter changed)
         texturesurface_->setTextureIndex( filter_->texture() );
 
+        // detect resampling (change of resolution in filter)
+        if ( renderbuffer_->resolution() != filter_->resolution() ) {
+            renderbuffer_->resize( filter_->resolution() );
+//            FrameBuffer *renderbuffer = new FrameBuffer( filter_->resolution(), origin_->frame()->flags() );
+//            attach(renderbuffer);
+        }
+
         // render textured surface into frame buffer
+        // NB: this also applies the color correction shader
         renderbuffer_->begin();
         texturesurface_->draw(glm::identity<glm::mat4>(), renderbuffer_->projection());
         renderbuffer_->end();
@@ -147,6 +154,9 @@ void CloneSource::setFilter(FrameBufferFilter::Type T)
     case FrameBufferFilter::FILTER_DELAY:
         filter_ = new DelayFilter;
         break;
+    case FrameBufferFilter::FILTER_RESAMPLE:
+        filter_ = new ResampleFilter;
+        break;
     case FrameBufferFilter::FILTER_BLUR:
         filter_ = new BlurFilter;
         break;

FrameBuffer.cpp

@@ -235,10 +235,10 @@ glm::vec3 FrameBuffer::resolution() const
     return glm::vec3(attrib_.viewport.x, attrib_.viewport.y, 0.f);
 }
 
-void FrameBuffer::resize(int width, int height)
+void FrameBuffer::resize(glm::vec3 res)
 {
     if (framebufferid_) {
-        if (attrib_.viewport.x != width || attrib_.viewport.y != height)
+        if (attrib_.viewport.x != res.x || attrib_.viewport.y != res.y)
         {
             // de-init
             glDeleteFramebuffers(1, &framebufferid_);
@@ -257,7 +257,7 @@ void FrameBuffer::resize(int width, int height)
             multisampling_textureid_ = 0;
 
             // change resolution
-            attrib_.viewport = glm::ivec2(width, height);
+            attrib_.viewport = glm::ivec2(res);
             mem_usage_ = 0;
         }
     }

FrameBuffer.h

@@ -80,7 +80,7 @@ public:
     inline uint width() const { return attrib_.viewport.x; }
     inline uint height() const { return attrib_.viewport.y; }
     glm::vec3 resolution() const;
-    void resize(int width, int height);
+    void resize(glm::vec3 res);
     float aspectRatio() const;
     std::string info() const;
 

FrameBufferFilter.cpp

@@ -6,7 +6,7 @@
 #include "FrameBufferFilter.h"
 
 const char* FrameBufferFilter::type_label[FrameBufferFilter::FILTER_INVALID] = {
-    "None", "Delay", "Blur", "Sharpen", "Edge", "Shader code"
+    "None", "Delay", "Resample", "Blur", "Sharpen", "Edge", "Shader code"
 };
 
 FrameBufferFilter::FrameBufferFilter() : enabled_(true), input_(nullptr)

FrameBufferFilter.h

@@ -20,6 +20,7 @@ public:
     typedef enum {
         FILTER_PASSTHROUGH = 0,
         FILTER_DELAY,
+        FILTER_RESAMPLE,
         FILTER_BLUR,
         FILTER_SHARPEN,
         FILTER_EDGE,

ImGuiVisitor.cpp

@@ -772,6 +772,26 @@ void ImGuiVisitor::visit (DelayFilter& f)
     }
 }
 
+void ImGuiVisitor::visit (ResampleFilter& f)
+{
+    std::ostringstream oss;
+
+    // Resampling Factor selection
+    if (ImGuiToolkit::IconButton(8, 5)) {
+        f.setFactor( 0 );
+        oss << "Resample " << ResampleFilter::factor_label[0];
+        Action::manager().store(oss.str());
+    }
+    ImGui::SameLine(0, IMGUI_SAME_LINE);
+    ImGui::SetNextItemWidth(IMGUI_RIGHT_ALIGN);
+    int m = (int) f.factor();
+    if (ImGui::Combo("Factor", &m, ResampleFilter::factor_label, IM_ARRAYSIZE(ResampleFilter::factor_label) )) {
+        f.setFactor( m );
+        oss << "Resample " << ResampleFilter::factor_label[m];
+        Action::manager().store(oss.str());
+    }
+}
+
 void ImGuiVisitor::visit (BlurFilter& f)
 {
     std::ostringstream oss;

ImGuiVisitor.h

@@ -39,6 +39,7 @@ public:
     void visit (FrameBufferFilter&) override;
     void visit (PassthroughFilter&) override;
     void visit (DelayFilter&) override;
+    void visit (ResampleFilter&) override;
     void visit (BlurFilter&) override;
     void visit (SharpenFilter&) override;
     void visit (EdgeFilter&) override;

ImageFilter.cpp

@@ -60,7 +60,7 @@ std::string fragmentFooter = "void main() {\n"
                              "    iChannelResolution[0] = vec3(textureSize(iChannel0, 0), 0.f);\n"
                              "    iChannelResolution[1] = vec3(textureSize(iChannel1, 0), 0.f);\n"
                              "    vec4 texcoord = iTransform * vec4(vertexUV.x, vertexUV.y, 0.0, 1.0);\n"
-                             "    mainImage( FragColor, texcoord.xy * iChannelResolution[0].xy );\n"
+                             "    mainImage( FragColor, texcoord.xy * iResolution.xy );\n"
                              "}\n";
 
 //std::string fragmentFooter = "void main() {\n"
@@ -293,6 +293,8 @@ void ImageFilteringShader::setCode(const std::string &code, std::promise<std::st
     if (code != code_)
     {
         code_ = code;
+        if (code_.empty())
+            code_ = filterDefault;
         shader_code_ = fragmentHeader + code_ + fragmentFooter;
         custom_shading_.setShaders("shaders/image.vs", shader_code_, ret);
     }
@@ -468,6 +470,108 @@ void ImageFilter::setProgramParameter(const std::string &p, float value)
     updateParameters();
 }
 
+////////////////////////////////////////
+/////                                 //
+////  RESAMPLING FILTERS             ///
+///                                 ////
+////////////////////////////////////////
+
+const char* ResampleFilter::factor_label[ResampleFilter::RESAMPLE_INVALID] = {
+    "Double x2", "Half   1/2", "Quarter 1/4"
+};
+
+std::vector< FilteringProgram > ResampleFilter::programs_ = {
+    FilteringProgram("Double",   "shaders/filters/resample_double.glsl",  "",   { }),
+    FilteringProgram("Half",     "shaders/filters/resample_half.glsl",  "",     { }),
+    FilteringProgram("Quarter",  "", "shaders/filters/resample_half.glsl",      { })
+};
+
+ResampleFilter::ResampleFilter (): ImageFilter(), factor_(RESAMPLE_INVALID)
+{
+}
+
+void ResampleFilter::setFactor(int factor)
+{
+    factor_ = (ResampleFactor) CLAMP(factor, RESAMPLE_DOUBLE, RESAMPLE_INVALID-1);
+    setProgram( programs_[ (int) factor_] );
+
+    // force re-init
+    input_ = nullptr;
+}
+
+void ResampleFilter::draw (FrameBuffer *input)
+{
+    // Default
+    if (factor_ == RESAMPLE_INVALID)
+        setFactor( RESAMPLE_DOUBLE );
+
+    // if input changed (typically on first draw)
+    if (input_ != input) {
+        // keep reference to input framebuffer
+        input_ = input;
+
+        // create first-pass surface and shader, taking as texture the input framebuffer
+        surfaces_.first->setTextureIndex( input_->texture() );
+        shaders_.first->mask_texture = input_->texture();
+        // (re)create framebuffer for result of first-pass
+        if (buffers_.first != nullptr)
+            delete buffers_.first;
+        // set resolution depending on resample factor
+        glm::vec3 res = input_->resolution();
+        switch (factor_) {
+        case RESAMPLE_DOUBLE:
+            res *= 2.;
+            break;
+        case RESAMPLE_HALF:
+        case RESAMPLE_QUARTER:
+            res /= 2.;
+            break;
+        default:
+        case RESAMPLE_INVALID:
+            break;
+        }
+        buffers_.first = new FrameBuffer( res, input_->flags() );
+        // enforce framebuffer if first-pass is created now, filled with input framebuffer
+        input_->blit( buffers_.first );
+
+        // SECOND PASS for QUARTER resolution (divide by 2 after first pass divide by 2)
+        // create second-pass surface and shader, taking as texture the first-pass framebuffer
+        surfaces_.second->setTextureIndex( buffers_.first->texture() );
+        shaders_.second->mask_texture = input_->texture();
+        // (re)create framebuffer for result of second-pass
+        if (buffers_.second != nullptr)
+            delete buffers_.second;
+        res /= 2.;
+        buffers_.second = new FrameBuffer( res, buffers_.first->flags() );
+    }
+
+    if ( enabled() )
+    {
+        // FIRST PASS
+        // render input surface into frame buffer
+        buffers_.first->begin();
+        surfaces_.first->draw(glm::identity<glm::mat4>(), buffers_.first->projection());
+        buffers_.first->end();
+
+        // SECOND PASS
+        if ( program().isTwoPass() ) {
+            // render filtered surface from first pass into frame buffer
+            buffers_.second->begin();
+            surfaces_.second->draw(glm::identity<glm::mat4>(), buffers_.second->projection());
+            buffers_.second->end();
+        }
+    }
+}
+
+void ResampleFilter::accept (Visitor& v)
+{
+    FrameBufferFilter::accept(v);
+    v.visit(*this);
+}
+
+
+
+
 ////////////////////////////////////////
 /////                                 //
 ////  BLURING FILTERS                ///

ImageFilter.h

@@ -102,6 +102,36 @@ protected:
 };
 
 
+class ResampleFilter : public ImageFilter
+{
+public:
+
+    ResampleFilter();
+
+    // Factors of resampling
+    typedef enum {
+        RESAMPLE_DOUBLE = 0,
+        RESAMPLE_HALF,
+        RESAMPLE_QUARTER,
+        RESAMPLE_INVALID
+    } ResampleFactor;
+    static const char* factor_label[RESAMPLE_INVALID];
+    ResampleFactor factor () const { return factor_; }
+    void setFactor(int factor);
+
+    // implementation of FrameBufferFilter
+    Type type() const override { return FrameBufferFilter::FILTER_RESAMPLE; }
+
+    void draw   (FrameBuffer *input) override;
+    void accept (Visitor& v) override;
+
+private:
+    ResampleFactor factor_;
+    static std::vector< FilteringProgram > programs_;
+};
+
+
+
 class BlurFilter : public ImageFilter
 {
 public:

SessionCreator.cpp

@@ -1225,6 +1225,13 @@ void SessionLoader::visit (DelayFilter& f)
     f.setDelay(d);
 }
 
+void SessionLoader::visit (ResampleFilter& f)
+{
+    int m = 0;
+    xmlCurrent_->QueryIntAttribute("factor", &m);
+    f.setFactor(m);
+}
+
 void SessionLoader::visit (BlurFilter& f)
 {
     int m = 0;

SessionCreator.h

@@ -64,6 +64,7 @@ public:
     void visit (CloneSource& s) override;
     void visit (FrameBufferFilter&) override;
     void visit (DelayFilter&) override;
+    void visit (ResampleFilter&) override;
     void visit (BlurFilter&) override;
     void visit (SharpenFilter&) override;
     void visit (EdgeFilter&) override;

SessionVisitor.cpp

@@ -694,6 +694,11 @@ void SessionVisitor::visit (DelayFilter& f)
     xmlCurrent_->SetAttribute("delay", f.delay());
 }
 
+void SessionVisitor::visit (ResampleFilter& f)
+{
+    xmlCurrent_->SetAttribute("factor", (int) f.factor());
+}
+
 void SessionVisitor::visit (BlurFilter& f)
 {
     xmlCurrent_->SetAttribute("method", (int) f.method());

SessionVisitor.h

@@ -72,6 +72,7 @@ public:
     void visit (CloneSource& s) override;
     void visit (FrameBufferFilter&) override;
     void visit (DelayFilter&) override;
+    void visit (ResampleFilter&) override;
     void visit (BlurFilter&) override;
     void visit (SharpenFilter&) override;
     void visit (EdgeFilter&) override;

Visitor.h

@@ -44,6 +44,7 @@ class MultiFileSource;
 class FrameBufferFilter;
 class PassthroughFilter;
 class DelayFilter;
+class ResampleFilter;
 class BlurFilter;
 class SharpenFilter;
 class EdgeFilter;
@@ -107,6 +108,7 @@ public:
     virtual void visit (FrameBufferFilter&) {}
     virtual void visit (PassthroughFilter&) {}
     virtual void visit (DelayFilter&) {}
+    virtual void visit (ResampleFilter&) {}
     virtual void visit (BlurFilter&) {}
     virtual void visit (SharpenFilter&) {}
     virtual void visit (EdgeFilter&) {}

rsc/shaders/filters/resample_double.glsl

@@ -0,0 +1,112 @@
+//=======================================================================================
+vec4 CubicHermite (vec4 A, vec4 B, vec4 C, vec4 D, float t)
+{
+    float t2 = t*t;
+    float t3 = t*t*t;
+    vec4 a = -A/2.0 + (3.0*B)/2.0 - (3.0*C)/2.0 + D/2.0;
+    vec4 b = A - (5.0*B)/2.0 + 2.0*C - D / 2.0;
+    vec4 c = -A/2.0 + C/2.0;
+    vec4 d = B;
+
+    return a*t3 + b*t2 + c*t + d;
+}
+
+//=======================================================================================
+vec4 BicubicHermiteTextureSample (vec2 P)
+{
+    float c_textureSize = iChannelResolution[0].x;
+    float c_onePixel = 1.0 / c_textureSize;
+    float c_twoPixels = 2.0 / c_textureSize;
+
+    vec2 pixel = P * c_textureSize + 0.5;
+
+    vec2 frac = fract(pixel);
+    pixel = floor(pixel) / c_textureSize - vec2(c_onePixel/2.0);
+
+    vec4 C00 = texture(iChannel0, pixel + vec2(-c_onePixel ,-c_onePixel));
+    vec4 C10 = texture(iChannel0, pixel + vec2( 0.0        ,-c_onePixel));
+    vec4 C20 = texture(iChannel0, pixel + vec2( c_onePixel ,-c_onePixel));
+    vec4 C30 = texture(iChannel0, pixel + vec2( c_twoPixels,-c_onePixel));
+
+    vec4 C01 = texture(iChannel0, pixel + vec2(-c_onePixel , 0.0));
+    vec4 C11 = texture(iChannel0, pixel + vec2( 0.0        , 0.0));
+    vec4 C21 = texture(iChannel0, pixel + vec2( c_onePixel , 0.0));
+    vec4 C31 = texture(iChannel0, pixel + vec2( c_twoPixels, 0.0));
+
+    vec4 C02 = texture(iChannel0, pixel + vec2(-c_onePixel , c_onePixel));
+    vec4 C12 = texture(iChannel0, pixel + vec2( 0.0        , c_onePixel));
+    vec4 C22 = texture(iChannel0, pixel + vec2( c_onePixel , c_onePixel));
+    vec4 C32 = texture(iChannel0, pixel + vec2( c_twoPixels, c_onePixel));
+
+    vec4 C03 = texture(iChannel0, pixel + vec2(-c_onePixel , c_twoPixels));
+    vec4 C13 = texture(iChannel0, pixel + vec2( 0.0        , c_twoPixels));
+    vec4 C23 = texture(iChannel0, pixel + vec2( c_onePixel , c_twoPixels));
+    vec4 C33 = texture(iChannel0, pixel + vec2( c_twoPixels, c_twoPixels));
+
+    vec4 CP0X = CubicHermite(C00, C10, C20, C30, frac.x);
+    vec4 CP1X = CubicHermite(C01, C11, C21, C31, frac.x);
+    vec4 CP2X = CubicHermite(C02, C12, C22, C32, frac.x);
+    vec4 CP3X = CubicHermite(C03, C13, C23, C33, frac.x);
+
+    return CubicHermite(CP0X, CP1X, CP2X, CP3X, frac.y);
+}
+
+// Insprired from https://www.shadertoy.com/view/MtVGWz
+// Reference http://www.decarpentier.nl/2d-catmull-rom-in-4-samples
+vec4 SampleTextureCatmullRom( vec2 uv, vec2 texSize )
+{
+    // We're going to sample a a 4x4 grid of texels surrounding the target UV coordinate. We'll do this by rounding
+    // down the sample location to get the exact center of our "starting" texel. The starting texel will be at
+    // location [1, 1] in the grid, where [0, 0] is the top left corner.
+    vec2 samplePos = uv * texSize;
+    vec2 texPos1 = floor(samplePos - 0.5) + 0.5;
+
+    // Compute the fractional offset from our starting texel to our original sample location, which we'll
+    // feed into the Catmull-Rom spline function to get our filter weights.
+    vec2 f = samplePos - texPos1;
+
+    // Compute the Catmull-Rom weights using the fractional offset that we calculated earlier.
+    // These equations are pre-expanded based on our knowledge of where the texels will be located,
+    // which lets us avoid having to evaluate a piece-wise function.
+    vec2 w0 = f * ( -0.5 + f * (1.0 - 0.5*f));
+    vec2 w1 = 1.0 + f * f * (-2.5 + 1.5*f);
+    vec2 w2 = f * ( 0.5 + f * (2.0 - 1.5*f) );
+    vec2 w3 = f * f * (-0.5 + 0.5 * f);
+
+    // Work out weighting factors and sampling offsets that will let us use bilinear filtering to
+    // simultaneously evaluate the middle 2 samples from the 4x4 grid.
+    vec2 w12 = w1 + w2;
+    vec2 offset12 = w2 / w12;
+
+    // Compute the final UV coordinates we'll use for sampling the texture
+    vec2 texPos0 = texPos1 - vec2(1.0);
+    vec2 texPos3 = texPos1 + vec2(2.0);
+    vec2 texPos12 = texPos1 + offset12;
+
+    texPos0 /= texSize;
+    texPos3 /= texSize;
+    texPos12 /= texSize;
+
+    vec4 result = vec4(0.0);
+    result += texture( iChannel0, vec2(texPos0.x,  texPos0.y)) * w0.x * w0.y;
+    result += texture( iChannel0, vec2(texPos12.x, texPos0.y)) * w12.x * w0.y;
+    result += texture( iChannel0, vec2(texPos3.x,  texPos0.y)) * w3.x * w0.y;
+
+    result += texture( iChannel0, vec2(texPos0.x,  texPos12.y)) * w0.x * w12.y;
+    result += texture( iChannel0, vec2(texPos12.x, texPos12.y)) * w12.x * w12.y;
+    result += texture( iChannel0, vec2(texPos3.x,  texPos12.y)) * w3.x * w12.y;
+
+    result += texture( iChannel0, vec2(texPos0.x,  texPos3.y)) * w0.x * w3.y;
+    result += texture( iChannel0, vec2(texPos12.x, texPos3.y)) * w12.x * w3.y;
+    result += texture( iChannel0, vec2(texPos3.x,  texPos3.y)) * w3.x * w3.y;
+
+    return result;
+}
+
+void mainImage( out vec4 fragColor, in vec2 fragCoord )
+{
+    vec2 uv = fragCoord.xy / iResolution.xy;
+    fragColor = SampleTextureCatmullRom(uv, iChannelResolution[0].xy );
+
+//    fragColor = BicubicHermiteTextureSample(uv);
+}

rsc/shaders/filters/resample_half.glsl

@@ -0,0 +1,20 @@
+## inpired by https://www.shadertoy.com/view/fsjBWm (License: MIT)
+
+void mainImage( out vec4 fragColor, in vec2 fragCoord )
+{
+    vec4 col = vec4(0.0);
+    vec2 uv = fragCoord.xy / iResolution.xy;
+
+    // optimized lowpass 2X downsampling filter.
+    col += 0.37487566 * texture(iChannel0, uv + vec2(-0.75777156,-0.75777156)/iChannelResolution[0].xy);
+    col += 0.37487566 * texture(iChannel0, uv + vec2(0.75777156,-0.75777156)/iChannelResolution[0].xy);
+    col += 0.37487566 * texture(iChannel0, uv + vec2(0.75777156,0.75777156)/iChannelResolution[0].xy);
+    col += 0.37487566 * texture(iChannel0, uv + vec2(-0.75777156,0.75777156)/iChannelResolution[0].xy);
+    col += -0.12487566 * texture(iChannel0, uv + vec2(-2.90709914,0.0)/iChannelResolution[0].xy);
+    col += -0.12487566 * texture(iChannel0, uv + vec2(2.90709914,0.0)/iChannelResolution[0].xy);
+    col += -0.12487566 * texture(iChannel0, uv + vec2(0.0,-2.90709914)/iChannelResolution[0].xy);
+    col += -0.12487566 * texture(iChannel0, uv + vec2(0.0,2.90709914)/iChannelResolution[0].xy);
+
+//    col = texture( iChannel0, uv );
+    fragColor = col;
+}

rsc/shaders/filters/resample_quarter.glsl

@@ -0,0 +1,32 @@
+
+void mainImage( out vec4 fragColor, in vec2 fragCoord )
+{
+    vec2 uv = fragCoord.xy / iResolution.xy;
+    fragColor = texture( iChannel0, uv );
+
+////    vec2 uv = fragCoord.xy / iChannelResolution[0].xy;
+
+////        float textureResolution = iChannelResolution[0].x;
+////        uv = uv*textureResolution + 0.5;
+////        vec2 iuv = floor( uv );
+////        vec2 fuv = fract( uv );
+////        uv = iuv + fuv*fuv*(3.0-2.0*fuv); // fuv*fuv*fuv*(fuv*(fuv*6.0-15.0)+10.0);;
+////        uv = (uv - 0.5)/textureResolution;
+
+//    vec4 col2  = texture(iChannel1, uv);
+
+//    // optimized lowpass 2X downsampling filter.
+//    vec4 col = vec4(0.0);
+////    col += 0.37487566 * texture(iChannel0, uv + vec2(-0.75777156,-0.75777156)/iChannelResolution[0].xy);
+////    col += 0.37487566 * texture(iChannel0, uv + vec2(0.75777156,-0.75777156)/iChannelResolution[0].xy);
+////    col += 0.37487566 * texture(iChannel0, uv + vec2(0.75777156,0.75777156)/iChannelResolution[0].xy);
+////    col += 0.37487566 * texture(iChannel0, uv + vec2(-0.75777156,0.75777156)/iChannelResolution[0].xy);
+////    col += -0.12487566 * texture(iChannel0, uv + vec2(-2.90709914,0.0)/iChannelResolution[0].xy);
+////    col += -0.12487566 * texture(iChannel0, uv + vec2(2.90709914,0.0)/iChannelResolution[0].xy);
+////    col += -0.12487566 * texture(iChannel0, uv + vec2(0.0,-2.90709914)/iChannelResolution[0].xy);
+////    col += -0.12487566 * texture(iChannel0, uv + vec2(0.0,2.90709914)/iChannelResolution[0].xy);
+
+//col = texture( iChannel0, uv );
+
+//    fragColor = 0.5 * col + 0.5 * col2;
+}