From 64b2a18ff3277d7755381f480e7a79cbe572fb1d Mon Sep 17 00:00:00 2001 From: Bruno Herbelin Date: Sat, 24 Jun 2023 23:28:13 +0200 Subject: [PATCH] New feature: Texture view Mask Source Enable use of a source as mask for another source. Improved Mask mechanism in Source class, with various flags for update of source (avoid repeated mask update (GPU costly). Using SourceLink to link source to mask (improved robustness of SourceLink). --- src/ImageShader.cpp | 10 ++-- src/ImageShader.h | 7 ++- src/SessionCreator.cpp | 7 ++- src/SessionVisitor.cpp | 7 ++- src/Source.cpp | 82 +++++++++++++++----------- src/Source.h | 21 +++++-- src/SourceList.cpp | 19 ++++--- src/TextureView.cpp | 114 ++++++++++++++++++++++++++++--------- src/UserInterfaceManager.h | 2 +- 9 files changed, 183 insertions(+), 86 deletions(-) diff --git a/src/ImageShader.cpp b/src/ImageShader.cpp index 0450540..cab9ef2 100644 --- a/src/ImageShader.cpp +++ b/src/ImageShader.cpp @@ -38,8 +38,8 @@ std::vector< ShadingProgram > maskPrograms = { ShadingProgram("shaders/simple.vs", "shaders/mask_vertical.fs") }; -const char* MaskShader::mask_icons[3] = { ICON_FA_SQUARE, ICON_FA_EDIT, ICON_FA_SHAPES }; -const char* MaskShader::mask_names[3] = { "No mask", "Paint mask", "Shape mask" }; +const char* MaskShader::mask_icons[4] = { ICON_FA_WINDOW_CLOSE, ICON_FA_EDIT, ICON_FA_SHAPES, ICON_FA_CLONE }; +const char* MaskShader::mask_names[4] = { "No mask", "Paint mask", "Shape mask", "Source mask" }; const char* MaskShader::mask_shapes[5] = { "Ellipse", "Oblong", "Rectangle", "Horizontal", "Vertical" }; ImageShader::ImageShader(): Shader(), mask_texture(0), stipple(0.f) @@ -113,9 +113,9 @@ MaskShader::MaskShader(): Shader(), mode(0) void MaskShader::use() { // select program to use - mode = MINI(mode, 2); - shape = MINI(shape, 4); - program_ = mode < 2 ? &maskPrograms[mode] : &maskPrograms[shape+2] ; + mode = MINI(mode, MaskShader::SOURCE); + shape = MINI(shape, MaskShader::VERTICAL); + program_ = mode > 2 ? &maskPrograms[0] : mode < 2 ? &maskPrograms[mode] : &maskPrograms[shape+2] ; // actual use of shader program Shader::use(); diff --git a/src/ImageShader.h b/src/ImageShader.h index 7ead953..8e9dde0 100644 --- a/src/ImageShader.h +++ b/src/ImageShader.h @@ -47,7 +47,8 @@ public: enum Modes { NONE = 0, PAINT = 1, - SHAPE = 2 + SHAPE = 2, + SOURCE = 3 }; uint mode; @@ -69,8 +70,8 @@ public: glm::vec4 cursor; glm::vec3 brush; - static const char* mask_icons[3]; - static const char* mask_names[3]; + static const char* mask_icons[4]; + static const char* mask_names[4]; static const char* mask_shapes[5]; }; diff --git a/src/SessionCreator.cpp b/src/SessionCreator.cpp index ff3e86d..bc8e530 100644 --- a/src/SessionCreator.cpp +++ b/src/SessionCreator.cpp @@ -1014,7 +1014,12 @@ void SessionLoader::visit (Source& s) if (xmlCurrent_) { // read the mask shader attributes s.maskShader()->accept(*this); - // set the mask from jpeg + // set id of source used as mask (if exists) + uint64_t id__ = 0; + xmlCurrent_->QueryUnsigned64Attribute("source", &id__); + s.maskSource()->connect(id__, session_); + s.touch(Source::SourceUpdate_Mask); + // set the mask from jpeg (if exists) s.setMask( SessionLoader::XMLToImage(xmlCurrent_) ); } diff --git a/src/SessionVisitor.cpp b/src/SessionVisitor.cpp index ebcca24..50e7dbd 100644 --- a/src/SessionVisitor.cpp +++ b/src/SessionVisitor.cpp @@ -608,13 +608,18 @@ void SessionVisitor::visit (Source& s) xmlCurrent_ = xmlDoc_->NewElement( "Mask" ); sourceNode->InsertEndChild(xmlCurrent_); s.maskShader()->accept(*this); - // if we are saving a pain mask + // if we are saving a paint mask if (s.maskShader()->mode == MaskShader::PAINT) { // get the mask previously stored XMLElement *imageelement = SessionVisitor::ImageToXML(s.getMask(), xmlDoc_); if (imageelement) xmlCurrent_->InsertEndChild(imageelement); } + // if we are saving a source mask + else if (s.maskShader()->mode == MaskShader::SOURCE) { + // get the id of the source used as mask + xmlCurrent_->SetAttribute("source", s.maskSource()->id()); + } xmlCurrent_ = xmlDoc_->NewElement( "ImageProcessing" ); xmlCurrent_->SetAttribute("enabled", s.imageProcessingEnabled()); diff --git a/src/Source.cpp b/src/Source.cpp index 76b4530..77beaae 100644 --- a/src/Source.cpp +++ b/src/Source.cpp @@ -116,7 +116,7 @@ SourceCore& SourceCore::operator= (SourceCore const& other) Source::Source(uint64_t id) : SourceCore(), id_(id), ready_(false), symbol_(nullptr), - active_(true), locked_(false), need_update_(true), dt_(16.f), workspace_(STAGE) + active_(true), locked_(false), need_update_(SourceUpdate_None), dt_(16.f), workspace_(STAGE) { // create unique id if (id_ == 0) @@ -318,7 +318,7 @@ Source::Source(uint64_t id) : SourceCore(), id_(id), ready_(false), symbol_(null activesurface_ = nullptr; maskbuffer_ = nullptr; maskimage_ = nullptr; - mask_need_update_ = false; + masksource_ = new SourceLink; } @@ -342,6 +342,7 @@ Source::~Source() delete maskimage_; if (masksurface_) delete masksurface_; // deletes maskshader_ + delete masksource_; delete texturesurface_; @@ -567,7 +568,7 @@ void Source::attach(FrameBuffer *renderbuffer) setMode(VISIBLE); // request update - need_update_ = true; + need_update_ |= Source::SourceUpdate_Render; } void Source::setActive (bool on) @@ -581,7 +582,8 @@ void Source::setActive (bool on) } // request update - need_update_ |= active_ != on; + if (active_ != on) + need_update_ |= Source::SourceUpdate_Render; // activate active_ = on; @@ -705,7 +707,7 @@ void Source::updateCallbacks(float dt) // call update on callbacks callback->update(this, dt); - need_update_ = true; + need_update_ |= Source::SourceUpdate_Render; // remove and delete finished callbacks if (callback->finished()) { @@ -742,8 +744,11 @@ void Source::update(float dt) updateCallbacks(dt); // update nodes if needed - if (need_update_) + if (need_update_ & SourceUpdate_Render) { + // do not update next frame + need_update_ &= ~SourceUpdate_Render; + // ADJUST alpha based on MIXING node // read position of the mixing node and interpret this as transparency of render output glm::vec2 dist = glm::vec2(groups_[View::MIXING]->translation_); @@ -822,15 +827,40 @@ void Source::update(float dt) // 7. switch back to UV coordinate system texturesurface_->shader()->iTransform = glm::inverse(UVtoScene) * glm::inverse(Sca) * glm::inverse(Ar) * Rot * Tra * Ar * UVtoScene; - // if a mask image was given to be updated - if (mask_need_update_) { - // fill the mask buffer (once) - maskbuffer_->fill(maskimage_); - mask_need_update_ = false; + // inform mixing group + if (mixinggroup_) + mixinggroup_->setAction(MixingGroup::ACTION_UPDATE); + } + + if (need_update_ & SourceUpdate_Mask_fill) { + + // do not update Mask fill next frame + need_update_ &= ~SourceUpdate_Mask_fill; + + // fill the mask buffer (once) + maskbuffer_->fill(maskimage_); + } + + if (need_update_ & SourceUpdate_Mask) { + + // do not update Mask next frame + need_update_ &= ~SourceUpdate_Mask; + + // MODIFY Mask based on mask shader mode + // if MaskShader::SOURCE available, set a source as mask for blending + if (maskshader_->mode == MaskShader::SOURCE && masksource_->connected()) { + Source *ref_source = masksource_->source(); + if (ref_source != nullptr) { + if (ref_source->ready()) + // set mask texture to mask source + blendingshader_->mask_texture = ref_source->frame()->texture(); + else + // retry for when source will be ready + need_update_ |= SourceUpdate_Mask; + } } - // otherwise, render the mask buffer - else - { + // all other MaskShader types to update + else { // draw mask in mask frame buffer maskbuffer_->begin(false); // loopback maskbuffer texture for painting @@ -838,17 +868,9 @@ void Source::update(float dt) // fill surface with mask texture masksurface_->draw(glm::identity(), maskbuffer_->projection()); maskbuffer_->end(); + // set mask texture to mask buffer + blendingshader_->mask_texture = maskbuffer_->texture(); } - - // set the rendered mask as mask for blending - blendingshader_->mask_texture = maskbuffer_->texture(); - - // inform mixing group - if (mixinggroup_) - mixinggroup_->setAction(MixingGroup::ACTION_UPDATE); - - // do not update next frame - need_update_ = false; } if (processingshader_link_.connected() && imageProcessingEnabled()) { @@ -872,7 +894,7 @@ FrameBuffer *Source::frame() const return renderbuffer_; } else { - static FrameBuffer *black = new FrameBuffer(320,180); + static FrameBuffer *black = new FrameBuffer(64,64); return black; } } @@ -886,7 +908,6 @@ bool Source::contains(Node *node) const return tester(this); } - void Source::storeMask(FrameBufferImage *img) { // free the output mask storage @@ -919,14 +940,9 @@ void Source::setMask(FrameBufferImage *img) // NB: will be freed when replaced storeMask(img); - // ask Source::update to use it at next update for filling mask buffer - mask_need_update_ = true; - - // ask to update the source - touch(); + // ask to update the source mask + touch(Source::SourceUpdate_Mask_fill); } - else - mask_need_update_ = false; } bool Source::hasNode::operator()(const Source* elem) const diff --git a/src/Source.h b/src/Source.h index a50802a..8deb9d9 100644 --- a/src/Source.h +++ b/src/Source.h @@ -136,11 +136,15 @@ public: // every Source has a frame buffer from the renderbuffer virtual FrameBuffer *frame () const; - // a Source has a shader used to render mask - inline MaskShader *maskShader () const { return maskshader_; } - // touch to request update - inline void touch () { need_update_ = true; } + enum UpdateFlags_ { + SourceUpdate_None = 0, + SourceUpdate_Render = 1 << 1, + SourceUpdate_Mask = 1 << 2, + SourceUpdate_Mask_fill = 1 << 3 + }; + typedef int UpdateFlags; + inline void touch (UpdateFlags f = SourceUpdate_Render) { need_update_ |= f; } // informs if its ready (i.e. initialized) inline bool ready () const { return ready_; } @@ -197,9 +201,14 @@ public: virtual void accept (Visitor& v); // operations on mask + // get shader used to render mask + inline MaskShader *maskShader () const { return maskshader_; } + // set/get framebuffer used for mask painting inline FrameBufferImage *getMask () const { return maskimage_; } void setMask (FrameBufferImage *img); void storeMask (FrameBufferImage *img = nullptr); + // link to source used as mask + inline SourceLink *maskSource() const { return masksource_; } // get properties float depth () const; @@ -300,8 +309,8 @@ protected: MaskShader *maskshader_; FrameBuffer *maskbuffer_; Surface *masksurface_; - bool mask_need_update_; FrameBufferImage *maskimage_; + SourceLink *masksource_; // surface to draw on Surface *texturesurface_; @@ -321,7 +330,7 @@ protected: // update bool active_; bool locked_; - bool need_update_; + UpdateFlags need_update_; float dt_; Workspace workspace_; diff --git a/src/SourceList.cpp b/src/SourceList.cpp index 5fb7252..e734c2f 100644 --- a/src/SourceList.cpp +++ b/src/SourceList.cpp @@ -198,8 +198,10 @@ void SourceLink::connect(uint64_t id, Session *se) if (connected()) disconnect(); - id_ = id; - host_ = se; + if (se != nullptr && id > 0) { + id_ = id; + host_ = se; + } } void SourceLink::connect(Source *s) @@ -207,11 +209,13 @@ void SourceLink::connect(Source *s) if (connected()) disconnect(); - target_ = s; - target_->links_.push_back(this); + if (s != nullptr) { + target_ = s; + target_->links_.push_back(this); - id_ = s->id(); - // TODO veryfy circular dependency recursively ? + id_ = s->id(); + // TODO veryfy circular dependency recursively ? + } } void SourceLink::disconnect() @@ -242,9 +246,6 @@ Source *SourceLink::source() target_ = *it; target_->links_.push_back(this); } -// // not found: invalidate link -// else -// disconnect(); } // no host: invalidate link else diff --git a/src/TextureView.cpp b/src/TextureView.cpp index 70796a9..c924119 100644 --- a/src/TextureView.cpp +++ b/src/TextureView.cpp @@ -436,7 +436,7 @@ void TextureView::adjustBackground() preview_surface_->setTextureIndex( Resource::getTextureTransparent() ); // if its a valid index - if (edit_source_ != nullptr) { + if (edit_source_ != nullptr && edit_source_->ready()) { // update rendering frame to match edit source AR image_original_width = edit_source_->frame()->aspectRatio(); scale = edit_source_->mixingsurface_->scale_; @@ -444,7 +444,7 @@ void TextureView::adjustBackground() preview_shader_->mask_texture = edit_source_->blendingShader()->mask_texture; preview_surface_->scale_ = scale; // mask appearance - mask_node_->visible_ = edit_source_->maskShader()->mode > MaskShader::PAINT && mask_cursor_shape_ > 0; + mask_node_->visible_ = edit_source_->maskShader()->mode == MaskShader::SHAPE && mask_cursor_shape_ > 0; int shape = edit_source_->maskShader()->shape; mask_circle_->visible_ = shape == MaskShader::ELLIPSE; @@ -591,25 +591,40 @@ void TextureView::draw() ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImVec4(0.24f, 0.24f, 0.24f, 0.46f)); - int mode = edit_source_->maskShader()->mode; + int maskmode = edit_source_->maskShader()->mode; ImGui::SetNextItemWidth( ImGui::GetTextLineHeight() * 2.6); - if (ImGui::BeginCombo("##Mask", MaskShader::mask_icons[mode])) { + if (ImGui::BeginCombo("##Mask", MaskShader::mask_icons[maskmode])) { - for (int m = 0; m < 3; ++m){ + for (int m = MaskShader::NONE; m <= MaskShader::SOURCE; ++m){ if (ImGui::Selectable( MaskShader::mask_icons[m] )) { - mode = m; - edit_source_->maskShader()->mode = mode; - if (mode == MaskShader::NONE) - Mixer::manager().setCurrentSource(edit_source_); - else if (mode == MaskShader::PAINT) - edit_source_->storeMask(); - edit_source_->touch(); - need_edit_update_ = true; - // store action history - std::ostringstream oss; - oss << edit_source_->name() << ": " << MaskShader::mask_names[mode]; - Action::manager().store(oss.str()); + // on change of mode + if (maskmode != m) { + // cancel previous source mask + if (maskmode == MaskShader::SOURCE) { + // store source image as mask before painting + if (edit_source_->maskSource()->connected() && m == MaskShader::PAINT) + edit_source_->setMask( edit_source_->maskSource()->source()->frame()->image() ); + // cancel source mask + edit_source_->maskSource()->disconnect(); + } + // store current mask before painting + else if (m == MaskShader::PAINT) + edit_source_->storeMask(); + // set new mode + maskmode = m; + edit_source_->maskShader()->mode = maskmode; + // force update source and view + edit_source_->touch(Source::SourceUpdate_Mask); + need_edit_update_ = true; + // store action history + std::ostringstream oss; + oss << edit_source_->name() << ": " << MaskShader::mask_names[maskmode]; + Action::manager().store(oss.str()); + // force take control of source for NONE and SOURCE modes + if (maskmode == MaskShader::NONE || maskmode == MaskShader::SOURCE) + Mixer::manager().setCurrentSource(edit_source_); + } } if (ImGui::IsItemHovered()) ImGuiToolkit::ToolTip(MaskShader::mask_names[m]); @@ -617,8 +632,53 @@ void TextureView::draw() ImGui::EndCombo(); } + + // GUI for selecting source mask + if (maskmode == MaskShader::SOURCE) { + + ImGui::SameLine(0, 60); + bool on = true; + ImGuiToolkit::ButtonToggle(ICON_FA_MOUSE_POINTER, &on, "Edit texture"); + + // List of sources + ImGui::SameLine(0, 60); + std::string label = "Select source"; + Source *ref_source = nullptr; + if (edit_source_->maskSource()->connected()) { + ref_source = edit_source_->maskSource()->source(); + if (ref_source != nullptr) + label = std::string("Source ") + ref_source->initials() + " - " + ref_source->name(); + } + if (ImGui::BeginCombo("##SourceMask", label.c_str())) { + SourceList::iterator iter; + for (iter = Mixer::manager().session()->begin(); iter != Mixer::manager().session()->end(); ++iter) + { + label = std::string("Source ") + (*iter)->initials() + " - " + (*iter)->name(); + if (ImGui::Selectable( label.c_str(), *iter == ref_source )) { + edit_source_->maskSource()->connect( *iter ); + edit_source_->touch(Source::SourceUpdate_Mask); + need_edit_update_ = true; + } + } + ImGui::EndCombo(); + } + if (ImGui::IsItemHovered()) + ImGuiToolkit::ToolTip("Source used as mask"); + // reset button + if (ref_source){ + ImGui::SameLine(); + if (ImGui::Button(ICON_FA_BACKSPACE)){ + edit_source_->maskSource()->disconnect(); + edit_source_->touch(Source::SourceUpdate_Mask); + need_edit_update_ = true; + } + if (ImGui::IsItemHovered()) + ImGuiToolkit::ToolTip("Reset"); + } + + } // GUI for drawing mask - if (edit_source_->maskShader()->mode == MaskShader::PAINT) { + else if (maskmode == MaskShader::PAINT) { // select cursor static bool on = true; @@ -739,7 +799,7 @@ void TextureView::draw() if (e>0) { edit_source_->maskShader()->effect = e; edit_source_->maskShader()->cursor = glm::vec4(100.0, 100.0, 0.f, 0.f); - edit_source_->touch(); + edit_source_->touch(Source::SourceUpdate_Mask); Action::manager().store(oss.str()); } ImGui::PopFont(); @@ -774,8 +834,8 @@ void TextureView::draw() } } - // GUI for all other masks - else if (edit_source_->maskShader()->mode == MaskShader::SHAPE) { + // GUI for shape masks + else if (maskmode == MaskShader::SHAPE) { // select cursor static bool on = true; @@ -804,7 +864,7 @@ void TextureView::draw() ImGui::SetNextItemWidth( ImGui::GetTextLineHeight() * 6.5f); if ( ImGui::Combo("##MaskShape", &shape, MaskShader::mask_shapes, IM_ARRAYSIZE(MaskShader::mask_shapes) ) ) { edit_source_->maskShader()->shape = shape; - edit_source_->touch(); + edit_source_->touch(Source::SourceUpdate_Mask); need_edit_update_ = true; // store action history std::ostringstream oss; @@ -826,7 +886,7 @@ void TextureView::draw() ImGuiToolkit::Indication("Blured ", 7, 16, ICON_FA_ARROW_UP); if (ImGui::VSliderInt("##shapeblur", ImVec2(30,260), &blur_percent, 0, 100, "") ){ edit_source_->maskShader()->blur = float(blur_percent) / 100.f; - edit_source_->touch(); + edit_source_->touch(Source::SourceUpdate_Mask); need_edit_update_ = true; smoothchanged = true; } @@ -856,7 +916,7 @@ void TextureView::draw() ImGui::TextDisabled( "mask"); } } - else {// mode == MaskShader::NONE + else {// maskmode == MaskShader::NONE // always active mouse pointer ImGui::SameLine(0, 60); bool on = true; @@ -951,7 +1011,7 @@ View::Cursor TextureView::grab (Source *s, glm::vec2 from, glm::vec2 to, std::pa // inform shader of a cursor action : coordinates and crop scaling edit_source_->maskShader()->cursor = glm::vec4(scene_to.x, scene_to.y, edit_source_->mixingsurface_->scale_.x, edit_source_->mixingsurface_->scale_.y); - edit_source_->touch(); + edit_source_->touch(Source::SourceUpdate_Mask); // action label info << MASK_PAINT_ACTION_LABEL; // cursor indication - no info, just cursor @@ -985,7 +1045,7 @@ View::Cursor TextureView::grab (Source *s, glm::vec2 from, glm::vec2 to, std::pa else edit_source_->maskShader()->size = glm::max(glm::abs(glm::vec2(val)), glm::vec2(0.2)); // edit_source_->maskShader()->size = glm::max( glm::min( glm::vec2(val), glm::vec2(2.f)), glm::vec2(hv?-2.f:0.2f)); - edit_source_->touch(); + edit_source_->touch(Source::SourceUpdate_Mask); // update need_edit_update_ = true; // action label @@ -1425,7 +1485,7 @@ void TextureView::arrow (glm::vec2 movement) if (mask_cursor_shape_ > 0) { float b = -0.02f * movement.y; edit_source_->maskShader()->blur = CLAMP(edit_source_->maskShader()->blur+b, SHAPE_MIN_BLUR, SHAPE_MAX_BLUR); - edit_source_->touch(); + edit_source_->touch(Source::SourceUpdate_Mask); } } } diff --git a/src/UserInterfaceManager.h b/src/UserInterfaceManager.h index 8fb8957..e77a287 100644 --- a/src/UserInterfaceManager.h +++ b/src/UserInterfaceManager.h @@ -79,7 +79,7 @@ #define MENU_NOTE ICON_FA_STICKY_NOTE " Add sticky note" #define MENU_METRICS ICON_FA_TACHOMETER_ALT " Metrics" -#define MENU_SOURCE_TOOL ICON_FA_LIST_ALT " Source toolbar" +#define MENU_SOURCE_TOOL ICON_FA_WRENCH " Source toolbar" #define MENU_HELP ICON_FA_LIFE_RING " Help" #define SHORTCUT_HELP CTRL_MOD "H" #define MENU_LOGS ICON_FA_LIST_UL " Logs"