New Evaluation of MediaPlayer; fast parsing of video, display info in source panel, disable backward playback if necessary, adjust timeline first and last PTS.

This commit is contained in:
brunoherbelin
2026-06-07 13:17:35 +02:00
parent 15f29006f9
commit 9f72b4dda5
7 changed files with 176 additions and 103 deletions
+100 -81
View File
@@ -101,6 +101,13 @@ MediaPlayer::~MediaPlayer()
{
close();
// cancel evaluator if still running
if (evaluator_.valid()) {
if (evaluator_cancel_)
evaluator_cancel_->store(true);
evaluator_.wait();
}
// cleanup opengl texture
if (textureindex_) {
glDeleteTextures(1, &textureindex_);
@@ -314,86 +321,107 @@ MediaEvaluation MediaPlayer::UriEvaluator(const std::string &uri, std::shared_pt
return eval;
}
// probe data filled by the GStreamer streaming thread; safe to read after pipeline stops
// probe data filled by the GStreamer streaming thread
struct ProbeData {
guint64 frame_count = 0;
guint64 keyframe_count = 0;
guint64 last_keyframe_frame = 0;
guint discontinuity_count = 0;
guint corrupted_count = 0;
gint error_code = 0;
GstClockTime pts_first = GST_CLOCK_TIME_NONE;
GstClockTime pts_last = GST_CLOCK_TIME_NONE;
std::vector<GstClockTime> keyframe_pts;
std::vector<guint64> gop_sizes;
std::atomic<bool> decoder_probed{false};
} probe_data;
// headless pipeline: decode at max speed, never sync to clock
std::string desc = "uridecodebin uri=" + uri + " ! videoconvert ! fakesink name=sink sync=false";
GError *error = NULL;
GstElement *pipeline = gst_parse_launch(desc.c_str(), &error);
if (error != NULL || pipeline == NULL) {
eval.log = error ? std::string(error->message) : "Pipeline construction failed";
g_clear_error(&error);
// Context for the pad-added callback
struct PadContext {
GstElement *pipeline;
ProbeData *data;
std::atomic<bool> video_probed{false};
} pad_ctx = { nullptr, &probe_data };
// Build pipeline : filesrc (raw bytes) -> parsebin (demux+parse, no decoding).
// Parsers set GST_BUFFER_FLAG_DELTA_UNIT on encoded
// packets, giving fast and accurate keyframe detection without decoding
GstElement *pipeline = gst_pipeline_new("evaluator");
GstElement *filesrc = gst_element_factory_make("filesrc", "src");
GstElement *parsebin = gst_element_factory_make("parsebin", "pb");
if (!pipeline || !filesrc || !parsebin) {
eval.log = "Failed to create pipeline elements";
if (pipeline) gst_object_unref(pipeline);
if (filesrc) gst_object_unref(filesrc);
if (parsebin) gst_object_unref(parsebin);
return eval;
}
g_clear_error(&error);
gchar *path = gst_uri_get_location(uri.c_str());
g_object_set(filesrc, "location", path, NULL);
g_free(path);
gst_bin_add_many(GST_BIN(pipeline), filesrc, parsebin, NULL);
gst_element_link(filesrc, parsebin);
pad_ctx.pipeline = pipeline;
// Probe the video DECODER'S SINK PAD (encoded input), not the decoded output.
// GST_BUFFER_FLAG_DELTA_UNIT is cleared by decoders on all output frames;
// it is only reliable on encoded buffers flowing into the decoder.
// deep-element-added fires when uridecodebin autoplugs the decoder element.
g_signal_connect(pipeline, "deep-element-added",
G_CALLBACK(+[](GstBin *, GstBin *, GstElement *element, gpointer ud) {
ProbeData *d = static_cast<ProbeData *>(ud);
// parsebin exposes one dynamic pad per stream; connect all pads to fakesinks (avoid
// unlinked-pad errors) and install the buffer probe on the first video pad.
g_signal_connect(parsebin, "pad-added",
G_CALLBACK(+[](GstElement *, GstPad *pad, gpointer ud) {
PadContext *ctx = static_cast<PadContext *>(ud);
// Identify the first video decoder added to the pipeline
GstElementFactory *factory = gst_element_get_factory(element);
if (!factory) return;
if (!gst_element_factory_list_is_type(factory,
GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO))
return;
// atomic swap: ensure only the first video decoder is probed
if (d->decoder_probed.exchange(true)) return;
// Install the probe that gets video buffers (BEFORE linking the pad).
if (!ctx->video_probed) {
GstCaps *caps = gst_pad_get_current_caps(pad);
if (!caps) caps = gst_pad_query_caps(pad, NULL);
if (caps) {
bool is_video = g_str_has_prefix(
gst_structure_get_name(gst_caps_get_structure(caps, 0)), "video/");
gst_caps_unref(caps);
if (is_video) {
ctx->video_probed.exchange(true);
ctx->data->error_code = gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_BUFFER,
[](GstPad *, GstPadProbeInfo *info, gpointer user_data) -> GstPadProbeReturn {
ProbeData *d = static_cast<ProbeData *>(user_data);
GstBuffer *buf = GST_PAD_PROBE_INFO_BUFFER(info);
d->frame_count++;
if (!GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_DELTA_UNIT)) {
if (d->keyframe_count > 0)
d->gop_sizes.push_back(d->frame_count - d->last_keyframe_frame);
d->last_keyframe_frame = d->frame_count;
d->keyframe_count++;
if (d->keyframe_pts.size() < MAX_KEYFRAME_STORED)
d->keyframe_pts.push_back(buf->pts);
}
if (GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_DISCONT))
d->discontinuity_count++;
if (GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_CORRUPTED))
d->corrupted_count++;
if (d->pts_first == GST_CLOCK_TIME_NONE && GST_CLOCK_TIME_IS_VALID(buf->pts))
d->pts_first = buf->pts;
if (GST_CLOCK_TIME_IS_VALID(buf->pts))
d->pts_last = buf->pts;
return GST_PAD_PROBE_OK;
},
ctx->data, NULL);
} // if (is_video)
} // if (caps)
} // if (!video_probed)
GstPad *sinkpad = gst_element_get_static_pad(element, "sink");
if (!sinkpad) return;
gst_pad_add_probe(sinkpad, GST_PAD_PROBE_TYPE_BUFFER,
[](GstPad *, GstPadProbeInfo *info, gpointer user_data) -> GstPadProbeReturn {
ProbeData *d = static_cast<ProbeData *>(user_data);
GstBuffer *buf = GST_PAD_PROBE_INFO_BUFFER(info);
d->frame_count++;
if (!GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_DELTA_UNIT)) {
if (d->keyframe_count > 0)
d->gop_sizes.push_back(d->frame_count - d->last_keyframe_frame);
d->last_keyframe_frame = d->frame_count;
d->keyframe_count++;
if (d->keyframe_pts.size() < MAX_KEYFRAME_STORED)
d->keyframe_pts.push_back(buf->pts);
}
if (GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_DISCONT))
d->discontinuity_count++;
if (GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_CORRUPTED))
d->corrupted_count++;
if (d->pts_first == GST_CLOCK_TIME_NONE && GST_CLOCK_TIME_IS_VALID(buf->pts))
d->pts_first = buf->pts;
if (GST_CLOCK_TIME_IS_VALID(buf->pts))
d->pts_last = buf->pts;
return GST_PAD_PROBE_OK;
},
d, NULL);
GstElement *fakesink = gst_element_factory_make("fakesink", NULL);
g_object_set(fakesink, "sync", FALSE, "async", FALSE, NULL);
gst_bin_add(GST_BIN(ctx->pipeline), fakesink);
gst_element_sync_state_with_parent(fakesink);
GstPad *sinkpad = gst_element_get_static_pad(fakesink, "sink");
gst_pad_link(pad, sinkpad);
gst_object_unref(sinkpad);
}),
&probe_data);
&pad_ctx);
// run pipeline, polling in 200 ms chunks to allow cancellation
GstBus *bus = gst_element_get_bus(pipeline);
gst_element_set_state(pipeline, GST_STATE_PLAYING);
bool cancelled_flag = false;
bool error_flag = false;
GstClockTime elapsed = 0;
GstClockTime elapsed = 0;
const GstClockTime chunk = 200 * GST_MSECOND;
const GstClockTime timeout_ns = (GstClockTime)EVALUATE_TIMEOUT * GST_SECOND;
@@ -410,7 +438,6 @@ MediaEvaluation MediaPlayer::UriEvaluator(const std::string &uri, std::shared_pt
gst_message_parse_error(msg, &err, NULL);
eval.log = err ? std::string(err->message) : "Pipeline error";
g_clear_error(&err);
error_flag = true;
}
gst_message_unref(msg);
break;
@@ -424,12 +451,15 @@ MediaEvaluation MediaPlayer::UriEvaluator(const std::string &uri, std::shared_pt
gst_object_unref(bus);
gst_object_unref(pipeline);
// process is done
eval.done = true;
if (cancelled_flag) {
eval.log = "Cancelled";
return eval;
}
if (elapsed >= timeout_ns && eval.log.empty())
eval.log = "Evaluation timed out (partial results)";
eval.log = "Evaluation incomplete";
// fill evaluation from probe data (pipeline fully stopped, no concurrent access)
eval.frame_count = probe_data.frame_count;
@@ -437,24 +467,20 @@ MediaEvaluation MediaPlayer::UriEvaluator(const std::string &uri, std::shared_pt
eval.keyframe_pts = std::move(probe_data.keyframe_pts);
eval.pts_first = probe_data.pts_first;
eval.pts_last = probe_data.pts_last;
eval.discontinuity_count = probe_data.discontinuity_count;
eval.discontinuity_count = probe_data.discontinuity_count == 0 ? 0 : probe_data.discontinuity_count - 1;
eval.corrupted_count = probe_data.corrupted_count;
if (!probe_data.gop_sizes.empty()) {
guint64 min_g = probe_data.gop_sizes[0];
guint64 max_g = probe_data.gop_sizes[0];
guint64 sum = 0;
for (guint64 g : probe_data.gop_sizes) {
if (g < min_g) min_g = g;
if (g > max_g) max_g = g;
sum += g;
}
eval.gop_size_min = (guint)min_g;
eval.gop_size_max = (guint)max_g;
eval.gop_size_average = (double)sum / (double)probe_data.gop_sizes.size();
}
eval.valid = (eval.frame_count > 0) && !error_flag;
return eval;
}
@@ -1023,23 +1049,10 @@ void MediaPlayer::close()
if (discoverer_.valid())
if ( discoverer_.wait_for(std::chrono::seconds(DISCOVER_TIMOUT)) == std::future_status::timeout )
failed_ = true;
// cancel evaluator if it was started before execute_open failed
if (evaluator_.valid()) {
if (evaluator_cancel_)
evaluator_cancel_->store(true);
evaluator_.wait();
}
// nothing else to change
return;
}
// cancel evaluator if still running
if (evaluator_.valid()) {
if (evaluator_cancel_)
evaluator_cancel_->store(true);
evaluator_.wait();
}
// un-ready the media player
opened_ = false;
failed_ = false;
@@ -1632,14 +1645,20 @@ void MediaPlayer::update()
if (evaluator_.valid()) {
if (evaluator_.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
evaluation_ = evaluator_.get();
if (evaluation_.valid)
Log::Info("MediaPlayer %s Evaluated: %lu frames, %lu keyframes, GOP %d-%d avg %.1f (%s)",
if (evaluation_.done) {
if (!evaluation_.log.empty())
Log::Warning("MediaPlayer %s Evaluation: %s", std::to_string(id_).c_str(), evaluation_.log.c_str());
else {
Log::Info("MediaPlayer %s Evaluation: %lu frames, %lu keyframes, GOP %d-%d",
std::to_string(id_).c_str(),
evaluation_.frame_count, evaluation_.keyframe_count,
evaluation_.gop_size_min, evaluation_.gop_size_max,
evaluation_.gop_size_average, evaluation_.log.c_str());
else if (!evaluation_.log.empty())
Log::Warning("MediaPlayer %s Evaluation: %s", std::to_string(id_).c_str(), evaluation_.log.c_str());
evaluation_.gop_size_min, evaluation_.gop_size_max);
// adjust timeline to media frames range
timeline_.setFirst(evaluation_.pts_first);
timeline_.setLast(evaluation_.pts_last);
}
}
}
}
+3 -5
View File
@@ -23,7 +23,7 @@ class Visitor;
#define MIN_PLAY_SPEED 0.1
#define N_VFRAME 10
#define DISCOVER_TIMOUT 15
#define EVALUATE_TIMEOUT 120
#define EVALUATE_TIMEOUT 5
#define MAX_KEYFRAME_STORED 10000
struct MediaInfo {
@@ -64,7 +64,7 @@ struct MediaInfo {
struct MediaEvaluation {
bool valid;
bool done;
std::string log;
// Frame-level timing & structure
@@ -75,7 +75,6 @@ struct MediaEvaluation {
// GOP size distribution (in frames between consecutive keyframes)
guint gop_size_min;
guint gop_size_max;
double gop_size_average;
// PTS range
GstClockTime pts_first;
@@ -86,12 +85,11 @@ struct MediaEvaluation {
guint corrupted_count;
MediaEvaluation() {
valid = false;
done = false;
frame_count = 0;
keyframe_count = 0;
gop_size_min = 0;
gop_size_max = 0;
gop_size_average = 0.0;
pts_first = GST_CLOCK_TIME_NONE;
pts_last = GST_CLOCK_TIME_NONE;
discontinuity_count = 0;
+4 -4
View File
@@ -683,15 +683,15 @@ bool renderTranscodingPanel(guint64 id, MediaPlayer *mp)
// Transcoding options
ImGuiToolkit::ButtonSwitch( "Backward playback", &Settings::application.transcode_options[0],
"Optimize the video for backward playback by adding more keyframes.", transcoder == nullptr);
"Optimize for backward playback (1 keyframe every 15-30 frames).", transcoder == nullptr);
ImGuiToolkit::ButtonSwitch( "Animation content", &Settings::application.transcode_options[1],
"Optimize the video encoding for animation content (cartoons, "
"Optimize image encoding for animation content (cartoons, "
"drawings, computer graphics) to preserve more details.", transcoder == nullptr);
ImGuiToolkit::ButtonSwitch( "Constant Quality", &Settings::application.transcode_options[2],
"Use Constant Quality encoding to preserve more visual details "
"(produces larger files).", transcoder == nullptr);
"(might produce larger files).", transcoder == nullptr);
ImGuiToolkit::ButtonSwitch( "Remove audio", &Settings::application.transcode_options[3],
"Remove the audio track from the video during transcoding.", transcoder == nullptr);
"Remove audio tracks from the video.", transcoder == nullptr);
// Start transcoding if not already started for current source
if (transcoder == nullptr) {
+11 -1
View File
@@ -81,6 +81,8 @@ Timeline& Timeline::operator = (const Timeline& b)
this->step_ = b.step_;
if (b.first_ != GST_CLOCK_TIME_NONE)
this->first_ = b.first_;
if (b.last_ != GST_CLOCK_TIME_NONE)
this->last_ = b.last_;
this->gaps_ = b.gaps_;
this->gaps_array_need_update_ = b.gaps_array_need_update_;
memcpy( this->gapsArray_, b.gapsArray_, MAX_TIMELINE_ARRAY * sizeof(float));
@@ -99,6 +101,7 @@ void Timeline::reset()
timing_.reset();
timing_.begin = 0;
first_ = GST_CLOCK_TIME_NONE;
last_ = GST_CLOCK_TIME_NONE;
step_ = GST_CLOCK_TIME_NONE;
clearGaps();
@@ -113,7 +116,8 @@ bool Timeline::is_valid() const
void Timeline::setFirst(GstClockTime first)
{
first_ = first;
if (first != GST_CLOCK_TIME_NONE && first > 0)
first_ = first;
}
void Timeline::setEnd(GstClockTime end)
@@ -121,6 +125,12 @@ void Timeline::setEnd(GstClockTime end)
timing_.end = end;
}
void Timeline::setLast(GstClockTime last)
{
if (last != GST_CLOCK_TIME_NONE && last > 0)
last_ = last;
}
void Timeline::setStep(GstClockTime dt)
{
step_ = dt;
+4 -2
View File
@@ -116,6 +116,7 @@ public:
// global properties of the timeline
void setEnd(GstClockTime end);
void setLast(GstClockTime end);
void setStep(GstClockTime dt);
void setFirst(GstClockTime first);
void setTiming(TimeInterval interval, GstClockTime step = GST_CLOCK_TIME_NONE);
@@ -125,9 +126,9 @@ public:
inline GstClockTime end() const { return timing_.end; }
inline GstClockTime duration() const { return timing_.duration(); }
inline GstClockTime first() const { return first_; }
inline GstClockTime last() const { return timing_.end - step_; }
inline GstClockTime last() const { return last_ != GST_CLOCK_TIME_NONE ? last_ : timing_.end - step_; }
inline GstClockTime step() const { return step_; }
inline size_t numFrames() const { if (step_) return duration() / step_; else return 1; }
inline size_t numFrames() const { if (step_) return end() / step_; else return 1; }
inline TimeInterval interval() const { return timing_; }
GstClockTime next(GstClockTime time) const;
GstClockTime previous(GstClockTime time) const;
@@ -203,6 +204,7 @@ private:
// global information on the timeline
TimeInterval timing_;
GstClockTime first_;
GstClockTime last_;
GstClockTime step_;
// main data structure containing list of gaps in the timeline
+33 -6
View File
@@ -17,6 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
**/
#include <ostream>
#include <vector>
#include <sstream>
#include <iomanip>
@@ -27,6 +28,7 @@
#include <glm/gtc/matrix_access.hpp>
#include "IconsFontAwesome5.h"
#include "Scene/Scene.h"
#include "MediaPlayer.h"
#include "Source/MediaSource.h"
@@ -97,7 +99,36 @@ void InfoVisitor::visit(MediaPlayer &mp)
oss << mp.media().codec_name.substr(0, mp.media().codec_name.find_first_of(" (,")) << ", ";
oss << mp.width() << " x " << mp.height();
if (!mp.singleFrame() && mp.frameRate() > 0.)
oss << ", " << std::fixed << std::setprecision(0) << mp.frameRate() << " fps";
oss << ", " << std::fixed << std::setprecision(1) << mp.frameRate() << " fps";
if (!mp.media().isimage) {
if (mp.evaluation().done) {
if (mp.evaluation().log.empty()) {
oss << ", Keyframes: " << mp.evaluation().keyframe_count;
oss << " / " << mp.evaluation().frame_count;
if (mp.evaluation().gop_size_max - mp.evaluation().gop_size_min > 1)
oss << " (1 key every " << mp.evaluation().gop_size_min << "-" << mp.evaluation().gop_size_max << " frames, ";
else
oss << " (1 key every " << mp.evaluation().gop_size_max << " frames, ";
if (mp.evaluation().gop_size_max < 1 )
oss << "NO backward playback)";
else if (mp.evaluation().gop_size_max * mp.height() > 35000 )
oss << "bad backward playback)";
else
oss << "OK backward playback)";
if (mp.evaluation().discontinuity_count > 0 )
oss << ", " << mp.evaluation().discontinuity_count << " discontinuities";
if (mp.evaluation().corrupted_count > 0 )
oss << ", " << mp.evaluation().corrupted_count << " corrupted frames";
}
else {
oss << ", " << mp.evaluation().log;
}
}
else {
static const char* animation[] = { ICON_FA_HOURGLASS_START,ICON_FA_HOURGLASS_HALF,ICON_FA_HOURGLASS_END,ICON_FA_HOURGLASS };
oss << ", Keyframes: " << animation[(g_get_monotonic_time() / 300000) % 4];
}
}
}
else {
oss << mp.filename() << std::endl;
@@ -111,7 +142,7 @@ void InfoVisitor::visit(MediaPlayer &mp)
information_ = oss.str();
// remember
if ( mp.isOpen() )
if ( mp.isOpen() && ( mp.evaluation().done || mp.media().isimage ) )
current_id_ = mp.id();
}
@@ -130,13 +161,9 @@ void InfoVisitor::visit(Stream &n)
void InfoVisitor::visit (MediaSource& s)
{
if (current_id_ == s.id())
return;
s.mediaplayer()->accept(*this);
if (s.ready())
current_id_ = s.id();
}
void InfoVisitor::visit (SessionFileSource& s)
+21 -4
View File
@@ -2407,6 +2407,14 @@ void SourceControlWindow::RenderMediaPlayer(MediaSource *ms)
current_loop = (int) mediaplayer_active_->loop();
if ( ImGuiToolkit::IconMultistate(icons_loop, &current_loop, tooltips_loop) )
mediaplayer_active_->setLoop( (MediaPlayer::LoopMode) current_loop );
// disable bounce mode if GOP size is invalid
if (current_loop == (int) MediaPlayer::LOOP_BIDIRECTIONAL && mediaplayer_active_->evaluation().done &&
(mediaplayer_active_->evaluation().gop_size_min < 1 ||
mediaplayer_active_->evaluation().gop_size_max < 1 ) )
{
current_loop++;
mediaplayer_active_->setLoop( (MediaPlayer::LoopMode) current_loop );
}
// speed slider
ImGui::SameLine(0, h_space_);
@@ -2488,10 +2496,19 @@ void SourceControlWindow::RenderMediaPlayer(MediaSource *ms)
oss << ": Play forward";
Action::manager().store(oss.str());
}
if (ImGuiToolkit::MenuItemIcon(9,0, "Play backward", nullptr, current_play_speed<0)) {
mediaplayer_active_->setPlaySpeed( - ABS(mediaplayer_active_->playSpeed()) );
oss << ": Play backward";
Action::manager().store(oss.str());
// Enable backward play only if GOP size is valid
if (mediaplayer_active_->evaluation().done &&
mediaplayer_active_->evaluation().gop_size_min > 0 &&
mediaplayer_active_->evaluation().gop_size_max > 0)
{
if (ImGuiToolkit::MenuItemIcon(9,0, "Play backward", nullptr, current_play_speed<0)) {
mediaplayer_active_->setPlaySpeed( - ABS(mediaplayer_active_->playSpeed()) );
oss << ": Play backward";
Action::manager().store(oss.str());
}
}
else {
ImGuiToolkit::MenuItemIcon(9,0, "Play backward", nullptr, false, false);
}
if (ImGuiToolkit::MenuItemIcon(19,15, "Reset speed")) {
mediaplayer_active_->setPlaySpeed(1.0);