From d1f63b8501cd7ee2f4d28ce91edfd246985ebcb8 Mon Sep 17 00:00:00 2001 From: Vasilis Liaskovitis Date: Mon, 9 Feb 2015 21:44:47 +0100 Subject: [PATCH] Fix issue #114: Audio playback of the video files played We now create an audio pipeline with a volume gstreamer element. The "volume" property of the element has a range of 0.0 - 1.0 and is 0.0 by default (i.e. effectively muted input). The property can be seen and changed in the paint property browser (in %) --- MediaImpl.cpp | 57 +++++++++++++++++++++++++++++++++++++++++++++++---- MediaImpl.h | 16 +++++++++++---- Paint.cpp | 10 +++++++++ Paint.h | 5 +++++ PaintGui.cpp | 14 ++++++++++++- PaintGui.h | 1 + 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/MediaImpl.cpp b/MediaImpl.cpp index 84bb901..f401493 100644 --- a/MediaImpl.cpp +++ b/MediaImpl.cpp @@ -101,6 +101,18 @@ void MediaImpl::setRate(double rate) } } +void MediaImpl::setVolume(double volume) +{ + // Only update volume if needed. + if (_volume != volume) + { + _volume = volume; + + // Set volume element property. + g_object_set (_audiovolume0, "volume", _volume, NULL); + } +} + void MediaImpl::build() { qDebug() << "Building video impl"; @@ -193,14 +205,19 @@ GstFlowReturn MediaImpl::gstNewSampleCallback(GstElement*, MediaImpl *p) return GST_FLOW_OK; } + MediaImpl::MediaImpl(const QString uri, bool live) : _bus(NULL), _pipeline(NULL), _uridecodebin0(NULL), _queue0(NULL), _videoconvert0(NULL), -//_audioSink(NULL), _appsink0(NULL), +_audioqueue0(NULL), +_audioconvert0(NULL), +_audioresample0(NULL), +_audiovolume0(NULL), +_audiosink0(NULL), _currentFrameSample(NULL), _currentFrameBuffer(NULL), _bitsChanged(false), @@ -372,11 +389,21 @@ bool MediaImpl::loadMovie(QString filename) _padHandlerData.videoSink = _appsink0; _padHandlerData.videoIsConnected = false; + _audioqueue0 = gst_element_factory_make ("queue", "audioqueue0"); + _audioconvert0 = gst_element_factory_make ("audioconvert", "audioconvert0"); + _audioresample0 = gst_element_factory_make ("audioresample", "audioresample0"); + _audiovolume0 = gst_element_factory_make ("volume", "audiovolume0"); + _audiosink0 = gst_element_factory_make ("autoaudiosink", "audiosink0"); + + _padHandlerData.audioToConnect = _audioqueue0; + // Create the empty pipeline. _pipeline = gst_pipeline_new ( "video-source-pipeline" ); if (!_pipeline || - !_queue0 || !_videoconvert0 || ! videoscale0 || ! capsfilter0 || ! _appsink0) + !_queue0 || !_videoconvert0 || ! videoscale0 || ! capsfilter0 || + !_appsink0 || !_audioqueue0 || !_audioconvert0 || !_audioresample0 || + !_audiovolume0 || !_audiosink0) { g_printerr ("Not all elements could be created.\n"); @@ -386,6 +413,11 @@ bool MediaImpl::loadMovie(QString filename) if (! videoscale0) g_printerr("videoscale0"); if (! capsfilter0) g_printerr("capsfilter0"); if (! _appsink0) g_printerr("_appsink0"); + if (! _audioqueue0) g_printerr("_audioqueue0"); + if (! _audioconvert0) g_printerr("_audioconvert0"); + if (! _audioresample0) g_printerr("_audioresample0"); + if (! _audiovolume0) g_printerr("_audiovolume0"); + if (! _audiosink0) g_printerr("_audiosink0"); unloadMovie(); return -1; @@ -417,7 +449,9 @@ bool MediaImpl::loadMovie(QString filename) // point. We will do it later. gst_bin_add_many (GST_BIN (_pipeline), _isSharedMemorySource ? _shmsrc0 : _uridecodebin0, _queue0, - _videoconvert0, videoscale0, capsfilter0, _appsink0, NULL); + _videoconvert0, videoscale0, capsfilter0, _appsink0, + _audioqueue0, _audioconvert0, _audioresample0, _audiovolume0, _audiosink0, + NULL); // special case for shmsrc if (_isSharedMemorySource) @@ -443,6 +477,14 @@ bool MediaImpl::loadMovie(QString filename) return false; } + if (! gst_element_link_many (_audioqueue0, _audioconvert0, _audioresample0, + _audiovolume0, _audiosink0, NULL)) + { + g_printerr ("Could not link audio queue, converter, resampler and audio sink.\n"); + unloadMovie(); + return false; + } + // Process URI. QByteArray ba = filename.toLocal8Bit(); gchar* uri = (gchar*) filename.toUtf8().constData(); @@ -498,6 +540,7 @@ bool MediaImpl::loadMovie(QString filename) // gst_caps_unref (audioCaps); // g_free (audioCapsText); + // Configure video appsink. GstCaps *videoCaps = gst_caps_from_string ("video/x-raw,format=RGBA"); g_object_set (capsfilter0, "caps", videoCaps, NULL); @@ -509,6 +552,9 @@ bool MediaImpl::loadMovie(QString filename) g_signal_connect (_appsink0, "new-sample", G_CALLBACK (MediaImpl::gstNewSampleCallback), this); gst_caps_unref (videoCaps); + g_object_set (_audiovolume0, "mute", false, NULL); + g_object_set (_audiovolume0, "volume", 0.0, NULL); + // Listen to the bus. _bus = gst_element_get_bus (_pipeline); @@ -788,8 +834,11 @@ void MediaImpl::gstPadAddedCallback(GstElement *src, GstPad *newPad, MediaImpl:: gst_structure_get_int(newPadStruct, "width", &data->width); gst_structure_get_int(newPadStruct, "height", &data->height); } - else + else if (g_str_has_prefix (newPadType, "audio/x-raw")) { + sinkPad = gst_element_get_static_pad (data->audioToConnect, "sink"); + } + else { g_print (" It has type '%s' which is not raw audio/video. Ignoring.\n", newPadType); goto exit; } diff --git a/MediaImpl.h b/MediaImpl.h index 49a666c..d7c8a6b 100644 --- a/MediaImpl.h +++ b/MediaImpl.h @@ -132,6 +132,9 @@ public: void setRate(double rate=1.0); double getRate() const { return _rate; } + void setVolume(double rate=0.0); + double getVolume() const { return _volume; } + void resetMovie(); protected: @@ -171,6 +174,8 @@ public: int width; int height; + GstElement* audioToConnect; + GstPadHandlerData() : videoToConnect(NULL), videoSink(NULL), videoIsConnected(false), @@ -201,13 +206,14 @@ private: GstElement *_uridecodebin0; GstElement *_shmsrc0; GstElement *_gdpdepay0; - //GstElement *_audioQueue; - //GstElement *_audioConvert; - //GstElement *_audioResample; GstElement *_queue0; GstElement *_videoconvert0; - //GstElement *_audioSink; GstElement *_appsink0; + GstElement *_audioqueue0; + GstElement *_audioconvert0; + GstElement *_audioresample0; + GstElement *_audiovolume0; + GstElement *_audiosink0; /** * Temporary contains the image data of the last frame. @@ -237,6 +243,8 @@ private: /// Playback rate (negative ==> reverse). double _rate; + /// Audio playback volume (0.0 ==> 1.0). + double _volume; /// Whether or not we are reading video from a shmsrc. bool _isSharedMemorySource; diff --git a/Paint.cpp b/Paint.cpp index 2b76099..a4c954c 100644 --- a/Paint.cpp +++ b/Paint.cpp @@ -131,6 +131,16 @@ double Media::getRate() const return impl_->getRate() * 100.0; } +void Media::setVolume(double rate) +{ + impl_->setVolume(rate / 100.0); +} + +double Media::getVolume() const +{ + return impl_->getVolume() * 100.0; +} + bool Media::hasVideoSupport() { return MediaImpl::hasVideoSupport(); diff --git a/Paint.h b/Paint.h index 19b69f6..0a4e337 100644 --- a/Paint.h +++ b/Paint.h @@ -243,6 +243,11 @@ public: /// Returns playback rate. double getRate() const; + /// Sets audio playback volume (in %). + virtual void setVolume(double volume=0.0); + + /// Returns audio playback volume. + double getVolume() const; /** * Checks whether or not video is supported on this platform. diff --git a/PaintGui.cpp b/PaintGui.cpp index 9e4059c..ab9b279 100644 --- a/PaintGui.cpp +++ b/PaintGui.cpp @@ -108,11 +108,17 @@ MediaGui::MediaGui(Paint::ptr paint) _mediaRateItem = _variantManager->addProperty(QVariant::Double, tr("Speed (%)")); - double rate = media->getRate(); // we need to save it because the call to setAttribute will set it to minimum _mediaRateItem->setAttribute("minimum", 1); _mediaRateItem->setAttribute("decimals", 1); _mediaRateItem->setValue(rate); + + _mediaVolumeItem = _variantManager->addProperty(QVariant::Double, + tr("Volume (%)")); + _mediaVolumeItem->setAttribute("minimum", 0.0); + _mediaVolumeItem->setAttribute("maximum", 100.0); + _mediaVolumeItem->setAttribute("decimals", 1); + //_mediaVolumeItem->setValue(rate); // _mediaReverseItem = _variantManager->addProperty(QVariant::Bool, // tr("Reverse")); @@ -120,6 +126,7 @@ MediaGui::MediaGui(Paint::ptr paint) _topItem->addSubProperty(_mediaFileItem); _topItem->addSubProperty(_mediaRateItem); + _topItem->addSubProperty(_mediaVolumeItem); // _topItem->addSubProperty(_mediaReverseItem); } @@ -143,5 +150,10 @@ void MediaGui::setValue(QtProperty* property, const QVariant& value) // media->setRate( (value.toBool() ? -1 : +1) * absoluteRate ); // emit valueChanged(_paint); // } + else if (property == _mediaVolumeItem) + { + media->setVolume(value.toDouble()); + emit valueChanged(_paint); + } } } diff --git a/PaintGui.h b/PaintGui.h index 2b8a729..ff6c08e 100644 --- a/PaintGui.h +++ b/PaintGui.h @@ -132,6 +132,7 @@ protected: std::tr1::shared_ptr media; QtVariantProperty* _mediaFileItem; QtVariantProperty* _mediaRateItem; + QtVariantProperty* _mediaVolumeItem; // QtVariantProperty* _mediaReverseItem; };