Fix issue #114: Audio playback of the video files played

We now create an audio pipeline with a volume gstreamer element. The "volume"
property of the element has a range of 0.0 - 1.0 and is 0.0 by default (i.e.
effectively muted input). The property can be seen and changed in the paint
property browser (in %)

MediaImpl.cpp

@@ -101,6 +101,18 @@ void MediaImpl::setRate(double rate)
   }
 }
 
+void MediaImpl::setVolume(double volume)
+{
+  // Only update volume if needed.
+  if (_volume != volume)
+  {
+    _volume = volume;
+
+    // Set volume element property.
+    g_object_set (_audiovolume0, "volume", _volume, NULL);
+  }
+}
+
 void MediaImpl::build()
 {
   qDebug() << "Building video impl";
@@ -193,14 +205,19 @@ GstFlowReturn MediaImpl::gstNewSampleCallback(GstElement*, MediaImpl *p)
   return GST_FLOW_OK;
 }
 
+
 MediaImpl::MediaImpl(const QString uri, bool live) :
 _bus(NULL),
 _pipeline(NULL),
 _uridecodebin0(NULL),
 _queue0(NULL),
 _videoconvert0(NULL),
-//_audioSink(NULL),
 _appsink0(NULL),
+_audioqueue0(NULL),
+_audioconvert0(NULL),
+_audioresample0(NULL),
+_audiovolume0(NULL),
+_audiosink0(NULL),
 _currentFrameSample(NULL),
 _currentFrameBuffer(NULL),
 _bitsChanged(false),
@@ -372,11 +389,21 @@ bool MediaImpl::loadMovie(QString filename)
   _padHandlerData.videoSink = _appsink0;
   _padHandlerData.videoIsConnected = false;
 
+  _audioqueue0 = gst_element_factory_make ("queue", "audioqueue0");
+  _audioconvert0 = gst_element_factory_make ("audioconvert", "audioconvert0");
+  _audioresample0 = gst_element_factory_make ("audioresample", "audioresample0");
+  _audiovolume0 = gst_element_factory_make ("volume", "audiovolume0");
+  _audiosink0 = gst_element_factory_make ("autoaudiosink", "audiosink0");
+
+  _padHandlerData.audioToConnect = _audioqueue0;
+
   // Create the empty pipeline.
   _pipeline = gst_pipeline_new ( "video-source-pipeline" );
 
   if (!_pipeline ||
-      !_queue0 || !_videoconvert0 || ! videoscale0 || ! capsfilter0 || ! _appsink0)
+      !_queue0 || !_videoconvert0 || ! videoscale0 || ! capsfilter0 ||
+      !_appsink0 || !_audioqueue0 || !_audioconvert0 || !_audioresample0 ||
+      !_audiovolume0 || !_audiosink0)
   {
     g_printerr ("Not all elements could be created.\n");
 
@@ -386,6 +413,11 @@ bool MediaImpl::loadMovie(QString filename)
     if (! videoscale0) g_printerr("videoscale0");
     if (! capsfilter0) g_printerr("capsfilter0");
     if (! _appsink0) g_printerr("_appsink0");
+    if (! _audioqueue0) g_printerr("_audioqueue0");
+    if (! _audioconvert0) g_printerr("_audioconvert0");
+    if (! _audioresample0) g_printerr("_audioresample0");
+    if (! _audiovolume0) g_printerr("_audiovolume0");
+    if (! _audiosink0) g_printerr("_audiosink0");
 
     unloadMovie();
     return -1;
@@ -417,7 +449,9 @@ bool MediaImpl::loadMovie(QString filename)
   // point. We will do it later.
   gst_bin_add_many (GST_BIN (_pipeline),
     _isSharedMemorySource ? _shmsrc0 : _uridecodebin0, _queue0,
-    _videoconvert0, videoscale0, capsfilter0, _appsink0, NULL);
+    _videoconvert0, videoscale0, capsfilter0, _appsink0,
+    _audioqueue0, _audioconvert0, _audioresample0, _audiovolume0, _audiosink0,
+    NULL);
 
   // special case for shmsrc
   if (_isSharedMemorySource)
@@ -443,6 +477,14 @@ bool MediaImpl::loadMovie(QString filename)
     return false;
   }
 
+  if (! gst_element_link_many (_audioqueue0, _audioconvert0, _audioresample0,
+        _audiovolume0, _audiosink0, NULL))
+  {
+    g_printerr ("Could not link audio queue, converter, resampler and audio sink.\n");
+    unloadMovie();
+    return false;
+  }
+
   // Process URI.
   QByteArray ba = filename.toLocal8Bit();
   gchar* uri = (gchar*) filename.toUtf8().constData();
@@ -498,6 +540,7 @@ bool MediaImpl::loadMovie(QString filename)
 //  gst_caps_unref (audioCaps);
 //  g_free (audioCapsText);
 
+
   // Configure video appsink.
   GstCaps *videoCaps = gst_caps_from_string ("video/x-raw,format=RGBA");
   g_object_set (capsfilter0, "caps", videoCaps, NULL);
@@ -509,6 +552,9 @@ bool MediaImpl::loadMovie(QString filename)
   g_signal_connect (_appsink0, "new-sample", G_CALLBACK (MediaImpl::gstNewSampleCallback), this);
   gst_caps_unref (videoCaps);
 
+  g_object_set (_audiovolume0, "mute", false, NULL);
+  g_object_set (_audiovolume0, "volume", 0.0, NULL);
+
   // Listen to the bus.
   _bus = gst_element_get_bus (_pipeline);
 
@@ -788,8 +834,11 @@ void MediaImpl::gstPadAddedCallback(GstElement *src, GstPad *newPad, MediaImpl::
     gst_structure_get_int(newPadStruct, "width",  &data->width);
     gst_structure_get_int(newPadStruct, "height", &data->height);
   }
-  else
+  else if (g_str_has_prefix (newPadType, "audio/x-raw"))
   {
+    sinkPad = gst_element_get_static_pad (data->audioToConnect, "sink");
+  }
+  else {
     g_print ("  It has type '%s' which is not raw audio/video. Ignoring.\n", newPadType);
     goto exit;
   }

MediaImpl.h

@@ -132,6 +132,9 @@ public:
   void setRate(double rate=1.0);
   double getRate() const { return _rate; }
 
+  void setVolume(double rate=0.0);
+  double getVolume() const { return _volume; }
+
   void resetMovie();
 
 protected:
@@ -171,6 +174,8 @@ public:
     int width;
     int height;
 
+    GstElement* audioToConnect;
+
     GstPadHandlerData() :
       videoToConnect(NULL), videoSink(NULL),
       videoIsConnected(false),
@@ -201,13 +206,14 @@ private:
   GstElement *_uridecodebin0;
   GstElement *_shmsrc0;
   GstElement *_gdpdepay0;
-  //GstElement *_audioQueue;
-  //GstElement *_audioConvert;
-  //GstElement *_audioResample;
   GstElement *_queue0;
   GstElement *_videoconvert0;
-  //GstElement *_audioSink;
   GstElement *_appsink0;
+  GstElement *_audioqueue0;
+  GstElement *_audioconvert0;
+  GstElement *_audioresample0;
+  GstElement *_audiovolume0;
+  GstElement *_audiosink0;
 
   /**
    * Temporary contains the image data of the last frame.
@@ -237,6 +243,8 @@ private:
 
   /// Playback rate (negative ==> reverse).
   double _rate;
+  /// Audio playback volume (0.0 ==> 1.0).
+  double _volume;
 
   /// Whether or not we are reading video from a shmsrc.
   bool _isSharedMemorySource;

Paint.cpp

@@ -131,6 +131,16 @@ double Media::getRate() const
   return impl_->getRate() * 100.0;
 }
 
+void Media::setVolume(double rate)
+{
+  impl_->setVolume(rate / 100.0);
+}
+
+double Media::getVolume() const
+{
+  return impl_->getVolume() * 100.0;
+}
+
 bool Media::hasVideoSupport()
 {
   return MediaImpl::hasVideoSupport();

Paint.h

@@ -243,6 +243,11 @@ public:
   /// Returns playback rate.
   double getRate() const;
 
+  /// Sets audio playback volume (in %).
+  virtual void setVolume(double volume=0.0);
+
+  /// Returns audio playback volume.
+  double getVolume() const;
 
   /**
    * Checks whether or not video is supported on this platform.

PaintGui.cpp

@@ -108,11 +108,17 @@ MediaGui::MediaGui(Paint::ptr paint)
 
   _mediaRateItem = _variantManager->addProperty(QVariant::Double,
                                                 tr("Speed (%)"));
-
   double rate = media->getRate(); // we need to save it because the call to setAttribute will set it to minimum
   _mediaRateItem->setAttribute("minimum", 1);
   _mediaRateItem->setAttribute("decimals", 1);
   _mediaRateItem->setValue(rate);
+  
+  _mediaVolumeItem = _variantManager->addProperty(QVariant::Double,
+                                                tr("Volume (%)"));
+  _mediaVolumeItem->setAttribute("minimum", 0.0);
+  _mediaVolumeItem->setAttribute("maximum", 100.0);
+  _mediaVolumeItem->setAttribute("decimals", 1);
+  //_mediaVolumeItem->setValue(rate);
 
 //  _mediaReverseItem = _variantManager->addProperty(QVariant::Bool,
 //                                                tr("Reverse"));
@@ -120,6 +126,7 @@ MediaGui::MediaGui(Paint::ptr paint)
 
   _topItem->addSubProperty(_mediaFileItem);
   _topItem->addSubProperty(_mediaRateItem);
+  _topItem->addSubProperty(_mediaVolumeItem);
 //  _topItem->addSubProperty(_mediaReverseItem);
 }
 
@@ -143,5 +150,10 @@ void MediaGui::setValue(QtProperty* property, const QVariant& value)
 //      media->setRate( (value.toBool() ? -1 : +1) * absoluteRate );
 //      emit valueChanged(_paint);
 //    }
+    else if (property == _mediaVolumeItem)
+    {
+      media->setVolume(value.toDouble());
+      emit valueChanged(_paint);
+    }
   }
 }

PaintGui.h

@@ -132,6 +132,7 @@ protected:
   std::tr1::shared_ptr<Media> media;
   QtVariantProperty* _mediaFileItem;
   QtVariantProperty* _mediaRateItem;
+  QtVariantProperty* _mediaVolumeItem;
 //  QtVariantProperty* _mediaReverseItem;
 };