reimplemented pixel access in video library when using P2D/P3D

java/libraries/video/src/processing/video/Capture.java

@@ -110,6 +110,8 @@ public class Capture extends PImage implements PConstants {
   protected Object bufferSink;
   protected Method sinkCopyMethod;
   protected Method sinkSetMethod;
+  protected Method sinkDisposeMethod;
+  protected Method sinkGetMethod;  
   protected String copyMask;
   protected Buffer natBuffer = null;
   protected BufferDataAppSink natSink = null;
@@ -417,7 +419,29 @@ public class Capture extends PImage implements PConstants {
     newFrame = true;
   }
 
-
+  
+  public synchronized void loadPixels() {
+    super.loadPixels();
+    if (useBufferSink) {      
+      if (natBuffer != null) {
+        // This means that the OpenGL texture hasn't been created so far (the
+        // video frame not drawn using image()), but the user wants to use the
+        // pixel array, which we can just get from natBuffer.
+        IntBuffer buf = natBuffer.getByteBuffer().asIntBuffer();
+        buf.rewind();
+        buf.get(pixels);
+        Video.convertToARGB(pixels, width, height);        
+      } else if (sinkGetMethod != null) {
+        try {
+          sinkGetMethod.invoke(bufferSink, new Object[] { pixels });
+        } catch (Exception e) {
+          e.printStackTrace();
+        }        
+      }      
+    }
+  }
+  
+  
   ////////////////////////////////////////////////////////////
 
   // List methods.
@@ -800,6 +824,7 @@ public class Capture extends PImage implements PConstants {
 
     // register methods
     parent.registerMethod("dispose", this);
+    parent.registerMethod("post", this);
 
     setEventHandlerObject(parent);
 
@@ -976,6 +1001,13 @@ public class Capture extends PImage implements PConstants {
     available = true;
     bufWidth = w;
     bufHeight = h;
+    if (natBuffer != null) {
+      // To handle the situation where read() is not called in the sketch, so 
+      // that the native buffers are not being sent to the sinke, and therefore, not disposed
+      // by it.
+      System.out.println("  disposing nat buffer before reading new one");
+      natBuffer.dispose(); 
+    }    
     natBuffer = buffer;
 
     // Creates a movieEvent.
@@ -1129,6 +1161,22 @@ public class Capture extends PImage implements PConstants {
       throw new RuntimeException("Capture: provided sink object doesn't have "+
                                  "a setBufferSource method.");
     }
+    
+    try {
+      sinkDisposeMethod = bufferSink.getClass().getMethod("disposeSourceBuffer", 
+        new Class[] { });
+    } catch (Exception e) {
+      throw new RuntimeException("Capture: provided sink object doesn't have " +
+                                 "a disposeSourceBuffer method.");
+    }
+        
+    try {
+      sinkGetMethod = bufferSink.getClass().getMethod("getBufferPixels", 
+        new Class[] { int[].class });
+    } catch (Exception e) {
+      throw new RuntimeException("Capture: provided sink object doesn't have " +
+                                 "a getBufferPixels method.");
+    }    
   }
 
 
@@ -1139,4 +1187,15 @@ public class Capture extends PImage implements PConstants {
       copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
     }
   }
+  
+  
+  public synchronized void post() {
+    if (useBufferSink && sinkDisposeMethod != null) {
+      try {
+        sinkDisposeMethod.invoke(bufferSink, new Object[] {});
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+    }
+  }  
 }

java/libraries/video/src/processing/video/Movie.java

@@ -82,6 +82,8 @@ public class Movie extends PImage implements PConstants {
   protected Object bufferSink;
   protected Method sinkCopyMethod;
   protected Method sinkSetMethod;
+  protected Method sinkDisposeMethod;
+  protected Method sinkGetMethod;  
   protected String copyMask;
   protected Buffer natBuffer = null;
   protected BufferDataAppSink natSink = null;
@@ -542,6 +544,28 @@ public class Movie extends PImage implements PConstants {
   }
 
 
+  public synchronized void loadPixels() {
+    super.loadPixels();
+    if (useBufferSink) {      
+      if (natBuffer != null) {
+        // This means that the OpenGL texture hasn't been created so far (the
+        // video frame not drawn using image()), but the user wants to use the
+        // pixel array, which we can just get from natBuffer.
+        IntBuffer buf = natBuffer.getByteBuffer().asIntBuffer();
+        buf.rewind();
+        buf.get(pixels);
+        Video.convertToARGB(pixels, width, height);        
+      } else if (sinkGetMethod != null) {
+        try {
+          sinkGetMethod.invoke(bufferSink, new Object[] { pixels });
+        } catch (Exception e) {
+          e.printStackTrace();
+        }        
+      }      
+    }
+  }
+  
+  
   ////////////////////////////////////////////////////////////
 
   // Initialization methods.
@@ -609,11 +633,11 @@ public class Movie extends PImage implements PConstants {
 
     // we've got a valid movie! let's rock.
     try {
-      // PApplet.println("we've got a valid movie! let's rock.");
       this.filename = filename; // for error messages
 
       // register methods
       parent.registerMethod("dispose", this);
+      parent.registerMethod("post", this);
 
       setEventHandlerObject(parent);
 
@@ -742,6 +766,13 @@ public class Movie extends PImage implements PConstants {
     available = true;
     bufWidth = w;
     bufHeight = h;
+    if (natBuffer != null) {
+      // To handle the situation where read() is not called in the sketch, so 
+      // that the native buffers are not being sent to the sinke, and therefore, not disposed
+      // by it.
+      System.out.println("  disposing nat buffer before reading new one");
+      natBuffer.dispose(); 
+    }
     natBuffer = buffer;
 
     if (playing) {
@@ -894,6 +925,22 @@ public class Movie extends PImage implements PConstants {
       throw new RuntimeException("Movie: provided sink object doesn't have a " +
                                  "setBufferSource method.");
     }
+    
+    try {
+      sinkDisposeMethod = bufferSink.getClass().getMethod("disposeSourceBuffer", 
+        new Class[] { });
+    } catch (Exception e) {
+      throw new RuntimeException("Movie: provided sink object doesn't have " +
+                                 "a disposeSourceBuffer method.");
+    }
+        
+    try {
+      sinkGetMethod = bufferSink.getClass().getMethod("getBufferPixels", 
+        new Class[] { int[].class });
+    } catch (Exception e) {
+      throw new RuntimeException("Movie: provided sink object doesn't have " +
+                                 "a getBufferPixels method.");
+    }    
   }
 
 
@@ -902,6 +949,17 @@ public class Movie extends PImage implements PConstants {
       copyMask = "red_mask=(int)0xFF000000, green_mask=(int)0xFF0000, blue_mask=(int)0xFF00";
     } else {
       copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
+    }    
+  }    
+  
+  
+  public synchronized void post() {
+    if (useBufferSink && sinkDisposeMethod != null) {
+      try {
+        sinkDisposeMethod.invoke(bufferSink, new Object[] {});
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
     }
   }
 }

java/libraries/video/src/processing/video/Video.java

@@ -26,7 +26,9 @@ package processing.video;
 import org.gstreamer.*;
 import processing.core.PApplet;
 import processing.core.PConstants;
+
 import java.io.File;
+import java.nio.ByteOrder;
 import java.util.List;
 
 /**
@@ -192,5 +194,41 @@ public class Video implements PConstants {
   static protected long secToNanoLong(float sec) {
     Float f = new Float(sec * 1E9);
     return f.longValue();
+  }
+  
+  
+  /**
+   * Reorders an OpenGL pixel array (RGBA) into ARGB. The array must be
+   * of size width * height.
+   * @param pixels int[]
+   */
+  static protected void convertToARGB(int[] pixels, int width, int height) {
+    int t = 0;
+    int p = 0;
+    if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
+
+      // RGBA to ARGB conversion: shifting RGB 8 bits to the right,
+      // and placing A 24 bits to the left.
+      for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+          int pixel = pixels[p++];
+          pixels[t++] = (pixel >> 8) | ((pixel << 24) & 0xFF000000);
+        }
+      }
+
+    } else {
+
+      // We have to convert ABGR into ARGB, so R and B must be swapped,
+      // A and G just brought back in.
+      for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+          int pixel = pixels[p++];
+          pixels[t++] = ((pixel & 0xFF) << 16) |
+                        ((pixel & 0xFF0000) >> 16) |
+                        (pixel & 0xFF00FF00);
+
+        }
+      }
+    }
   }  
 }