reimplemented pixel access in video library when using P2D/P3D

core/src/processing/opengl/Texture.java

@@ -102,6 +102,7 @@ public class Texture implements PConstants {
 
   protected Object bufferSource;
   protected LinkedList<BufferData> bufferCache = null;
+  protected LinkedList<BufferData> usedBuffers = null;
   protected Method disposeBufferMethod;
   public static final int MAX_BUFFER_CACHE_SIZE = 3;
 
@@ -153,7 +154,6 @@ public class Texture implements PConstants {
   @Override
   protected void finalize() throws Throwable {
     try {
-//      PApplet.println("finalize texture");
       if (glName != 0) {
         PGraphicsOpenGL.finalizeTextureObject(glName, context);
       }
@@ -520,27 +520,6 @@ public class Texture implements PConstants {
   }
 
 
-  /**
-   * Copies the contents of the texture to the pixels array.
-   * @param pixels
-   */
-//  public void loadPixels(int[] pixels) {
-//    if (hasBuffers()) {
-//      // Updates the texture AND the pixels array of the image at the same time,
-//      // getting the pixels directly from the buffer data (and thus avoiding
-//      // expensive transfer between video and main memory).
-//      bufferUpdate(pixels);
-//    }
-//
-//    if (isModified()) {
-//      // Regular pixel copy from texture.
-//      get(pixels);
-//    }
-//
-//    setModified(false);
-//  }
-
-
   ////////////////////////////////////////////////////////////
 
   // Put methods (the source texture is not resized to cover the entire
@@ -861,9 +840,9 @@ public class Texture implements PConstants {
       bufferCache.add(new BufferData(natRef, byteBuf.asIntBuffer(), w, h));
     } else {
       // The buffer cache reached the maximum size, so we just dispose
-      // the new buffer.
+      // the new buffer by adding it to the list of used buffers.
       try {
-        disposeBufferMethod.invoke(bufferSource, new Object[] { natRef });
+        usedBuffers.add(new BufferData(natRef, byteBuf.asIntBuffer(), w, h));
       } catch (Exception e) {
         e.printStackTrace();
       }
@@ -871,6 +850,43 @@ public class Texture implements PConstants {
   }
 
 
+  public void disposeSourceBuffer() {
+    System.out.println(" in disposeSourceBuffer");
+    if (usedBuffers == null) return;
+
+    while (0 < usedBuffers.size()) {
+      BufferData data = null;
+      try {
+        data = usedBuffers.remove(0);
+        System.out.println("Disposing " + data + " in disposeSourceBuffer");
+      } catch (NoSuchElementException ex) {
+        PGraphics.showWarning("Cannot remove used buffer");
+      }
+      if (data != null) {
+        data.dispose();
+      }
+    }
+  }
+
+  public void getBufferPixels(int[] pixels) {
+    BufferData data = null;
+    if (usedBuffers != null && 0 < usedBuffers.size()) {
+      // the last used buffer is the one currently stored in the opengl the
+      // texture
+      data = usedBuffers.getLast();
+    } else if (bufferCache != null && 0 < bufferCache.size()) {
+      // The first buffer in the cache will be uploaded to the opengl texture
+      // the next time it is rendered
+      data = bufferCache.getFirst();
+    }
+    if (data != null) {
+      data.rgbBuf.rewind();
+      data.rgbBuf.get(pixels);
+      convertToARGB(pixels);
+    }
+  }
+
+
   public boolean hasBufferSource() {
     return bufferSource != null;
   }
@@ -894,35 +910,15 @@ public class Texture implements PConstants {
       if ((data.w != width) || (data.h != height)) {
         init(data.w, data.h);
       }
+      data.rgbBuf.rewind();
       setNative(data.rgbBuf, 0, 0, width, height);
 
-      data.dispose();
-
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-
-  protected boolean bufferUpdate(int[] pixels) {
-    BufferData data = null;
-    try {
-      data = bufferCache.remove(0);
-    } catch (NoSuchElementException ex) {
-      PGraphics.showWarning("Don't have pixel data to copy to texture");
-    }
-
-    if (data != null) {
-      if ((data.w != width) || (data.h != height)) {
-        init(data.w, data.h);
+      // Putting the buffer in the used buffers list to dispose at the end of
+      // draw.
+      if (usedBuffers == null) {
+        usedBuffers = new LinkedList<BufferData>();
       }
-      setNative(data.rgbBuf, 0, 0, width, height);
-
-      data.rgbBuf.get(pixels);
-      convertToARGB(pixels);
-
-      data.dispose();
+      usedBuffers.add(data);
 
       return true;
     } else {
@@ -1115,8 +1111,8 @@ public class Texture implements PConstants {
         for (int x = 0; x < width; x++) {
           int pixel = intArray[p++];
           intArray[t++] = ((pixel & 0xFF) << 16) |
-                           ((pixel & 0xFF0000) >> 16) |
-                           (pixel & 0xFF00FF00);
+                          ((pixel & 0xFF0000) >> 16) |
+                          (pixel & 0xFF00FF00);
 
         }
       }

java/libraries/video/src/processing/video/Capture.java

@@ -110,6 +110,8 @@ public class Capture extends PImage implements PConstants {
   protected Object bufferSink;
   protected Method sinkCopyMethod;
   protected Method sinkSetMethod;
+  protected Method sinkDisposeMethod;
+  protected Method sinkGetMethod;  
   protected String copyMask;
   protected Buffer natBuffer = null;
   protected BufferDataAppSink natSink = null;
@@ -417,7 +419,29 @@ public class Capture extends PImage implements PConstants {
     newFrame = true;
   }
 
-
+  
+  public synchronized void loadPixels() {
+    super.loadPixels();
+    if (useBufferSink) {      
+      if (natBuffer != null) {
+        // This means that the OpenGL texture hasn't been created so far (the
+        // video frame not drawn using image()), but the user wants to use the
+        // pixel array, which we can just get from natBuffer.
+        IntBuffer buf = natBuffer.getByteBuffer().asIntBuffer();
+        buf.rewind();
+        buf.get(pixels);
+        Video.convertToARGB(pixels, width, height);        
+      } else if (sinkGetMethod != null) {
+        try {
+          sinkGetMethod.invoke(bufferSink, new Object[] { pixels });
+        } catch (Exception e) {
+          e.printStackTrace();
+        }        
+      }      
+    }
+  }
+  
+  
   ////////////////////////////////////////////////////////////
 
   // List methods.
@@ -800,6 +824,7 @@ public class Capture extends PImage implements PConstants {
 
     // register methods
     parent.registerMethod("dispose", this);
+    parent.registerMethod("post", this);
 
     setEventHandlerObject(parent);
 
@@ -976,6 +1001,13 @@ public class Capture extends PImage implements PConstants {
     available = true;
     bufWidth = w;
     bufHeight = h;
+    if (natBuffer != null) {
+      // To handle the situation where read() is not called in the sketch, so 
+      // that the native buffers are not being sent to the sinke, and therefore, not disposed
+      // by it.
+      System.out.println("  disposing nat buffer before reading new one");
+      natBuffer.dispose(); 
+    }    
     natBuffer = buffer;
 
     // Creates a movieEvent.
@@ -1129,6 +1161,22 @@ public class Capture extends PImage implements PConstants {
       throw new RuntimeException("Capture: provided sink object doesn't have "+
                                  "a setBufferSource method.");
     }
+    
+    try {
+      sinkDisposeMethod = bufferSink.getClass().getMethod("disposeSourceBuffer", 
+        new Class[] { });
+    } catch (Exception e) {
+      throw new RuntimeException("Capture: provided sink object doesn't have " +
+                                 "a disposeSourceBuffer method.");
+    }
+        
+    try {
+      sinkGetMethod = bufferSink.getClass().getMethod("getBufferPixels", 
+        new Class[] { int[].class });
+    } catch (Exception e) {
+      throw new RuntimeException("Capture: provided sink object doesn't have " +
+                                 "a getBufferPixels method.");
+    }    
   }
 
 
@@ -1139,4 +1187,15 @@ public class Capture extends PImage implements PConstants {
       copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
     }
   }
+  
+  
+  public synchronized void post() {
+    if (useBufferSink && sinkDisposeMethod != null) {
+      try {
+        sinkDisposeMethod.invoke(bufferSink, new Object[] {});
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+    }
+  }  
 }

java/libraries/video/src/processing/video/Movie.java

@@ -82,6 +82,8 @@ public class Movie extends PImage implements PConstants {
   protected Object bufferSink;
   protected Method sinkCopyMethod;
   protected Method sinkSetMethod;
+  protected Method sinkDisposeMethod;
+  protected Method sinkGetMethod;  
   protected String copyMask;
   protected Buffer natBuffer = null;
   protected BufferDataAppSink natSink = null;
@@ -542,6 +544,28 @@ public class Movie extends PImage implements PConstants {
   }
 
 
+  public synchronized void loadPixels() {
+    super.loadPixels();
+    if (useBufferSink) {      
+      if (natBuffer != null) {
+        // This means that the OpenGL texture hasn't been created so far (the
+        // video frame not drawn using image()), but the user wants to use the
+        // pixel array, which we can just get from natBuffer.
+        IntBuffer buf = natBuffer.getByteBuffer().asIntBuffer();
+        buf.rewind();
+        buf.get(pixels);
+        Video.convertToARGB(pixels, width, height);        
+      } else if (sinkGetMethod != null) {
+        try {
+          sinkGetMethod.invoke(bufferSink, new Object[] { pixels });
+        } catch (Exception e) {
+          e.printStackTrace();
+        }        
+      }      
+    }
+  }
+  
+  
   ////////////////////////////////////////////////////////////
 
   // Initialization methods.
@@ -609,11 +633,11 @@ public class Movie extends PImage implements PConstants {
 
     // we've got a valid movie! let's rock.
     try {
-      // PApplet.println("we've got a valid movie! let's rock.");
       this.filename = filename; // for error messages
 
       // register methods
       parent.registerMethod("dispose", this);
+      parent.registerMethod("post", this);
 
       setEventHandlerObject(parent);
 
@@ -742,6 +766,13 @@ public class Movie extends PImage implements PConstants {
     available = true;
     bufWidth = w;
     bufHeight = h;
+    if (natBuffer != null) {
+      // To handle the situation where read() is not called in the sketch, so 
+      // that the native buffers are not being sent to the sinke, and therefore, not disposed
+      // by it.
+      System.out.println("  disposing nat buffer before reading new one");
+      natBuffer.dispose(); 
+    }
     natBuffer = buffer;
 
     if (playing) {
@@ -894,6 +925,22 @@ public class Movie extends PImage implements PConstants {
       throw new RuntimeException("Movie: provided sink object doesn't have a " +
                                  "setBufferSource method.");
     }
+    
+    try {
+      sinkDisposeMethod = bufferSink.getClass().getMethod("disposeSourceBuffer", 
+        new Class[] { });
+    } catch (Exception e) {
+      throw new RuntimeException("Movie: provided sink object doesn't have " +
+                                 "a disposeSourceBuffer method.");
+    }
+        
+    try {
+      sinkGetMethod = bufferSink.getClass().getMethod("getBufferPixels", 
+        new Class[] { int[].class });
+    } catch (Exception e) {
+      throw new RuntimeException("Movie: provided sink object doesn't have " +
+                                 "a getBufferPixels method.");
+    }    
   }
 
 
@@ -902,6 +949,17 @@ public class Movie extends PImage implements PConstants {
       copyMask = "red_mask=(int)0xFF000000, green_mask=(int)0xFF0000, blue_mask=(int)0xFF00";
     } else {
       copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
+    }    
+  }    
+  
+  
+  public synchronized void post() {
+    if (useBufferSink && sinkDisposeMethod != null) {
+      try {
+        sinkDisposeMethod.invoke(bufferSink, new Object[] {});
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
     }
   }
 }

java/libraries/video/src/processing/video/Video.java

@@ -26,7 +26,9 @@ package processing.video;
 import org.gstreamer.*;
 import processing.core.PApplet;
 import processing.core.PConstants;
+
 import java.io.File;
+import java.nio.ByteOrder;
 import java.util.List;
 
 /**
@@ -192,5 +194,41 @@ public class Video implements PConstants {
   static protected long secToNanoLong(float sec) {
     Float f = new Float(sec * 1E9);
     return f.longValue();
+  }
+  
+  
+  /**
+   * Reorders an OpenGL pixel array (RGBA) into ARGB. The array must be
+   * of size width * height.
+   * @param pixels int[]
+   */
+  static protected void convertToARGB(int[] pixels, int width, int height) {
+    int t = 0;
+    int p = 0;
+    if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
+
+      // RGBA to ARGB conversion: shifting RGB 8 bits to the right,
+      // and placing A 24 bits to the left.
+      for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+          int pixel = pixels[p++];
+          pixels[t++] = (pixel >> 8) | ((pixel << 24) & 0xFF000000);
+        }
+      }
+
+    } else {
+
+      // We have to convert ABGR into ARGB, so R and B must be swapped,
+      // A and G just brought back in.
+      for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+          int pixel = pixels[p++];
+          pixels[t++] = ((pixel & 0xFF) << 16) |
+                        ((pixel & 0xFF0000) >> 16) |
+                        (pixel & 0xFF00FF00);
+
+        }
+      }
+    }
   }  
 }