Added direct-buffer copy to video library

This commit is contained in:
codeanticode
2012-01-22 02:03:35 +00:00
parent f04373e11d
commit 50cebf56af
6 changed files with 383 additions and 185 deletions
+8
View File
@@ -13652,6 +13652,14 @@ public class PApplet extends Applet
}
/**
* Return true if this renderer does rendering through OpenGL. Defaults to false.
*/
public boolean isGL() {
return g.isGL();
}
public PShape createShape() {
return g.createShape();
}
+8
View File
@@ -7427,6 +7427,14 @@ public class PGraphics extends PImage implements PConstants {
return false;
}
/**
* Return true if this renderer does rendering through OpenGL. Defaults to false.
*/
public boolean isGL() {
return false;
}
//////////////////////////////////////////////////////////////
// New API:
@@ -3188,7 +3188,7 @@ public class PGraphicsOpenGL extends PGraphics {
* Implementation of actual drawing for a line of text.
*/
protected void textLineImpl(char buffer[], int start, int stop, float x, float y) {
textTex = (PFontTexture)textFont.getCache(renderer);
textTex = (PFontTexture)textFont.getCache(renderer);
if (textTex == null) {
textTex = new PFontTexture(parent, textFont, maxTextureSize, maxTextureSize);
textFont.setCache(this, textTex);
@@ -3201,7 +3201,7 @@ public class PGraphicsOpenGL extends PGraphics {
textTex = new PFontTexture(parent, textFont, maxTextureSize, maxTextureSize);
textFont.setCache(this, textTex);
}
}
}
textTex.setFirstTexture();
// Saving style parameters modified by text rendering.
@@ -3232,7 +3232,7 @@ public class PGraphicsOpenGL extends PGraphics {
blendMode(BLEND);
super.textLineImpl(buffer, start, stop, x, y);
// Restoring original style.
textureMode = savedTextureMode;
stroke = savedStroke;
@@ -4795,6 +4795,11 @@ public class PGraphicsOpenGL extends PGraphics {
public boolean is3D() {
return true;
}
public boolean isGL() {
return true;
}
//////////////////////////////////////////////////////////////
@@ -5666,7 +5671,10 @@ public class PGraphicsOpenGL extends PGraphics {
}
updateTexture(img, tex);
}
if (tex.hasBuffers()) {
tex.bufferUpdate();
}
}
return tex;
}
@@ -5844,7 +5852,7 @@ public class PGraphicsOpenGL extends PGraphics {
buffer.rewind();
pgl.enableTexturing(tex.glTarget);
pgl.bindTexture(tex.glTarget, tex.glID);
pgl.copyTexSubImage(buffer, tex.glTarget, x, y, w, h);
pgl.copyTexSubImage(buffer, tex.glTarget, 0, x, y, w, h);
pgl.bindTexture(tex.glTarget, 0);
pgl.disableTexturing(tex.glTarget);
}
@@ -23,6 +23,12 @@
package processing.opengl;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import javax.media.opengl.GLContext;
import processing.core.PApplet;
import processing.core.PConstants;
@@ -63,6 +69,11 @@ public class PTexture implements PConstants {
protected int[] tempPixels = null;
protected PFramebuffer tempFbo = null;
protected Object bufferSource;
protected LinkedList<BufferData> bufferCache = null;
protected Method disposeBufferMethod;
public static final int MAX_BUFFER_CACHE_SIZE = 3;
////////////////////////////////////////////////////////////
// Constructors.
@@ -246,7 +257,7 @@ public class PTexture implements PConstants {
int[] rgbaPixels = new int[w * h];
convertToRGBA(pixels, rgbaPixels, format, w, h);
pgl.enableTexMipmapGen(glTarget);
setTexels(x, y, w, h, rgbaPixels);
setTexels(rgbaPixels, x, y, w, h);
rgbaPixels = null;
} else {
// TODO: Manual mipmap generation.
@@ -256,7 +267,7 @@ public class PTexture implements PConstants {
} else {
int[] rgbaPixels = new int[w * h];
convertToRGBA(pixels, rgbaPixels, format, w, h);
setTexels(x, y, w, h, rgbaPixels);
setTexels(rgbaPixels, x, y, w, h);
rgbaPixels = null;
}
@@ -411,6 +422,73 @@ public class PTexture implements PConstants {
pgl.unbindTexture(glTarget);
}
////////////////////////////////////////////////////////////
// Buffer sink interface.
public void setBufferSource(Object source) {
bufferSource = source;
getSourceMethods();
}
public void copyBufferFromSource(Object natRef, ByteBuffer byteBuf, int w, int h) {
if (bufferCache == null) {
bufferCache = new LinkedList<BufferData>();
}
if (bufferCache.size() + 1 <= MAX_BUFFER_CACHE_SIZE) {
bufferCache.add(new BufferData(natRef, byteBuf.asIntBuffer(), w, h));
} else {
// The buffer cache reached the maximum size, so we just dispose the new buffer.
try {
disposeBufferMethod.invoke(bufferSource, new Object[] { natRef });
} catch (Exception e) {
e.printStackTrace();
}
}
}
public boolean hasBuffers() {
return bufferSource != null && bufferCache != null && 0 < bufferCache.size();
}
protected boolean bufferUpdate() {
BufferData data = null;
try {
data = bufferCache.remove(0);
} catch (NoSuchElementException ex) {
PGraphics.showWarning("PTexture: don't have pixel data to copy to texture");
}
if (data != null) {
if ((data.w != width) || (data.h != height)) {
init(data.w, data.h);
}
bind();
setTexels(data.rgbBuf, 0, 0, width, height);
unbind();
data.dispose();
return true;
} else {
return false;
}
}
protected void getSourceMethods() {
try {
disposeBufferMethod = bufferSource.getClass().getMethod("disposeBuffer", new Class[] { Object.class });
} catch (Exception e) {
throw new RuntimeException("PTexture: provided source object doesn't have a disposeBuffer method.");
}
}
////////////////////////////////////////////////////////////
// Utilities
@@ -714,7 +792,7 @@ public class PTexture implements PConstants {
// Once OpenGL knows the size of the new texture, we make sure it doesn't
// contain any garbage in the region of interest (0, 0, width, height):
int[] texels = new int[width * height];
setTexels(0, 0, width, height, texels);
setTexels(texels, 0, 0, width, height);
texels = null;
pgl.unbindTexture(glTarget);
@@ -768,13 +846,20 @@ public class PTexture implements PConstants {
renderer.popFramebuffer();
}
protected void setTexels(int x, int y, int w, int h, int[] pix) {
setTexels(0, x, y, w, h, pix);
protected void setTexels(int[] pix, int x, int y, int w, int h) {
setTexels(pix, 0, x, y, w, h);
}
protected void setTexels(int level, int x, int y, int w, int h, int[] pix) {
//getGl().glTexSubImage2D(glTarget, 0, x, y, w, h, GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, IntBuffer.wrap(pix));
pgl.copyTexSubPixels(pix, glTarget, x, y, w, h);
protected void setTexels(int[] pix, int level, int x, int y, int w, int h) {
pgl.copyTexSubImage(pix, glTarget, level, x, y, w, h);
}
protected void setTexels(IntBuffer buffer, int x, int y, int w, int h) {
setTexels(buffer, 0, x, y, w, h);
}
protected void setTexels(IntBuffer buffer, int level, int x, int y, int w, int h) {
pgl.copyTexSubImage(buffer, glTarget, level, x, y, w, h);
}
protected void copyObject(PTexture src) {
@@ -1014,4 +1099,34 @@ public class PTexture implements PConstants {
this.wrapV = src.wrapV;
}
}
/**
* This class stores a buffer copied from the buffer source.
*
*/
protected class BufferData {
int w, h;
// Native buffer object.
Object natBuf;
// Buffer viewed as int.
IntBuffer rgbBuf;
BufferData(Object nat, IntBuffer rgb, int w, int h) {
natBuf = nat;
rgbBuf = rgb;
this.w = w;
this.h = h;
}
void dispose() {
try {
// Disposing the native buffer.
disposeBufferMethod.invoke(bufferSource, new Object[] { natBuf });
natBuf = null;
rgbBuf = null;
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
@@ -94,30 +94,30 @@ public class Capture extends PImage implements PConstants {
protected Element gsource;
protected Method captureEventMethod;
protected Method copyBufferMethod;
protected Object eventHandler;
protected Object copyHandler;
protected boolean available;
protected boolean pipelineReady;
protected boolean newFrame;
protected RGBDataAppSink rgbSink = null;
protected int[] copyPixels = null;
protected BufferDataAppSink natSink = null;
protected Buffer natBuffer = null;
protected boolean copyBufferMode = false;
protected String copyMask;
protected boolean firstFrame = true;
protected ArrayList<Resolution> resolutions;
protected ArrayList<Resolution> resolutions;
protected int reqWidth;
protected int reqHeight;
protected boolean useBufferSink = false;
protected boolean useGLSink = true;
protected Object bufferSink;
protected Method sinkCopyMethod;
protected Method sinkSetMethod;
protected String copyMask;
protected Buffer natBuffer = null;
protected BufferDataAppSink natSink = null;
/**
* @param parent typically use "this"
* @param requestWidth width of the frame
@@ -343,27 +343,36 @@ public class Capture extends PImage implements PConstants {
// pixels array, even if without any valid image inside.
loadPixels();
if (copyBufferMode) {
// The native buffer from gstreamer is copies to the destination object.
if (natBuffer == null || copyBufferMethod == null) {
if (useBufferSink) { // The native buffer from gstreamer is copied to the buffer sink.
if (natBuffer == null) {
return;
}
}
if (firstFrame) {
super.init(bufWidth, bufHeight, RGB);
super.init(bufWidth, bufHeight, ARGB);
loadPixels();
firstFrame = false;
}
if (bufferSink == null) {
Object cache = getCache(parent.g);
if (cache == null) {
return;
}
setBufferSink(cache);
getSinkMethods();
}
IntBuffer rgbBuffer = natBuffer.getByteBuffer().asIntBuffer();
ByteBuffer byteBuffer = natBuffer.getByteBuffer();
try {
copyBufferMethod.invoke(copyHandler, new Object[] { natBuffer, rgbBuffer, bufWidth, bufHeight });
sinkCopyMethod.invoke(bufferSink, new Object[] { natBuffer, byteBuffer, bufWidth, bufHeight });
} catch (Exception e) {
e.printStackTrace();
}
natBuffer = null;
} else {
natBuffer = null;
} else { // The pixels just read from gstreamer are copied to the pixels array.
if (copyPixels == null) {
return;
}
@@ -505,35 +514,9 @@ public class Capture extends PImage implements PConstants {
}
}
/**
* Sets the object to use as destination for the frames read from the stream.
* The color conversion mask is automatically set to the one required to
* copy the frames to OpenGL.
*
* @param Object dest
*/
public void setPixelDest(Object dest) {
copyHandler = dest;
if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
copyMask = "red_mask=(int)0xFF000000, green_mask=(int)0xFF0000, blue_mask=(int)0xFF00";
} else {
copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
}
}
/**
* Sets the object to use as destination for the frames read from the stream.
*
* @param Object dest
* @param String mask
*/
public void setPixelDest(Object dest, String mask) {
copyHandler = dest;
copyMask = mask;
}
/**
* Uses a generic object as handler of the movie. This object should have a
* Uses a generic object as handler of the capture. This object should have a
* movieEvent method that receives a GSMovie argument. This method will
* be called upon a new frame read event.
*
@@ -644,47 +627,35 @@ public class Capture extends PImage implements PConstants {
fpsStr = ", framerate=" + fps;
}
if (copyHandler != null) {
try {
copyBufferMethod = copyHandler.getClass().getMethod("addPixelsToBuffer",
new Class[] { Object.class, IntBuffer.class, int.class, int.class });
copyBufferMode = true;
} catch (Exception e) {
// no such method, or an error.. which is fine, just ignore
copyBufferMode = false;
if (bufferSink != null || (useGLSink && parent.g.isGL())) {
useBufferSink = true;
if (bufferSink != null) {
getSinkMethods();
}
if (copyMask == null || copyMask.equals("")) {
initCopyMask();
}
if (copyBufferMode) {
String caps = "width=" + reqWidth + ", height=" + reqHeight + fpsStr + ", " + copyMask;
try {
Method meth = copyHandler.getClass().getMethod("setPixelSource", new Class[] { Object.class});
meth.invoke(copyHandler, new Object[] { this });
} catch (Exception e) {
copyBufferMode = false;
}
if (copyBufferMode) {
String caps = "width=" + reqWidth + ", height=" + reqHeight + ", " + copyMask;
natSink = new BufferDataAppSink("nat", caps,
new BufferDataAppSink.Listener() {
public void bufferFrame(int w, int h, Buffer buffer) {
invokeEvent(w, h, buffer);
}
});
natSink.setAutoDisposeBuffer(false);
// No need for rgbSink.dispose(), because the addMany() doesn't increment the
// refcount of the videoSink object.
gpipeline.addMany(gsource, natSink);
Element.linkMany(gsource, natSink);
}
}
}
natSink = new BufferDataAppSink("nat", caps,
new BufferDataAppSink.Listener() {
public void bufferFrame(int w, int h, Buffer buffer) {
invokeEvent(w, h, buffer);
}
});
if (!copyBufferMode) {
natSink.setAutoDisposeBuffer(false);
// No need for rgbSink.dispose(), because the addMany() doesn't increment the
// refcount of the videoSink object.
gpipeline.addMany(gsource, natSink);
Element.linkMany(gsource, natSink);
} else {
Element conv = ElementFactory.make("ffmpegcolorspace", "ColorConverter");
Element videofilter = ElementFactory.make("capsfilter", "ColorFilter");
@@ -833,7 +804,67 @@ public class Capture extends PImage implements PConstants {
}
}
////////////////////////////////////////////////////////////
// Buffer source interface.
/**
* Disables automatic use of hardware acceleration to play video for OpenGL-based
* renderers.
*
*/
public void noGL() {
useGLSink = false;
}
/**
* Sets the object to use as destination for the frames read from the stream.
* The color conversion mask is automatically set to the one required to
* copy the frames to OpenGL.
*
* @param Object dest
*/
public void setBufferSink(Object sink) {
bufferSink = sink;
initCopyMask();
}
/**
* Sets the object to use as destination for the frames read from the stream.
*
* @param Object dest
* @param String mask
*/
public void setBufferSink(Object sink, String mask) {
bufferSink = sink;
copyMask = mask;
}
public synchronized void disposeBuffer(Object buf) {
((Buffer)buf).dispose();
}
protected void getSinkMethods() {
try {
sinkCopyMethod = bufferSink.getClass().getMethod("copyBufferFromSource",
new Class[] { Object.class, ByteBuffer.class, int.class, int.class });
} catch (Exception e) {
throw new RuntimeException("Movie: provided sink object doesn't have a copyBufferFromSource method.");
}
try {
sinkSetMethod = bufferSink.getClass().getMethod("setBufferSource", new Class[] { Object.class });
sinkSetMethod.invoke(bufferSink, new Object[] { this });
} catch (Exception e) {
throw new RuntimeException("Movie: provided sink object doesn't have a setBufferSource method.");
}
}
protected void initCopyMask() {
if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
copyMask = "red_mask=(int)0xFF000000, green_mask=(int)0xFF0000, blue_mask=(int)0xFF00";
} else {
copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
}
}
}
@@ -66,10 +66,7 @@ public class Movie extends PImage implements PConstants {
protected PlayBin2 gplayer;
protected Method movieEventMethod;
protected Method copyBufferMethod;
protected Object eventHandler;
protected Object copyHandler;
protected boolean available;
protected boolean sinkReady;
@@ -78,14 +75,18 @@ public class Movie extends PImage implements PConstants {
protected RGBDataAppSink rgbSink = null;
protected int[] copyPixels = null;
protected BufferDataAppSink natSink = null;
protected Buffer natBuffer = null;
protected boolean copyBufferMode = false;
protected String copyMask;
protected boolean firstFrame = true;
protected boolean seeking = false;
protected boolean useBufferSink = false;
protected boolean useGLSink = true;
protected Object bufferSink;
protected Method sinkCopyMethod;
protected Method sinkSetMethod;
protected String copyMask;
protected Buffer natBuffer = null;
protected BufferDataAppSink natSink = null;
/**
* Creates an instance of GSMovie loading the movie from filename.
*
@@ -531,29 +532,36 @@ public class Movie extends PImage implements PConstants {
// pixels array, even if without any valid image inside.
loadPixels();
if (copyBufferMode) {
// The native buffer from gstreamer is copies to the destination object.
if (natBuffer == null || copyBufferMethod == null) {
if (useBufferSink) { // The native buffer from gstreamer is copied to the buffer sink.
if (natBuffer == null) {
return;
}
}
if (firstFrame) {
super.init(bufWidth, bufHeight, RGB);
super.init(bufWidth, bufHeight, ARGB);
loadPixels();
firstFrame = false;
}
if (bufferSink == null) {
Object cache = getCache(parent.g);
if (cache == null) {
return;
}
setBufferSink(cache);
getSinkMethods();
}
IntBuffer rgbBuffer = natBuffer.getByteBuffer().asIntBuffer();
ByteBuffer byteBuffer = natBuffer.getByteBuffer();
try {
copyBufferMethod.invoke(copyHandler, new Object[] { natBuffer, rgbBuffer, bufWidth, bufHeight });
sinkCopyMethod.invoke(bufferSink, new Object[] { natBuffer, byteBuffer, bufWidth, bufHeight });
} catch (Exception e) {
e.printStackTrace();
}
natBuffer = null;
} else {
// Normal operation mode: the pixels just read from gstreamer
// are copied to the pixels array.
} else { // The pixels just read from gstreamer are copied to the pixels array.
if (copyPixels == null) {
return;
}
@@ -620,33 +628,6 @@ public class Movie extends PImage implements PConstants {
}
}
/**
* Sets the object to use as destination for the frames read from the stream.
* The color conversion mask is automatically set to the one required to
* copy the frames to OpenGL.
*
* @param Object dest
*/
public void setPixelDest(Object dest) {
copyHandler = dest;
if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
copyMask = "red_mask=(int)0xFF000000, green_mask=(int)0xFF0000, blue_mask=(int)0xFF00";
} else {
copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
}
}
/**
* Sets the object to use as destination for the frames read from the stream.
*
* @param Object dest
* @param String mask
*/
public void setPixelDest(Object dest, String mask) {
copyHandler = dest;
copyMask = mask;
}
/**
* Uses a generic object as handler of the movie. This object should have a
* movieEvent method that receives a GSMovie argument. This method will
@@ -741,44 +722,31 @@ public class Movie extends PImage implements PConstants {
}
protected void initSink() {
if (copyHandler != null) {
try {
copyBufferMethod = copyHandler.getClass().getMethod("addPixelsToBuffer",
new Class[] { Object.class, IntBuffer.class, int.class, int.class });
copyBufferMode = true;
} catch (Exception e) {
// no such method, or an error.. which is fine, just ignore
copyBufferMode = false;
}
if (copyBufferMode) {
try {
Method meth = copyHandler.getClass().getMethod("setPixelSource", new Class[] { Object.class});
meth.invoke(copyHandler, new Object[] { this });
} catch (Exception e) {
copyBufferMode = false;
}
if (copyBufferMode) {
natSink = new BufferDataAppSink("nat", copyMask,
new BufferDataAppSink.Listener() {
public void bufferFrame(int w, int h, Buffer buffer) {
invokeEvent(w, h, buffer);
}
});
natSink.setAutoDisposeBuffer(false);
gplayer.setVideoSink(natSink);
// The setVideoSink() method sets the videoSink as a property of the PlayBin,
// which increments the refcount of the videoSink element. Disposing here once
// to decrement the refcount.
natSink.dispose();
}
}
}
if (bufferSink != null || (useGLSink && parent.g.isGL())) {
useBufferSink = true;
if (!copyBufferMode) {
if (bufferSink != null) {
getSinkMethods();
}
if (copyMask == null || copyMask.equals("")) {
initCopyMask();
}
natSink = new BufferDataAppSink("nat", copyMask,
new BufferDataAppSink.Listener() {
public void bufferFrame(int w, int h, Buffer buffer) {
invokeEvent(w, h, buffer);
}
});
natSink.setAutoDisposeBuffer(false);
gplayer.setVideoSink(natSink);
// The setVideoSink() method sets the videoSink as a property of the PlayBin,
// which increments the refcount of the videoSink element. Disposing here once
// to decrement the refcount.
natSink.dispose();
} else {
rgbSink = new RGBDataAppSink("rgb",
new RGBDataAppSink.Listener() {
public void rgbFrame(int w, int h, IntBuffer buffer) {
@@ -860,10 +828,6 @@ public class Movie extends PImage implements PConstants {
}
}
public synchronized void disposeBuffer(Object buf) {
((Buffer)buf).dispose();
}
protected void eosEvent() {
if (repeat) {
goToBeginning();
@@ -871,4 +835,68 @@ public class Movie extends PImage implements PConstants {
playing = false;
}
}
////////////////////////////////////////////////////////////
// Buffer source interface.
/**
* Disables automatic use of hardware acceleration to play video for OpenGL-based
* renderers.
*
*/
public void noGL() {
useGLSink = false;
}
/**
* Sets the object to use as destination for the frames read from the stream.
* The color conversion mask is automatically set to the one required to
* copy the frames to OpenGL.
*
* @param Object dest
*/
public void setBufferSink(Object sink) {
bufferSink = sink;
initCopyMask();
}
/**
* Sets the object to use as destination for the frames read from the stream.
*
* @param Object dest
* @param String mask
*/
public void setBufferSink(Object sink, String mask) {
bufferSink = sink;
copyMask = mask;
}
public synchronized void disposeBuffer(Object buf) {
((Buffer)buf).dispose();
}
protected void getSinkMethods() {
try {
sinkCopyMethod = bufferSink.getClass().getMethod("copyBufferFromSource",
new Class[] { Object.class, ByteBuffer.class, int.class, int.class });
} catch (Exception e) {
throw new RuntimeException("Movie: provided sink object doesn't have a copyBufferFromSource method.");
}
try {
sinkSetMethod = bufferSink.getClass().getMethod("setBufferSource", new Class[] { Object.class });
sinkSetMethod.invoke(bufferSink, new Object[] { this });
} catch (Exception e) {
throw new RuntimeException("Movie: provided sink object doesn't have a setBufferSource method.");
}
}
protected void initCopyMask() {
if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
copyMask = "red_mask=(int)0xFF000000, green_mask=(int)0xFF0000, blue_mask=(int)0xFF00";
} else {
copyMask = "red_mask=(int)0xFF, green_mask=(int)0xFF00, blue_mask=(int)0xFF0000";
}
}
}