my current problem is, that I want to find a way to capture a frame/screenshot during a webrtc connection on Android. I know there are already some solutions to this here but none were working for me.
With my current approach I followed this Gist.
The problem is, that the it returns a black bitmap. I will append my approach but it basically is the same as the Gist. If anybody has any ideas how to solve this. Thanks in advance.
Activity SingleFrameCapturer.BitmapListener gotFrameListener = new SingleFrameCapturer.BitmapListener() {
@Override
public void gotBitmap(Bitmap theBitmap) {
Log.e(TAG, "got bitmap!");
ImageView imageView = findViewById(R.id.object_preview);
imageView.setImageBitmap(theBitmap);
imageView.setVisibility(View.VISIBLE);
}
};
MediaStream stream = contextManager.getStream();
SingleFrameCapturer.toBitmap(this, stream, gotFrameListener);
}
SingleFrameCapturer
import android.graphics.Bitmap;
import android.util.Base64;
import android.util.Log;
import java.nio.ByteBuffer;
import org.webrtc.VideoTrack;
import org.webrtc.MediaStream;
import org.webrtc.EglBase;
import org.webrtc.RendererCommon;
import javax.microedition.khronos.egl.EGL10;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.egl.EGLDisplay;
public class SingleFrameCapturer {
public interface BitmapListener {
public void gotBitmap(Bitmap theBitmap);
}
private static boolean firstTimeOnly = true;
// the below pixelBuffer code is based on from
// https://github.com/CyberAgent/android-gpuimage/blob/master/library/src/jp/co/cyberagent/android/gpuimage/PixelBuffer.java
//
class PixelBuffer implements org.webrtc.VideoRenderer.Callbacks {
final static String TAG = "PixelBuffer";
final static boolean LIST_CONFIGS = false;
int mWidth, mHeight;
EGL10 mEGL;
EGLDisplay mEGLDisplay;
boolean gotFrame = false;
String mThreadOwner;
BitmapListener listener;
android.app.Activity activity;
public PixelBuffer(android.app.Activity activity, BitmapListener listener) {
this.listener = listener;
this.activity = activity;
}
private static final String VERTEX_SHADER_STRING =
"varying vec2 interp_tc;\n"
+ "attribute vec4 in_pos;\n"
+ "attribute vec4 in_tc;\n"
+ "\n"
+ "uniform mat4 texMatrix;\n"
+ "\n"
+ "void main() {\n"
+ " gl_Position = in_pos;\n"
+ " interp_tc = (texMatrix * in_tc).xy;\n"
+ "}\n";
@Override
public void renderFrame(final org.webrtc.VideoRenderer.I420Frame i420Frame) {
Log.d(TAG, "entered renderFrame");
//
// we only want to grab a single frame but our method may get called
// a few times before we're done.
//
if (gotFrame || i420Frame.width == 0 || i420Frame.height == 0) {
Log.d(TAG, "Already got frame so taking honourable exit");
org.webrtc.VideoRenderer.renderFrameDone(i420Frame);
return;
}
activity.runOnUiThread(new Runnable() {
public void run() {
int width = i420Frame.width;
int height = i420Frame.height;
Log.d(TAG, "about to call initWithSize");
initWithSize(width, height);
Bitmap bitmap = toBitmap(i420Frame);
org.webrtc.VideoRenderer.renderFrameDone(i420Frame);
gotFrame = true;
listener.gotBitmap(bitmap);
destroy();
}
});
}
private int buildARGB(int r, int g, int b) {
return (0xff << 24) |(r << 16) | (g << 8) | b;
}
private Bitmap toBitmap(org.webrtc.VideoRenderer.I420Frame frame) {
if (frame.yuvFrame) {
//EglBase eglBase = EglBase.create();
EglBase eglBase = StreamActivity.rootEglBase;
if(firstTimeOnly) {
eglBase.createDummyPbufferSurface();
firstTimeOnly = false;
}
eglBase.makeCurrent();
TextureToRGB textureToRGB = new TextureToRGB();
int numPixels = mWidth *mHeight;
final int bytesPerPixel = 4;
ByteBuffer framebuffer = ByteBuffer.allocateDirect(numPixels*bytesPerPixel);
final float frameAspectRatio = (float) frame.rotatedWidth() / (float) frame.rotatedHeight();
final float[] rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
final float[] layoutMatrix = RendererCommon.getLayoutMatrix(
false, frameAspectRatio, (float) mWidth / mHeight);
final float[] texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
textureToRGB.convert(framebuffer, mWidth, mHeight, frame.textureId, texMatrix);
byte [] frameBytes = framebuffer.array();
int [] dataARGB = new int[numPixels];
for(int i = 0, j = 0; j < numPixels; i+=bytesPerPixel, j++) {
//
// data order in frameBytes is red, green, blue, alpha, red, green, ....
//
dataARGB[j] = buildARGB(frameBytes[i] & 0xff,frameBytes[i+1] &0xff,frameBytes[i+2] &0xff);
}
Bitmap bitmap = Bitmap.createBitmap(dataARGB, mWidth, mHeight, Bitmap.Config.ARGB_8888);
return bitmap;
}
else {
return null;
}
}
private void initWithSize(final int width, final int height) {
mWidth = width;
mHeight = height;
// Record thread owner of OpenGL context
mThreadOwner = Thread.currentThread().getName();
}
public void destroy() {
}
private int getConfigAttrib(final EGLConfig config, final int attribute) {
int[] value = new int[1];
return mEGL.eglGetConfigAttrib(mEGLDisplay, config,
attribute, value) ? value[0] : 0;
}
}
final private static String TAG = "SingleFrameCapturer";
org.webrtc.VideoRenderer renderer;
private SingleFrameCapturer(final android.app.Activity activity, MediaStream mediaStream, final BitmapListener gotFrameListener) {
if( mediaStream.videoTracks.size() == 0) {
Log.e(TAG, "No video track to capture from");
return;
}
final VideoTrack videoTrack = mediaStream.videoTracks.get(0);
final PixelBuffer vg = new PixelBuffer(activity, new BitmapListener() {
@Override
public void gotBitmap(final Bitmap bitmap) {
activity.runOnUiThread(new Runnable(){
public void run() {
videoTrack.removeRenderer(renderer);
try {
gotFrameListener.gotBitmap(bitmap);
} catch( Exception e1) {
Log.e(TAG, "Exception in gotBitmap callback:" + e1.getMessage());
e1.printStackTrace(System.err);
}
}
});
}
});
renderer = new org.webrtc.VideoRenderer(vg);
videoTrack.addRenderer(renderer);
}
/**
* This constructor builds an object which captures a frame from mediastream to a Bitmap.
* @param mediaStream The input media mediaStream.
* @param gotFrameListener A callback which will receive the Bitmap.
*/
public static void toBitmap(android.app.Activity activity, MediaStream mediaStream, final BitmapListener gotFrameListener) {
new SingleFrameCapturer(activity, mediaStream, gotFrameListener);
}
/**
* This method captures a frame from the supplied media stream to a jpeg file written to the supplied outputStream.
* @param mediaStream the source media stream
* @param quality the quality of the jpeq 0 to 100.
* @param outputStream the output stream the jpeg file will be written to.
* @param done a runnable that will be invoked when the outputstream has been written to.
* @return The frame capturer. You should keep a reference to the frameCapturer until the done object is invoked.
*/
public static void toOutputStream(android.app.Activity activity, MediaStream mediaStream, final int quality, final java.io.OutputStream outputStream, final Runnable done) {
BitmapListener gotFrameListener = new BitmapListener() {
@Override
public void gotBitmap(Bitmap theBitmap) {
theBitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream);
try {
done.run();
} catch( Exception e1) {
Log.e(TAG, "Exception in toOutputStream done callback:" + e1.getMessage());
e1.printStackTrace(System.err);
}
}
};
toBitmap(activity, mediaStream, gotFrameListener);
}
/**
* This method captures a frame from the supplied mediastream to a dataurl written to a StringBuilder.
* @param mediaStream the source media stream
* @param quality the quality of the jpeq 0 to 100.
* @param output a StringBuilder which will be the recipient of the dataurl.
* @param done a runnable that will be invoked when the dataurl is built.
* @return The frame capturer. You should keep a reference to the frameCapturer until the done object is invoked.
*/
public static void toDataUrl(android.app.Activity activity, MediaStream mediaStream, final int quality, final StringBuilder output, final Runnable done) {
final java.io.ByteArrayOutputStream outputStream = new java.io.ByteArrayOutputStream();
Runnable convertToUrl = new Runnable() {
@Override
public void run() {
output.append("data:image/jpeg;base64,");
output.append(Base64.encodeToString(outputStream.toByteArray(), Base64.DEFAULT));
try {
done.run();
} catch( Exception e1) {
Log.e(TAG, "Exception in toDataUrl done callback:" + e1.getMessage());
e1.printStackTrace(System.err);
}
}
};
toOutputStream(activity, mediaStream, quality, outputStream, convertToUrl);
}
}
TextureToRGB
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
import org.webrtc.*;
/**
* Class for converting OES textures RGBA. It should be constructed on a thread with
* an active EGL context, and only be used from that thread. It is used by the EasyrtcSingleFrameCapturer.
*/
public class TextureToRGB {
// Vertex coordinates in Normalized Device Coordinates, i.e.
// (-1, -1) is bottom-left and (1, 1) is top-right.
private static final FloatBuffer DEVICE_RECTANGLE = GlUtil.createFloatBuffer(new float[] {
-1.0f, -1.0f, // Bottom left.
1.0f, -1.0f, // Bottom right.
-1.0f, 1.0f, // Top left.
1.0f, 1.0f, // Top right.
});
// Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right.
private static final FloatBuffer TEXTURE_RECTANGLE = GlUtil.createFloatBuffer(new float[] {
0.0f, 0.0f, // Bottom left.
1.0f, 0.0f, // Bottom right.
0.0f, 1.0f, // Top left.
1.0f, 1.0f // Top right.
});
private static final String VERTEX_SHADER =
"varying vec2 interp_tc;\n"
+ "attribute vec4 in_pos;\n"
+ "attribute vec4 in_tc;\n"
+ "\n"
+ "uniform mat4 texMatrix;\n"
+ "\n"
+ "void main() {\n"
+ " gl_Position = in_pos;\n"
+ " interp_tc = (texMatrix * in_tc).xy;\n"
+ "}\n";
private static final String FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "varying vec2 interp_tc;\n"
+ "\n"
+ "uniform samplerExternalOES oesTex;\n"
+ "\n"
+ "void main() {\n"
+ " gl_FragColor = texture2D(oesTex, interp_tc);\n"
+ "}\n";
// clang-format on
private final GlTextureFrameBuffer textureFrameBuffer;
private final GlShader shader;
private final int texMatrixLoc;
private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker();
private boolean released = false;
/**
* This class should be constructed on a thread that has an active EGL context.
*/
public TextureToRGB() {
threadChecker.checkIsOnValidThread();
textureFrameBuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER);
shader.useProgram();
texMatrixLoc = shader.getUniformLocation("texMatrix");
GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0);
GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values.");
// Initialize vertex shader attributes.
shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE);
// If the width is not a multiple of 4 pixels, the texture
// will be scaled up slightly and clipped at the right border.
shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE);
}
public void convert(ByteBuffer buf, int width, int height, int srcTextureId,
float[] transformMatrix) {
threadChecker.checkIsOnValidThread();
if (released) {
throw new IllegalStateException("TextureToRGB.convert called on released object");
}
int size = width * height;
if (buf.capacity() < size) {
throw new IllegalArgumentException("TextureToRGB.convert called with too small buffer");
}
// Produce a frame buffer starting at top-left corner, not
// bottom-left.
transformMatrix =
RendererCommon.multiplyMatrices(transformMatrix, RendererCommon.verticalFlipMatrix());
final int frameBufferWidth = width;
final int frameBufferHeight =height;
textureFrameBuffer.setSize(frameBufferWidth, frameBufferHeight);
// Bind our framebuffer.
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, textureFrameBuffer.getFrameBufferId());
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, srcTextureId);
GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0);
GLES20.glViewport(0, 0, width, height);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
GLES20.glReadPixels(
0, 0, frameBufferWidth, frameBufferHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf);
// Restore normal framebuffer.
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
// Unbind texture. Reportedly needed on some devices to get
// the texture updated from the camera.
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
GlUtil.checkNoGLES2Error("TextureToRGB.convert");
}
public void release() {
threadChecker.checkIsOnValidThread();
released = true;
shader.release();
textureFrameBuffer.release();
}
}
from WebRTC, Capture Screen
