I am creating a face detector app which will detect faces in real time and identify landmarks on faces. The landmarks for the faces are working perfectly fine, however my real time face detection isn't working at all.
I followed the instructions in Google's ML Kit(https://developers.google.com/ml-kit/vision/face-detection/android), but am really struggling to obtain the functionality in real time face detection.
In my debugger, the code crashes at facedetector.process(image).addOnSuccessListener() and instead goes into the onFailure()
This is my code for the realtime face detection part(I have commented some parts to make it readable + taken out the redundant parts at the end):
@Override
//process method to detect frame by frame in real time face detection
public void process(@NonNull Frame frame) {
int width = frame.getSize().getWidth();
int height = frame.getSize().getHeight();
InputImage image = InputImage.fromByteArray(
frame.getData(),
/* image width */width,
/* image height */height,
//if camera is facing front rotate image 90, else 270 degrees
(cameraFacing != Facing.FRONT) ? 90 : 270,
InputImage.IMAGE_FORMAT_YUV_420_888 // or IMAGE_FORMAT_YV12
);
FaceDetectorOptions faceDetectorOptions = new FaceDetectorOptions.Builder()
.setContourMode(FaceDetectorOptions.CONTOUR_MODE_ALL)
//setting contour mode to detect all facial contours in real time
.build();
FaceDetector faceDetector = FaceDetection.getClient(faceDetectorOptions);
faceDetector.process(image).addOnSuccessListener(new OnSuccessListener<List<Face>>() {
@Override
public void onSuccess(@NonNull List<Face> faces) {
imageView.setImageBitmap(null);
Bitmap bitmap = Bitmap.createBitmap(height, width, Bitmap.Config.ARGB_8888);
Canvas canvas = new Canvas(bitmap);
Paint dotPaint = new Paint();
dotPaint.setColor(Color.YELLOW);
dotPaint.setStyle(Paint.Style.FILL);
dotPaint.setStrokeWidth(6f);
Paint linePaint = new Paint();
linePaint.setColor(Color.GREEN);
linePaint.setStyle(Paint.Style.STROKE);
linePaint.setStrokeWidth(4f);
//looping through each face to detect each contour
for (Face face : faces) {
List<PointF> faceContours = face.getContour(
FaceContour.FACE
).getPoints();
for (int i = 0; i < faceContours.size(); i++) {
PointF faceContour = null;
if (i != (faceContours.size() - 1)) {
faceContour = faceContours.get(i);
canvas.drawLine(
faceContour.x, faceContour.y, faceContours.get(i + 1).x, faceContours.get(i + 1).y, linePaint
);
} else {//if at the last point, draw to the first point
canvas.drawLine(faceContour.x, faceContour.y, faceContours.get(0).x, faceContours.get(0).y, linePaint);
}
canvas.drawCircle(faceContour.x, faceContour.y, 4f, dotPaint);
}//end inner loop
List<PointF> leftEyebrowTopCountours = face.getContour(
FaceContour.LEFT_EYEBROW_TOP).getPoints();
for (int i = 0; i < leftEyebrowTopCountours.size(); i++) {
PointF leftEyebrowTopContour = leftEyebrowTopCountours.get(i);
if (i != (leftEyebrowTopCountours.size() - 1))
canvas.drawLine(leftEyebrowTopContour.x, leftEyebrowTopContour.y, leftEyebrowTopCountours.get(i + 1).x, leftEyebrowTopCountours.get(i + 1).y, linePaint);
canvas.drawCircle(leftEyebrowTopContour.x, leftEyebrowTopContour.y, 4f, dotPaint);
}
}
}
Side note: I am using Pixel 2 API 29 in my emulator. I left out the repetitive code since the rest is just going through a bunch of more Contours.
If someone could help to identify where I went wrong, I would really appreciate it. I've been struggling with this for several weeks now.
Full code for reference:
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.PointF;
import android.graphics.Typeface;
import android.net.Uri;
import android.os.Bundle;
import android.provider.MediaStore;
import android.view.View;
import android.widget.Button;
import android.widget.FrameLayout;
import android.widget.ImageView;
import android.widget.Toast;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AppCompatActivity;
import androidx.appcompat.widget.Toolbar;
import androidx.recyclerview.widget.LinearLayoutManager;
import androidx.recyclerview.widget.RecyclerView;
import com.google.android.gms.tasks.OnFailureListener;
import com.google.android.gms.tasks.OnSuccessListener;
import com.google.android.material.bottomsheet.BottomSheetBehavior;
import com.google.mlkit.vision.common.InputImage;
import com.google.mlkit.vision.face.Face;
import com.google.mlkit.vision.face.FaceContour;
import com.google.mlkit.vision.face.FaceDetection;
import com.google.mlkit.vision.face.FaceDetector;
import com.google.mlkit.vision.face.FaceDetectorOptions;
import com.google.mlkit.vision.face.FaceLandmark;
import com.otaliastudios.cameraview.CameraView;
import com.otaliastudios.cameraview.controls.Facing;
import com.otaliastudios.cameraview.frame.Frame;
import com.otaliastudios.cameraview.frame.FrameProcessor;
import com.theartofdev.edmodo.cropper.CropImage;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class MainActivity extends AppCompatActivity implements FrameProcessor {
private Facing cameraFacing = Facing.FRONT;
private ImageView imageView;
private CameraView faceDetectionCameraView;
private RecyclerView bottomSheetRecyclerView;
private BottomSheetBehavior bottomSheetBehavior;
private ArrayList<FaceDetectionModel> faceDetectionModels;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
Toolbar toolbar = findViewById(R.id.toolbar);
setSupportActionBar(toolbar);
faceDetectionModels = new ArrayList<>();
bottomSheetBehavior = BottomSheetBehavior.from(findViewById(R.id.bottom_sheet));
imageView = findViewById(R.id.face_detection_image_view);
faceDetectionCameraView = findViewById(R.id.face_detection_camera_view);
Button toggle = findViewById(R.id.face_detection_cam_toggle_button);
FrameLayout bottomSheetButton = findViewById(R.id.bottom_sheet_button);
bottomSheetRecyclerView = findViewById(R.id.bottom_sheet_recycler_view);
faceDetectionCameraView.setFacing(cameraFacing);
faceDetectionCameraView.setLifecycleOwner(MainActivity.this);
faceDetectionCameraView.addFrameProcessor(MainActivity.this);
toggle.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
cameraFacing = (cameraFacing == Facing.FRONT) ? Facing.BACK : Facing.FRONT;
faceDetectionCameraView.setFacing(cameraFacing);
}
});
bottomSheetButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
CropImage.activity().start(MainActivity.this);
}
});
bottomSheetRecyclerView.setLayoutManager(new LinearLayoutManager(MainActivity.this));
bottomSheetRecyclerView.setAdapter(new FaceDetectionAdapter(faceDetectionModels, MainActivity.this));
}
@Override
protected void onActivityResult(int requestCode, int resultCode, @Nullable Intent data) {
super.onActivityResult(requestCode, resultCode, data);
if(requestCode == CropImage.CROP_IMAGE_ACTIVITY_REQUEST_CODE){
CropImage.ActivityResult result = CropImage.getActivityResult(data);
if(resultCode == RESULT_OK){
Uri imageUri = result.getUri();
try {
analyseImage(MediaStore.Images.Media.getBitmap(getContentResolver(), imageUri));
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
private void analyseImage(Bitmap bitmap) {
if(bitmap == null){
Toast.makeText(this, "There was an error", Toast.LENGTH_SHORT).show();
}
//imageView.setImageBitmap(null);
faceDetectionModels.clear();
Objects.requireNonNull(bottomSheetRecyclerView.getAdapter()).notifyDataSetChanged();
bottomSheetBehavior.setState(BottomSheetBehavior.STATE_COLLAPSED);
showProgress();
InputImage firebaseInputImage = InputImage.fromBitmap(bitmap, 0);
FaceDetectorOptions options =
new FaceDetectorOptions.Builder()
.setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE)
.setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL)
.setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL)
.build();
FaceDetector faceDetector = FaceDetection.getClient(options);
faceDetector.process(firebaseInputImage)
.addOnSuccessListener(new OnSuccessListener<List<Face>>() {
@Override
public void onSuccess(@NonNull List<Face> faces) {
Bitmap mutableImage = bitmap.copy(Bitmap.Config.ARGB_8888, true);
detectFaces(faces, mutableImage);
imageView.setImageBitmap(mutableImage);
hideProgress();
bottomSheetRecyclerView.getAdapter().notifyDataSetChanged();
bottomSheetBehavior.setState(BottomSheetBehavior.STATE_EXPANDED);
}
})
.addOnFailureListener(new OnFailureListener() {
@Override
public void onFailure(@NonNull Exception e) {
Toast.makeText(MainActivity.this, "There was an error", Toast.LENGTH_SHORT).show();
hideProgress();
}
});
}
private void detectFaces(List<Face> faces, Bitmap bitmap) {
if(faces == null || bitmap == null) {
Toast.makeText(this, "There was an error", Toast.LENGTH_SHORT).show();
return;
}
Canvas canvas = new Canvas(bitmap);
Paint facePaint = new Paint();
facePaint.setColor(Color.GREEN);
facePaint.setStyle(Paint.Style.STROKE);
facePaint.setStrokeWidth(5f);
Paint faceTextPaint = new Paint();
faceTextPaint.setColor(Color.BLUE);
faceTextPaint.setTextSize(30f);
faceTextPaint.setTypeface(Typeface.SANS_SERIF);
Paint landmarkPaint = new Paint();
landmarkPaint.setColor(Color.YELLOW);
landmarkPaint.setStyle(Paint.Style.FILL);
landmarkPaint.setStrokeWidth(8f);
for(int i = 0; i < faces.size(); i++){
canvas.drawRect(faces.get(i).getBoundingBox(), facePaint);
canvas.drawText("Face" + i,
(faces.get(i).getBoundingBox().centerX()
-(faces.get(i).getBoundingBox().width() >> 1) + 8f),
(faces.get(i).getBoundingBox().centerY() + (faces.get(i).getBoundingBox().height() >> 1) - 8f), facePaint);
Face face = faces.get(i); //get one face
if(face.getLandmark(FaceLandmark.LEFT_EYE) != null){
FaceLandmark leftEye = face.getLandmark(FaceLandmark.LEFT_EYE);
//Now we have our left eye, we draw a little circle
canvas.drawCircle(leftEye.getPosition().x, leftEye.getPosition().y, 8f, landmarkPaint);
}
if(face.getLandmark(FaceLandmark.RIGHT_EYE) != null){
FaceLandmark rightEye = face.getLandmark(FaceLandmark.RIGHT_EYE);
//Now we have our left eye, we draw a little circle
canvas.drawCircle(rightEye.getPosition().x, rightEye.getPosition().y, 8f, landmarkPaint);
}
if(face.getLandmark(FaceLandmark.NOSE_BASE) != null){
FaceLandmark noseBase = face.getLandmark(FaceLandmark.NOSE_BASE);
//Now we have our left eye, we draw a little circle
canvas.drawCircle(noseBase.getPosition().x, noseBase.getPosition().y, 8f, landmarkPaint);
}
if(face.getLandmark(FaceLandmark.LEFT_EAR) != null){
FaceLandmark leftEar = face.getLandmark(FaceLandmark.LEFT_EAR);
//Now we have our left eye, we draw a little circle
canvas.drawCircle(leftEar.getPosition().x, leftEar.getPosition().y, 8f, landmarkPaint);
}
if(face.getLandmark(FaceLandmark.RIGHT_EAR) != null){
FaceLandmark rightEar = face.getLandmark(FaceLandmark.RIGHT_EAR);
//Now we have our left eye, we draw a little circle
canvas.drawCircle(rightEar.getPosition().x, rightEar.getPosition().y, 8f, landmarkPaint);
}
if(face.getLandmark(FaceLandmark.MOUTH_LEFT) != null && face.getLandmark(FaceLandmark.MOUTH_BOTTOM) != null && face.getLandmark(FaceLandmark.MOUTH_RIGHT) != null){
FaceLandmark mouthLeft = face.getLandmark(FaceLandmark.MOUTH_LEFT);
FaceLandmark mouthRight = face.getLandmark(FaceLandmark.MOUTH_RIGHT);
FaceLandmark mouthBottom = face.getLandmark(FaceLandmark.MOUTH_BOTTOM);
//Now we have our left eye, we draw a little circle
canvas.drawLine(mouthLeft.getPosition().x, mouthLeft.getPosition().y, mouthBottom.getPosition().x, mouthBottom.getPosition().y, landmarkPaint);
canvas.drawLine(mouthBottom.getPosition().x, mouthBottom.getPosition().y, mouthRight.getPosition().x, mouthRight.getPosition().y, landmarkPaint);
}
faceDetectionModels.add(new FaceDetectionModel(i, "Smiling probability"
+ face.getSmilingProbability()));
faceDetectionModels.add(new FaceDetectionModel(i, "Left eye open probability"
+ face.getLeftEyeOpenProbability()));
faceDetectionModels.add(new FaceDetectionModel(i, "Right eye open probability"
+ face.getRightEyeOpenProbability()));
}
}
private void showProgress() {
findViewById(R.id.bottom_sheet_button_img).setVisibility(View.GONE);
findViewById(R.id.bottom_sheet_butotn_progress_bar).setVisibility(View.VISIBLE);
}
private void hideProgress() {
findViewById(R.id.bottom_sheet_button_img).setVisibility(View.VISIBLE);
findViewById(R.id.bottom_sheet_butotn_progress_bar).setVisibility(View.GONE);
}
//real-time detection starts HERE
@Override
public void process(@NonNull Frame frame) {
//setting up width and frame height
int width = frame.getSize().getWidth();
int height = frame.getSize().getHeight();
byte[] byteArray = frame.getData();
InputImage image = InputImage.fromByteArray(
//frame.getData()
byteArray,
width,
height,
//rotation
(cameraFacing == Facing.FRONT) ? 90 : 270,
//image format
InputImage.IMAGE_FORMAT_YV12 // or IMAGE_FORMAT_YV12
);
//Contour mode all is real time contour detection
FaceDetectorOptions realTimeOpts = new FaceDetectorOptions.Builder()
.setContourMode(FaceDetectorOptions.CONTOUR_MODE_ALL)
.build();
FaceDetector faceDetector = FaceDetection.getClient(realTimeOpts);
faceDetector.process(image).addOnSuccessListener(new OnSuccessListener<List<Face>>() {
@Override
public void onSuccess(@NonNull List<Face> faces) {
//don't have image yet set to null first
imageView.setImageBitmap(null);
//bitmap stores pixels of image
Bitmap bitmap = Bitmap.createBitmap(height, width, Bitmap.Config.ARGB_8888);
//canvas hold the draw calls -- write into the bitmap
Canvas canvas = new Canvas(bitmap);
//paint specifies what the canvas should draw
Paint dotPaint = new Paint();
dotPaint.setColor(Color.YELLOW);
dotPaint.setStyle(Paint.Style.FILL);
dotPaint.setStrokeWidth(6f);
Paint linePaint = new Paint();
linePaint.setColor(Color.GREEN);
linePaint.setStyle(Paint.Style.STROKE);
linePaint.setStrokeWidth(4f);
for (Face face : faces) {
//fetching contours
List<PointF> faceContours = face.getContour(
FaceContour.FACE
).getPoints();
for (int i = 0; i < faceContours.size(); i++) {
PointF faceContour = faceContours.get(i);
if (i != (faceContours.size() - 1)) {
canvas.drawLine(
//if not at last index, continue drawing to next index
faceContour.x, faceContour.y, faceContours.get(i + 1).x, faceContours.get(i + 1).y, linePaint
);
} else {
return;
}
//always draw circle
canvas.drawCircle(faceContour.x, faceContour.y, 4f, dotPaint);
}//end inner loop
List<PointF> leftEyebrowTopCountours = face.getContour(
FaceContour.LEFT_EYEBROW_TOP).getPoints();
for (int i = 0; i < leftEyebrowTopCountours.size(); i++) {
PointF leftEyebrowTopContour = leftEyebrowTopCountours.get(i);
if (i != (leftEyebrowTopCountours.size() - 1)) {
canvas.drawLine(leftEyebrowTopContour.x, leftEyebrowTopContour.y, leftEyebrowTopCountours.get(i + 1).x, leftEyebrowTopCountours.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(leftEyebrowTopContour.x, leftEyebrowTopContour.y, 4f, dotPaint);
}
List<PointF> rightEyebrowTopCountours = face.getContour(
FaceContour.RIGHT_EYEBROW_TOP).getPoints();
for (int i = 0; i < rightEyebrowTopCountours.size(); i++) {
PointF rightEyebrowContour = rightEyebrowTopCountours.get(i);
if (i != (rightEyebrowTopCountours.size() - 1)) {
canvas.drawLine(rightEyebrowContour.x, rightEyebrowContour.y, rightEyebrowTopCountours.get(i + 1).x, rightEyebrowTopCountours.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(rightEyebrowContour.x, rightEyebrowContour.y, 4f, dotPaint);
}
List<PointF> rightEyebrowBottomCountours = face.getContour(
FaceContour.RIGHT_EYEBROW_BOTTOM).getPoints();
for (int i = 0; i < rightEyebrowBottomCountours.size(); i++) {
PointF rightEyebrowBottomContour = rightEyebrowBottomCountours.get(i);
if (i != (rightEyebrowBottomCountours.size() - 1)) {
canvas.drawLine(rightEyebrowBottomContour.x, rightEyebrowBottomContour.y, rightEyebrowBottomCountours.get(i + 1).x, rightEyebrowBottomCountours.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(rightEyebrowBottomContour.x, rightEyebrowBottomContour.y, 4f, dotPaint);
}
List<PointF> leftEyeContours = face.getContour(
FaceContour.LEFT_EYE).getPoints();
for (int i = 0; i < leftEyeContours.size(); i++) {
PointF leftEyeContour = leftEyeContours.get(i);
if (i != (leftEyeContours.size() - 1)) {
canvas.drawLine(leftEyeContour.x, leftEyeContour.y, leftEyeContours.get(i + 1).x, leftEyeContours.get(i + 1).y, linePaint);
} else {
return;
}
canvas.drawCircle(leftEyeContour.x, leftEyeContour.y, 4f, dotPaint);
}
List<PointF> rightEyeContours = face.getContour(
FaceContour.RIGHT_EYE).getPoints();
for (int i = 0; i < rightEyeContours.size(); i++) {
PointF rightEyeContour = rightEyeContours.get(i);
if (i != (rightEyeContours.size() - 1)) {
canvas.drawLine(rightEyeContour.x, rightEyeContour.y, rightEyeContours.get(i + 1).x, rightEyeContours.get(i + 1).y, linePaint);
} else {
return;
}
canvas.drawCircle(rightEyeContour.x, rightEyeContour.y, 4f, dotPaint);
}
List<PointF> upperLipTopContour = face.getContour(
FaceContour.UPPER_LIP_TOP).getPoints();
for (int i = 0; i < upperLipTopContour.size(); i++) {
PointF upperLipContour = upperLipTopContour.get(i);
if (i != (upperLipTopContour.size() - 1)) {
canvas.drawLine(upperLipContour.x, upperLipContour.y,
upperLipTopContour.get(i + 1).x,
upperLipTopContour.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(upperLipContour.x, upperLipContour.y, 4f, dotPaint);
}
List<PointF> upperLipBottomContour = face.getContour(
FaceContour.UPPER_LIP_BOTTOM).getPoints();
for (int i = 0; i < upperLipBottomContour.size(); i++) {
PointF upBottom = upperLipBottomContour.get(i);
if (i != (upperLipBottomContour.size() - 1)) {
canvas.drawLine(upBottom.x, upBottom.y, upperLipBottomContour.get(i + 1).x, upperLipBottomContour.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(upBottom.x, upBottom.y, 4f, dotPaint);
}
List<PointF> lowerLipTopContour = face.getContour(
FaceContour.LOWER_LIP_TOP).getPoints();
for (int i = 0; i < lowerLipTopContour.size(); i++) {
PointF lowerTop = lowerLipTopContour.get(i);
if (i != (lowerLipTopContour.size() - 1)) {
canvas.drawLine(lowerTop.x, lowerTop.y, lowerLipTopContour.get(i + 1).x, lowerLipTopContour.get(i + 1).y, linePaint);
}
else{
return;
}
canvas.drawCircle(lowerTop.x, lowerTop.y, 4f, dotPaint);
}
List<PointF> lowerLipBottomContour = face.getContour(
FaceContour.LOWER_LIP_BOTTOM).getPoints();
for (int i = 0; i < lowerLipBottomContour.size(); i++) {
PointF lowerBottom = lowerLipBottomContour.get(i);
if (i != (lowerLipBottomContour.size() - 1)) {
canvas.drawLine(lowerBottom.x, lowerBottom.y, lowerLipBottomContour.get(i + 1).x, lowerLipBottomContour.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(lowerBottom.x, lowerBottom.y, 4f, dotPaint);
}
List<PointF> noseBridgeContours = face.getContour(
FaceContour.NOSE_BRIDGE).getPoints();
for (int i = 0; i < noseBridgeContours.size(); i++) {
PointF noseBridge = noseBridgeContours.get(i);
if (i != (noseBridgeContours.size() - 1)) {
canvas.drawLine(noseBridge.x, noseBridge.y, noseBridgeContours.get(i + 1).x, noseBridgeContours.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(noseBridge.x, noseBridge.y, 4f, dotPaint);
}
List<PointF> noseBottomContours = face.getContour(
FaceContour.NOSE_BOTTOM).getPoints();
for (int i = 0; i < noseBottomContours.size(); i++) {
PointF noseBottom = noseBottomContours.get(i);
if (i != (noseBottomContours.size() - 1)) {
canvas.drawLine(noseBottom.x, noseBottom.y, noseBottomContours.get(i + 1).x, noseBottomContours.get(i + 1).y, linePaint);
}else{
return;
}
canvas.drawCircle(noseBottom.x, noseBottom.y, 4f, dotPaint);
//facing front flip image
if (cameraFacing == Facing.FRONT) {
//Flip image!
Matrix matrix = new Matrix();
matrix.preScale(-1f, 1f);
Bitmap flippedBitmap = Bitmap.createBitmap(bitmap, 0, 0,
bitmap.getWidth(), bitmap.getHeight(),
matrix, true);
imageView.setImageBitmap(flippedBitmap);
} else {
imageView.setImageBitmap(bitmap);
}
}//end outer loop
canvas.save();
}
}
}).addOnFailureListener(new OnFailureListener() {
@Override
public void onFailure (@NonNull Exception e){
imageView.setImageBitmap(null);
}
});
}
}
Thank you so much!!
from Why is Google's ML Face Detection Kit crashing on .process()
No comments:
Post a Comment