ffmpeg+opencv从视频中获取人脸工具javaCV

文章字数：827,阅读全文大约需要3分钟

• maven

      <dependency>
	<groupId>org.bytedeco</groupId>
	<artifactId>javacpp</artifactId>
	<version>1.5.5</version>
</dependency>
<dependency>
	<groupId>org.bytedeco</groupId>
	<artifactId>javacv</artifactId>
	<version>1.5.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.bytedeco/ffmpeg -->
<dependency>
	<groupId>org.bytedeco</groupId>
	<artifactId>ffmpeg</artifactId>
	<version>4.3.2-1.5.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.bytedeco/ffmpeg-platform -->
<dependency>
	<groupId>org.bytedeco</groupId>
	<artifactId>ffmpeg-platform</artifactId>
	<version>4.3.2-1.5.5</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.bytedeco/opencv -->
<dependency>
	<groupId>org.bytedeco</groupId>
	<artifactId>opencv</artifactId>
	<version>4.5.1-1.5.5</version>
</dependency>
<dependency>
	<groupId>org.bytedeco</groupId>
	<artifactId>opencv-platform</artifactId>
	<version>4.5.1-1.5.5</version>
</dependency>

package org.example.util;

import org.bytedeco.ffmpeg.*;
import org.bytedeco.ffmpeg.global.avcodec;
import org.bytedeco.javacv.*;
import org.bytedeco.javacv.Frame;
import org.bytedeco.opencv.*;
import org.opencv.core.*;
import org.opencv.core.Point;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.opencv.objdetect.CascadeClassifier;

import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.*;
import java.util.*;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;

/**
 * 视频工具
 *
 * @author colin.cheng
 * @since 1.0.0
 */
public class VidUtil {
    /** 人脸分类器地址 */
    static final String HAARCASCADES = "F:\\test\\vid\\haarcascade_frontalface_alt_tree.xml";
    /** 分类器 */
    static CascadeClassifier cascade;

    static {
        new opencv_java();
        new ffmpeg();
        cascade = new CascadeClassifier(HAARCASCADES);
    }

    /**
     * 根据视频获取帧信息
     *
     * @param vidPath 地址
     * @param interval 获取帧的间隔
     * @param doFunc 每帧如何操作
     * @param indexFunc 当前执行帧数回调
     * @param totalFunc 总帧数回调
     * @throws FrameGrabber.Exception
     */
    public static void getFrameMatByVideo(String vidPath, int interval, Consumer<Mat> doFunc, Consumer<Integer> indexFunc, Consumer<Integer> totalFunc) throws FrameGrabber.Exception {
        try(FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(vidPath)) {
            fFmpegFrameGrabber.start();
            int totalIndex = fFmpegFrameGrabber.getLengthInFrames();
            if(totalFunc != null) {
                totalFunc.accept(totalIndex);
            }
            Frame frame;
            final OpenCVFrameConverter.ToOrgOpenCvCoreMat converter = new OpenCVFrameConverter.ToOrgOpenCvCoreMat();
            for (int i = 0; i < totalIndex; i += interval) {
                if(indexFunc != null) {
                    indexFunc.accept(i);
                }
                fFmpegFrameGrabber.setFrameNumber(i);
                frame = fFmpegFrameGrabber.grabImage();
                doFunc.accept(converter.convert(frame));
            }
        }
    }

    /**
     * 根据视频获取帧信息
     *
     * @param vidPath 地址
     * @param interval 获取帧的间隔
     * @param doFunc 每帧如何操作
     * @throws FrameGrabber.Exception
     */
    public static void getFrameMatByVideo(String vidPath, int interval, Consumer<Mat> doFunc) throws FrameGrabber.Exception {
        getFrameMatByVideo(vidPath, interval, doFunc, null, null);
    }

    /**
     * 将Mat保存成图片
     *
     * @param path 图片路径 + 名称
     * @param mat Mat信息
     */
    public static void saveMatAsImg(String path, Mat mat) {
        Imgcodecs.imwrite(path, mat);
    }

    /**
     * 根据图片获取Mat
     *
     * @param path
     * @return
     */
    public static Mat getMatFromImg(String path) {
        return Imgcodecs.imread(path);
    }

    /**
     * 复制Mat
     *
     * @param from
     * @return
     */
    public static Mat copy(Mat from) {
        Mat to = new Mat();
        from.copyTo(to);
        return to;
    }

    /**
     * 检测人脸，并返回位置
     *
     * @param srcImage
     * @return
     */
    public static MatOfRect faceDetect(Mat srcImage) {
        Mat grayImage = new Mat();
        Imgproc.cvtColor(srcImage, grayImage, Imgproc.COLOR_BGR2GRAY);
        MatOfRect faceDetections = new MatOfRect();
        cascade.detectMultiScale(grayImage, faceDetections);
        return faceDetections;
    }

    /**
     * 三帧法检测运动区域
     * @param previousFrame 第一帧
     * @param currentFrame 第二帧
     * @param frame3 第三帧
     * @return 运动区域
     */
    public static Mat mobileMonitoring(Mat previousFrame, Mat currentFrame, Mat frame3) {
        Imgproc.cvtColor(previousFrame, previousFrame, Imgproc.COLOR_BGR2GRAY);
        Imgproc.cvtColor(currentFrame, currentFrame, Imgproc.COLOR_BGR2GRAY);
        Imgproc.cvtColor(frame3, frame3, Imgproc.COLOR_BGR2GRAY);

        Mat difFrame = new Mat(), difFrame2 = new Mat(), difFrame3 = new Mat(), tempFrame = new Mat();

        Core.absdiff(currentFrame, previousFrame, difFrame);
        Core.absdiff(previousFrame, frame3, difFrame2);
        Core.bitwise_and(difFrame, difFrame2, difFrame3);

        Imgproc.threshold(difFrame3, tempFrame, 20, 255.0, Imgproc.THRESH_BINARY);
        Imgproc.dilate(tempFrame, tempFrame, new Mat());
        Imgproc.erode(tempFrame, tempFrame, new Mat());

        return tempFrame;
    }

    /**
     * 三帧法运动检测
     * @param previousFrame
     * @param currentFrame
     * @param frame3
     * @return
     */
    public static boolean isMove(Mat previousFrame, Mat currentFrame, Mat frame3) {
        final ArrayList<MatOfPoint> list = new ArrayList<>();
        Mat hierarchy = new Mat();
        Mat moveMat = mobileMonitoring(previousFrame, currentFrame, frame3);
        /**
         * mode:
         * RETR_EXTERNAL	只检测最外围的轮廓。
         * RETR_LIST	检测所有轮廓，不建立等级关系，彼此独立。
         * RETR_CCOMP	检测所有轮廓，但所有轮廓都只建立两个等级关系 。
         * RETR_TREE	检测所有轮廓，并且所有轮廓建立一个树结构，层次完整。
         * RETR_FLOODFILL	洪水填充法
         *
         * method:
         * CHAIN_APPROX_NONE 	保存物体边界上所有连续的轮廓点
         * CHAIN_APPROX_SIMPLE 	压缩水平方向，垂直方向，对角线方向的元素，只保留该方向的终点坐标，例如一个矩形轮廓只需4个点来保存轮廓信息
         * CV_CHAIN_APPROX_TC89_L1	使用Teh-Chin 链近似算法
         * CV_CHAIN_APPROX_TC89_KCOS	使用Teh-Chin 链近似算法
         */
        Imgproc.findContours(moveMat, list, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
        for (MatOfPoint point : list) {
            final double contour = Imgproc.contourArea(point);
            if(contour > 1000) {
                return true;
            }
        }
        return false;
    }

    /**
     * 三帧法运动检测并画框
     * @param previousFrame
     * @param currentFrame
     * @param frame3
     * @param path
     */
    public static void moveRectangle(Mat previousFrame, Mat currentFrame, Mat frame3, String path) {
        Mat baseFrame = new Mat();
        currentFrame.copyTo(baseFrame);
        final ArrayList<MatOfPoint> list = new ArrayList<>();
        Mat hierarchy = new Mat();
        Mat moveMat = mobileMonitoring(previousFrame, currentFrame, frame3);
        Imgproc.findContours(moveMat, list, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
        for (MatOfPoint point : list) {
            final MatOfPoint2f matOfPoint2f = new MatOfPoint2f(point.toArray());
            final RotatedRect rotatedRect = Imgproc.minAreaRect(matOfPoint2f);
            final Rect rect = rotatedRect.boundingRect();
            final double contour = Imgproc.contourArea(point);
            if(contour > 1000) {
                Imgproc.rectangle(baseFrame, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height),new Scalar(0, 0, 255), 2);
            }
        }
        saveMatAsImg(path, baseFrame);
    }

    /**
     * 人脸画框
     *
     * @param srcImage
     */
    public static Mat faceRectangle(Mat srcImage) {
        MatOfRect faceDetections = faceDetect(srcImage);
        Mat dstImage = new Mat();
        srcImage.copyTo(dstImage);
        for (Rect rect : faceDetections.toArray()) {
            Imgproc.rectangle(dstImage, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height),new Scalar(0, 0, 255), 2);
        }
        return dstImage;
    }

    /**
     * 切割人脸
     *
     * @param srcImage
     * @return
     */
    public static List<Mat> cutFace(Mat srcImage) {
        Rect[] rectArr = faceDetect(srcImage).toArray();
        final LinkedList<Mat> res = new LinkedList<>();
        if(rectArr.length > 0) {
            for (Rect rect : rectArr) {
                res.add(new Mat(srcImage, rect));
            }
        }
        return res;
    }

    /**
     * 切割第一张人脸
     *
     * @param srcImage
     * @return
     */
    public static Mat cutFirstFace(Mat srcImage) {
        Rect[] rectArr = faceDetect(srcImage).toArray();
        if(rectArr.length > 0) {
            return new Mat(srcImage, rectArr[0]);
        }
        return null;
    }


    /**
     * 使用直方图比对照片
     * @param img1
     * @param img2
     * @return
     */
    public static int compareHistogram(Mat img1, Mat img2) {
        int retVal = 0;
        if (null != img1 && null != img2) {
            Mat hsvImg1 = new Mat();
            Mat hsvImg2 = new Mat();
            // Convert to HSV 转换为彩色模型
            Imgproc.cvtColor(img1, hsvImg1, Imgproc.COLOR_BGR2HSV);
            Imgproc.cvtColor(img2, hsvImg2, Imgproc.COLOR_BGR2HSV);
            // Set configuration for calchist()
            List<Mat> listImg1 = new ArrayList<Mat>();
            List<Mat> listImg2 = new ArrayList<Mat>();
            listImg1.add(hsvImg1);
            listImg2.add(hsvImg2);
            MatOfFloat ranges = new MatOfFloat(0, 255);
            MatOfInt histSize = new MatOfInt(50);
            MatOfInt channels = new MatOfInt(0);
            // Histograms 直方图
            Mat histImg1 = new Mat();
            Mat histImg2 = new Mat();
            // Calculate the histogram for the HSV imgaes
            // 计算HSV imgaes的直方图。
            Imgproc.calcHist(listImg1, channels, new Mat(), histImg1, histSize, ranges);
            Imgproc.calcHist(listImg2, channels, new Mat(), histImg2, histSize, ranges);
            Core.normalize(histImg1, histImg1, 0, 1, Core.NORM_MINMAX, -1, new Mat());
            Core.normalize(histImg2, histImg2, 0, 1, Core.NORM_MINMAX, -1, new Mat());
            // Apply the histogram comparison methods
            // 0 - correlation: the higher the metric, the more accurate the
            // match
            // "> 0.9"
            // 1 - chi-square: the lower the metric, the more accurate the match
            // "<
            // 0.1"
            // 2 - intersection: the higher the metric, the more accurate the
            // match
            // "> 1.5"
            // 3 - bhattacharyya: the lower the metric, the more accurate the
            // match
            // "< 0.3"
            double result0, result1, result2, result3;
            result0 = Imgproc.compareHist(histImg1, histImg2, 0);
            result1 = Imgproc.compareHist(histImg1, histImg2, 1);
            result2 = Imgproc.compareHist(histImg1, histImg2, 2);
            result3 = Imgproc.compareHist(histImg1, histImg2, 3);
            // If the count that it is satisfied with the condition is over 3,
            // two
            // images is same.
            int count = 0;
            if (result0 > 0.9) {
                count++;
            }
            if (result1 < 0.1) {
                count++;
            }
            if (result2 > 1.5) {
                count++;
            }
            if (result3 < 0.3) {
                count++;
            }
            // 对相似度进行评判
            if (count >= 3) {
                retVal = 1;
            }
        }

        return retVal;
    }

    /**
     * Mat转换成BufferedImage
     *
     * @param matrix
     *            要转换的Mat
     * @param fileExtension
     *            格式为 ".jpg", ".png", etc
     * @return
     */
    public static BufferedImage Mat2BufImg (Mat matrix, String fileExtension) {
        MatOfByte mob = new MatOfByte();
        Imgcodecs.imencode(fileExtension, matrix, mob);
        byte[] byteArray = mob.toArray();
        BufferedImage bufImage = null;
        try {
            InputStream in = new ByteArrayInputStream(byteArray);
            bufImage = ImageIO.read(in);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return bufImage;
    }

    /**
     * BufferedImage转换成Mat
     *
     * @param original
     *            要转换的BufferedImage
     * @param imgType
     *            bufferedImage的类型 如 BufferedImage.TYPE_3BYTE_BGR
     * @param matType
     *            转换成mat的type 如 CvType.CV_8UC3
     */
    public static Mat BufImg2Mat (BufferedImage original, int imgType, int matType) {
        if (original == null) {
            throw new IllegalArgumentException("original == null");
        }
        // Don't convert if it already has correct type
        if (original.getType() != imgType) {
            // Create a buffered image
            BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), imgType);
            // Draw the image onto the new buffer
            Graphics2D g = image.createGraphics();
            try {
                g.setComposite(AlphaComposite.Src);
                g.drawImage(original, 0, 0, null);
            } finally {
                g.dispose();
            }
        }
        byte[] pixels = ((DataBufferByte) original.getRaster().getDataBuffer()).getData();
        Mat mat = Mat.eye(original.getHeight(), original.getWidth(), matType);
        mat.put(0, 0, pixels);
        return mat;
    }



    public static void main(String[] args) {
        VidUtil util = new VidUtil();
        util.getMove("F:\\test\\vid\\2.mp4", "F:\\test\\vid\\img\\res.flv");

    }

    /**
     * 提取移动的信息
     * @param vidPath
     * @param outputFile
     */
    private void getMove(String vidPath, String outputFile) {
        try(FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(vidPath)) {
            fFmpegFrameGrabber.start();

            FrameRecorder recorder = FrameRecorder.createDefault(outputFile, fFmpegFrameGrabber.getImageWidth(), fFmpegFrameGrabber.getImageHeight());
            recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
            recorder.setFormat("flv");
            recorder.setFrameRate(fFmpegFrameGrabber.getFrameRate());
            recorder.start();

            final OpenCVFrameConverter.ToOrgOpenCvCoreMat converter = new OpenCVFrameConverter.ToOrgOpenCvCoreMat();
            Mat frame1 = null, frame2 = null;
            final int total = fFmpegFrameGrabber.getLengthInFrames();
            for (int i = 0; i < total; i++) {
                System.out.println(i + "/" + total);
                Frame frame = fFmpegFrameGrabber.grab();
                Mat mat = converter.convert(frame);

                if(frame1 == null) {
                    frame1 = new Mat();
                    mat.copyTo(frame1);
                } else if(frame2 == null) {
                    frame2 = new Mat();
                    mat.copyTo(frame2);
                } else {
                    Mat f1 = new Mat(), f2 = new Mat(), f3 = new Mat();
                    frame1.copyTo(f1);
                    frame2.copyTo(f2);
                    mat.copyTo(f3);
                    boolean move = VidUtil.isMove(f1, f2, f3);
                    if(move) {
                        recorder.record(frame);
                    }
                    frame2.copyTo(frame1);
                    mat.copyTo(frame2);
                }
            }
            recorder.stop();
            recorder.release();
            fFmpegFrameGrabber.stop();
            fFmpegFrameGrabber.release();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    
    /**
     * 推送rmtp服务器
     * 
     * @param outputFile 服务器地址
     * @param frameRate
     * @throws Exception
     * @throws InterruptedException
     * @throws org.bytedeco.javacv.FrameRecorder.Exception
     */
    public static void recordCamera(String outputFile, double frameRate)throws Exception {
        Loader.load(opencv_objdetect.class);
        FrameGrabber grabber = FrameGrabber.createDefault(0);
        grabber.start();//开启抓取器

        OpenCVFrameConverter.ToIplImage converter = new OpenCVFrameConverter.ToIplImage();
        IplImage grabbedImage = converter.convert(grabber.grab());
        int width = grabbedImage.width();
        int height = grabbedImage.height();

        FrameRecorder recorder = FrameRecorder.createDefault(outputFile, width, height);
        recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
        recorder.setFormat("flv");
        recorder.setFrameRate(frameRate);

        recorder.start();//开启录制器
        long startTime=0;
        long videoTS=0;
        CanvasFrame frame = new CanvasFrame("camera", CanvasFrame.getDefaultGamma() / grabber.getGamma());
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        frame.setAlwaysOnTop(true);
        Frame rotatedFrame=converter.convert(grabbedImage);
        while (frame.isVisible() && (grabbedImage = converter.convert(grabber.grab())) != null) {
            rotatedFrame = converter.convert(grabbedImage);
            frame.showImage(rotatedFrame);
            if (startTime == 0) {
                startTime = System.currentTimeMillis();
            }
            videoTS = 1000 * (System.currentTimeMillis() - startTime);
            recorder.setTimestamp(videoTS);
            recorder.record(rotatedFrame);
            Thread.sleep(40);
        }
        frame.dispose();
        recorder.stop();
        recorder.release();
        grabber.stop();
    }
}

---