之前初步做好了相机预览的功能，但回头越想越不对劲，预览的延时感觉有点高了，于是切出系统的相机应用对比了一下：
可以看到系统的相机都跑到26.55了，我开发的相机才跑到26.42，慢了一百多毫秒，感觉需要狠狠的优化一下。
稍微分析了一下，感觉那段NV12转RGB的算法问题最大，不过我自己也不是什么算法工程师，还是直接用别人的吧，直接上OpenCV！
是这三个包，尤其时OpenCvSharp4.runtime.win，很容易被忽略。
改一下数据转换的代码
private void Reader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{var frame = sender.TryAcquireLatestFrame()?.VideoMediaFrame?.SoftwareBitmap;if (frame is null) return;//创建缓冲using var lockbuf = frame.LockBuffer(Windows.Graphics.Imaging.BitmapBufferAccessMode.Read);if (lockbuf is null) return;using var n = lockbuf.CreateReference();var buf = new Windows.Storage.Streams.Buffer(n.Capacity);//读取数据frame.CopyToBuffer(buf);using var reader = Windows.Storage.Streams.DataReader.FromBuffer(buf);var datas = new byte[buf.Length];reader.ReadBytes(datas);//NV12转Rbg24var nv12 = new Mat(frame.PixelHeight * 3 / 2, frame.PixelWidth, MatType.CV_8UC1);var bgr = new Mat();Marshal.Copy(datas, 0, nv12.Data, datas.Length);Cv2.CvtColor(nv12, bgr, ColorConversionCodes.YUV2BGR_NV12);Dispatcher.InvokeAsync(() => RendererFrame(bgr, frame.PixelWidth, frame.PixelHeight));
}
图片数据的写入也顺便改了
private void RendererFrame()
{FrameSource.Dispatcher.InvokeAsync(() =>{if (FrameSource.Source is not WriteableBitmap bmp || bmp.PixelWidth != FrameWidth || bmp.PixelHeight != FrameHeight){FrameSource.Source = new WriteableBitmap(FrameWidth, FrameHeight, 96, 96, PixelFormats.Bgr24, null);return;}WriteableBitmapConverter.ToWriteableBitmap(FrameBgr, bmp);});
}
可以看到，简单的改动后从之前的一百多毫秒的差距降到了三十，算是比较成功的优化。
不过既然还有三十毫米的差距，那就努努力把它平掉。
经过仔细分析后，认为每一帧都new几个实体开销也挺大的，尝试复用：
/// <summary>
/// 帧数据缓冲区
/// </summary>
Windows.Storage.Streams.Buffer FrameBuf { get; set; }
/// <summary>
/// 帧数据
/// </summary>
public byte[] FrameData { get; private set; }
/// <summary>
/// NV12帧数据
/// </summary>
Mat FrameNV12 { get; set; }
/// <summary>
/// Bgr帧数据
/// </summary>
Mat FrameBgr { get; set; }private void Reader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{var frame = sender.TryAcquireLatestFrame()?.VideoMediaFrame?.SoftwareBitmap;if (frame is null) return;//创建缓冲if (FrameNV12 is null || FrameNV12.Width != frame.PixelWidth || FrameNV12.Height != frame.PixelHeight){using var lockbuf = frame.LockBuffer(Windows.Graphics.Imaging.BitmapBufferAccessMode.Read);if (lockbuf is null) return;using var n = lockbuf.CreateReference();var t = lockbuf.GetPlaneDescription(0);FrameBuf = new Windows.Storage.Streams.Buffer(n.Capacity);FrameNV12 = new Mat(frame.PixelHeight * 3 / 2, frame.PixelWidth, MatType.CV_8UC1);FrameBgr = new Mat();}//读取数据frame.CopyToBuffer(FrameBuf);if (FrameData is null || FrameData.Length != FrameBuf.Length) FrameData = new byte[FrameBuf.Length];using var reader = DataReader.FromBuffer(FrameBuf);reader.ReadBytes(FrameData);//NV12转Rbg24Marshal.Copy(FrameData, 0, FrameNV12.Data, FrameData.Length);Cv2.CvtColor(FrameNV12, FrameBgr, ColorConversionCodes.YUV2BGR_NV12);Dispatcher.InvokeAsync(() => RendererFrame(FrameBgr, frame.PixelWidth, frame.PixelHeight));
}
可以看到，目前已经看不到有什么差距了，收工！
整体的代码也贴一下：
using OpenCvSharp;
using OpenCvSharp.WpfExtensions;
using System.DirectoryServices.ActiveDirectory;
using System.Reflection.PortableExecutable;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using Windows.Media.Capture;
using Windows.Media.Capture.Frames;
using Windows.Media.MediaProperties;
using Windows.Storage.Streams;
using Window = System.Windows.Window;namespace VideoCapture
{/// <summary>/// Interaction logic for MainWindow.xaml/// </summary>public partial class MainWindow : Window{private MediaFrameReader? FrameReader = null;private MediaCapture? Capture = null;public MainWindow(){InitializeComponent();}protected override void OnSourceInitialized(EventArgs e){base.OnSourceInitialized(e);Task.Run(CaptureInit);}private async Task CaptureInit(){//查找摄像头设备,这里仅作示例，只取第一个。实际开发请根据实际情况变通。var device = (await MediaFrameSourceGroup.FindAllAsync()).FirstOrDefault();if (device is null) return;//初始化 MediaCapture 对象var capture = new MediaCapture();var settings = new MediaCaptureInitializationSettings(){SourceGroup = device,SharingMode = MediaCaptureSharingMode.SharedReadOnly,//只获取画面，不控制设备。实际开发请换成可控制设备。StreamingCaptureMode = StreamingCaptureMode.Video,MemoryPreference = MediaCaptureMemoryPreference.Cpu,//这里不要用auto，不然可能会读不到帧数据。};await capture.InitializeAsync(settings);//获取预览流foreach (MediaFrameSource source in capture.FrameSources.Values){MediaFrameSourceKind kind = source.Info.SourceKind;if (kind != MediaFrameSourceKind.Color) continue;//为了方便，这里只要Nv12的，实际开发可根据情况变通if (source.CurrentFormat.Subtype.ToUpper() != "NV12"){//查找指定的格式foreach(var f in source.SupportedFormats){if (f.Subtype.ToUpper() != "NV12") continue;//尝试设置格式try { await source.SetFormatAsync(f); }catch { continue; }}}//创建帧数据读取设备MediaFrameReader frameReader = await capture.CreateFrameReaderAsync(source, MediaEncodingSubtypes.Nv12);frameReader.FrameArrived += Reader_FrameArrived;MediaFrameReaderStartStatus status = await frameReader.StartAsync();if (status == MediaFrameReaderStartStatus.Success){FrameReader = frameReader;Capture = capture;break;}//没能正确创建，将其释放掉await frameReader.StopAsync();frameReader.Dispose();}}/// <summary>/// 帧数据缓冲区/// </summary>Windows.Storage.Streams.Buffer FrameBuf { get; set; }/// <summary>/// 帧数据/// </summary>public byte[] FrameData { get; private set; }/// <summary>/// NV12帧数据/// </summary>Mat FrameNV12 { get; set; }/// <summary>/// Bgr帧数据/// </summary>Mat FrameBgr { get; set; }private void Reader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args){var frame = sender.TryAcquireLatestFrame()?.VideoMediaFrame?.SoftwareBitmap;if (frame is null) return;//创建缓冲if (FrameNV12 is null || FrameNV12.Width != frame.PixelWidth || FrameNV12.Height != frame.PixelHeight){using var lockbuf = frame.LockBuffer(Windows.Graphics.Imaging.BitmapBufferAccessMode.Read);if (lockbuf is null) return;using var n = lockbuf.CreateReference();var t = lockbuf.GetPlaneDescription(0);FrameBuf = new Windows.Storage.Streams.Buffer(n.Capacity);FrameNV12 = new Mat(frame.PixelHeight * 3 / 2, frame.PixelWidth, MatType.CV_8UC1);FrameBgr = new Mat();}//读取数据frame.CopyToBuffer(FrameBuf);if (FrameData is null || FrameData.Length != FrameBuf.Length) FrameData = new byte[FrameBuf.Length];using var reader = DataReader.FromBuffer(FrameBuf);reader.ReadBytes(FrameData);//NV12转Rbg24Marshal.Copy(FrameData, 0, FrameNV12.Data, FrameData.Length);Cv2.CvtColor(FrameNV12, FrameBgr, ColorConversionCodes.YUV2BGR_NV12);Dispatcher.InvokeAsync(() => RendererFrame(FrameBgr, frame.PixelWidth, frame.PixelHeight));}private void RendererFrame(Mat bgr, int width, int height){if (bgr is null || width <= 0 || height <= 0 || FrameBgr.Width != width || FrameBgr.Height != height) return;if (FrameImage.Source is not WriteableBitmap bmp || bmp.PixelWidth != width || bmp.PixelHeight != height){bmp = new WriteableBitmap(width, height, 96, 96, PixelFormats.Bgr24, default);FrameImage.Source = bmp;}WriteableBitmapConverter.ToWriteableBitmap(bgr, bmp);}}
}