import 'dart:async'; import 'dart:convert'; import 'package:camera/camera.dart'; import 'package:fis_jsonrpc/rpc.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:get/get.dart'; import 'package:google_mlkit_face_detection/google_mlkit_face_detection.dart'; import 'package:image_gallery_saver/image_gallery_saver.dart'; import 'package:vitalapp/architecture/storage/storage.dart'; import 'package:vitalapp/architecture/utils/prompt_box.dart'; import 'package:vitalapp/components/alert_dialog.dart'; import 'package:vitalapp/managers/interfaces/cache.dart'; import 'package:fis_common/logger/logger.dart'; import 'package:vitalapp/rpc.dart'; import 'package:vitalapp/store/store.dart'; import 'dart:ui' as ui; import 'index.dart'; import 'package:fis_common/index.dart'; class FacialRecognitionController extends GetxController with WidgetsBindingObserver { FacialRecognitionController({ required this.mode, this.patientInfo, }); final FacialRecognitionMode mode; /// 当前需要录入的身份信息 final PatientDTO? patientInfo; final state = FacialRecognitionState(); List _cameras = []; List get cameras => _cameras; CameraController? kCameraController; double _minAvailableExposureOffset = 0.0; double _maxAvailableExposureOffset = 0.0; double _minAvailableZoom = 1.0; double _maxAvailableZoom = 1.0; final double _currentScale = 1.0; double _baseScale = 1.0; // 屏幕上手指数量 int pointers = 0; // 当前需要返回的身份信息 IdCardInfoModel idCardInfo = IdCardInfoModel(); /// 开始缩放 void handleScaleStart(ScaleStartDetails details) { _baseScale = _currentScale; } /// 当前捕获帧的人脸列表 List kFrameFacesResult = []; /// 当前捕获帧大小 Size kFrameImageSize = Size.zero; /// 缩放更新 Future handleScaleUpdate(ScaleUpdateDetails details) async { // When there are not exactly two fingers on screen don't scale if (kCameraController == null || pointers != 2) { return; } // 屏蔽缩放 // _currentScale = (_baseScale * details.scale) // .clamp(_minAvailableZoom, _maxAvailableZoom); // await kCameraController!.setZoomLevel(_currentScale); } /// 修改对焦点 [暂不执行] void onViewFinderTap(TapDownDetails details, BoxConstraints constraints) { if (kCameraController == null) { return; } final CameraController cameraController = kCameraController!; final Offset offset = Offset( details.localPosition.dx / constraints.maxWidth, details.localPosition.dy / constraints.maxHeight, ); cameraController.setExposurePoint(offset); cameraController.setFocusPoint(offset); } /// 初始化相机 Future initAvailableCameras() async { try { _cameras = await availableCameras(); if (_cameras.isNotEmpty) { // state.isCameraReady = true; } // print("cameras: ${_cameras.length}"); } on CameraException catch (e) { logger.e("cameras: ${e.code} ${e.description}"); } } /// 启动指定相机 Future openNewCamera(CameraDescription cameraDescription) async { final CameraController? oldController = kCameraController; if (oldController != null) { // `kCameraController` needs to be set to null before getting disposed, // to avoid a race condition when we use the kCameraController that is being // disposed. This happens when camera permission dialog shows up, // which triggers `didChangeAppLifecycleState`, which disposes and // re-creates the kCameraController. kCameraController = null; await oldController.dispose(); } final CameraController cameraController = CameraController( cameraDescription, ResolutionPreset.max, enableAudio: false, imageFormatGroup: ImageFormatGroup.jpeg, ); kCameraController = cameraController; // If the kCameraController is updated then update the UI. cameraController.addListener(() { if (cameraController.value.hasError) { PromptBox.toast( "Camera error ${cameraController.value.errorDescription}"); } }); try { await cameraController.initialize(); await Future.wait(>[ // The exposure mode is currently not supported on the web. ...!kIsWeb ? >[ cameraController.getMinExposureOffset().then( (double value) => _minAvailableExposureOffset = value), cameraController .getMaxExposureOffset() .then((double value) => _maxAvailableExposureOffset = value) ] : >[], cameraController .getMaxZoomLevel() .then((double value) => _maxAvailableZoom = value), cameraController .getMinZoomLevel() .then((double value) => _minAvailableZoom = value), ]); } on CameraException catch (e) { switch (e.code) { case 'CameraAccessDenied': PromptBox.toast('You have denied camera access.'); break; case 'CameraAccessDeniedWithoutPrompt': // iOS only PromptBox.toast('Please go to Settings app to enable camera access.'); break; case 'CameraAccessRestricted': // iOS only PromptBox.toast('Camera access is restricted.'); break; case 'AudioAccessDenied': PromptBox.toast('You have denied audio access.'); break; case 'AudioAccessDeniedWithoutPrompt': // iOS only PromptBox.toast('Please go to Settings app to enable audio access.'); break; case 'AudioAccessRestricted': // iOS only PromptBox.toast('Audio access is restricted.'); break; default: PromptBox.toast('Error: ${e.code}\n${e.description}'); break; } } } /// 遍历当前相机列表并启动后置相机 void openBackCamera() async { logger.i("openBackCamera 启动前置摄像头"); if (_cameras.isEmpty) { PromptBox.toast('Error: No cameras found.'); } else { for (CameraDescription cameraDescription in _cameras) { if (cameraDescription.lensDirection == CameraLensDirection.back) { await openNewCamera(cameraDescription); lockCaptureOrientation(); state.isUsingFrontCamera = false; update(); state.isCameraReady = true; break; } } } } /// 遍历当前相机列表并启动前置相机 void openFrontCamera() async { logger.i("openFrontCamera 启动后置摄像头"); if (_cameras.isEmpty) { PromptBox.toast('Error: No cameras found.'); } else { for (CameraDescription cameraDescription in _cameras) { if (cameraDescription.lensDirection == CameraLensDirection.front) { await openNewCamera(cameraDescription); state.isUsingFrontCamera = true; lockCaptureOrientation(); update(); state.isCameraReady = true; break; } } } } /// 前后置摄像头切换 void switchCameraLens() async { if (state.isUsingFrontCamera) { openBackCamera(); } else { openFrontCamera(); } } /// 相机锁定旋转 Future lockCaptureOrientation() async { final CameraController? cameraController = kCameraController; if (cameraController == null || !cameraController.value.isInitialized) { PromptBox.toast('Error: select a camera first.'); return; } if (!cameraController.value.isCaptureOrientationLocked) { try { await cameraController .lockCaptureOrientation(DeviceOrientation.landscapeLeft); } on CameraException catch (e) { PromptBox.toast('Error: ${e.code}\n${e.description}'); } } else { PromptBox.toast('Rotation lock is already enabled.'); } } /// 执行一次拍摄 Future takePicture() async { final CameraController? cameraController = kCameraController; if (cameraController == null || !cameraController.value.isInitialized) { PromptBox.toast('Error: select a camera first.'); return null; } if (cameraController.value.isTakingPicture) { // A capture is already pending, do nothing. return null; } try { final XFile file = await cameraController.takePicture(); return file; } on CameraException catch (e) { PromptBox.toast('Error: ${e.code}\n${e.description}'); return null; } } /// 测试图像文件缓存,print 遍历输出 void debugShowCache() async { final cacheManager = Get.find(); double cacheSize = await cacheManager.getCacheSize(); double imageCacheSize = await cacheManager.getImageCacheSize(); debugPrint('cacheSize = $cacheSize : ${formatSize(cacheSize)}'); debugPrint( 'imageCacheSize = $imageCacheSize : ${formatSize(imageCacheSize)}'); } /// 文件大小转为可读 Str static String formatSize(double value) { List unitArr = ['B', 'K', 'M', 'G']; int index = 0; while (value > 1024) { index++; value = value / 1024; } String size = value.toStringAsFixed(2); return size + unitArr[index]; } /// 保存到相册 void saveImageToGallery(XFile image) async { // 获取图像的字节数据 Uint8List bytes = await image.readAsBytes(); // 将图像保存到相册 await ImageGallerySaver.saveImage(bytes, quality: 100); } /// 处理图像裁切 Future clipLocalImage(XFile soureceImage, double scale) async { assert(scale >= 1, 'scale must be greater than 1'); // 获取图像的字节数据 Uint8List bytes = await soureceImage.readAsBytes(); var codec = await ui.instantiateImageCodec(bytes); var nextFrame = await codec.getNextFrame(); var image = nextFrame.image; Rect src = Rect.fromLTWH( (scale - 1) / 2 / scale * image.width.toDouble(), (scale - 1) / 2 / scale * image.height.toDouble(), image.width.toDouble() / scale, image.height.toDouble() / scale, ); Rect dst = Rect.fromLTWH(0, 0, image.width.toDouble(), image.height.toDouble()); ui.Image croppedImage = await getCroppedImage(image, src, dst); ByteData? newImageBytes = await croppedImage.toByteData(format: ui.ImageByteFormat.png); if (newImageBytes == null) { return ''; } Uint8List newImageUint8List = newImageBytes.buffer.asUint8List(); /// FIXME 不要存到相册而是存到临时目录 Map result = await ImageGallerySaver.saveImage(newImageUint8List, quality: 100); String jsonString = jsonEncode(result); Map json = jsonDecode(jsonString); String? filePath = json['filePath']; return filePath ?? ''; } /// 获取图像文件的图像尺寸 Future getImageSize(XFile soureceImage) async { // 获取图像的字节数据 Uint8List bytes = await soureceImage.readAsBytes(); var codec = await ui.instantiateImageCodec(bytes); var nextFrame = await codec.getNextFrame(); var image = nextFrame.image; return Size(image.width.toDouble(), image.height.toDouble()); } /// 获取裁切后的图像 Future getCroppedImage(ui.Image image, Rect src, Rect dst) { var pictureRecorder = ui.PictureRecorder(); Canvas canvas = Canvas(pictureRecorder); canvas.drawImageRect(image, src, dst, Paint()); return pictureRecorder.endRecording().toImage( dst.width.floor(), dst.height.floor(), ); } /// 发生开始人脸识别/人像采集事件 void onCaptureFaceButtonPressed() { if (mode == FacialRecognitionMode.faceRecognition) { doFacialRecognitionTimes = 0; state.isRunningFaceRecognition = true; doFacialRecognition(); // 人脸识别 } else { doFaceInput(); // 人像采集 } } /// 人脸识别执行次数 int doFacialRecognitionTimes = 0; /// 人脸识别逻辑 void doFacialRecognition() async { logger.i("doFacialRecognition 进行人脸识别 $doFacialRecognitionTimes"); doFacialRecognitionTimes++; if (doFacialRecognitionTimes >= 10) { // 尝试十次后宣告失败 state.isRunningFaceRecognition = false; PromptBox.toast('人脸识别失败,请先录入人脸信息或更新人脸信息'); return; } if (kCameraController == null) { state.isRunningFaceRecognition = false; return; } final XFile? file = await takePicture(); state.processingImageLocalPath = ''; if (file != null) { faceDetector = FaceDetector(options: FaceDetectorOptions()); // faceDetector = // FaceDetector(options: FaceDetectorOptions(enableContours: true)); int faceNum = await doDetection(faceDetector, file.path); // max 分辨率下检测用时大约 100ms if (doFacialRecognitionTimes >= 10) { // 手动取消,中途跳出循环 state.isRunningFaceRecognition = false; return; } if (faceNum == 0) { PromptBox.toast('请将面部保持在识别框内'); await Future.delayed(const Duration(seconds: 2)); } else if (faceNum > 1) { PromptBox.toast('请保持只有一张面部在识别范围内'); await Future.delayed(const Duration(seconds: 2)); } else { final String fileType = file.path.split('.').last; if (!['png', 'jpg'].contains(fileType)) { PromptBox.toast('上传的图像类型错误'); return; } state.processingImageLocalPath = file.path; final url = await rpc.storage.upload( file, fileType: fileType, ); print('⭐⭐⭐⭐⭐⭐⭐⭐ url: $url'); try { if (url == null || url.isEmpty) { PromptBox.toast('上传失败'); throw Exception('图像上传超时'); } PatientBaseDTO result = await rpc.patient.getPatientBaseByFaceImageAsync( GetPatientBaseByFaceImageRequest( token: Store.user.token, image: url, ), ); state.processingImageLocalPath = ''; if (doFacialRecognitionTimes >= 10) { // 手动取消,中途跳出循环 state.isRunningFaceRecognition = false; return; } if (result.faceScanErrorType == FaceScanErrorTypeEnum.Success) { finishFaceDetection(result); // 识别成功,阻断循环 doFacialRecognitionTimes = 10; } else if (result.faceScanErrorType == FaceScanErrorTypeEnum.NoCreated) { PromptBox.toast('识别到未采集过的人脸信息'); /// 如果返回结果告知不在档,则快进到失败十次提示重新建档 doFacialRecognitionTimes = 10; } else { if (result.errorMessage.isNotNullOrEmpty) { logger.e(result.errorMessage!); } if (kDebugMode) { PromptBox.toast('无法识别面部信息,请确保面部清晰可见: $doFacialRecognitionTimes'); } else { PromptBox.toast('无法识别面部信息,请确保面部清晰可见'); } await Future.delayed(const Duration(seconds: 1)); } } catch (e) { logger.e("getPatientBaseByFaceImageAsync failed: $e", e); state.processingImageLocalPath = ''; } state.processingImageLocalPath = ''; } } if (doFacialRecognitionTimes >= 10) { // 手动取消,中途跳出循环 state.isRunningFaceRecognition = false; return; } doFacialRecognition(); } /// 人像采集逻辑 void doFaceInput() async { if (kCameraController == null) { return; } if (patientInfo != null) { idCardInfo.idCardNumber = patientInfo!.cardNo!; } final XFile? file = await takePicture(); state.processingImageLocalPath = ''; if (file != null) { faceDetector = FaceDetector(options: FaceDetectorOptions()); // faceDetector = // FaceDetector(options: FaceDetectorOptions(enableContours: true)); int faceNum = await doDetection(faceDetector, file.path); // max 分辨率下检测用时大约 100ms if (faceNum == 0) { PromptBox.toast('请将面部保持在识别框内'); return; } else if (faceNum > 1) { PromptBox.toast('请保持只有一张面部在识别范围内'); return; } final String fileType = file.path.split('.').last; if (!['png', 'jpg'].contains(fileType)) { PromptBox.toast('上传的图像类型错误'); return; } state.processingImageLocalPath = file.path; final url = await rpc.storage.upload( file, fileType: fileType, ); print('⭐⭐⭐⭐⭐⭐⭐⭐ url: $url'); try { if (url == null || url.isEmpty) { PromptBox.toast('上传失败'); throw Exception('图像上传超时'); } SavePersonDTO result = await rpc.patient.savePatientBaseByFaceImageAsync( SavePatientBaseByFaceImageRequest( cardNo: idCardInfo.idCardNumber, token: Store.user.token, image: url, ), ); state.processingImageLocalPath = ''; if (result.success) { Get.back(result: true); } else { // 如果失败且存在 bindCardNo ,则说明已录入过 if (result.bindCardNo != null) { if (result.bindCardNo == idCardInfo.idCardNumber) { Get.back(result: true); } else { /// 询问是否需要解绑原身份证并绑定当前身份证 /// 原身份证:result.bindCardNo /// 当前身份证:idCardInfo.idCardNumber bool? dialogResult = await Get.dialog( VAlertDialog( title: '提示', width: 600, content: Container( margin: const EdgeInsets.only(bottom: 20, left: 20, right: 20), child: Text( '该人脸已绑定身份证(${result.bindCardNo})\n是否解绑并绑定当前身份证(${idCardInfo.idCardNumber})?', style: const TextStyle(fontSize: 20), textAlign: TextAlign.left, ), ), showCancel: true, onConfirm: () async { bool success = await unbindAndCreateByFaceImageAsync( result.bindCardNo!, idCardInfo.idCardNumber, url, ); if (success) { Get.back(result: true); } else { Get.back(result: false); } }, ), ); if (dialogResult != null && dialogResult) { Get.back(result: true); } else if (dialogResult != null && !dialogResult) { PromptBox.toast('人脸数据存入失败'); } else { PromptBox.toast('人脸数据存入取消'); } } } else { PromptBox.toast('人脸数据存入失败: ${result.errMessage}'); } } } catch (e) { state.processingImageLocalPath = ''; logger.e("savePatientBaseByFaceImageAsync failed: $e", e); } state.processingImageLocalPath = ''; } } /// 取消采集 void doCancelCapture() { doFacialRecognitionTimes = 10; state.isRunningFaceRecognition = false; } /// 重新绑定并创建新档案 Future unbindAndCreateByFaceImageAsync( String oldId, String newId, String url) async { DeletePersonDTO result = await rpc.patient.unbindAndCreateByFaceImageAsync( UnbindAndCreateByFaceImageRequest( oldCardNo: oldId, newCardNo: newId, token: Store.user.token, image: url, ), ); return result.success; } /// 在 widget 内存中分配后立即调用。 @override void onInit() async { // await initCamera(); super.onInit(); WidgetsBinding.instance.addObserver(this); } /// 在 onInit() 之后调用 1 帧。这是进入的理想场所 @override void onReady() async { super.onReady(); logger.i( "onReady 进入人脸识别/采集页面,当前模式:${mode == FacialRecognitionMode.faceRecognition ? '人脸识别' : '人像采集'}"); await initAvailableCameras(); if (state.isUsingFrontCamera) { openFrontCamera(); } else { openBackCamera(); } } /// 在 [onDelete] 方法之前调用。 @override void onClose() { logger.i("onClose 离开人脸识别/采集页面"); super.onClose(); // 关闭人脸采集 doCancelCapture(); final cacheManager = Get.find(); cacheManager.clearApplicationImageCache(); closeDetector(); WidgetsBinding.instance.removeObserver(this); final CameraController? cameraController = kCameraController; if (cameraController != null) { kCameraController = null; cameraController.dispose(); } } /// dispose 释放内存 @override void dispose() { super.dispose(); } @override void didChangeAppLifecycleState(AppLifecycleState state) async { super.didChangeAppLifecycleState(state); final CameraController? cameraController = kCameraController; // App state changed before we got the chance to initialize. if (cameraController == null || !cameraController.value.isInitialized) { return; } if (state == AppLifecycleState.inactive) { cameraController.dispose(); this.state.isCameraReady = false; } else if (state == AppLifecycleState.resumed) { await openNewCamera(cameraController.description); this.state.isCameraReady = true; } } /// 完成人脸识别 void finishFaceDetection(PatientBaseDTO patient) { Get.back( result: FaceRecognitionResult( success: true, patientInfo: patient, ), ); } /// 面部识别 基于 Google's ML Kit InputImage inputImage = InputImage.fromFilePath(''); FaceDetector faceDetector = FaceDetector(options: FaceDetectorOptions()); // 进行一次人脸检测 (返回人脸数量) Future doDetection( FaceDetector faceDetector, String imagePath, ) async { inputImage = InputImage.fromFilePath(imagePath); // inputImage = image; final List faces = await faceDetector.processImage(inputImage); kFrameFacesResult = []; kFrameFacesResult.addAll(faces); return kFrameFacesResult.length; } /// 销毁检测器 void closeDetector() { faceDetector.close(); } } class FaceRecognitionResult { bool success; /// 身份信息 PatientBaseDTO patientInfo; FaceRecognitionResult({ required this.success, required this.patientInfo, }); } /// 运行模式 enum FacialRecognitionMode { /// 人脸识别(用于登录) faceRecognition, /// 人像采集 faceInput, }