diff --git a/.gitignore b/.gitignore index 964bfa4e48ee8e7c9387339d5775a3df90c63eb4..45aebfbe19533d4a9e436d2b227c067f5425742a 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ SwiftProtobuf.framework paddle-mobile.xcworkspace metal/models/ metal/images/ +*.a +metal/paddle-mobile/paddle-mobile/CPU/libpaddle-mobile.a diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj index 4036ea7722f78ca1e14ac79dc496b0f437948df1..5cdb9dfd7b19790dc3e6de7f15850a213ef93da5 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj +++ b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj @@ -17,7 +17,6 @@ FC27991321343A3A000B6BAD /* CPUCompute.mm in Sources */ = {isa = PBXBuildFile; fileRef = FC27991221343A3A000B6BAD /* CPUCompute.mm */; }; FC3C800F2133F46600D1295E /* MobileNetSSD.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC3C800E2133F46600D1295E /* MobileNetSSD.swift */; }; FC3C80112133F4AB00D1295E /* MobileNet.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC3C80102133F4AB00D1295E /* MobileNet.swift */; }; - FC4FD95121402B610073E130 /* PaddleMobile.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC4FD95021402B610073E130 /* PaddleMobile.swift */; }; FC8CFEE2213524EA0094D569 /* Genet.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC8CFEE1213524EA0094D569 /* Genet.swift */; }; FC8CFEE62135452C0094D569 /* genet_params in Resources */ = {isa = PBXBuildFile; fileRef = FC8CFEE42135452B0094D569 /* genet_params */; }; FC8CFEE72135452C0094D569 /* genet_model in Resources */ = {isa = PBXBuildFile; fileRef = FC8CFEE52135452B0094D569 /* genet_model */; }; @@ -66,7 +65,7 @@ FC27991421343A46000B6BAD /* CPUCompute.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CPUCompute.h; sourceTree = ""; }; FC3C800E2133F46600D1295E /* MobileNetSSD.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MobileNetSSD.swift; sourceTree = ""; }; FC3C80102133F4AB00D1295E /* MobileNet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MobileNet.swift; sourceTree = ""; }; - FC4FD95021402B610073E130 /* PaddleMobile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = PaddleMobile.swift; path = ../../../../../../Desktop/PaddleMobile.swift; sourceTree = ""; }; + FC4FD97B2140EE250073E130 /* libc++.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = "libc++.tbd"; path = "usr/lib/libc++.tbd"; sourceTree = SDKROOT; }; FC8CFEE1213524EA0094D569 /* Genet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Genet.swift; sourceTree = ""; }; FC8CFEE42135452B0094D569 /* genet_params */ = {isa = PBXFileReference; lastKnownFileType = file; path = genet_params; sourceTree = ""; }; FC8CFEE52135452B0094D569 /* genet_model */ = {isa = PBXFileReference; lastKnownFileType = file; path = genet_model; sourceTree = ""; }; @@ -108,6 +107,7 @@ 7B7DED984E9EE7BFB45E24E8 /* Frameworks */ = { isa = PBXGroup; children = ( + FC4FD97B2140EE250073E130 /* libc++.tbd */, 18896810981724F8A0FED62A /* Pods_paddle_mobile_demo.framework */, ); name = Frameworks; @@ -176,7 +176,6 @@ FC8CFED2213519540094D569 /* Net */ = { isa = PBXGroup; children = ( - FC4FD95021402B610073E130 /* PaddleMobile.swift */, FC013927210204A3008100E3 /* PreProcessKernel.metal */, FCBCCC542122EF5400D94F7E /* MetalHelper.swift */, FC3C800E2133F46600D1295E /* MobileNetSSD.swift */, @@ -346,7 +345,6 @@ FCBCCC552122EF5500D94F7E /* MetalHelper.swift in Sources */, FC27991321343A3A000B6BAD /* CPUCompute.mm in Sources */, FC3C80112133F4AB00D1295E /* MobileNet.swift in Sources */, - FC4FD95121402B610073E130 /* PaddleMobile.swift in Sources */, FC3C800F2133F46600D1295E /* MobileNetSSD.swift in Sources */, FC039B8220E11C550081E9F8 /* AppDelegate.swift in Sources */, ); @@ -497,6 +495,7 @@ CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = A798K58VVL; + ENABLE_BITCODE = NO; INFOPLIST_FILE = "paddle-mobile-demo/Info.plist"; IPHONEOS_DEPLOYMENT_TARGET = 9.0; LD_RUNPATH_SEARCH_PATHS = ( @@ -523,6 +522,7 @@ CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = A798K58VVL; + ENABLE_BITCODE = NO; INFOPLIST_FILE = "paddle-mobile-demo/Info.plist"; IPHONEOS_DEPLOYMENT_TARGET = 9.0; LD_RUNPATH_SEARCH_PATHS = ( diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/AppDelegate.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/AppDelegate.swift index 54dad2b5bf721f3d132bad2502d30b34ca0773ab..537fb06ed9e5b9100bea43b7acae9c014e0f4a78 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/AppDelegate.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/AppDelegate.swift @@ -19,7 +19,6 @@ class AppDelegate: UIResponder, UIApplicationDelegate { var window: UIWindow? - func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplicationLaunchOptionsKey: Any]?) -> Bool { // Override point for customization after application launch. return true diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/Genet.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/Genet.swift index 9f1a31920de671350649091e184dc5d452f331a6..a7c678ea184ef1d53565c9fb8c1cd60159174d38 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/Genet.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/Genet.swift @@ -17,6 +17,10 @@ import paddle_mobile class Genet: Net { + var means: [Float] = [128.0, 128.0, 128.0] + + var scale: Float = 0.017 + let except: Int = 0 class GenetPreProccess: CusomKernel { diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNet.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNet.swift index 4579ee07356c3c599f58060c9dc11d8401992817..2766c14bfd4f2910efa96b87aec8750c0a4db83a 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNet.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNet.swift @@ -18,7 +18,11 @@ import paddle_mobile class MobileNet: Net{ - + + var means: [Float] = [123.68, 116.78, 103.94] + + var scale: Float = 0.017 + let except: Int = 0 class MobilenetPreProccess: CusomKernel { diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift index 83767f0bc3ebb3def795dd45643a4a77d8f59dd1..4483f5882fd0725047a7cd3b0c79d450dbd618d0 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift @@ -16,6 +16,11 @@ import Foundation import paddle_mobile class MobileNet_ssd_hand: Net{ + + var means: [Float] = [123.68, 116.78, 103.94] + + var scale: Float = 0.017 + let except: Int = 2 class MobilenetssdPreProccess: CusomKernel { init(device: MTLDevice) { @@ -27,7 +32,7 @@ class MobileNet_ssd_hand: Net{ func resultStr(res: [Float]) -> String { return " \(res)" } - + func fetchResult(paddleMobileRes: ResultHolder) -> [Float32] { guard let interRes = paddleMobileRes.intermediateResults else { diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift index cecf14411634e93c44ee7624ee1872dc2539938d..69fb820cd5ff34633d0084d697aa8d986f0117d1 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift @@ -17,10 +17,11 @@ import MetalKit import paddle_mobile import MetalPerformanceShaders +let platform: Platform = .CPU let threadSupport = [1] -//.mobilenet : MobileNet.init(), -let modelHelperMap: [SupportModel : Net] = [.mobilenet_ssd : MobileNet_ssd_hand.init(), .genet : Genet.init()] +let modelHelperMap: [SupportModel : Runner] = [.mobilenet_ssd : Runner.init(inNet: MobileNet_ssd_hand.init(), commandQueue: MetalHelper.shared.queue, inPlatform: platform), + .genet : Runner.init(inNet: Genet.init(), commandQueue: MetalHelper.shared.queue, inPlatform: platform)] //, .genet : Genet.init() //let modelHelperMap: [SupportModel : Net] = [.mobilenet : MobileNet.init(), .mobilenet_ssd : MobileNet_ssd_hand.init()] @@ -30,7 +31,7 @@ enum SupportModel: String{ case genet = "genet" static func supportedModels() -> [SupportModel] { //.mobilenet, - return [.mobilenet_ssd ,.genet] + return [.mobilenet_ssd, .genet] } } @@ -40,12 +41,13 @@ class ViewController: UIViewController { @IBOutlet weak var elapsedTimeLabel: UILabel! @IBOutlet weak var modelPickerView: UIPickerView! @IBOutlet weak var threadPickerView: UIPickerView! - var runnner: Runner! + var selectImage: UIImage? + var inputPointer: UnsafeMutablePointer? var modelType: SupportModel = SupportModel.supportedModels()[0] var toPredictTexture: MTLTexture? - var net: Net { + var runner: Runner { get { return modelHelperMap[modelType] ?! " has no this type " } @@ -56,8 +58,7 @@ class ViewController: UIViewController { var threadNum = 1 @IBAction func loadAct(_ sender: Any) { - - if runnner.load() { + if runner.load() { print(" load success ! ") } else { print(" load error ! ") @@ -72,27 +73,61 @@ class ViewController: UIViewController { } @IBAction func clearAct(_ sender: Any) { - runnner.clear() + runner.clear() } @IBAction func predictAct(_ sender: Any) { - guard let inTexture = toPredictTexture else { - resultTextView.text = "请选择图片 ! " - return - } let max = 50 - let startDate = Date.init() - for i in 0.. +#import + +@interface PaddleMobile : NSObject + +/* + 创建对象 +*/ +- (instancetype)init; + +/* + load 模型, 开辟内存 +*/ +- (BOOL)load:(NSString *)modelPath andWeightsPath:(NSString *)weighsPath; + +/* + 加载散开形式的模型, 需传入模型的目录 +*/ +- (BOOL)load:(NSString *)modelAndWeightPath; + +/* + * 从内存中加载模型 + * */ +- (BOOL)LoadCombinedMemory:(size_t)modelLen + andModelBuf:(const uint8_t *)modelBuf + andModelParamsLen:(size_t)combinedParamsLen + andCombinedParamsBuf:(const uint8_t *)combinedParamsBuf; + + +/* + * 进行预测, means 和 scale 为训练模型时的预处理参数, 如训练时没有做这些预处理则直接使用 predict + */ +- (NSArray *)predict:(CGImageRef)image + dim:(NSArray *)dim + means:(NSArray *)means + scale:(float)scale; + +/* + * 预测输入 + * */ +- (NSArray *)predictInput:(float *)input + dim:(NSArray *)dim + means:(NSArray *)means + scale:(float)scale; + +/* + * 对图像进行预处理 + * */ +-(void)preprocess:(CGImageRef)image + output:(float *)output + means:(NSArray *)means + scale:(float)scale + dim:(NSArray *)dim; + +/* + 清理内存 +*/ +- (void)clear; + +@end diff --git a/metal/paddle-mobile/paddle-mobile/CPU/libpaddle-mobile.a b/metal/paddle-mobile/paddle-mobile/CPU/libpaddle-mobile.a new file mode 100644 index 0000000000000000000000000000000000000000..9185441567a1ff772ac12aaf7b1f4e8ff1e64cc3 Binary files /dev/null and b/metal/paddle-mobile/paddle-mobile/CPU/libpaddle-mobile.a differ diff --git a/metal/paddle-mobile/paddle-mobile/PaddleMobile.swift b/metal/paddle-mobile/paddle-mobile/PaddleMobile.swift index 8e28fb2c2d77c4324cb38bbc9c0e178cc1780697..44cccea90fe3b78f7e230507b6d97d78c310d655 100644 --- a/metal/paddle-mobile/paddle-mobile/PaddleMobile.swift +++ b/metal/paddle-mobile/paddle-mobile/PaddleMobile.swift @@ -22,6 +22,8 @@ class ScaleKernel: CusomKernel { public protocol Net { var except: Int { get } + var means: [Float] { get } + var scale: Float { get } var dim: (n: Int, h: Int, w: Int, c: Int) { get } var preprocessKernel: CusomKernel { get } // var paramPointer: UnsafeMutableRawPointer { get } @@ -36,7 +38,7 @@ public protocol Net { } extension Net { - func fetchResult(paddleMobileRes: ResultHolder) -> [Float32] { + public func fetchResult(paddleMobileRes: ResultHolder) -> [Float32] { return paddleMobileRes.resultArr } } @@ -46,9 +48,15 @@ public class Runner { var executor: Executor? var queue: MTLCommandQueue? var textureLoader: MTKTextureLoader? - let net: Net + public let net: Net let device: MTLDevice? let platform: Platform + var cpuPaddleMobile: PaddleMobile? + let numel: Int + let meansNumber: [NSNumber] + + // dims num nchw + let dimsNum: [NSNumber] /** * inNet: 需要运行的网络 * commandQueue: GPU 是需要传入 @@ -62,6 +70,15 @@ public class Runner { if let inDevice = device { textureLoader = MTKTextureLoader.init(device: inDevice) } + if platform == .CPU { + cpuPaddleMobile = PaddleMobile.init() + } + numel = net.dim.n * net.dim.c * net.dim.h * net.dim.w + meansNumber = net.means.map { NSNumber.init(value: $0) } + dimsNum = [NSNumber.init(value: net.dim.n), + NSNumber.init(value: net.dim.c), + NSNumber.init(value: net.dim.h), + NSNumber.init(value: net.dim.w)] } /** @@ -82,56 +99,83 @@ public class Runner { return false } } else { - print(" need implementation ") - return false + return cpuPaddleMobile?.load(net.modelPath, andWeightsPath: net.paramPath) ?? false } return true } - /** - * CPU GPU 通用版本 predict - * cgImage: 需要预测的图片 - * ( _ success: Bool, _ time:TimeInterval, _ resultArray: [Float32]) -> Void : 回调闭包, 三个参数分别为: 是否成功, 预测耗时, 结果数组 - */ - public func predict(cgImage: CGImage, completion: @escaping ( _ success: Bool, _ time:TimeInterval, _ resultArray: [Float32]) -> Void) { - if platform == .GPU { - getTexture(image: cgImage) { [weak self] (texture) in - guard let SSelf = self else { - fatalError() - } - SSelf.predict(texture: texture, completion: completion) - } - } else if platform == .CPU { - + public func predict(inputPointer: UnsafeMutablePointer, completion: @escaping ( _ success: Bool, _ resultArray: [Float32]) -> Void) { + guard let res = cpuPaddleMobile?.predictInput(inputPointer, dim: dimsNum, means: meansNumber, scale: net.scale) else { + completion(false, []) + return } + completion(true, res.map { ($0 as! NSNumber).floatValue }) } + /** * GPU 版本 predict * texture: 需要预测的 texture 需要做过预处理 * ( _ success: Bool, _ time:TimeInterval, _ resultArray: [Float32]) -> Void : 回调闭包, 三个参数分别为: 是否成功, 预测耗时, 结果数组 */ - public func predict(texture: MTLTexture, completion: @escaping ( _ success: Bool, _ time:TimeInterval, _ resultArray: [Float32]) -> Void) { + public func predict(texture: MTLTexture, completion: @escaping ( _ success: Bool, _ resultArray: [Float32]) -> Void) { do { try self.executor?.predict(input: texture, dim: [self.net.dim.n, self.net.dim.h, self.net.dim.w, self.net.dim.c], completionHandle: { [weak self] (res) in guard let SSelf = self else { fatalError( " self nil " ) } let resultArray = SSelf.net.fetchResult(paddleMobileRes: res) - completion(true, res.elapsedTime, resultArray) + completion(true, resultArray) }, preProcessKernle: self.net.preprocessKernel, except: self.net.except) } catch let error { print(error) - completion(false, 0.0, []) + completion(false, []) return } } + + /** + * CPU GPU 通用版本 predict + * cgImage: 需要预测的图片 + * ( _ success: Bool, _ time:TimeInterval, _ resultArray: [Float32]) -> Void : 回调闭包, 三个参数分别为: 是否成功, 预测耗时, 结果数组 + */ + public func predict(cgImage: CGImage, completion: @escaping ( _ success: Bool, _ resultArray: [Float32]) -> Void) { + if platform == .GPU { + getTexture(image: cgImage) { [weak self] (texture) in + guard let SSelf = self else { + fatalError( "" ) + } + SSelf.predict(texture: texture, completion: completion) + } + } else if platform == .CPU { + let input = preproccess(image: cgImage) + predict(inputPointer: input, completion: completion) + input.deinitialize(count: numel) + input.deallocate() + } + } + /* * 清理内存, 调用此函数后, 不能再使用, 需重新 load */ public func clear() { - executor?.clear() - executor = nil - program = nil + if platform == .GPU { + executor?.clear() + executor = nil + program = nil + } else if platform == .CPU { + cpuPaddleMobile?.clear() + } + } + + public func preproccess(image: CGImage) -> UnsafeMutablePointer { + let output = UnsafeMutablePointer.allocate(capacity: numel) + let means = net.means.map { NSNumber.init(value: $0) } + let dims = [NSNumber.init(value: net.dim.n), + NSNumber.init(value: net.dim.c), + NSNumber.init(value: net.dim.h), + NSNumber.init(value: net.dim.w)] + cpuPaddleMobile?.preprocess(image, output: output, means: means, scale: net.scale, dim: dims) + return output } /* @@ -169,7 +213,3 @@ public class Runner { } - - - - diff --git a/metal/paddle-mobile/paddle-mobile/paddle_mobile.h b/metal/paddle-mobile/paddle-mobile/paddle_mobile.h index ffa44be38a4c3a1f3109c51b3d15506591f2de2e..b657679715be1389b3dafdf34a4d279508dbbd58 100644 --- a/metal/paddle-mobile/paddle-mobile/paddle_mobile.h +++ b/metal/paddle-mobile/paddle-mobile/paddle_mobile.h @@ -14,12 +14,14 @@ #pragma once + +#import "PaddleMobile.h" #import //! Project version number for paddle_mobile. -FOUNDATION_EXPORT double paddle_mobileVersionNumber; +//FOUNDATION_EXPORT double paddle_mobileVersionNumber; //! Project version string for paddle_mobile. -FOUNDATION_EXPORT const unsigned char paddle_mobileVersionString[]; +//FOUNDATION_EXPORT const unsigned char paddle_mobileVersionString[];