diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj index 749c8b2a92daf71ebbbdf7b0b6fa8b25073e9280..04bb0aa82a5ab9143b523a89cdc89702eab9ecf6 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj +++ b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj @@ -15,6 +15,7 @@ FC039B8720E11C550081E9F8 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8520E11C550081E9F8 /* Main.storyboard */; }; FC039B8920E11C560081E9F8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8820E11C560081E9F8 /* Assets.xcassets */; }; FC039B8C20E11C560081E9F8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8A20E11C560081E9F8 /* LaunchScreen.storyboard */; }; + FC1A1D552237EAC70047B7FD /* ImageTool.m in Sources */ = {isa = PBXBuildFile; fileRef = FC1A1D542237EAC70047B7FD /* ImageTool.m */; }; FC203FB221CBFDBA00B37166 /* test.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC203FA921CBFDBA00B37166 /* test.jpg */; }; FC2BFCBC21DF0A8600C262B2 /* 00001.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC2BFCBB21DF0A8600C262B2 /* 00001.jpg */; }; FC2BFCBE21DF15D900C262B2 /* 123.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC2BFCBD21DF15D900C262B2 /* 123.jpg */; }; @@ -42,6 +43,8 @@ FC9797C321D608E000F2FD90 /* mobilenet_params in Resources */ = {isa = PBXBuildFile; fileRef = FC9797C121D608DF00F2FD90 /* mobilenet_params */; }; FC9797C721D609FB00F2FD90 /* synset.txt in Resources */ = {isa = PBXBuildFile; fileRef = FC9797C621D609FB00F2FD90 /* synset.txt */; }; FC9797CF21D6506F00F2FD90 /* mingren.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC9797CE21D6506F00F2FD90 /* mingren.jpg */; }; + FC9B9F092236ABF300BB6DB0 /* combined_mobilenet_model_16 in Resources */ = {isa = PBXBuildFile; fileRef = FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */; }; + FC9B9F0A2236ABF500BB6DB0 /* combined_mobilenet_params_16 in Resources */ = {isa = PBXBuildFile; fileRef = FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */; }; FCAFD84B2231614200496A36 /* yolo_16_param in Resources */ = {isa = PBXBuildFile; fileRef = FCAFD8492231614200496A36 /* yolo_16_param */; }; FCAFD84C2231614200496A36 /* yolo_16_model in Resources */ = {isa = PBXBuildFile; fileRef = FCAFD84A2231614200496A36 /* yolo_16_model */; }; FCBCCC552122EF5500D94F7E /* MetalHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCBCCC542122EF5400D94F7E /* MetalHelper.swift */; }; @@ -83,6 +86,8 @@ FC039B8820E11C560081E9F8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; FC039B8B20E11C560081E9F8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; FC039B8D20E11C560081E9F8 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + FC1A1D532237EAC70047B7FD /* ImageTool.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ImageTool.h; sourceTree = ""; }; + FC1A1D542237EAC70047B7FD /* ImageTool.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ImageTool.m; sourceTree = ""; }; FC203FA921CBFDBA00B37166 /* test.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = test.jpg; sourceTree = ""; }; FC27991121343A39000B6BAD /* paddle-mobile-demo-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "paddle-mobile-demo-Bridging-Header.h"; sourceTree = ""; }; FC2BFCBB21DF0A8600C262B2 /* 00001.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = 00001.jpg; sourceTree = ""; }; @@ -115,6 +120,8 @@ FC9797C121D608DF00F2FD90 /* mobilenet_params */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_params; sourceTree = ""; }; FC9797C621D609FB00F2FD90 /* synset.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = synset.txt; sourceTree = ""; }; FC9797CE21D6506F00F2FD90 /* mingren.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = mingren.jpg; sourceTree = ""; }; + FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */ = {isa = PBXFileReference; lastKnownFileType = file; path = combined_mobilenet_model_16; sourceTree = ""; }; + FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */ = {isa = PBXFileReference; lastKnownFileType = file; path = combined_mobilenet_params_16; sourceTree = ""; }; FCAFD8492231614200496A36 /* yolo_16_param */ = {isa = PBXFileReference; lastKnownFileType = file; path = yolo_16_param; sourceTree = ""; }; FCAFD84A2231614200496A36 /* yolo_16_model */ = {isa = PBXFileReference; lastKnownFileType = file; path = yolo_16_model; sourceTree = ""; }; FCBCCC542122EF5400D94F7E /* MetalHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MetalHelper.swift; sourceTree = ""; }; @@ -183,6 +190,7 @@ FC039B8020E11C550081E9F8 /* paddle-mobile-demo */ = { isa = PBXGroup; children = ( + FC1A1D522237EAA10047B7FD /* OC */, FC2BFD4F21DF892500C262B2 /* Resources */, FCBCCC542122EF5400D94F7E /* MetalHelper.swift */, FC2BFD3F21DF5DDF00C262B2 /* OCInterface */, @@ -201,6 +209,15 @@ path = "paddle-mobile-demo"; sourceTree = ""; }; + FC1A1D522237EAA10047B7FD /* OC */ = { + isa = PBXGroup; + children = ( + FC1A1D532237EAC70047B7FD /* ImageTool.h */, + FC1A1D542237EAC70047B7FD /* ImageTool.m */, + ); + path = OC; + sourceTree = ""; + }; FC203FA821CBFDBA00B37166 /* images */ = { isa = PBXGroup; children = ( @@ -269,6 +286,7 @@ FC704C1B21D237FC00F98BAB /* vision_model */ = { isa = PBXGroup; children = ( + FC9B9F062236AB4800BB6DB0 /* mobilenet_16 */, FCE834AB2232A4AE0057BF43 /* vision_mobilenet */, FCAFD8482231614200496A36 /* yolo_16 */, FC704C1F21D237FC00F98BAB /* yolo */, @@ -320,6 +338,15 @@ path = mobilenet; sourceTree = ""; }; + FC9B9F062236AB4800BB6DB0 /* mobilenet_16 */ = { + isa = PBXGroup; + children = ( + FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */, + FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */, + ); + path = mobilenet_16; + sourceTree = ""; + }; FCAFD8482231614200496A36 /* yolo_16 */ = { isa = PBXGroup; children = ( @@ -419,10 +446,12 @@ FCAFD84C2231614200496A36 /* yolo_16_model in Resources */, FC2BFCC021DF279900C262B2 /* classify-img-output.png in Resources */, FC203FB221CBFDBA00B37166 /* test.jpg in Resources */, + FC9B9F0A2236ABF500BB6DB0 /* combined_mobilenet_params_16 in Resources */, FCC15E15221E716500DC3CB2 /* paddle-mobile-metallib.metallib in Resources */, FC9797C321D608E000F2FD90 /* mobilenet_params in Resources */, FC704C2421D237FC00F98BAB /* yolo_params in Resources */, FCE834AE2232A4AE0057BF43 /* combined_mobilenet_params in Resources */, + FC9B9F092236ABF300BB6DB0 /* combined_mobilenet_model_16 in Resources */, FC2BFCBC21DF0A8600C262B2 /* 00001.jpg in Resources */, FC9797BE21D6045B00F2FD90 /* banana.jpeg in Resources */, FC704C2521D237FC00F98BAB /* yolo_model in Resources */, @@ -490,6 +519,7 @@ C2E67E5E21524E460013F575 /* LoadPointerViewController.m in Sources */, FC2BFD3121DF3FEA00C262B2 /* Genet.swift in Sources */, FC039B8220E11C550081E9F8 /* AppDelegate.swift in Sources */, + FC1A1D552237EAC70047B7FD /* ImageTool.m in Sources */, FC2BFD4421DF5E1E00C262B2 /* SuperResolutionNet.swift in Sources */, FC2BFD3E21DF5CE800C262B2 /* PreProcessKernel.metal in Sources */, FC2BFD3821DF46DE00C262B2 /* OCDemoViewController.m in Sources */, diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift index 3afe84fde90c343ae0fdf4dc1fef2a1bc1cd8dd8..7412b41466a128ebc53a317ffbed266f528fbefb 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift @@ -19,13 +19,14 @@ public class MobileNetCombined: Net { @objc public override init(device: MTLDevice) { super.init(device: device) except = 0 - modelPath = Bundle.main.path(forResource: "combined_mobilenet_model", ofType: nil) ?! "model null" - paramPath = Bundle.main.path(forResource: "combined_mobilenet_params", ofType: nil) ?! "para null" + modelPath = Bundle.main.path(forResource: "combined_mobilenet_model_16", ofType: nil) ?! "model null" + paramPath = Bundle.main.path(forResource: "combined_mobilenet_params_16", ofType: nil) ?! "para null" inputDim = Dim.init(inDim: [1, 224, 224, 3]) metalLoadMode = .LoadMetalInCustomMetalLib let paddleMobileMetallib = Bundle.main.path(forResource: "paddle-mobile-metallib", ofType: "metallib") metalLibPath = paddleMobileMetallib useMPS = true + paramPrecision = .Float16 preprocessKernel = ScaleKernel.init(device: device, shape: Shape.init(inWidth: 224, inHeight: 224, inChannel: 3), metalLoadMode: .LoadMetalInCustomMetalLib, metalLibPath: paddleMobileMetallib) } diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h new file mode 100644 index 0000000000000000000000000000000000000000..6a1e52b5400f8b216d6c626c5eff5f41783c6e6c --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h @@ -0,0 +1,22 @@ +// +// ImageTool.h +// paddle-mobile-demo +// +// Created by liuRuiLong on 2019/3/12. +// Copyright © 2019 orange. All rights reserved. +// + +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ImageTool : NSObject + ++ (CVPixelBufferRef)imageToRGBPixelBuffer:(UIImage *)image; + + +@end + +NS_ASSUME_NONNULL_END diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m new file mode 100644 index 0000000000000000000000000000000000000000..e45ee919574c2ec61092935442c7ca8f0d032fcc --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m @@ -0,0 +1,38 @@ +// +// ImageTool.m +// paddle-mobile-demo +// +// Created by liuRuiLong on 2019/3/12. +// Copyright © 2019 orange. All rights reserved. +// + +#import "ImageTool.h" + +@implementation ImageTool + ++ (CVPixelBufferRef)imageToRGBPixelBuffer:(UIImage *)image { + CVPixelBufferRef pxbuffer = NULL; +#if defined(__arm__) || defined(__arm64__) + CGSize frameSize = CGSizeMake(CGImageGetWidth(image.CGImage),CGImageGetHeight(image.CGImage)); + //Metal渲染纹理需要IOSurface属性 + NSDictionary *options = + [NSDictionary dictionaryWithObjectsAndKeys: + [NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey, + [NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey, + [NSNumber numberWithBool:YES], kCVPixelBufferIOSurfaceOpenGLESTextureCompatibilityKey, + [NSNumber numberWithBool:YES], kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey,nil]; + CVPixelBufferCreate(kCFAllocatorDefault, frameSize.width, frameSize.height,kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef)options, &pxbuffer); + CVPixelBufferLockBaseAddress(pxbuffer, 0); + void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer); + CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(pxdata, frameSize.width, frameSize.height,8, CVPixelBufferGetBytesPerRow(pxbuffer),rgbColorSpace,(CGBitmapInfo)kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst); + CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image.CGImage),CGImageGetHeight(image.CGImage)), image.CGImage); + CGColorSpaceRelease(rgbColorSpace); + CGContextRelease(context); + CVPixelBufferUnlockBaseAddress(pxbuffer, 0); +#endif + + return pxbuffer; +} + +@end diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift index 53c417e88d64926160a95829498425646c2ba1ad..ba0948a189912b28fe0829e496dab7395950ac93 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift @@ -16,6 +16,7 @@ import UIKit import MetalKit import CoreMedia import paddle_mobile +import paddle_mobile_demo import MetalPerformanceShaders class FileReader { @@ -75,8 +76,10 @@ class ViewController: UIViewController { @IBOutlet weak var modelPickerView: UIPickerView! @IBOutlet weak var threadPickerView: UIPickerView! @IBOutlet weak var videoView: UIView! + var inputImageSize: CGSize = CGSize.init(width: 0, height: 0) // var videoCapture: VideoCapture! + var textureCache: CVMetalTextureCache? var selectImage: UIImage? var inputPointer: UnsafeMutablePointer? var modelType: SupportModel = SupportModel.supportedModels()[0] @@ -102,7 +105,9 @@ class ViewController: UIViewController { if self.toPredictTexture == nil { let beforeDate = Date.init() if modelType == .mobilenet_combined || modelType == .yolo { - self.toPredictTexture = try! MetalHelper.shared.textureLoader.newTexture(cgImage: selectImage!.cgImage!, options: nil) + let buffer = ImageTool.image(toRGBPixelBuffer: selectImage!) + let texture = convertToMTLTexture(imageBuffer: buffer.takeRetainedValue()) + self.toPredictTexture = texture } else { runner.getTexture(image: selectImage!.cgImage!) { [weak self] (texture) in let timeUse = Date.init().timeIntervalSince(beforeDate) @@ -150,6 +155,8 @@ class ViewController: UIViewController { } if success, let inResultHolderArr = resultHolder { +// writeToLibrary(fileName: "00001_result_32_new_new", buffer: UnsafeBufferPointer.init(start: inResultHolderArr[0].result, count: inResultHolderArr[0].capacity)) + let inResultHolder = inResultHolderArr[0] if i == max - 1 { let time = Date.init().timeIntervalSince(startDate) @@ -173,6 +180,8 @@ class ViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() + CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, MetalHelper.shared.device, nil, &textureCache) + GlobalConfig.shared.computePrecision = .Float16 GlobalConfig.shared.debug = false @@ -259,5 +268,28 @@ extension ViewController: VideoCaptureDelegate{ } +extension ViewController { + private func convertToMTLTexture(imageBuffer: CVPixelBuffer?) -> MTLTexture? { + if let textureCache = textureCache, let imageBuffer = imageBuffer { + CVPixelBufferLockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0)) + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) + inputImageSize = CGSize(width: width, height: height); + let pixelFormat: MTLPixelFormat = .bgra8Unorm + var texture: CVMetalTexture? + + CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, + imageBuffer, nil, pixelFormat, width, height, 0, &texture) + + CVPixelBufferUnlockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0)) + + if let texture = texture { + return CVMetalTextureGetTexture(texture) + } + } + return nil + } +} + diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h b/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h index 92de82860ccd372ba0eae962edd1b271986f1862..cb5511cc294c5508a5484a2c275fefa03b58489e 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h @@ -3,3 +3,4 @@ // #import +#import "ImageTool.h" diff --git a/metal/paddle-mobile/paddle-mobile/API/Net.swift b/metal/paddle-mobile/paddle-mobile/API/Net.swift index 35cd09eb49cde78a958ab019e69b03d4dfe35d0d..c0300e45cac703913ea9e0c38753a93f7942d669 100644 --- a/metal/paddle-mobile/paddle-mobile/API/Net.swift +++ b/metal/paddle-mobile/paddle-mobile/API/Net.swift @@ -56,10 +56,10 @@ import Foundation /// 输入维度，按照 n h w c 方式传入 @objc public var inputDim: Dim = Dim.init(inDim: []) - /// 是否使用 MetalPerformanceShaders 进行运算 + /// 是否使用 MetalPerformanceShaders 进行运算, 运算精度为 32 位时不支持开启 MPS @objc public var useMPS: Bool = false - /// 模型精度 - 当使用模型精度为 Float 16 时 不要开启 useMPS, 暂不支持 + /// 模型精度 @objc public var paramPrecision: Precision = .Float32 @objc public init(device: MTLDevice, inParamPointer: UnsafeMutableRawPointer, inParamSize:Int, inModelPointer: UnsafeMutableRawPointer, inModelSize: Int) { diff --git a/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift b/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift index 6128aa87768aaefddb782cf29ce0056a67284b37..e7e88dd00d8c9c7c1ded358767056106c9ebf9fb 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift @@ -25,3 +25,22 @@ func writeToLibrary(fileName: String, array: [P]) { fileHandler.closeFile() } +public func writeToLibrary(fileName: String, buffer: UnsafeBufferPointer

) { + let libraryPath = NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true).last ?! " library path get error " + let filePath = libraryPath + "/" + fileName + let fileManager = FileManager.init() + fileManager.createFile(atPath: filePath, contents: nil, attributes: nil) + let fileHandler = FileHandle.init(forWritingAtPath: filePath) ?! " file handler nil " + let data = Data.init(buffer: buffer) + fileHandler.write(data) + fileHandler.closeFile() +} + +func createFile(fileName: String) -> String { + let libraryPath = NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true).last ?! " library path get error " + let filePath = libraryPath + "/" + fileName + let fileManager = FileManager.init() + fileManager.createFile(atPath: filePath, contents: nil, attributes: nil) + return filePath +} + diff --git a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Base/Kernel.swift b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Base/Kernel.swift index 43ce7927ebf90c5ccc2ae1acf7df8f3f6b681863..f59afaab7304d5614726ccc6ab2b9ee1f11a6fa3 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Base/Kernel.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Base/Kernel.swift @@ -38,11 +38,21 @@ protocol KernelProtocol { } @objc open class Kernel: NSObject{ - let pipline: MTLComputePipelineState - let functionName: String - public init(device: MTLDevice, inFunctionName: String, usePaddleMobileLib: Bool = false, initContext: InitContext) { - pipline = device.pipeLine(funcName: inFunctionName, metalLoadMode: initContext.metalLoadMode, metalLibPath: initContext.metalLibPath) + + private var _pipline: MTLComputePipelineState? = nil + + var pipline: MTLComputePipelineState { + get { + return _pipline ?! " pipeline can't be nil " + } + } + + let functionName: String? + public init(device: MTLDevice, inFunctionName: String?, usePaddleMobileLib: Bool = false, initContext: InitContext) { functionName = inFunctionName + if let funcName = inFunctionName { + _pipline = device.pipeLine(funcName: funcName, metalLoadMode: initContext.metalLoadMode, metalLibPath: initContext.metalLibPath) + } } } @@ -104,7 +114,7 @@ open class BufferToTextureKernel: Kernel { @objc open class CusomKernel: Kernel { public let outputTexture: MTLTexture - public init(device: MTLDevice, inFunctionName: String, outputDim: Shape, metalLoadModel: MetalLoadMode, metalLibPath: String?) { + public init(device: MTLDevice, inFunctionName: String?, outputDim: Shape, metalLoadModel: MetalLoadMode, metalLibPath: String?) { let textureDesc = MTLTextureDescriptor.init() textureDesc.textureType = .type2D textureDesc.width = outputDim.width diff --git a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/ConvAddKernel.swift b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/ConvAddKernel.swift index 70c3379e8a97edd36e47d35a708a0ea2fa4b2d9e..a4d88814c0255b2242dbf02f01aaa36ebe76c431 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/ConvAddKernel.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/ConvAddKernel.swift @@ -145,7 +145,7 @@ class ConvAddKernel: Kernel, Computable { convDic[key] = conv imageDic[identifyingKey + "_input"] = MPSImage.init(texture: param.input.metalTexture, featureChannels: param.input.tensorDim[1]) imageDic[identifyingKey + "_output"] = MPSImage.init(texture: param.output.metalTexture, featureChannels: param.output.tensorDim[1]) - super.init(device: device, inFunctionName: "place_holder", initContext: initContext) + super.init(device: device, inFunctionName: nil, initContext: initContext) return } } diff --git a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift index cab3f3b0db922612f4fc4752ba0c95a554c05ec1..36fecd3369efd2c58d85a39762e23d5fcd1fb2eb 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift @@ -13,16 +13,24 @@ limitations under the License. */ import Foundation +import MetalPerformanceShaders public class ScaleKernel: CusomKernel { + var lanczos: MPSImageLanczosScale! public init(device: MTLDevice, shape: Shape, metalLoadMode: MetalLoadMode, metalLibPath: String?) { + lanczos = MPSImageLanczosScale(device: device) if GlobalConfig.shared.computePrecision == .Float32 { - super.init(device: device, inFunctionName: "scale", outputDim: shape, metalLoadModel: metalLoadMode, metalLibPath: metalLibPath) + super.init(device: device, inFunctionName: nil, outputDim: shape, metalLoadModel: metalLoadMode, metalLibPath: metalLibPath) } else if GlobalConfig.shared.computePrecision == .Float16 { - super.init(device: device, inFunctionName: "scale_half", outputDim: shape, metalLoadModel: metalLoadMode, metalLibPath: metalLibPath) + super.init(device: device, inFunctionName: nil, outputDim: shape, metalLoadModel: metalLoadMode, metalLibPath: metalLibPath) } else { fatalError(" unsupport ") } } + + public override func compute(inputTexuture: MTLTexture, commandBuffer: MTLCommandBuffer) throws { + lanczos.encode(commandBuffer: commandBuffer, sourceTexture: inputTexuture, destinationTexture: outputTexture) + } + } diff --git a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Texture2DTo2DArrayKernel.swift b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Texture2DTo2DArrayKernel.swift index 022d31df8f569a1a055c580f3ae02b48ed43ff24..65ea94905e2b23941580fc43e02a0ff04ab993a4 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Texture2DTo2DArrayKernel.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Texture2DTo2DArrayKernel.swift @@ -23,15 +23,6 @@ struct Texture2DTo2DArrayParam { } class Texture2DTo2DArrayKernel: Kernel, Computable{ - func compute(commandBuffer: MTLCommandBuffer, param: FeedParam

) throws { - guard let encoder = commandBuffer.makeComputeCommandEncoder() else { - throw PaddleMobileError.predictError(message: " encode is nil") - } - encoder.setTexture(param.input.mtlTexture, index: 0) - encoder.setTexture(param.output.metalTexture, index: 1) - encoder.dispatch(computePipline: pipline, outTexture: param.input.mtlTexture) - encoder.endEncoding() - } required init(device: MTLDevice, param: FeedParam

, initContext: InitContext) { param.output.initTexture(device: device, inTranspose: [0, 2, 3, 1], computePrecision: GlobalConfig.shared.computePrecision) @@ -42,6 +33,15 @@ class Texture2DTo2DArrayKernel: Kernel, Computable{ } else { fatalError() } - + } + + func compute(commandBuffer: MTLCommandBuffer, param: FeedParam

) throws { + guard let encoder = commandBuffer.makeComputeCommandEncoder() else { + throw PaddleMobileError.predictError(message: " encode is nil") + } + encoder.setTexture(param.input.mtlTexture, index: 0) + encoder.setTexture(param.output.metalTexture, index: 1) + encoder.dispatch(computePipline: pipline, outTexture: param.input.mtlTexture) + encoder.endEncoding() } } diff --git a/src/framework/executor.cpp b/src/framework/executor.cpp index 06e19ea093b639eb150c42fa113d7279cb458342..89253188951e8bc8268999c15298ceeb2ae0d69f 100644 --- a/src/framework/executor.cpp +++ b/src/framework/executor.cpp @@ -465,11 +465,12 @@ void Executor::FeedData(const Tensor &t) { template void Executor::FeedData(const std::vector &v) { auto input_size = v.size(); - auto vars = program_.scope->VarContain("feed"); + int index = 0; + auto vars = program_.scope->VarContain("feed", &index); PADDLE_MOBILE_ENFORCE(input_size == vars.size(), "input data number not correct"); for (int i = 0; i < input_size; i++) { - auto var = vars[i]; + auto var = program_.scope->Var("feed", i + index); auto feed_tensor = var->template GetMutable(); feed_tensor->external_data = v[i]; } @@ -478,11 +479,12 @@ void Executor::FeedData(const std::vector &v) { template void Executor::FeedTensorData(const vector &v) { auto input_size = v.size(); - auto vars = program_.scope->VarContain("feed"); + int index = 0; + auto vars = program_.scope->VarContain("feed", &index); PADDLE_MOBILE_ENFORCE(input_size == vars.size(), "input data number not correct"); for (int i = 0; i < input_size; i++) { - auto var = vars[i]; + auto var = program_.scope->Var("feed", i + index); auto feed_tensor = var->template GetMutable(); feed_tensor->ShareDataWith(v[i]); } @@ -492,12 +494,13 @@ template void Executor::GetResults(std::vector *v) { auto output_size = v->size(); PADDLE_MOBILE_ENFORCE(output_size > 0, "Empty output"); - auto vars = program_.scope->VarContain("fetch"); + int index = 0; + auto vars = program_.scope->VarContain("fetch", &index); PADDLE_MOBILE_ENFORCE(output_size == vars.size(), "output data number not correct"); for (int i = 0; i < output_size; i++) { - auto var = vars[i]; + auto var = program_.scope->Var("fetch", i + index); auto fetch_tensor = var->template GetMutable(); (*v)[i] = fetch_tensor->template data(); } @@ -506,11 +509,11 @@ void Executor::GetResults(std::vector *v) { template void Executor::GetTensorResults( std::vector *v) { - auto vars = program_.scope->VarContain("fetch"); + int index = 0; + auto vars = program_.scope->VarContain("fetch", &index); auto output_size = vars.size(); - for (int i = 0; i < output_size; i++) { - auto var = vars[i]; + auto var = program_.scope->Var("fetch", i + index); auto fetch_tensor = var->template GetMutable(); v->push_back(fetch_tensor); } diff --git a/src/framework/scope.cpp b/src/framework/scope.cpp index db263081446f9804e5352588063a23f72a8bf163..026b1e6b72dcca081554152ea356204ba311b359 100644 --- a/src/framework/scope.cpp +++ b/src/framework/scope.cpp @@ -116,18 +116,26 @@ Variable *Scope::Var(const std::string &name, const int id) { return Var(name + std::to_string(id)); } -std::vector Scope::VarContain(const std::string substring) { +std::vector Scope::VarContain(const std::string substring, + int *min) { std::vector v; + + int temp = 9999; + auto len0 = substring.length(); for (auto pair : vars_) { if (pair.first.find(substring) == 0) { v.push_back(pair.second); + auto len1 = pair.first.length(); + int index = std::stoi(pair.first.substr(len0, len1)); + if (index < temp) { + temp = index; + } } } + *min = temp; return v; } -void Scope::InsertVar(const std::string str, Variable *var) {} - void Scope::print_vars() { DLOG << "====================start to print variables================="; for (auto pair : vars_) { diff --git a/src/framework/scope.h b/src/framework/scope.h index d9e3a179e0aae9f93947df60cea410d3eb5cb128..2bf9ed85bd36013814a6adb80884693d7a944c90 100644 --- a/src/framework/scope.h +++ b/src/framework/scope.h @@ -85,8 +85,7 @@ class Scope { #ifdef PADDLE_MOBILE_FPGA Variable *Var(const std::string &name, const int id); - std::vector VarContain(const std::string substring); - void InsertVar(const std::string str, Variable *var); + std::vector VarContain(const std::string substring, int *min); void print_vars(); #endif diff --git a/src/io/api_paddle_mobile.cc b/src/io/api_paddle_mobile.cc index d644dc4804b6daa1447aa68084283d345b2dc6f8..b7a8ebba9c1d9adac3993d22aa33fb5445235349 100644 --- a/src/io/api_paddle_mobile.cc +++ b/src/io/api_paddle_mobile.cc @@ -171,18 +171,6 @@ void PaddleMobilePredictor::GetPaddleTensor(const std::string &name, ConvertTensors(*t, output); }; -template -void PaddleMobilePredictor::FeedData( - const std::vector &inputs) { - paddle_mobile_->FeedData(inputs); -} - -template -void PaddleMobilePredictor::GetResults( - std::vector *outputs) { - paddle_mobile_->GetResults(outputs); -} - template void PaddleMobilePredictor::Predict_From_To(int start, int end) { paddle_mobile_->Predict_From_To(start, end); diff --git a/src/io/api_paddle_mobile.h b/src/io/api_paddle_mobile.h index 2e3d8dc69b60138eb5f82c1b1faa9c182c51442b..aa0050ca057baa94c457dc0eb313b2bd7f8355be 100644 --- a/src/io/api_paddle_mobile.h +++ b/src/io/api_paddle_mobile.h @@ -32,8 +32,6 @@ class PaddleMobilePredictor : public PaddlePredictor { std::vector* output_data, int batch_size = -1) override; #ifdef PADDLE_MOBILE_FPGA - void FeedData(const std::vector& inputs) override; - void GetResults(std::vector* outputs) override; void Predict_From_To(int start, int end) override; void FeedPaddleTensors(const std::vector& inputs) override; void FetchPaddleTensors(std::vector* outputs) override; diff --git a/src/io/paddle_inference_api.h b/src/io/paddle_inference_api.h index 81779da1eafa892bd800641e133014988dab13e8..36435e07e6bad84527b58c0216cc12df557a8ab7 100644 --- a/src/io/paddle_inference_api.h +++ b/src/io/paddle_inference_api.h @@ -113,6 +113,7 @@ class PaddlePredictor { // `inputs`. `inputs` should be available until Run returns. Caller should be // responsible for the output tensor's buffer, either allocated or passed from // outside. + virtual bool Run(const std::vector& inputs, std::vector* output_data, int batch_size = -1) = 0; @@ -126,8 +127,6 @@ class PaddlePredictor { std::string param_file; }; #ifdef PADDLE_MOBILE_FPGA - virtual void FeedData(const std::vector& inputs) = 0; - virtual void GetResults(std::vector* outputs) = 0; virtual void Predict_From_To(int start, int end) = 0; virtual void FeedPaddleTensors(const std::vector& inputs) = 0; virtual void FetchPaddleTensors(std::vector* outputs) = 0; diff --git a/test/fpga/test_rfcn_api.cpp b/test/fpga/test_rfcn_api.cpp index 74e99ff5ffe553101a07c407cd05c177bd6287b2..2268fc46e5eb98eeff781e6de3a57e5efb911d3d 100644 --- a/test/fpga/test_rfcn_api.cpp +++ b/test/fpga/test_rfcn_api.cpp @@ -55,8 +55,21 @@ PaddleMobileConfig GetConfig() { return config; } +PaddleMobileConfig GetConfig1() { + PaddleMobileConfig config; + config.precision = PaddleMobileConfig::FP32; + config.device = PaddleMobileConfig::kFPGA; + config.model_dir = "../models/resnet50"; + config.thread_num = 1; + config.batch_size = 1; + config.optimize = true; + config.quantification = false; + return config; +} + int main() { open_device(); + PaddleMobileConfig config = GetConfig(); auto predictor = CreatePaddlePredictor(fpga_malloc(img_length * sizeof(float))); readStream(g_image, reinterpret_cast(img)); std::cout << "Finishing initializing data" << std::endl; - /* - predictor->FeedData({img_info, img}); - predictor->Predict_From_To(0, -1); - std::cout << " Finishing predicting " << std::endl; - std::vector v(3, nullptr); - predictor->GetResults(&v); - int post_nms = 300; - for (int num = 0; num < post_nms; num ++){ - for (int i = 0; i < 8; i ++){ - std:: cout << ((float*)(v[0]))[num * 8 + i] << std::endl; - } - } - for (int num = 0; num < post_nms; num ++){ - for (int i = 0; i < 8; i ++){ - std:: cout << ((float*)(v[1]))[num * 8 + i] << std::endl; - } - } - for (int num = 0; num < post_nms; num ++){ - for (int i = 0; i < 4; i ++){ - std:: cout << ((float*)(v[2]))[num * 4 + i] << std::endl; - } - } - */ - struct PaddleTensor t_img_info, t_img; - t_img_info.dtype = FLOAT32; + t_img.dtypeid = typeid(float); t_img_info.layout = LAYOUT_HWC; t_img_info.shape = std::vector({1, 3}); t_img_info.name = "Image information"; t_img_info.data.Reset(img_info, 3 * sizeof(float)); - t_img.dtype = FLOAT32; + t_img.dtypeid = typeid(float); t_img.layout = LAYOUT_HWC; - t_img.shape = std::vector({1, 768, 1536, 3}); + t_img.shape = std::vector({1, 432, 1280, 3}); t_img.name = "Image information"; t_img.data.Reset(img, img_length * sizeof(float)); predictor->FeedPaddleTensors({t_img_info, t_img}); @@ -116,6 +105,9 @@ int main() { std::vector v; // No need to initialize v predictor->FetchPaddleTensors(&v); // Old data in v will be cleared std::cout << "Output number is " << v.size() << std::endl; + std::cout << "out[0] length " << v[0].data.length() << std::endl; + std::cout << "out[1] length " << v[1].data.length() << std::endl; + std::cout << "out[2] length " << v[2].data.length() << std::endl; auto post_nms = v[0].data.length() / sizeof(float) / 8; for (int num = 0; num < post_nms; num++) { @@ -138,6 +130,8 @@ int main() { } std::cout << "Finish getting vector values" << std::endl; + //////////////////////////////////////////////////// + PaddleTensor tensor; predictor->GetPaddleTensor("fetch2", &tensor); for (int i = 0; i < post_nms; i++) { @@ -145,5 +139,36 @@ int main() { std::cout << p[+i] << std::endl; } + ////////////////////////////////////////////////////// + + PaddleMobileConfig config1 = GetConfig1(); + auto predictor1 = + CreatePaddlePredictor(config1); + + std::cout << "Finishing loading model" << std::endl; + + int img_length1 = 224 * 224 * 3; + auto img1 = + reinterpret_cast(fpga_malloc(img_length1 * sizeof(float))); + + std::cout << "Finishing initializing data" << std::endl; + + struct PaddleTensor t_img1; + + t_img1.dtypeid = typeid(float); + t_img1.layout = LAYOUT_HWC; + t_img1.shape = std::vector({1, 224, 224, 3}); + t_img1.name = "Image information"; + t_img1.data.Reset(img1, img_length1 * sizeof(float)); + predictor1->FeedPaddleTensors({t_img1}); + predictor1->Predict_From_To(0, -1); + std::cout << "Finishing predicting " << std::endl; + + std::vector v1; // No need to initialize v + predictor1->FetchPaddleTensors(&v1); // Old data in v will be cleared + std::cout << "Output number is " << v1.size() << std::endl; + std::cout << "out[0] length " << v1[0].data.length() << std::endl; + return 0; }