From 7d5d941e82aa96697e4cd9cffba9da7983d50979 Mon Sep 17 00:00:00 2001 From: liuruilong Date: Tue, 12 Mar 2019 21:33:20 +0800 Subject: [PATCH] relpace scale imp --- .../project.pbxproj | 30 +++++++++++++++ .../Net/MobileNetCombined.swift | 5 ++- .../paddle-mobile-demo/OC/ImageTool.h | 22 +++++++++++ .../paddle-mobile-demo/OC/ImageTool.m | 38 +++++++++++++++++++ .../paddle-mobile-demo/ViewController.swift | 34 ++++++++++++++++- .../paddle-mobile-demo-Bridging-Header.h | 1 + .../paddle-mobile/Src/Common/Tools.swift | 19 ++++++++++ .../Src/Operators/Kernels/Scale.swift | 8 ++++ 8 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h create mode 100644 metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj index 749c8b2a92..04bb0aa82a 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj +++ b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj @@ -15,6 +15,7 @@ FC039B8720E11C550081E9F8 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8520E11C550081E9F8 /* Main.storyboard */; }; FC039B8920E11C560081E9F8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8820E11C560081E9F8 /* Assets.xcassets */; }; FC039B8C20E11C560081E9F8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8A20E11C560081E9F8 /* LaunchScreen.storyboard */; }; + FC1A1D552237EAC70047B7FD /* ImageTool.m in Sources */ = {isa = PBXBuildFile; fileRef = FC1A1D542237EAC70047B7FD /* ImageTool.m */; }; FC203FB221CBFDBA00B37166 /* test.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC203FA921CBFDBA00B37166 /* test.jpg */; }; FC2BFCBC21DF0A8600C262B2 /* 00001.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC2BFCBB21DF0A8600C262B2 /* 00001.jpg */; }; FC2BFCBE21DF15D900C262B2 /* 123.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC2BFCBD21DF15D900C262B2 /* 123.jpg */; }; @@ -42,6 +43,8 @@ FC9797C321D608E000F2FD90 /* mobilenet_params in Resources */ = {isa = PBXBuildFile; fileRef = FC9797C121D608DF00F2FD90 /* mobilenet_params */; }; FC9797C721D609FB00F2FD90 /* synset.txt in Resources */ = {isa = PBXBuildFile; fileRef = FC9797C621D609FB00F2FD90 /* synset.txt */; }; FC9797CF21D6506F00F2FD90 /* mingren.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC9797CE21D6506F00F2FD90 /* mingren.jpg */; }; + FC9B9F092236ABF300BB6DB0 /* combined_mobilenet_model_16 in Resources */ = {isa = PBXBuildFile; fileRef = FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */; }; + FC9B9F0A2236ABF500BB6DB0 /* combined_mobilenet_params_16 in Resources */ = {isa = PBXBuildFile; fileRef = FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */; }; FCAFD84B2231614200496A36 /* yolo_16_param in Resources */ = {isa = PBXBuildFile; fileRef = FCAFD8492231614200496A36 /* yolo_16_param */; }; FCAFD84C2231614200496A36 /* yolo_16_model in Resources */ = {isa = PBXBuildFile; fileRef = FCAFD84A2231614200496A36 /* yolo_16_model */; }; FCBCCC552122EF5500D94F7E /* MetalHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCBCCC542122EF5400D94F7E /* MetalHelper.swift */; }; @@ -83,6 +86,8 @@ FC039B8820E11C560081E9F8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; FC039B8B20E11C560081E9F8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; FC039B8D20E11C560081E9F8 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + FC1A1D532237EAC70047B7FD /* ImageTool.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ImageTool.h; sourceTree = ""; }; + FC1A1D542237EAC70047B7FD /* ImageTool.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ImageTool.m; sourceTree = ""; }; FC203FA921CBFDBA00B37166 /* test.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = test.jpg; sourceTree = ""; }; FC27991121343A39000B6BAD /* paddle-mobile-demo-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "paddle-mobile-demo-Bridging-Header.h"; sourceTree = ""; }; FC2BFCBB21DF0A8600C262B2 /* 00001.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = 00001.jpg; sourceTree = ""; }; @@ -115,6 +120,8 @@ FC9797C121D608DF00F2FD90 /* mobilenet_params */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_params; sourceTree = ""; }; FC9797C621D609FB00F2FD90 /* synset.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = synset.txt; sourceTree = ""; }; FC9797CE21D6506F00F2FD90 /* mingren.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = mingren.jpg; sourceTree = ""; }; + FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */ = {isa = PBXFileReference; lastKnownFileType = file; path = combined_mobilenet_model_16; sourceTree = ""; }; + FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */ = {isa = PBXFileReference; lastKnownFileType = file; path = combined_mobilenet_params_16; sourceTree = ""; }; FCAFD8492231614200496A36 /* yolo_16_param */ = {isa = PBXFileReference; lastKnownFileType = file; path = yolo_16_param; sourceTree = ""; }; FCAFD84A2231614200496A36 /* yolo_16_model */ = {isa = PBXFileReference; lastKnownFileType = file; path = yolo_16_model; sourceTree = ""; }; FCBCCC542122EF5400D94F7E /* MetalHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MetalHelper.swift; sourceTree = ""; }; @@ -183,6 +190,7 @@ FC039B8020E11C550081E9F8 /* paddle-mobile-demo */ = { isa = PBXGroup; children = ( + FC1A1D522237EAA10047B7FD /* OC */, FC2BFD4F21DF892500C262B2 /* Resources */, FCBCCC542122EF5400D94F7E /* MetalHelper.swift */, FC2BFD3F21DF5DDF00C262B2 /* OCInterface */, @@ -201,6 +209,15 @@ path = "paddle-mobile-demo"; sourceTree = ""; }; + FC1A1D522237EAA10047B7FD /* OC */ = { + isa = PBXGroup; + children = ( + FC1A1D532237EAC70047B7FD /* ImageTool.h */, + FC1A1D542237EAC70047B7FD /* ImageTool.m */, + ); + path = OC; + sourceTree = ""; + }; FC203FA821CBFDBA00B37166 /* images */ = { isa = PBXGroup; children = ( @@ -269,6 +286,7 @@ FC704C1B21D237FC00F98BAB /* vision_model */ = { isa = PBXGroup; children = ( + FC9B9F062236AB4800BB6DB0 /* mobilenet_16 */, FCE834AB2232A4AE0057BF43 /* vision_mobilenet */, FCAFD8482231614200496A36 /* yolo_16 */, FC704C1F21D237FC00F98BAB /* yolo */, @@ -320,6 +338,15 @@ path = mobilenet; sourceTree = ""; }; + FC9B9F062236AB4800BB6DB0 /* mobilenet_16 */ = { + isa = PBXGroup; + children = ( + FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */, + FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */, + ); + path = mobilenet_16; + sourceTree = ""; + }; FCAFD8482231614200496A36 /* yolo_16 */ = { isa = PBXGroup; children = ( @@ -419,10 +446,12 @@ FCAFD84C2231614200496A36 /* yolo_16_model in Resources */, FC2BFCC021DF279900C262B2 /* classify-img-output.png in Resources */, FC203FB221CBFDBA00B37166 /* test.jpg in Resources */, + FC9B9F0A2236ABF500BB6DB0 /* combined_mobilenet_params_16 in Resources */, FCC15E15221E716500DC3CB2 /* paddle-mobile-metallib.metallib in Resources */, FC9797C321D608E000F2FD90 /* mobilenet_params in Resources */, FC704C2421D237FC00F98BAB /* yolo_params in Resources */, FCE834AE2232A4AE0057BF43 /* combined_mobilenet_params in Resources */, + FC9B9F092236ABF300BB6DB0 /* combined_mobilenet_model_16 in Resources */, FC2BFCBC21DF0A8600C262B2 /* 00001.jpg in Resources */, FC9797BE21D6045B00F2FD90 /* banana.jpeg in Resources */, FC704C2521D237FC00F98BAB /* yolo_model in Resources */, @@ -490,6 +519,7 @@ C2E67E5E21524E460013F575 /* LoadPointerViewController.m in Sources */, FC2BFD3121DF3FEA00C262B2 /* Genet.swift in Sources */, FC039B8220E11C550081E9F8 /* AppDelegate.swift in Sources */, + FC1A1D552237EAC70047B7FD /* ImageTool.m in Sources */, FC2BFD4421DF5E1E00C262B2 /* SuperResolutionNet.swift in Sources */, FC2BFD3E21DF5CE800C262B2 /* PreProcessKernel.metal in Sources */, FC2BFD3821DF46DE00C262B2 /* OCDemoViewController.m in Sources */, diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift index 3afe84fde9..7412b41466 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift @@ -19,13 +19,14 @@ public class MobileNetCombined: Net { @objc public override init(device: MTLDevice) { super.init(device: device) except = 0 - modelPath = Bundle.main.path(forResource: "combined_mobilenet_model", ofType: nil) ?! "model null" - paramPath = Bundle.main.path(forResource: "combined_mobilenet_params", ofType: nil) ?! "para null" + modelPath = Bundle.main.path(forResource: "combined_mobilenet_model_16", ofType: nil) ?! "model null" + paramPath = Bundle.main.path(forResource: "combined_mobilenet_params_16", ofType: nil) ?! "para null" inputDim = Dim.init(inDim: [1, 224, 224, 3]) metalLoadMode = .LoadMetalInCustomMetalLib let paddleMobileMetallib = Bundle.main.path(forResource: "paddle-mobile-metallib", ofType: "metallib") metalLibPath = paddleMobileMetallib useMPS = true + paramPrecision = .Float16 preprocessKernel = ScaleKernel.init(device: device, shape: Shape.init(inWidth: 224, inHeight: 224, inChannel: 3), metalLoadMode: .LoadMetalInCustomMetalLib, metalLibPath: paddleMobileMetallib) } diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h new file mode 100644 index 0000000000..6a1e52b540 --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h @@ -0,0 +1,22 @@ +// +// ImageTool.h +// paddle-mobile-demo +// +// Created by liuRuiLong on 2019/3/12. +// Copyright © 2019 orange. All rights reserved. +// + +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ImageTool : NSObject + ++ (CVPixelBufferRef)imageToRGBPixelBuffer:(UIImage *)image; + + +@end + +NS_ASSUME_NONNULL_END diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m new file mode 100644 index 0000000000..e45ee91957 --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m @@ -0,0 +1,38 @@ +// +// ImageTool.m +// paddle-mobile-demo +// +// Created by liuRuiLong on 2019/3/12. +// Copyright © 2019 orange. All rights reserved. +// + +#import "ImageTool.h" + +@implementation ImageTool + ++ (CVPixelBufferRef)imageToRGBPixelBuffer:(UIImage *)image { + CVPixelBufferRef pxbuffer = NULL; +#if defined(__arm__) || defined(__arm64__) + CGSize frameSize = CGSizeMake(CGImageGetWidth(image.CGImage),CGImageGetHeight(image.CGImage)); + //Metal渲染纹理需要IOSurface属性 + NSDictionary *options = + [NSDictionary dictionaryWithObjectsAndKeys: + [NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey, + [NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey, + [NSNumber numberWithBool:YES], kCVPixelBufferIOSurfaceOpenGLESTextureCompatibilityKey, + [NSNumber numberWithBool:YES], kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey,nil]; + CVPixelBufferCreate(kCFAllocatorDefault, frameSize.width, frameSize.height,kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef)options, &pxbuffer); + CVPixelBufferLockBaseAddress(pxbuffer, 0); + void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer); + CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(pxdata, frameSize.width, frameSize.height,8, CVPixelBufferGetBytesPerRow(pxbuffer),rgbColorSpace,(CGBitmapInfo)kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst); + CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image.CGImage),CGImageGetHeight(image.CGImage)), image.CGImage); + CGColorSpaceRelease(rgbColorSpace); + CGContextRelease(context); + CVPixelBufferUnlockBaseAddress(pxbuffer, 0); +#endif + + return pxbuffer; +} + +@end diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift index 53c417e88d..ba0948a189 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift @@ -16,6 +16,7 @@ import UIKit import MetalKit import CoreMedia import paddle_mobile +import paddle_mobile_demo import MetalPerformanceShaders class FileReader { @@ -75,8 +76,10 @@ class ViewController: UIViewController { @IBOutlet weak var modelPickerView: UIPickerView! @IBOutlet weak var threadPickerView: UIPickerView! @IBOutlet weak var videoView: UIView! + var inputImageSize: CGSize = CGSize.init(width: 0, height: 0) // var videoCapture: VideoCapture! + var textureCache: CVMetalTextureCache? var selectImage: UIImage? var inputPointer: UnsafeMutablePointer? var modelType: SupportModel = SupportModel.supportedModels()[0] @@ -102,7 +105,9 @@ class ViewController: UIViewController { if self.toPredictTexture == nil { let beforeDate = Date.init() if modelType == .mobilenet_combined || modelType == .yolo { - self.toPredictTexture = try! MetalHelper.shared.textureLoader.newTexture(cgImage: selectImage!.cgImage!, options: nil) + let buffer = ImageTool.image(toRGBPixelBuffer: selectImage!) + let texture = convertToMTLTexture(imageBuffer: buffer.takeRetainedValue()) + self.toPredictTexture = texture } else { runner.getTexture(image: selectImage!.cgImage!) { [weak self] (texture) in let timeUse = Date.init().timeIntervalSince(beforeDate) @@ -150,6 +155,8 @@ class ViewController: UIViewController { } if success, let inResultHolderArr = resultHolder { +// writeToLibrary(fileName: "00001_result_32_new_new", buffer: UnsafeBufferPointer.init(start: inResultHolderArr[0].result, count: inResultHolderArr[0].capacity)) + let inResultHolder = inResultHolderArr[0] if i == max - 1 { let time = Date.init().timeIntervalSince(startDate) @@ -173,6 +180,8 @@ class ViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() + CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, MetalHelper.shared.device, nil, &textureCache) + GlobalConfig.shared.computePrecision = .Float16 GlobalConfig.shared.debug = false @@ -259,5 +268,28 @@ extension ViewController: VideoCaptureDelegate{ } +extension ViewController { + private func convertToMTLTexture(imageBuffer: CVPixelBuffer?) -> MTLTexture? { + if let textureCache = textureCache, let imageBuffer = imageBuffer { + CVPixelBufferLockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0)) + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) + inputImageSize = CGSize(width: width, height: height); + let pixelFormat: MTLPixelFormat = .bgra8Unorm + var texture: CVMetalTexture? + + CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, + imageBuffer, nil, pixelFormat, width, height, 0, &texture) + + CVPixelBufferUnlockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0)) + + if let texture = texture { + return CVMetalTextureGetTexture(texture) + } + } + return nil + } +} + diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h b/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h index 92de82860c..cb5511cc29 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h @@ -3,3 +3,4 @@ // #import +#import "ImageTool.h" diff --git a/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift b/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift index 6128aa8776..e7e88dd00d 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift @@ -25,3 +25,22 @@ func writeToLibrary(fileName: String, array: [P]) { fileHandler.closeFile() } +public func writeToLibrary(fileName: String, buffer: UnsafeBufferPointer

) { + let libraryPath = NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true).last ?! " library path get error " + let filePath = libraryPath + "/" + fileName + let fileManager = FileManager.init() + fileManager.createFile(atPath: filePath, contents: nil, attributes: nil) + let fileHandler = FileHandle.init(forWritingAtPath: filePath) ?! " file handler nil " + let data = Data.init(buffer: buffer) + fileHandler.write(data) + fileHandler.closeFile() +} + +func createFile(fileName: String) -> String { + let libraryPath = NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true).last ?! " library path get error " + let filePath = libraryPath + "/" + fileName + let fileManager = FileManager.init() + fileManager.createFile(atPath: filePath, contents: nil, attributes: nil) + return filePath +} + diff --git a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift index cab3f3b0db..c024c8ec2e 100644 --- a/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift +++ b/metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift @@ -13,9 +13,12 @@ limitations under the License. */ import Foundation +import MetalPerformanceShaders public class ScaleKernel: CusomKernel { + var lanczos: MPSImageLanczosScale! public init(device: MTLDevice, shape: Shape, metalLoadMode: MetalLoadMode, metalLibPath: String?) { + lanczos = MPSImageLanczosScale(device: device) if GlobalConfig.shared.computePrecision == .Float32 { super.init(device: device, inFunctionName: "scale", outputDim: shape, metalLoadModel: metalLoadMode, metalLibPath: metalLibPath) } else if GlobalConfig.shared.computePrecision == .Float16 { @@ -24,5 +27,10 @@ public class ScaleKernel: CusomKernel { fatalError(" unsupport ") } } + + public override func compute(inputTexuture: MTLTexture, commandBuffer: MTLCommandBuffer) throws { + lanczos.encode(commandBuffer: commandBuffer, sourceTexture: inputTexuture, destinationTexture: outputTexture) + } + } -- GitLab