relpace scale imp

metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj

@@ -15,6 +15,7 @@
 		FC039B8720E11C550081E9F8 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8520E11C550081E9F8 /* Main.storyboard */; };
 		FC039B8920E11C560081E9F8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8820E11C560081E9F8 /* Assets.xcassets */; };
 		FC039B8C20E11C560081E9F8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8A20E11C560081E9F8 /* LaunchScreen.storyboard */; };
+		FC1A1D552237EAC70047B7FD /* ImageTool.m in Sources */ = {isa = PBXBuildFile; fileRef = FC1A1D542237EAC70047B7FD /* ImageTool.m */; };
 		FC203FB221CBFDBA00B37166 /* test.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC203FA921CBFDBA00B37166 /* test.jpg */; };
 		FC2BFCBC21DF0A8600C262B2 /* 00001.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC2BFCBB21DF0A8600C262B2 /* 00001.jpg */; };
 		FC2BFCBE21DF15D900C262B2 /* 123.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC2BFCBD21DF15D900C262B2 /* 123.jpg */; };
@@ -42,6 +43,8 @@
 		FC9797C321D608E000F2FD90 /* mobilenet_params in Resources */ = {isa = PBXBuildFile; fileRef = FC9797C121D608DF00F2FD90 /* mobilenet_params */; };
 		FC9797C721D609FB00F2FD90 /* synset.txt in Resources */ = {isa = PBXBuildFile; fileRef = FC9797C621D609FB00F2FD90 /* synset.txt */; };
 		FC9797CF21D6506F00F2FD90 /* mingren.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC9797CE21D6506F00F2FD90 /* mingren.jpg */; };
+		FC9B9F092236ABF300BB6DB0 /* combined_mobilenet_model_16 in Resources */ = {isa = PBXBuildFile; fileRef = FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */; };
+		FC9B9F0A2236ABF500BB6DB0 /* combined_mobilenet_params_16 in Resources */ = {isa = PBXBuildFile; fileRef = FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */; };
 		FCAFD84B2231614200496A36 /* yolo_16_param in Resources */ = {isa = PBXBuildFile; fileRef = FCAFD8492231614200496A36 /* yolo_16_param */; };
 		FCAFD84C2231614200496A36 /* yolo_16_model in Resources */ = {isa = PBXBuildFile; fileRef = FCAFD84A2231614200496A36 /* yolo_16_model */; };
 		FCBCCC552122EF5500D94F7E /* MetalHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCBCCC542122EF5400D94F7E /* MetalHelper.swift */; };
@@ -83,6 +86,8 @@
 		FC039B8820E11C560081E9F8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
 		FC039B8B20E11C560081E9F8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
 		FC039B8D20E11C560081E9F8 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		FC1A1D532237EAC70047B7FD /* ImageTool.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ImageTool.h; sourceTree = "<group>"; };
+		FC1A1D542237EAC70047B7FD /* ImageTool.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ImageTool.m; sourceTree = "<group>"; };
 		FC203FA921CBFDBA00B37166 /* test.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = test.jpg; sourceTree = "<group>"; };
 		FC27991121343A39000B6BAD /* paddle-mobile-demo-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "paddle-mobile-demo-Bridging-Header.h"; sourceTree = "<group>"; };
 		FC2BFCBB21DF0A8600C262B2 /* 00001.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = 00001.jpg; sourceTree = "<group>"; };
@@ -115,6 +120,8 @@
 		FC9797C121D608DF00F2FD90 /* mobilenet_params */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_params; sourceTree = "<group>"; };
 		FC9797C621D609FB00F2FD90 /* synset.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = synset.txt; sourceTree = "<group>"; };
 		FC9797CE21D6506F00F2FD90 /* mingren.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = mingren.jpg; sourceTree = "<group>"; };
+		FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */ = {isa = PBXFileReference; lastKnownFileType = file; path = combined_mobilenet_model_16; sourceTree = "<group>"; };
+		FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */ = {isa = PBXFileReference; lastKnownFileType = file; path = combined_mobilenet_params_16; sourceTree = "<group>"; };
 		FCAFD8492231614200496A36 /* yolo_16_param */ = {isa = PBXFileReference; lastKnownFileType = file; path = yolo_16_param; sourceTree = "<group>"; };
 		FCAFD84A2231614200496A36 /* yolo_16_model */ = {isa = PBXFileReference; lastKnownFileType = file; path = yolo_16_model; sourceTree = "<group>"; };
 		FCBCCC542122EF5400D94F7E /* MetalHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MetalHelper.swift; sourceTree = "<group>"; };
@@ -183,6 +190,7 @@
 		FC039B8020E11C550081E9F8 /* paddle-mobile-demo */ = {
 			isa = PBXGroup;
 			children = (
+				FC1A1D522237EAA10047B7FD /* OC */,
 				FC2BFD4F21DF892500C262B2 /* Resources */,
 				FCBCCC542122EF5400D94F7E /* MetalHelper.swift */,
 				FC2BFD3F21DF5DDF00C262B2 /* OCInterface */,
@@ -201,6 +209,15 @@
 			path = "paddle-mobile-demo";
 			sourceTree = "<group>";
 		};
+		FC1A1D522237EAA10047B7FD /* OC */ = {
+			isa = PBXGroup;
+			children = (
+				FC1A1D532237EAC70047B7FD /* ImageTool.h */,
+				FC1A1D542237EAC70047B7FD /* ImageTool.m */,
+			);
+			path = OC;
+			sourceTree = "<group>";
+		};
 		FC203FA821CBFDBA00B37166 /* images */ = {
 			isa = PBXGroup;
 			children = (
@@ -269,6 +286,7 @@
 		FC704C1B21D237FC00F98BAB /* vision_model */ = {
 			isa = PBXGroup;
 			children = (
+				FC9B9F062236AB4800BB6DB0 /* mobilenet_16 */,
 				FCE834AB2232A4AE0057BF43 /* vision_mobilenet */,
 				FCAFD8482231614200496A36 /* yolo_16 */,
 				FC704C1F21D237FC00F98BAB /* yolo */,
@@ -320,6 +338,15 @@
 			path = mobilenet;
 			sourceTree = "<group>";
 		};
+		FC9B9F062236AB4800BB6DB0 /* mobilenet_16 */ = {
+			isa = PBXGroup;
+			children = (
+				FC9B9F072236ABD200BB6DB0 /* combined_mobilenet_model_16 */,
+				FC9B9F082236ABD300BB6DB0 /* combined_mobilenet_params_16 */,
+			);
+			path = mobilenet_16;
+			sourceTree = "<group>";
+		};
 		FCAFD8482231614200496A36 /* yolo_16 */ = {
 			isa = PBXGroup;
 			children = (
@@ -419,10 +446,12 @@
 				FCAFD84C2231614200496A36 /* yolo_16_model in Resources */,
 				FC2BFCC021DF279900C262B2 /* classify-img-output.png in Resources */,
 				FC203FB221CBFDBA00B37166 /* test.jpg in Resources */,
+				FC9B9F0A2236ABF500BB6DB0 /* combined_mobilenet_params_16 in Resources */,
 				FCC15E15221E716500DC3CB2 /* paddle-mobile-metallib.metallib in Resources */,
 				FC9797C321D608E000F2FD90 /* mobilenet_params in Resources */,
 				FC704C2421D237FC00F98BAB /* yolo_params in Resources */,
 				FCE834AE2232A4AE0057BF43 /* combined_mobilenet_params in Resources */,
+				FC9B9F092236ABF300BB6DB0 /* combined_mobilenet_model_16 in Resources */,
 				FC2BFCBC21DF0A8600C262B2 /* 00001.jpg in Resources */,
 				FC9797BE21D6045B00F2FD90 /* banana.jpeg in Resources */,
 				FC704C2521D237FC00F98BAB /* yolo_model in Resources */,
@@ -490,6 +519,7 @@
 				C2E67E5E21524E460013F575 /* LoadPointerViewController.m in Sources */,
 				FC2BFD3121DF3FEA00C262B2 /* Genet.swift in Sources */,
 				FC039B8220E11C550081E9F8 /* AppDelegate.swift in Sources */,
+				FC1A1D552237EAC70047B7FD /* ImageTool.m in Sources */,
 				FC2BFD4421DF5E1E00C262B2 /* SuperResolutionNet.swift in Sources */,
 				FC2BFD3E21DF5CE800C262B2 /* PreProcessKernel.metal in Sources */,
 				FC2BFD3821DF46DE00C262B2 /* OCDemoViewController.m in Sources */,

metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetCombined.swift

@@ -19,13 +19,14 @@ public class MobileNetCombined: Net {
     @objc public override init(device: MTLDevice) {
         super.init(device: device)
         except = 0
-        modelPath = Bundle.main.path(forResource: "combined_mobilenet_model", ofType: nil) ?! "model null"
-        paramPath = Bundle.main.path(forResource: "combined_mobilenet_params", ofType: nil) ?! "para null"
+        modelPath = Bundle.main.path(forResource: "combined_mobilenet_model_16", ofType: nil) ?! "model null"
+        paramPath = Bundle.main.path(forResource: "combined_mobilenet_params_16", ofType: nil) ?! "para null"
         inputDim = Dim.init(inDim: [1, 224, 224, 3])
         metalLoadMode = .LoadMetalInCustomMetalLib
         let paddleMobileMetallib = Bundle.main.path(forResource: "paddle-mobile-metallib", ofType: "metallib")
         metalLibPath = paddleMobileMetallib
         useMPS = true
+        paramPrecision = .Float16
         preprocessKernel = ScaleKernel.init(device: device, shape: Shape.init(inWidth: 224, inHeight: 224, inChannel: 3), metalLoadMode: .LoadMetalInCustomMetalLib, metalLibPath: paddleMobileMetallib)
         
     }

metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.h

+//
+//  ImageTool.h
+//  paddle-mobile-demo
+//
+//  Created by liuRuiLong on 2019/3/12.
+//  Copyright © 2019 orange. All rights reserved.
+//
+
+#import <UIKit/UIKit.h>
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface ImageTool : NSObject
+
++ (CVPixelBufferRef)imageToRGBPixelBuffer:(UIImage *)image;
+
+
+@end
+
+NS_ASSUME_NONNULL_END

metal/paddle-mobile-demo/paddle-mobile-demo/OC/ImageTool.m

+//
+//  ImageTool.m
+//  paddle-mobile-demo
+//
+//  Created by liuRuiLong on 2019/3/12.
+//  Copyright © 2019 orange. All rights reserved.
+//
+
+#import "ImageTool.h"
+
+@implementation ImageTool
+
++ (CVPixelBufferRef)imageToRGBPixelBuffer:(UIImage *)image {
+    CVPixelBufferRef pxbuffer = NULL;
+#if defined(__arm__) || defined(__arm64__)
+    CGSize frameSize = CGSizeMake(CGImageGetWidth(image.CGImage),CGImageGetHeight(image.CGImage));
+    //Metal渲染纹理需要IOSurface属性
+    NSDictionary *options =
+    [NSDictionary dictionaryWithObjectsAndKeys:
+     [NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey,
+     [NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey,
+     [NSNumber numberWithBool:YES], kCVPixelBufferIOSurfaceOpenGLESTextureCompatibilityKey,
+     [NSNumber numberWithBool:YES], kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey,nil];
+    CVPixelBufferCreate(kCFAllocatorDefault, frameSize.width, frameSize.height,kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef)options, &pxbuffer);
+    CVPixelBufferLockBaseAddress(pxbuffer, 0);
+    void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);
+    CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
+    CGContextRef context = CGBitmapContextCreate(pxdata, frameSize.width, frameSize.height,8, CVPixelBufferGetBytesPerRow(pxbuffer),rgbColorSpace,(CGBitmapInfo)kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst);
+    CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image.CGImage),CGImageGetHeight(image.CGImage)), image.CGImage);
+    CGColorSpaceRelease(rgbColorSpace);
+    CGContextRelease(context);
+    CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
+#endif
+    
+    return pxbuffer;
+}
+
+@end

metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift

@@ -16,6 +16,7 @@ import UIKit
 import MetalKit
 import CoreMedia
 import paddle_mobile
+import paddle_mobile_demo
 import MetalPerformanceShaders
 
 class FileReader {
@@ -75,8 +76,10 @@ class ViewController: UIViewController {
     @IBOutlet weak var modelPickerView: UIPickerView!
     @IBOutlet weak var threadPickerView: UIPickerView!
     @IBOutlet weak var videoView: UIView!
+    var inputImageSize: CGSize = CGSize.init(width: 0, height: 0)
     //  var videoCapture: VideoCapture!
     
+    var textureCache: CVMetalTextureCache?
     var selectImage: UIImage?
     var inputPointer: UnsafeMutablePointer<Float32>?
     var modelType: SupportModel = SupportModel.supportedModels()[0]
@@ -102,7 +105,9 @@ class ViewController: UIViewController {
             if self.toPredictTexture == nil {
                 let beforeDate = Date.init()
                 if modelType == .mobilenet_combined || modelType == .yolo {
-                    self.toPredictTexture = try! MetalHelper.shared.textureLoader.newTexture(cgImage: selectImage!.cgImage!, options: nil)
+                    let buffer = ImageTool.image(toRGBPixelBuffer: selectImage!)
+                    let texture = convertToMTLTexture(imageBuffer: buffer.takeRetainedValue())
+                    self.toPredictTexture = texture
                 } else {
                     runner.getTexture(image: selectImage!.cgImage!) { [weak self] (texture) in
                         let timeUse = Date.init().timeIntervalSince(beforeDate)
@@ -150,6 +155,8 @@ class ViewController: UIViewController {
                     }
                     
                     if success, let inResultHolderArr = resultHolder {
+//                        writeToLibrary(fileName: "00001_result_32_new_new", buffer: UnsafeBufferPointer<Float32>.init(start: inResultHolderArr[0].result, count: inResultHolderArr[0].capacity))
+
                         let inResultHolder = inResultHolderArr[0]
                         if i == max - 1 {
                             let time = Date.init().timeIntervalSince(startDate)
@@ -173,6 +180,8 @@ class ViewController: UIViewController {
     override func viewDidLoad() {
         super.viewDidLoad()
         
+        CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, MetalHelper.shared.device, nil, &textureCache)
+
         GlobalConfig.shared.computePrecision = .Float16
         GlobalConfig.shared.debug = false
         
@@ -259,5 +268,28 @@ extension ViewController: VideoCaptureDelegate{
 }
 
 
+extension ViewController {
+    private func convertToMTLTexture(imageBuffer: CVPixelBuffer?) -> MTLTexture? {
+        if let textureCache = textureCache, let imageBuffer = imageBuffer {
+            CVPixelBufferLockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0))
+            let width = CVPixelBufferGetWidth(imageBuffer)
+            let height = CVPixelBufferGetHeight(imageBuffer)
+            inputImageSize = CGSize(width: width, height: height);
+            let pixelFormat: MTLPixelFormat = .bgra8Unorm
+            var texture: CVMetalTexture?
+            
+            CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache,
+                                                      imageBuffer, nil, pixelFormat, width, height, 0, &texture)
+            
+            CVPixelBufferUnlockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0))
+            
+            if let texture = texture {
+                return CVMetalTextureGetTexture(texture)
+            }
+        }
+        return nil
+    }
+}
+
 
 

metal/paddle-mobile-demo/paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h

@@ -3,3 +3,4 @@
 //
 
 #import <paddle_mobile/paddle_mobile.h>
+#import "ImageTool.h"

metal/paddle-mobile/paddle-mobile/Src/Common/Tools.swift

@@ -25,3 +25,22 @@ func writeToLibrary<P: PrecisionProtocol>(fileName: String, array: [P]) {
     fileHandler.closeFile()
 }
 
+public func writeToLibrary<P: PrecisionProtocol>(fileName: String, buffer: UnsafeBufferPointer<P>) {
+    let libraryPath = NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true).last ?! " library path get error "
+    let filePath = libraryPath + "/" + fileName
+    let fileManager = FileManager.init()
+    fileManager.createFile(atPath: filePath, contents: nil, attributes: nil)
+    let fileHandler = FileHandle.init(forWritingAtPath: filePath) ?! " file handler nil "
+    let data = Data.init(buffer: buffer)
+    fileHandler.write(data)
+    fileHandler.closeFile()
+}
+
+func createFile(fileName: String) -> String {
+    let libraryPath = NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true).last ?! " library path get error "
+    let filePath = libraryPath + "/" + fileName
+    let fileManager = FileManager.init()
+    fileManager.createFile(atPath: filePath, contents: nil, attributes: nil)
+    return filePath
+}
+

metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/Scale.swift

@@ -13,9 +13,12 @@
  limitations under the License. */
 
 import Foundation
+import MetalPerformanceShaders
 
 public class ScaleKernel: CusomKernel {
+    var lanczos: MPSImageLanczosScale!
     public init(device: MTLDevice, shape: Shape, metalLoadMode: MetalLoadMode, metalLibPath: String?) {
+        lanczos = MPSImageLanczosScale(device: device)        
         if GlobalConfig.shared.computePrecision == .Float32 {
             super.init(device: device, inFunctionName: "scale", outputDim: shape, metalLoadModel: metalLoadMode, metalLibPath: metalLibPath)
         } else if GlobalConfig.shared.computePrecision == .Float16 {
@@ -24,5 +27,10 @@ public class ScaleKernel: CusomKernel {
             fatalError(" unsupport ")
         }
     }
+    
+    public override func compute(inputTexuture: MTLTexture, commandBuffer: MTLCommandBuffer) throws {
+        lanczos.encode(commandBuffer: commandBuffer, sourceTexture: inputTexuture, destinationTexture: outputTexture)
+    }
+    
 }