Merge pull request #1475 from codeWorm2015/develop

use mps

metal/MobileNetDemo/MobileNetDemo/MobileNet.swift

@@ -61,6 +61,7 @@ public class MobileNet: Net{
         inputDim = Dim.init(inDim: [1, 224, 224, 3])
         metalLoadMode = .LoadMetalInCustomMetalLib
         metalLibPath = Bundle.main.path(forResource: "paddle-mobile-metallib", ofType: "metallib")
+        useMPS = true
     }
 }
 

metal/paddle-mobile-demo/paddle-mobile-demo/Net/YoloNet.swift

@@ -25,6 +25,7 @@ public class YoloNet: Net {
         inputDim = Dim.init(inDim: [1, 416, 416, 3])
         metalLoadMode = .LoadMetalInCustomMetalLib
         metalLibPath = Bundle.main.path(forResource: "paddle-mobile-metallib", ofType: "metallib")
+        useMPS = true
     }
     
     override  public func resultStr(res: [ResultHolder]) -> String {

metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift

@@ -181,7 +181,7 @@ class ViewController: UIViewController {
         modelPickerView.dataSource = self
         threadPickerView.delegate = self
         threadPickerView.dataSource = self
-        if let image = UIImage.init(named: "test.jpg") {
+        if let image = UIImage.init(named: "00001.jpg") {
             selectImage = image
             selectImageView.image = image
         } else {

metal/paddle-mobile/paddle-mobile/API/Net.swift

@@ -56,6 +56,8 @@ import Foundation
     /// 输入维度，按照 n h w c 方式传入
     @objc public var inputDim: Dim = Dim.init(inDim: [])
     
+    /// 是否使用 MetalPerformanceShaders 进行运算
+    @objc public var useMPS: Bool = false
     
     @objc public init(device: MTLDevice, inParamPointer: UnsafeMutableRawPointer, inParamSize:Int, inModelPointer: UnsafeMutableRawPointer, inModelSize: Int) {
         self.paramPointer = inParamPointer

metal/paddle-mobile/paddle-mobile/API/Runner.swift

@@ -87,6 +87,8 @@ import Foundation
             let initContext: InitContext = InitContext.init()
             initContext.metalLoadMode = net.metalLoadMode
             initContext.metalLibPath = net.metalLibPath
+            initContext.useMPS = net.useMPS
+
             executor = try Executor<Float32>.init(inDevice: inDevice, inQueue: inQueue, inProgram: program!, initContext: initContext)
             net.updateProgram(program: program!)
         } catch let error {

metal/paddle-mobile/paddle-mobile/Src/Framework/Tensor.swift

@@ -28,23 +28,80 @@ extension Tensorial {
     }
 }
 
+class DataConverter<P: PrecisionType> {
+    func convert(from: UnsafeMutablePointer<P>, to: UnsafeMutablePointer<P>, fromDim: Dim) {
+        fatalError(" need imp")
+    }
+    
+    func getToDim(fromDim: Dim, layout: DataLayout) -> (dim: Dim, layout: DataLayout) {
+        fatalError(" need imp")
+    }
+}
 
+/// [ outputChannels ][ inputChannels ][ kernelHeight ][ kernelWidth ] ->
+/// [ outputChannels ][ kernelHeight ][ kernelWidth ][ inputChannels ]
+class MPSPointerConverter<P: PrecisionType>: DataConverter<P>{
+    
+    /// [ outputChannels ][ inputChannels ][ kernelHeight ][ kernelWidth ] ->
+    /// [ outputChannels ][ kernelHeight ][ kernelWidth ][ inputChannels ]
+    /// - Parameters:
+    ///   - from: from pointer
+    ///   - to: to pointer
+    override func convert(from: UnsafeMutablePointer<P>, to: UnsafeMutablePointer<P>, fromDim: Dim) {
+        let outputChannels = fromDim[0]
+        let inputChannels = fromDim[1]
+        let kernelHeight = fromDim[2]
+        let kernelWidth = fromDim[3]
+        
+        for outChannel in 0..<outputChannels {
+            for kernelH in 0..<kernelHeight {
+                for kernelW in 0..<kernelWidth {
+                    for inChannel in 0..<inputChannels {
+                        to[outChannel * inputChannels * kernelHeight * kernelWidth + kernelH * kernelWidth * inputChannels + kernelW * inputChannels + inChannel] =
+                        from[outChannel * inputChannels * kernelHeight * kernelWidth + inChannel * kernelHeight * kernelWidth + kernelH * kernelWidth + kernelW]
+                    }
+                }
+            }
+        }
+    }
+    
+    override func getToDim(fromDim: Dim, layout: DataLayout) -> (dim: Dim, layout: DataLayout) {
+        
+        if layout != DataLayout.NCHW() {
+            fatalError("not support")
+        }
+        
+        let outputChannels = fromDim[0]
+        let inputChannels = fromDim[1]
+        let kernelHeight = fromDim[2]
+        let kernelWidth = fromDim[3]
+        let toDim = Dim.init(inDim: [outputChannels, kernelHeight, kernelWidth, inputChannels])
+        
+        return (dim: toDim, layout: DataLayout.NHWC())
+    }
+}
 
 class Tensor<P: PrecisionType>: Tensorial {
     
     var data: Data
     var dim: Dim
+    
+    /// 模型中的维度: 未经过转换 paddle 模型维度为 N C H W
+    var tensorDim: Dim
     var buffer: MTLBuffer!
     private(set) var layout: DataLayout
     
     class Data {
-        init(inSize: Int, inPointer: UnsafeMutablePointer<P>) {
-            size = inSize
+        private var released = false
+        let count: Int
+        let size: Int
+        init(inCount: Int, inPointer: UnsafeMutablePointer<P>) {
+            count = inCount
+            size = inCount * MemoryLayout<P>.size
             pointer = inPointer
         }
-        let size: Int
-        var pointer: UnsafeMutablePointer<P>
-        subscript(index: Int) -> P{
+        internal private(set) var pointer: UnsafeMutablePointer<P>
+        subscript(index: Int) -> P {
             get {
                 return pointer[index]
             }
@@ -53,22 +110,40 @@ class Tensor<P: PrecisionType>: Tensorial {
             }
         }
         func release() {
-            pointer.deinitialize(count: size)
-            pointer.deallocate()
+            if !released {
+                pointer.deinitialize(count: count)
+                pointer.deallocate()
+                released = true
+            }
         }
+        
         deinit {
-            //            release()
+            if !released {
+                pointer.deinitialize(count: count)
+                pointer.deallocate()
+                released = true
+            }
         }
     }
     
     init(inDim: Dim, inLayout: DataLayout = DataLayout.NCHW()) {
+        tensorDim = inDim
         dim = inDim
-        let size = inDim.numel() * MemoryLayout<P>.size
-        let pointer = UnsafeMutablePointer<P>.allocate(capacity: size)
-        data = Data.init(inSize: size, inPointer: pointer)
+        let pointer = UnsafeMutablePointer<P>.allocate(capacity: inDim.numel())
+        data = Data.init(inCount: inDim.numel(), inPointer: pointer)
         layout = inLayout
     }
     
+    func convert(converter: DataConverter<P>) -> UnsafeMutablePointer<P> {
+        let to = UnsafeMutablePointer<P>.allocate(capacity: numel())
+        converter.convert(from: data.pointer, to: to, fromDim: dim)
+        data = Data.init(inCount: numel(), inPointer: to)
+        let dimAndLayout = converter.getToDim(fromDim: dim, layout: layout)
+        dim = dimAndLayout.dim
+        layout = dimAndLayout.layout
+        return to
+    }
+    
     func convert(to: DataLayout) {
         guard to != layout else {
             return
@@ -82,14 +157,15 @@ class Tensor<P: PrecisionType>: Tensorial {
             // other not support
             return
         }
-        let newPointer = UnsafeMutablePointer<P>.allocate(capacity: data.size)
+        
+        let newPointer = UnsafeMutablePointer<P>.allocate(capacity: numel())
         
         if layout == DataLayout.NCHW() {
             NCHW2NHWC(newPtr: newPointer)
         }
         
         data.release()
-        data.pointer = newPointer
+        data = Data.init(inCount: data.count, inPointer: newPointer)
         layout = to
     }
     
@@ -114,7 +190,7 @@ class Tensor<P: PrecisionType>: Tensorial {
             
             dim.swapeDimAt(index1: 0, index2: 3)
             data.release()
-            data.pointer = transposePointer
+            data = Data.init(inCount: data.count, inPointer: transposePointer)
         }
         
         guard let floatPointer = data.pointer as? UnsafeMutablePointer<Float32> else {
@@ -338,7 +414,7 @@ extension Tensor {
     func logDataPointer(header: String = "") {
         print(header)
         var str = ""
-        str += "data size: \(data.size) \n"
+        str += "data count: \(data.count) \n"
         str += "dim: \(dim) \n"
         for i in 0..<numel() {
             str += " \(data.pointer[i])"

metal/paddle-mobile/paddle-mobile/Src/Operators/Base/Operator.swift

@@ -60,10 +60,16 @@ extension Runable where Self: OperatorProtocol{
 }
 
 public class InitContext {
+    
     /// metal 代码加载方式
     var metalLoadMode: MetalLoadMode = .LoadMetalInDefaultLib
+    
     /// 当 metalLoadMode 为 LoadMetalInCustomMetalLib 时， metal library 路径不能为空
     var metalLibPath: String? = nil
+    
+    /// 是否使用 MetalPerformanceShaders 进行运算
+    var useMPS: Bool = false
+    
     init() {
         metalLoadMode = .LoadMetalInDefaultLib
         metalLibPath = nil

metal/paddle-mobile/paddle-mobile/Src/Operators/Kernels/ConvAddKernel.swift

@@ -13,11 +13,65 @@
  limitations under the License. */
 
 import Foundation
+import MetalPerformanceShaders
+
+@available(iOS 10.0, *)
+var convDic: [String : MPSCNNConvolution] = [:]
+@available(iOS 10.0, *)
+var imageDic: [String : MPSImage] = [:]
+
+/// 获取唯一字符串
+///
+/// - Returns: 唯一字符串
+func getUniqueKey() -> String {
+    return UUID.init().uuidString
+}
 
 class ConvAddKernel<P: PrecisionType>: Kernel, Computable {
     var metalParam: MetalConvParam!
+    
+    let identifyingKey: String = getUniqueKey()
+    
     required init(device: MTLDevice, param: ConvAddParam<P>, initContext: InitContext) {
+        
         param.output.initTexture(device: device, inTranspose: [0, 2, 3, 1], computePrecision: GlobalConfig.shared.computePrecision)
+        
+        let offsetY = (Int(param.dilations[1]) * (param.filter.tensorDim[2] - 1) + 1)/2 - Int(param.paddings[1])
+        let offsetX = (Int(param.dilations[0]) * (param.filter.tensorDim[3] - 1) + 1)/2 - Int(param.paddings[0])
+        
+        let key = identifyingKey
+        if initContext.useMPS {
+            if #available(iOS 10.0, *) {
+                if !(param.filter.tensorDim[1] == 1 && param.filter.tensorDim[0] == param.input.tensorDim[1]) && param.input.tensorDim[1] > 4 && param.output.tensorDim[1] > 4 {
+                    
+                    let desc = MPSCNNConvolutionDescriptor(kernelWidth: param.filter.tensorDim[3],
+                                                           kernelHeight: param.filter.tensorDim[2],
+                                                           inputFeatureChannels: param.input.tensorDim[1],
+                                                           outputFeatureChannels: param.output.tensorDim[1],
+                                                           neuronFilter: nil)
+                    desc.strideInPixelsX = Int(param.stride[0])
+                    desc.strideInPixelsY = Int(param.stride[1])
+                    
+                    let tensorPointer = param.filter.convert(converter: MPSPointerConverter<P>.init())
+                    let yPointer = param.y.data.pointer
+                    
+                    tensorPointer.withMemoryRebound(to: Float.self, capacity: param.filter.numel()) { (weightPointer: UnsafeMutablePointer<Float>) in
+                        yPointer.withMemoryRebound(to: Float.self, capacity: param.y.numel(), { (biasePointer: UnsafeMutablePointer<Float>) in
+                            let conv = MPSCNNConvolution.init(device: device, convolutionDescriptor: desc, kernelWeights: weightPointer, biasTerms: biasePointer, flags: .none)
+                            conv.offset = MPSOffset.init(x: offsetX, y: offsetY, z: 0)
+                            conv.edgeMode = .zero
+                            convDic[key] = conv
+                        })
+                    }
+                    
+                    imageDic[identifyingKey + "_input"] = MPSImage.init(texture: param.input.metalTexture, featureChannels: param.input.tensorDim[1])
+                    imageDic[identifyingKey + "_output"] = MPSImage.init(texture: param.output.metalTexture, featureChannels: param.output.tensorDim[1])
+                    super.init(device: device, inFunctionName: "place_holder", initContext: initContext)
+                    return
+                }
+            }
+        }
+        
         let padWhenOneC = !(param.filter.channel == 1 && param.filter.n == param.input.tensorDim[1])
         param.filter.initBuffer(device: device, precision: GlobalConfig.shared.computePrecision, padWhenOneC: padWhenOneC)
         param.y.initBuffer(device: device, precision: GlobalConfig.shared.computePrecision)
@@ -54,12 +108,6 @@ class ConvAddKernel<P: PrecisionType>: Kernel, Computable {
             fatalError()
         }
         
-        
-        
-        let offsetY = (Int(param.dilations[1]) * (param.filter.height - 1) + 1)/2 - Int(param.paddings[1])
-        
-        let offsetX = (Int(param.dilations[0]) * (param.filter.width - 1) + 1)/2 - Int(param.paddings[0])
-        
         //    print(" function: \(functionName)")
         //    print("offset x: \(offsetX)")
         //    print("offset y: \(offsetY)")
@@ -73,10 +121,16 @@ class ConvAddKernel<P: PrecisionType>: Kernel, Computable {
     }
     
     func compute(commandBuffer: MTLCommandBuffer, param: ConvAddParam<P>) throws {
+        if #available(iOS 10.0, *) {
+            if let conv = convDic[identifyingKey], let inputImage = imageDic[identifyingKey + "_input"], let outputImage = imageDic[identifyingKey + "_output"] {
+                conv.encode(commandBuffer: commandBuffer, sourceImage: inputImage, destinationImage: outputImage)
+                return;
+            }
+        }
+
         guard let encoder = commandBuffer.makeComputeCommandEncoder() else {
             throw PaddleMobileError.predictError(message: " encode is nil")
         }
-        
         encoder.setTexture(param.input.metalTexture, index: 0)
         encoder.setTexture(param.output.metalTexture, index: 1)
         encoder.setBytes(&metalParam, length: MemoryLayout<MetalConvParam>.size, index: 0)
@@ -85,4 +139,13 @@ class ConvAddKernel<P: PrecisionType>: Kernel, Computable {
         encoder.dispatch(computePipline: pipline, outTexture: param.output.metalTexture)
         encoder.endEncoding()
     }
+    
+    deinit {
+        if #available(iOS 10.0, *) {
+            convDic.removeValue(forKey: identifyingKey)
+            imageDic.removeValue(forKey: identifyingKey + "_input")
+            imageDic.removeValue(forKey: identifyingKey + "_output")
+        }
+    }
 }
+