diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj index aa98fce5d07f4eb231303816d7165b9e95d7050c..17af3762b15a622b046d0cc65eeca70b76b28705 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj +++ b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/project.pbxproj @@ -15,8 +15,11 @@ FC039B8920E11C560081E9F8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8820E11C560081E9F8 /* Assets.xcassets */; }; FC039B8C20E11C560081E9F8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = FC039B8A20E11C560081E9F8 /* LaunchScreen.storyboard */; }; FC27991021341CE5000B6BAD /* Net.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC27990F21341CE5000B6BAD /* Net.swift */; }; + FC27991321343A3A000B6BAD /* CPUCompute.mm in Sources */ = {isa = PBXBuildFile; fileRef = FC27991221343A3A000B6BAD /* CPUCompute.mm */; }; FC3C800F2133F46600D1295E /* MobileNetSSD.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC3C800E2133F46600D1295E /* MobileNetSSD.swift */; }; FC3C80112133F4AB00D1295E /* MobileNet.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC3C80102133F4AB00D1295E /* MobileNet.swift */; }; + FC8CFEDB21351F5D0094D569 /* params in Resources */ = {isa = PBXBuildFile; fileRef = FC8CFED921351F5D0094D569 /* params */; }; + FC8CFEDC21351F5D0094D569 /* model in Resources */ = {isa = PBXBuildFile; fileRef = FC8CFEDA21351F5D0094D569 /* model */; }; FC918191211DBC3500B6F354 /* paddle-mobile.png in Resources */ = {isa = PBXBuildFile; fileRef = FC918190211DBC3500B6F354 /* paddle-mobile.png */; }; FC918193211DC70500B6F354 /* iphone.JPG in Resources */ = {isa = PBXBuildFile; fileRef = FC918192211DC70500B6F354 /* iphone.JPG */; }; FCA3A16121313E1F00084FE5 /* hand.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FCA3A16021313E1F00084FE5 /* hand.jpg */; }; @@ -58,8 +61,13 @@ FC039B8B20E11C560081E9F8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; FC039B8D20E11C560081E9F8 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; FC27990F21341CE5000B6BAD /* Net.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Net.swift; sourceTree = ""; }; + FC27991121343A39000B6BAD /* paddle-mobile-demo-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "paddle-mobile-demo-Bridging-Header.h"; sourceTree = ""; }; + FC27991221343A3A000B6BAD /* CPUCompute.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = CPUCompute.mm; sourceTree = ""; }; + FC27991421343A46000B6BAD /* CPUCompute.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CPUCompute.h; sourceTree = ""; }; FC3C800E2133F46600D1295E /* MobileNetSSD.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MobileNetSSD.swift; sourceTree = ""; }; FC3C80102133F4AB00D1295E /* MobileNet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MobileNet.swift; sourceTree = ""; }; + FC8CFED921351F5D0094D569 /* params */ = {isa = PBXFileReference; lastKnownFileType = file; path = params; sourceTree = ""; }; + FC8CFEDA21351F5D0094D569 /* model */ = {isa = PBXFileReference; lastKnownFileType = file; path = model; sourceTree = ""; }; FC918190211DBC3500B6F354 /* paddle-mobile.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "paddle-mobile.png"; sourceTree = ""; }; FC918192211DC70500B6F354 /* iphone.JPG */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = iphone.JPG; sourceTree = ""; }; FCA3A16021313E1F00084FE5 /* hand.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = hand.jpg; sourceTree = ""; }; @@ -125,6 +133,7 @@ FC039B8020E11C550081E9F8 /* paddle-mobile-demo */ = { isa = PBXGroup; children = ( + FC8CFED2213519540094D569 /* Net */, FC0E2C2020EDC03B009C1FAC /* models */, FC0E2C1D20EDC030009C1FAC /* images */, FC039B8120E11C550081E9F8 /* AppDelegate.swift */, @@ -134,10 +143,7 @@ FC039B8820E11C560081E9F8 /* Assets.xcassets */, FC039B8A20E11C560081E9F8 /* LaunchScreen.storyboard */, FC039B8D20E11C560081E9F8 /* Info.plist */, - FCBCCC542122EF5400D94F7E /* MetalHelper.swift */, - FC3C800E2133F46600D1295E /* MobileNetSSD.swift */, - FC3C80102133F4AB00D1295E /* MobileNet.swift */, - FC27990F21341CE5000B6BAD /* Net.swift */, + FC27991121343A39000B6BAD /* paddle-mobile-demo-Bridging-Header.h */, ); path = "paddle-mobile-demo"; sourceTree = ""; @@ -158,6 +164,7 @@ FC0E2C2020EDC03B009C1FAC /* models */ = { isa = PBXGroup; children = ( + FC8CFED821351F5D0094D569 /* enet */, FCBCCC4F2122EEDC00D94F7E /* mobilenet_ssd_hand */, FCD04E6020F3146A0007374F /* mobilenet */, ); @@ -165,6 +172,28 @@ path = ../../models; sourceTree = ""; }; + FC8CFED2213519540094D569 /* Net */ = { + isa = PBXGroup; + children = ( + FCBCCC542122EF5400D94F7E /* MetalHelper.swift */, + FC3C800E2133F46600D1295E /* MobileNetSSD.swift */, + FC3C80102133F4AB00D1295E /* MobileNet.swift */, + FC27990F21341CE5000B6BAD /* Net.swift */, + FC27991221343A3A000B6BAD /* CPUCompute.mm */, + FC27991421343A46000B6BAD /* CPUCompute.h */, + ); + path = Net; + sourceTree = ""; + }; + FC8CFED821351F5D0094D569 /* enet */ = { + isa = PBXGroup; + children = ( + FC8CFED921351F5D0094D569 /* params */, + FC8CFEDA21351F5D0094D569 /* model */, + ); + path = enet; + sourceTree = ""; + }; FCBCCC4F2122EEDC00D94F7E /* mobilenet_ssd_hand */ = { isa = PBXGroup; children = ( @@ -218,6 +247,7 @@ TargetAttributes = { FC039B7D20E11C550081E9F8 = { CreatedOnToolsVersion = 9.3.1; + LastSwiftMigration = 0940; }; }; }; @@ -244,7 +274,9 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + FC8CFEDB21351F5D0094D569 /* params in Resources */, FCD04E6320F3146B0007374F /* params in Resources */, + FC8CFEDC21351F5D0094D569 /* model in Resources */, FC039B8C20E11C560081E9F8 /* LaunchScreen.storyboard in Resources */, FC918191211DBC3500B6F354 /* paddle-mobile.png in Resources */, FC039B8920E11C560081E9F8 /* Assets.xcassets in Resources */, @@ -309,6 +341,7 @@ FC013928210204A3008100E3 /* PreProcessKernel.metal in Sources */, FC27991021341CE5000B6BAD /* Net.swift in Sources */, FCBCCC552122EF5500D94F7E /* MetalHelper.swift in Sources */, + FC27991321343A3A000B6BAD /* CPUCompute.mm in Sources */, FC3C80112133F4AB00D1295E /* MobileNet.swift in Sources */, FC3C800F2133F46600D1295E /* MobileNetSSD.swift in Sources */, FC039B8220E11C550081E9F8 /* AppDelegate.swift in Sources */, @@ -456,19 +489,22 @@ baseConfigurationReference = 878829884E1A14D7044721D5 /* Pods-paddle-mobile-demo.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - CODE_SIGN_IDENTITY = "iPhone Developer"; - CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = A798K58VVL; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_IDENTITY = "iPhone Distribution"; + CODE_SIGN_STYLE = Manual; + DEVELOPMENT_TEAM = 6T9LLJKSM4; INFOPLIST_FILE = "paddle-mobile-demo/Info.plist"; IPHONEOS_DEPLOYMENT_TARGET = 9.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - PRODUCT_BUNDLE_IDENTIFIER = com.paddlemobile.metal; + PRODUCT_BUNDLE_IDENTIFIER = com.baidu.mms.qa; PRODUCT_NAME = "$(TARGET_NAME)"; - PROVISIONING_PROFILE = ""; - PROVISIONING_PROFILE_SPECIFIER = ""; + PROVISIONING_PROFILE = "ba9c4b24-7bd0-49c5-93cd-e3051e775d6c"; + PROVISIONING_PROFILE_SPECIFIER = Distribution_MMS; + SWIFT_OBJC_BRIDGING_HEADER = "paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 4.0; TARGETED_DEVICE_FAMILY = "1,2"; }; @@ -479,19 +515,21 @@ baseConfigurationReference = 081C9CF10DB06C58B8B6B039 /* Pods-paddle-mobile-demo.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - CODE_SIGN_IDENTITY = "iPhone Developer"; - CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = A798K58VVL; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_IDENTITY = "iPhone Distribution"; + CODE_SIGN_STYLE = Manual; + DEVELOPMENT_TEAM = 6T9LLJKSM4; INFOPLIST_FILE = "paddle-mobile-demo/Info.plist"; IPHONEOS_DEPLOYMENT_TARGET = 9.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - PRODUCT_BUNDLE_IDENTIFIER = com.paddlemobile.metal; + PRODUCT_BUNDLE_IDENTIFIER = com.baidu.mms.qa; PRODUCT_NAME = "$(TARGET_NAME)"; - PROVISIONING_PROFILE = ""; - PROVISIONING_PROFILE_SPECIFIER = ""; + PROVISIONING_PROFILE = "ba9c4b24-7bd0-49c5-93cd-e3051e775d6c"; + PROVISIONING_PROFILE_SPECIFIER = Distribution_MMS; + SWIFT_OBJC_BRIDGING_HEADER = "paddle-mobile-demo/paddle-mobile-demo-Bridging-Header.h"; SWIFT_VERSION = 4.0; TARGETED_DEVICE_FAMILY = "1,2"; }; diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/xcuserdata/liuruilong.xcuserdatad/xcschemes/paddle-mobile-demo.xcscheme b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/xcuserdata/liuruilong.xcuserdatad/xcschemes/paddle-mobile-demo.xcscheme index 46c65bd36a9ab7027b1cb7a81533dcd553ccb62e..de579675e07b8ffb8869464bebe203b591fa7778 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/xcuserdata/liuruilong.xcuserdatad/xcschemes/paddle-mobile-demo.xcscheme +++ b/metal/paddle-mobile-demo/paddle-mobile-demo.xcodeproj/xcuserdata/liuruilong.xcuserdatad/xcschemes/paddle-mobile-demo.xcscheme @@ -42,7 +42,7 @@ ? - - let except: Int = 2 - class MobilenetssdPreProccess: CusomKernel { - init(device: MTLDevice) { - let s = CusomKernel.Shape.init(inWidth: 300, inHeight: 300, inChannel: 3) - super.init(device: device, inFunctionName: "mobilenet_ssd_preprocess", outputDim: s, usePaddleMobileLib: false) - } - } - - func resultStr(res: [Float]) -> String { - return "哈哈哈, 还没好" - } - - func bboxArea(box: [Float32], normalized: Bool) -> Float32 { - if box[2] < box[0] || box[3] < box[1] { - return 0.0 - } else { - let w = box[2] - box[0] - let h = box[3] - box[1] - if normalized { - return w * h - } else { - return (w + 1) * (h + 1) - } - } - } - - func jaccardOverLap(box1: [Float32], box2: [Float32], normalized: Bool) -> Float32 { - if box2[0] > box1[2] || box2[2] < box1[0] || box2[1] > box1[3] || - box2[3] < box1[1] { - return 0.0 - } else { - let interXmin = max(box1[0], box2[0]) - let interYmin = max(box1[1], box2[1]) - let interXmax = min(box1[2], box2[2]) - let interYmax = min(box1[3], box2[3]) - let interW = interXmax - interXmin - let interH = interYmax - interYmin - let interArea = interW * interH - let bbox1Area = bboxArea(box: box1, normalized: normalized) - let bbox2Area = bboxArea(box: box2, normalized: normalized) - return interArea / (bbox1Area + bbox2Area - interArea) - } - } - - func fetchResult(paddleMobileRes: ResultHolder) -> [Float32]{ - guard let interRes = paddleMobileRes.intermediateResults else { - fatalError(" need have inter result ") - } - - guard let scores = interRes["Scores"], scores.count > 0, let score = scores[0] as? Texture else { - fatalError(" need score ") - } - - guard let bboxs = interRes["BBoxes"], bboxs.count > 0, let bbox = bboxs[0] as? Texture else { - fatalError() - } - - let score_thredshold: Float32 = 0.01 - let nms_top_k = 400 - let keep_top_k = 200 - let nms_eta: Float32 = 1.0 - var nms_threshold: Float32 = 0.45 - - let bboxArr = bbox.metalTexture.floatArray { (f) -> Float32 in - return f - } - - let scoreFormatArr: [Float32] = score.metalTexture.realNHWC(dim: (n: score.originDim[0], h: score.originDim[1], w: score.originDim[2], c: score.originDim[3])) - var outputArr: [Float32] = [] - let cNumOfOneClass = score.tensorDim[2] // 1917 - let boxSize = bbox.tensorDim[2] // 4 - let classNum = score.tensorDim[1] // 7 - - var selectedIndexs: [Int : [(Int, Float32)]] = [:] - var numDet: Int = 0 - for i in 0..(scoreFormatArr[(i * cNumOfOneClass)..<((i + 1) * cNumOfOneClass)]) - - var scoreThresholdArr: [(Float32, Int)] = [] - - for j in 0.. score_thredshold { - scoreThresholdArr.append((sliceScore[j], j)) - } - } - - scoreThresholdArr.sort { $0 > $1 } - - if scoreThresholdArr.count > nms_top_k { - scoreThresholdArr.removeLast(scoreThresholdArr.count - nms_top_k) - } - - var selectedIndex: [(Int, Float32)] = [] - - while scoreThresholdArr.count > 0 { - let idx = scoreThresholdArr[0].1 - let score = scoreThresholdArr[0].0 - var keep = true - for j in 0..(bboxArr[(idx * boxSize)..<(idx * boxSize + 4)]) - let box2 = Array(bboxArr[(keptIdx * boxSize)..<(keptIdx * boxSize + 4)]) - - let overlap = jaccardOverLap(box1: box1, box2: box2, normalized: true) - keep = (overlap <= nms_threshold) - } else { - break - } - } - - if keep { - selectedIndex.append((idx, score)) - } - - scoreThresholdArr.removeFirst() - if keep && nms_eta < 1.0 && nms_threshold > 0.5 { - nms_threshold *= nms_eta - } - } - selectedIndexs[i] = selectedIndex - numDet += selectedIndex.count - } - - var scoreIndexPairs: [(Float32, (Int, Int))] = [] - for selected in selectedIndexs { - for scoreIndex in selected.value { - scoreIndexPairs.append((scoreIndex.1, (selected.key, scoreIndex.0))) - } - } - - scoreIndexPairs.sort { $0.0 > $1.0 } - - if scoreIndexPairs.count > keep_top_k { - scoreIndexPairs.removeLast(scoreIndexPairs.count - keep_top_k) - } - - var newIndices: [Int : [(Int, Float32)]] = [:] - for scoreIndexPair in scoreIndexPairs { - // label: scoreIndexPair.1.0 - let label = scoreIndexPair.1.0 - if newIndices[label] != nil { - newIndices[label]?.append((scoreIndexPair.1.1, scoreIndexPair.0)) - } else { - newIndices[label] = [(scoreIndexPair.1.1, scoreIndexPair.0)] - } - } - - for indice in newIndices { - let selectedIndexAndScore = indice.value - for indexAndScore in selectedIndexAndScore { - outputArr.append(Float32(indice.key)) // label - outputArr.append(indexAndScore.1) // score - let subBox = bboxArr[(indexAndScore.0 * boxSize)..<(indexAndScore.0 * boxSize + 4)] - outputArr.append(contentsOf: subBox) - } - } - print(" fuck success !") - return outputArr - } - - var preprocessKernel: CusomKernel - let dim = [1, 300, 300, 3] - let modelPath: String - let paramPath: String - let modelDir: String - - init() { - modelPath = Bundle.main.path(forResource: "ssd_hand_model", ofType: nil) ?! "model null" - paramPath = Bundle.main.path(forResource: "ssd_hand_params", ofType: nil) ?! "para null" - modelDir = "" - preprocessKernel = MobilenetssdPreProccess.init(device: MetalHelper.shared.device) - } -} diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/CPUCompute.h b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/CPUCompute.h new file mode 100644 index 0000000000000000000000000000000000000000..0b0bc61602bc6383cd3a3dcad5163c030b4dcf7a --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/CPUCompute.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#import + +@interface NMSCompute: NSObject + +@property (assign, nonatomic) float scoreThredshold; + +@property (assign, nonatomic) int nmsTopK; + +@property (assign, nonatomic) int keepTopK; + +@property (assign, nonatomic) float nmsEta; + +@property (assign, nonatomic) float nmsThreshold; + +@property (assign, nonatomic) int background_label; + +@property (strong, nonatomic) NSArray *scoreDim; + +@property (strong, nonatomic) NSArray *bboxDim; + +-(NSArray *)computeWithScore:(float *)score andBBoxs:(float *)bbox; + +@end diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/CPUCompute.mm b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/CPUCompute.mm new file mode 100644 index 0000000000000000000000000000000000000000..2dce4d675e3d3cdc621e2faecf38c52adcc2c117 --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/CPUCompute.mm @@ -0,0 +1,333 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + + +#import "CPUCompute.h" + +#import +#import +#import +#import + + +struct NMSParam { + + float *score_data; + + float *box_data; + + float *output; + + int output_size; + + std::vector score_dim; + + std::vector box_dim; + + float scoreThredshold; + + int nmsTopK; + + int keepTopK; + + float nmsEta; + + float nmsThreshold; + + int background_label; +}; + + +constexpr int kOutputDim = 6; +constexpr int kBBoxSize = 4; + +template +bool SortScorePairDescend(const std::pair& pair1, + const std::pair& pair2) { + return pair1.first > pair2.first; +} + +template +static inline void GetMaxScoreIndex( + const std::vector& scores, const T threshold, int top_k, + std::vector>* sorted_indices) { + for (size_t i = 0; i < scores.size(); ++i) { + if (scores[i] > threshold) { + sorted_indices->push_back(std::make_pair(scores[i], i)); + } + } + // Sort the score pair according to the scores in descending order + std::stable_sort(sorted_indices->begin(), sorted_indices->end(), + SortScorePairDescend); + // Keep top_k scores if needed. + if (top_k > -1 && top_k < static_cast(sorted_indices->size())) { + sorted_indices->resize(top_k); + } +} + +template +static inline T BBoxArea(const T* box, const bool normalized) { + if (box[2] < box[0] || box[3] < box[1]) { + // If coordinate values are is invalid + // (e.g. xmax < xmin or ymax < ymin), return 0. + return static_cast(0.); + } else { + const T w = box[2] - box[0]; + const T h = box[3] - box[1]; + if (normalized) { + return w * h; + } else { + // If coordinate values are not within range [0, 1]. + return (w + 1) * (h + 1); + } + } +} + +template +static inline T JaccardOverlap(const T* box1, const T* box2, + const bool normalized) { + if (box2[0] > box1[2] || box2[2] < box1[0] || box2[1] > box1[3] || + box2[3] < box1[1]) { + return static_cast(0.); + } else { + const T inter_xmin = std::max(box1[0], box2[0]); + const T inter_ymin = std::max(box1[1], box2[1]); + const T inter_xmax = std::min(box1[2], box2[2]); + const T inter_ymax = std::min(box1[3], box2[3]); + const T inter_w = inter_xmax - inter_xmin; + const T inter_h = inter_ymax - inter_ymin; + const T inter_area = inter_w * inter_h; + const T bbox1_area = BBoxArea(box1, normalized); + const T bbox2_area = BBoxArea(box2, normalized); + return inter_area / (bbox1_area + bbox2_area - inter_area); + } +} + +template +static inline void NMSFast( + const T *bbox_data, + std::vector bbox_dim, + const T *score_data, + const T score_threshold, const T nms_threshold, + const T eta, const int top_k, + std::vector* selected_indices) { + // The total boxes for each instance. + int num_boxes = bbox_dim[0]; + // 4: [xmin ymin xmax ymax] + int box_size = bbox_dim[1]; + + std::vector scores_data(num_boxes); + std::copy_n(score_data, num_boxes, scores_data.begin()); + std::vector> sorted_indices; + GetMaxScoreIndex(scores_data, score_threshold, top_k, &sorted_indices); + + selected_indices->clear(); + T adaptive_threshold = nms_threshold; + + while (sorted_indices.size() != 0) { + const int idx = sorted_indices.front().second; + bool keep = true; + for (size_t k = 0; k < selected_indices->size(); ++k) { + if (keep) { + const int kept_idx = (*selected_indices)[k]; + T overlap = JaccardOverlap(bbox_data + idx * box_size, + bbox_data + kept_idx * box_size, true); + keep = overlap <= adaptive_threshold; + } else { + break; + } + } + if (keep) { + selected_indices->push_back(idx); + } + sorted_indices.erase(sorted_indices.begin()); + if (keep && eta < 1 && adaptive_threshold > 0.5) { + adaptive_threshold *= eta; + } + } +} + +template +void MultiClassNMS(const T *boxes_data, + const std::vector &box_dim, + const T *scores_data, + const std::vector &score_dim, + std::map>* indices, int* num_nmsed_out, + const int& background_label, const int& nms_top_k, + const int& keep_top_k, const T& nms_threshold, + const T& nms_eta, const T& score_threshold) { + + int64_t class_num = score_dim[0]; + int64_t predict_dim = score_dim[1]; + int num_det = 0; + for (int c = 0; c < class_num; ++c) { + if (c == background_label) continue; + const T *score_data = scores_data + c * predict_dim; + + /// [c] is key + NMSFast(boxes_data, box_dim, score_data, score_threshold, nms_threshold, nms_eta, + nms_top_k, &((*indices)[c])); + num_det += (*indices)[c].size(); + } + + *num_nmsed_out = num_det; + if (keep_top_k > -1 && num_det > keep_top_k) { + std::vector>> score_index_pairs; + for (const auto& it : *indices) { + int label = it.first; + const T* sdata = scores_data + label * predict_dim; + const std::vector& label_indices = it.second; + for (size_t j = 0; j < label_indices.size(); ++j) { + int idx = label_indices[j]; + // PADDLE_ENFORCE_LT(idx, predict_dim); + score_index_pairs.push_back(std::make_pair(sdata[idx], std::make_pair(label, idx))); + } + } + // Keep top k results per image. + std::stable_sort(score_index_pairs.begin(), score_index_pairs.end(), + SortScorePairDescend>); + score_index_pairs.resize(keep_top_k); + + // Store the new indices. + std::map> new_indices; + for (size_t j = 0; j < score_index_pairs.size(); ++j) { + int label = score_index_pairs[j].second.first; + int idx = score_index_pairs[j].second.second; + new_indices[label].push_back(idx); + } + new_indices.swap(*indices); + *num_nmsed_out = keep_top_k; + } +} + +template +void MultiClassOutput(const T *scores_data, + const std::vector &score_dim, + const T *bboxes_data, + T *outputs_data, + const std::map>& selected_indices) { + int predict_dim = score_dim[1]; + int count = 0; + for (const auto& it : selected_indices) { + /// one batch + int label = it.first; + const T* sdata = scores_data + label * predict_dim; + const std::vector& indices = it.second; + for (size_t j = 0; j < indices.size(); ++j) { + int idx = indices[j]; + const T* bdata = bboxes_data + idx * kBBoxSize; + outputs_data[count * kOutputDim] = label; // label + outputs_data[count * kOutputDim + 1] = sdata[idx]; // score + // xmin, ymin, xmax, ymax + std::memcpy(outputs_data + count * kOutputDim + 2, bdata, 4 * sizeof(T)); + count++; + } + } +} + +void MultiClassNMSCompute(NMSParam *param) { + assert(param->score_dim[0] == 1); + assert(param->box_dim[0] == 1); + assert (param->score_dim.size() == 3); + assert(param->box_dim.size() == 3); + + float* outputs; + auto background_label = param->background_label; + auto nms_top_k = param->nmsTopK; + auto keep_top_k = param->keepTopK; + auto nms_threshold = param->nmsThreshold; + auto nms_eta = param->nmsEta; + auto score_threshold = param->scoreThredshold; + + std::vector score_dim_one_batch = {param->score_dim[1], param->score_dim[2]}; + std::vector box_dim_one_batch = {param->box_dim[1], param->box_dim[2]}; + + std::vector batch_starts = {0}; + + std::map> indices; + int num_nmsed_out = 0; + + MultiClassNMS(param->box_data, box_dim_one_batch, param->score_data, score_dim_one_batch, &indices, &num_nmsed_out, + background_label, nms_top_k, keep_top_k, nms_threshold, + nms_eta, score_threshold); + batch_starts.push_back(batch_starts.back() + num_nmsed_out); + + int output_size = 0; + int num_kept = batch_starts.back(); + if (num_kept == 0) { + outputs = new float[1]; + outputs[0] = -1; + output_size = 1; + } else { + outputs = new float[num_kept * kOutputDim]; + int64_t s = batch_starts[0]; + int64_t e = batch_starts[1]; + if (e > s) { + MultiClassOutput(param->score_data, score_dim_one_batch, param->box_data, outputs, indices); + } + output_size = num_kept * kOutputDim; + } + param->output = outputs; + param->output_size = output_size; +} + +@implementation NMSCompute + +-(NSArray *)computeWithScore:(float *)score andBBoxs:(float *)bbox { + NMSParam param; + param.box_data = bbox; + param.score_data = score; + param.background_label = self.background_label; + param.scoreThredshold = self.scoreThredshold; + param.nmsTopK = self.nmsTopK; + param.keepTopK = self.keepTopK; + param.nmsEta = self.nmsEta; + param.nmsThreshold = self.nmsThreshold; + std::vector score_dim; + for (int i = 0; i < self.scoreDim.count; ++i) { + score_dim.push_back(self.scoreDim[i].intValue); + } + param.score_dim = score_dim; + + std::vector box_dim; + for (int i = 0; i < self.bboxDim.count; ++i) { + box_dim.push_back(self.bboxDim[i].intValue); + } + param.box_dim = box_dim; + MultiClassNMSCompute(¶m); + NSMutableArray *output = [NSMutableArray arrayWithCapacity:param.output_size]; + for (int i = 0; i < param.output_size; ++i) { + [output addObject:[NSNumber numberWithFloat:param.output[i]]]; + } + return output; +} + +@end + + + + + + + + + + + + + + + + + diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/MetalHelper.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MetalHelper.swift similarity index 100% rename from metal/paddle-mobile-demo/paddle-mobile-demo/MetalHelper.swift rename to metal/paddle-mobile-demo/paddle-mobile-demo/Net/MetalHelper.swift diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/MobileNet.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNet.swift similarity index 100% rename from metal/paddle-mobile-demo/paddle-mobile-demo/MobileNet.swift rename to metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNet.swift diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift new file mode 100644 index 0000000000000000000000000000000000000000..10805a2b3ba468dc4b48bd30cb837fd1ead0f636 --- /dev/null +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/MobileNetSSD.swift @@ -0,0 +1,88 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +import Foundation +import paddle_mobile +//import + +class MobileNet_ssd_hand: Net{ + + var program: Program? + var executor: Executor? + + let except: Int = 2 + class MobilenetssdPreProccess: CusomKernel { + init(device: MTLDevice) { + let s = CusomKernel.Shape.init(inWidth: 300, inHeight: 300, inChannel: 3) + super.init(device: device, inFunctionName: "mobilenet_ssd_preprocess", outputDim: s, usePaddleMobileLib: false) + } + } + + func resultStr(res: [Float]) -> String { + return " \(res)" + } + + func fetchResult(paddleMobileRes: ResultHolder) -> [Float32]{ + + guard let interRes = paddleMobileRes.intermediateResults else { + fatalError(" need have inter result ") + } + + guard let scores = interRes["Scores"], scores.count > 0, let score = scores[0] as? Texture else { + fatalError(" need score ") + } + + guard let bboxs = interRes["BBoxes"], bboxs.count > 0, let bbox = bboxs[0] as? Texture else { + fatalError() + } + + var scoreFormatArr: [Float32] = score.metalTexture.realNHWC(dim: (n: score.originDim[0], h: score.originDim[1], w: score.originDim[2], c: score.originDim[3])) + var bboxArr = bbox.metalTexture.floatArray { (f) -> Float32 in + return f + } + + let nmsCompute = NMSCompute.init() + nmsCompute.scoreThredshold = 0.01 + nmsCompute.nmsTopK = 200 + nmsCompute.keepTopK = 200 + nmsCompute.nmsEta = 1.0 + nmsCompute.nmsThreshold = 0.45 + nmsCompute.background_label = 0; + + nmsCompute.scoreDim = [NSNumber.init(value: score.tensorDim[0]), NSNumber.init(value: score.tensorDim[1]), NSNumber.init(value: score.tensorDim[2])] + + nmsCompute.bboxDim = [NSNumber.init(value: bbox.tensorDim[0]), NSNumber.init(value: bbox.tensorDim[1]), NSNumber.init(value: bbox.tensorDim[2])] + guard let result = nmsCompute.compute(withScore: &scoreFormatArr, andBBoxs: &bboxArr) else { + fatalError( " result error " ) + } + + let output: [Float32] = result.map { $0.floatValue } + + + return output + } + + var preprocessKernel: CusomKernel + let dim = [1, 300, 300, 3] + let modelPath: String + let paramPath: String + let modelDir: String + + init() { + modelPath = Bundle.main.path(forResource: "ssd_hand_model", ofType: nil) ?! "model null" + paramPath = Bundle.main.path(forResource: "ssd_hand_params", ofType: nil) ?! "para null" + modelDir = "" + preprocessKernel = MobilenetssdPreProccess.init(device: MetalHelper.shared.device) + } +} diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/Net.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/Net.swift similarity index 60% rename from metal/paddle-mobile-demo/paddle-mobile-demo/Net.swift rename to metal/paddle-mobile-demo/paddle-mobile-demo/Net/Net.swift index 2112af94437a147282fb8f84a2b438da4c5ff039..6aada07d5c7e97dd6d8e3470d59babd38b0171f5 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/Net.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/Net/Net.swift @@ -1,10 +1,16 @@ -// -// Net.swift -// paddle-mobile-demo -// -// Created by liuRuiLong on 2018/8/27. -// Copyright © 2018年 orange. All rights reserved. -// +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ import Foundation @@ -14,13 +20,16 @@ import Foundation import paddle_mobile import MetalPerformanceShaders -let modelHelperMap: [SupportModel : Net] = [.mobilenet : MobileNet.init(), .mobilenet_ssd : MobileNet_ssd_hand.init()] + +let modelHelperMap: [SupportModel : Net] = [.mobilenet_ssd : MobileNet_ssd_hand.init()] +//let modelHelperMap: [SupportModel : Net] = [.mobilenet : MobileNet.init(), .mobilenet_ssd : MobileNet_ssd_hand.init()] enum SupportModel: String{ - case mobilenet = "mobilenet" +// case mobilenet = "mobilenet" case mobilenet_ssd = "mobilenetssd" static func supportedModels() -> [SupportModel] { - return [.mobilenet, .mobilenet_ssd] + //.mobilenet, + return [.mobilenet_ssd] } } @@ -37,7 +46,8 @@ protocol Net { func resultStr(res: [Float]) -> String func fetchResult(paddleMobileRes: ResultHolder) -> [Float32] mutating func load() throws - func predict(inTexture: MTLTexture, completion: @escaping ([Float32]) -> Void) throws + + func predict(inTexture: MTLTexture, completion: @escaping ((time:TimeInterval, resultArray: [Float32])) -> Void) throws mutating func clear() } @@ -54,13 +64,16 @@ extension Net { } } - func predict(inTexture: MTLTexture, completion: @escaping ([Float32]) -> Void) throws { + func predict(inTexture: MTLTexture, completion: @escaping ((time:TimeInterval, resultArray: [Float32])) -> Void) throws { guard let inExecutor = executor else { fatalError(" 请先 load ") } try inExecutor.predict(input: inTexture, dim: dim, completionHandle: { (result) in - let resultArr = self.fetchResult(paddleMobileRes: result) - completion(resultArr) + + var resultArr:[Float32] = [] + resultArr = self.fetchResult(paddleMobileRes: result) + completion((time: TimeInterval(result.elapsedTime), resultArray: resultArr)) + }, preProcessKernle: preprocessKernel, except: except) } @@ -81,6 +94,4 @@ extension Net { return paddleMobileRes.resultArr } - // func predict() - } diff --git a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift index 9266f6fab88c897fe7a169f11ba2f6458af01b20..d910905bc544f50a149386ce3191a77f311ffa13 100644 --- a/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift +++ b/metal/paddle-mobile-demo/paddle-mobile-demo/ViewController.swift @@ -28,6 +28,7 @@ class ViewController: UIViewController { var selectImage: UIImage? var modelType: SupportModel = SupportModel.supportedModels()[0] var toPredictTexture: MTLTexture? + var net: Net { get { return modelHelperMap[modelType] ?! " has no this type " @@ -35,6 +36,7 @@ class ViewController: UIViewController { set { } } + var threadNum = 1 @IBAction func loadAct(_ sender: Any) { @@ -63,53 +65,26 @@ class ViewController: UIViewController { return } do { - try net.predict(inTexture: inTexture) { [weak self] (result) in - guard let sSelf = self else { - fatalError() - } - print(result) - let resultStr = sSelf.net.resultStr(res: result) - DispatchQueue.main.async { - sSelf.resultTextView.text = resultStr + let max = 1 + let startDate = Date.init() + for i in 0.. [(Int, Element)] { + public func strideArray(inCount: Int = 20) -> [(Int, Element)] { if count < inCount { return (0.. [Float32] { - print("origin dim: \(dim)") - print("texture: ") - print(self) +// print("origin dim: \(dim)") +// print("texture: ") +// print(self) let textureArray = floatArray { (i : Float32) -> Float32 in return i @@ -394,7 +394,7 @@ public extension MTLTexture { } } } - print(" tensor count -- \(output.count)") +// print(" tensor count -- \(output.count)") return output } diff --git a/metal/paddle-mobile/paddle-mobile/Executor.swift b/metal/paddle-mobile/paddle-mobile/Executor.swift index d5e2249caaf5c2010f851718aa7a1ad07d6f4aa8..5c6e2c299c2aa5a5185019f4b64be272214286c1 100644 --- a/metal/paddle-mobile/paddle-mobile/Executor.swift +++ b/metal/paddle-mobile/paddle-mobile/Executor.swift @@ -125,9 +125,9 @@ public class Executor { // print(stridableInput) // let _: Flo? = input.logDesc(header: "input: ", stridable: true) - for op in self.ops { +// for op in self.ops { // op.delogOutput() - } +// } // return // self.ops[testTo].delogOutput() diff --git a/metal/paddle-mobile/paddle-mobile/Operators/Kernels/PriorBoxKernel.swift b/metal/paddle-mobile/paddle-mobile/Operators/Kernels/PriorBoxKernel.swift index 77a931a0f58fd99b3c364530d36d6b4e5d57ee50..e2363e44d3a3d81b430f82303b2b1017ddfc5200 100644 --- a/metal/paddle-mobile/paddle-mobile/Operators/Kernels/PriorBoxKernel.swift +++ b/metal/paddle-mobile/paddle-mobile/Operators/Kernels/PriorBoxKernel.swift @@ -98,16 +98,6 @@ class PriorBoxKernel: Kernel, Computable{ guard let encoder = commandBuffer.makeComputeCommandEncoder() else { throw PaddleMobileError.predictError(message: " encode is nil") } - print("metalParam: \(metalParam)") - - print(" newAspectRatios ") - print(param.newAspectRatios!) - - print(" clip: \(metalParam.clip)") - - print(" metalParam.numPriors: \(metalParam.numPriors)") - - print(" aspecRatiosSize: \(metalParam.aspecRatiosSize)") encoder.setTexture(param.input.metalTexture, index: 0) encoder.setTexture(param.output.metalTexture, index: 1) diff --git a/metal/paddle-mobile/paddle-mobile/framework/Tensor.swift b/metal/paddle-mobile/paddle-mobile/framework/Tensor.swift index 45e9eeb0009c2f3daeb11f6f879343d4790cc4e9..d2c8228ed9d9dd7e439aa1f688c60579f615b8a1 100644 --- a/metal/paddle-mobile/paddle-mobile/framework/Tensor.swift +++ b/metal/paddle-mobile/paddle-mobile/framework/Tensor.swift @@ -174,7 +174,7 @@ class Tensor: Tensorial { fatalError(" not support !") } //TODO: release -// data.release() + data.release() } var width: Int {