add some test case for profiler

tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0

+op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
+op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
+op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
+op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"

tests/ut/data/profiler_data/JOB1/Framework.host.vm.task_desc_info.0.slice_0

+Default/Cast-op6 32 51517 0
+Default/TransData-op7 32 51518 0
+Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
+Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0

tests/ut/data/profiler_data/JOB1/pipeline_profiling_0.json

+{
+  "sampling_interval": 10,
+  "op_info": [
+    {
+      "op_id": 4,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": null,
+      "children": [3]
+    },
+    {
+      "op_id": 3,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": null
+    },
+    {
+      "op_id": 2,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": null
+    },
+    {
+      "op_id": 1,
+      "op_type": "Shuffle",
+      "num_workers": 1,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": [2, 4]
+    },
+    {
+      "op_id": 0,
+      "op_type": "Batch",
+      "num_workers": 4,
+      "metrics": null,
+      "children": [1]
+    }
+  ]
+}
\ No newline at end of file

tests/ut/data/profiler_data/JOB2/pipeline_profiling_0.json

+{
+  "sampling_interval": 10,
+  "op_info": [
+    {
+      "op_id": 3,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": null
+    },
+    {
+      "op_id": 2,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": null
+    },
+    {
+      "op_id": 1,
+      "op_type": "Shuffle",
+      "num_workers": 1,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": [2, 3]
+    },
+    {
+      "op_id": 0,
+      "op_type": "Batch",
+      "num_workers": 4,
+      "metrics": null,
+      "children": [1]
+    }
+  ]
+}
\ No newline at end of file

tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.graph_desc_info.0.slice_0

+op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
+op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
+op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
+op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"

tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.point.0.slice_0

+1 Default/Cast-op6
+2 Default/TransData-op7

tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.task_desc_info.0.slice_0

+Default/Cast-op6 32 1 0
+Default/TransData-op7 32 2 0
+Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 3 0
+Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 4 0

tests/ut/data/profiler_data/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt

+serial_number node_type_name total_time(ms) dispatch_time(ms) run_start run_end
+1 InitData 1.567 0.1 2298200409 2298200538
+2 GetNext 0.989 0.087 2302769932 2302769980
+3 TruncatedNormal 1.566 0.105 4098200409 4098200538
+AI CPU Total Time(ms): 4.122000

tests/ut/data/profiler_data/container/0/data/Framework.host.vm.graph_desc_info.0.JOB2.slice_0

+op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
+op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
+op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
+op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"

tests/ut/data/profiler_data/container/0/data/Framework.host.vm.task_desc_info.0.JOB2.slice_0

+Default/Cast-op6 32 51517 0
+Default/TransData-op7 32 51518 0
+Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
+Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0

tests/ut/data/profiler_data/profiler/aicore_intermediate_1_detail.csv

+full_op_time,execution_time
+Default/AtomicAddrClean-op104,0.00133
+Default/AtomicAddrClean-op105,0.000987
+Default/AtomicAddrClean-op106,0.001129
+Default/Cast-op10,0.00466
+Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,0.002366
+Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,0.004879
+Default/TransData-op11,0.006366
+Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,0.006782
+Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,0.05651
+Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,0.370864

tests/ut/data/profiler_data/profiler/aicore_intermediate_1_type.csv

+op_type,execution_time,execution_frequency,percent
+AtomicAddrClean,0.007283,6,0.49
+Cast,0.053395,13,3.63
+TransData,0.121800,5,8.23
+Conv2D,0.063656,2,4.33
+MatMul,1.085982,9,73.80

tests/ut/data/profiler_data/profiler/framework_raw_0.csv

+task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info
+51517,0,32,Default/Cast-op6,Cast-op6,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}}"
+51518,0,32,Default/TransData-op7,TransData-op7,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,224,224,16""}}"
+51519,0,32,Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5,Cast-op5,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""49,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""49,4,16,16""}}"
+51522,0,4,Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28,Cast-op28,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""4,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""4,4,16,16""}}"

tests/ut/data/profiler_data/profiler/framework_raw_1.csv

+task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info
+30290,0,1,Default/AtomicAddrClean-op104,AtomicAddrClean-op104,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": """"}}"
+30295,0,1,Default/AtomicAddrClean-op105,AtomicAddrClean-op105,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""10""}}"
+30300,0,1,Default/AtomicAddrClean-op106,AtomicAddrClean-op106,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}}"
+30268,0,32,Default/Cast-op10,Cast-op10,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}"
+30271,0,9,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,Cast-op12,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}}"
+30320,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,Cast-op53,Cast,Gradients,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,28,28,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}"
+30269,0,32,Default/TransData-op11,TransData-op11,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}"
+30308,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,TransData-op44,TransData,Gradients,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,16,5,5""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,5,5,16""}}"
+30272,0,32,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,Conv2D-op13,Conv2D,Default,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32,16""}, ""input_1"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}"
+30286,0,1,Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,MatMul-op9,MatMul,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,120""}, ""input_1"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84,120""}, ""input_2"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,84""}}"

tests/ut/data/profiler_data/profiler/minddata_pipeline_raw_0.csv

+op_id,op_type,num_workers,output_queue_size,output_queue_average_size,output_queue_length,output_queue_usage_rate,sample_interval,parent_id,children_id
+0,Batch,4,,,,,10,,[1]
+1,Shuffle,1,"[10, 20, 30]",20.0,64,0.3125,10,0,"[2, 3]"
+2,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1,
+3,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1,

tests/ut/data/profiler_data/profiler/pipeline_profiling_1.json

+{
+  "sampling_interval": 10,
+  "op_info": [
+    {
+      "op_id": 4,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": null,
+      "children": [3]
+    },
+    {
+      "op_id": 3,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": null
+    },
+    {
+      "op_id": 2,
+      "op_type": "TFReader",
+      "num_workers": 4,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": null
+    },
+    {
+      "op_id": 1,
+      "op_type": "Shuffle",
+      "num_workers": 1,
+      "metrics": {
+        "output_queue": {
+          "size": [10, 20, 30],
+          "length": 64
+        }
+      },
+      "children": [2, 4]
+    },
+    {
+      "op_id": 0,
+      "op_type": "Batch",
+      "num_workers": 4,
+      "metrics": null,
+      "children": [1]
+    }
+  ]
+}
\ No newline at end of file

tests/ut/data/profiler_data/profiler/step_trace_raw_0_detail_time.csv

+step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0
+1,45000030081,45004033128,4003047,45000030081,45001733025,0,1702944,2300103,45000042679,45000060275,17596,45001048152,45001346254,298102,45002247411,45002448354,200943,45000049687,45000075987,26300
+2,45004033128,45017085658,13052530,45013070937,45014785314,9037809,1714377,2300344,45013085379,45013105429,20050,45014087119,45014385136,298017,45015297166,45015504449,207283,45013084925,45013118334,33409
+3,45017085658,45030119392,13033734,45026116231,45027818443,9030573,1702212,2300949,45026131909,45026150554,18645,45027134392,45027430418,296026,45028337093,45028537767,200674,45026129217,45026160937,31720
+4,45030119392,45043158607,13039215,45039152348,45040856975,9032956,1704627,2301632,45039169890,45039188966,19076,45040169338,45040466770,297432,45041374122,45041567754,193632,45039171681,45039193865,22184
+5,45043158607,45056198128,13039521,45052190932,45053898028,9032325,1707096,2300100,45052207675,45052222642,14967,45053204442,45053505540,301098,45054413207,45054616536,203329,45052201931,45052237599,35668
+6,45056198128,45069239564,13041436,45065233106,45066939463,9034978,1706357,2300101,45065245482,45065272534,27052,45066248423,45066546419,297996,45067455113,45067659145,204032,45065245817,45065279896,34079
+7,45069239564,45082281383,13041819,45078274997,45079980193,9035433,1705196,2301190,45078293910,45078312935,19025,45079287754,45079593841,306087,45080492957,45080691395,198438,45078292067,45078322277,30210
+8,45082281383,45095336378,13054995,45091321488,45093036084,9040105,1714596,2300294,45091338628,45091359138,20510,45092338469,45092638994,300525,45093554195,45093747470,193275,45091341356,45091369667,28311
+9,45095336378,45108372225,13035847,45104363079,45106071009,9026701,1707930,2301216,45104374524,45104400088,25564,45105378751,45105683029,304278,45106587481,45106785336,197855,45104382131,45104410852,28721
+10,45108372225,45121412413,13040188,45117401873,45119111301,9029648,1709428,2301112,45117417721,45117439668,21947,45118413083,45118718050,304967,45119629347,45119829996,200649,45117421502,45117446718,25216
+11,45121412413,45134477662,13065249,45130459598,45132175723,9047185,1716125,2301939,45130478168,45130498936,20768,45131477957,45131775220,297263,45132691645,45132893707,202062,45130470285,45130501652,31367
+12,45134477662,45147533298,13055636,45143521860,45145232553,9044198,1710693,2300745,45143533787,45143557293,23506,45144533554,45144841545,307991,45145744997,45145952255,207258,45143537383,45143563466,26083
+13,45147533298,45160588134,13054836,45156570201,45158286694,9036903,1716493,2301440,45156581069,45156609506,28437,45157581617,45157880841,299224,45158806166,45158999875,193709,45156589050,45156615664,26614
+14,45160588134,45173640064,13051930,45169625906,45171339426,9037772,1713520,2300638,45169637432,45169661754,24322,45170639482,45170940949,301467,45171853721,45172056606,202885,45169644605,45169673410,28805
+15,45173640064,45186671634,13031570,45182666696,45184371430,9026632,1704734,2300204,45182678355,45182698471,20116,45183679568,45183981082,301514,45184887156,45185083035,195879,45182680062,45182708455,28393
+16,45186671634,45199720448,13048814,45195714716,45197420410,9043082,1705694,2300038,45195728993,45195754646,25653,45196732493,45197028048,295555,45197934921,45198139237,204316,45195733069,45195764102,31033
+17,45199720448,45212762605,13042157,45208758416,45210460864,9037968,1702448,2301741,45208771010,45208790367,19357,45209773548,45210074988,301440,45210978277,45211173577,195300,45208773143,45208803280,30137
+18,45212762605,45225814601,13051996,45221801814,45223514580,9039209,1712766,2300021,45221815911,45221839644,23733,45222819211,45223114544,295333,45224031469,45224234043,202574,45221812106,45221849103,36997
+19,45225814601,45238848430,13033829,45234842015,45236548356,9027414,1706341,2300074,45234855444,45234876469,21025,45235853358,45236160825,307467,45237063061,45237260964,197903,45234857141,45234882976,25835
+20,45238848430,45251899738,13051308,45247879385,45249598280,9030955,1718895,2301458,45247896725,45247917316,20591,45248896361,45249193681,297320,45250117916,45250315651,197735,45247894228,45247926723,32495
+-,45121436513,45134482124,13045611,45130471874,45132181322,9035360,1709449,2300802,45130486422,45130508229,21808,45131486785,45131787364,300579,45132697369,45132897305,199936,45130487458,45130517315,29857

tests/ut/data/profiler_data/profiler/step_trace_raw_10_detail_time.csv

+step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0
+1,45000025226,45004034753,4009527,45000025226,45001734362,0,1709136,2300391,45000044023,45000060886,16863,45001043581,45001343373,299792,45002254048,45002452830,198782,45000043807,45000065736,21929
+2,45004034753,45017091420,13056667,45013073790,45014789509,9039037,1715719,2301911,45013085205,45013104210,19005,45014086339,45014393261,306922,45015299546,45015501808,202262,45013085040,45013119810,34770
+3,45017091420,45030144372,13052952,45026123867,45027843651,9032447,1719784,2300721,45026138546,45026154524,15978,45027135742,45027437486,301744,45028363120,45028560901,197781,45026136046,45026171363,35317
+4,45030144372,45043184486,13040114,45039173149,45040883087,9028777,1709938,2301399,45039190927,45039209948,19021,45040185915,45040484897,298982,45041399754,45041594775,195021,45039192768,45039221423,28655
+5,45043184486,45056241064,13056578,45052223555,45053940709,9039069,1717154,2300355,45052241736,45052262186,20450,45053239605,45053540866,301261,45054452604,45054654505,201901,45052233932,45052265774,31842
+6,45056241064,45069291346,13050282,45065278144,45066991121,9037080,1712977,2300225,45065293660,45065316136,22476,45066289480,45066589910,300430,45067511002,45067701731,190729,45065293679,45065321296,27617
+7,45069291346,45082344927,13053581,45078335376,45080043268,9044030,1707892,2301659,45078353164,45078365382,12218,45079354748,45079648384,293636,45080557453,45080760374,202921,45078353030,45078384530,31500
+8,45082344927,45095382554,13037627,45091368697,45093080797,9023770,1712100,2301757,45091381244,45091405208,23964,45092382630,45092684285,301655,45093590961,45093796698,205737,45091381199,45091413840,32641
+9,45095382554,45108433947,13051393,45104419947,45106132133,9037393,1712186,2301814,45104432587,45104457476,24889,45105431458,45105735476,304018,45106651213,45106845305,194092,45104435207,45104466677,31470
+10,45108433947,45121486591,13052644,45117469353,45119185969,9035406,1716616,2300622,45117483627,45117504869,21242,45118483411,45118788540,305129,45119696660,45119898575,201915,45117485587,45117510985,25398
+11,45121486591,45134546571,13059980,45130528618,45132244809,9042027,1716191,2301762,45130539730,45130561122,21392,45131538695,45131846715,308020,45132759789,45132960848,201059,45130545378,45130569412,24034
+12,45134546571,45147608222,13061651,45143597023,45145307273,9050452,1710250,2300949,45143615771,45143631460,15689,45144610592,45144910736,300144,45145818642,45146024326,205684,45143613528,45143640223,26695
+13,45147608222,45160663790,13055568,45156648696,45158362923,9040474,1714227,2300867,45156663193,45156685466,22273,45157661576,45157963074,301498,45158881212,45159074431,193219,45156667038,45156694912,27874
+14,45160663790,45173707626,13043836,45169694535,45171407246,9030745,1712711,2300380,45169710667,45169727936,17269,45170705802,45171013806,308004,45171924100,45172120273,196173,45169708524,45169739038,30514
+15,45173707626,45186754860,13047234,45182750254,45184454036,9042628,1703782,2300824,45182765445,45182789799,24354,45183761335,45184065169,303834,45184973312,45185170444,197132,45182769451,45182799598,30147
+16,45186754860,45199798718,13043858,45195792271,45197497908,9037411,1705637,2300810,45195804771,45195827915,23144,45196804016,45197108243,304227,45198013357,45198209858,196501,45195806656,45195841674,35018
+17,45199798718,45212854993,13056275,45208834355,45210553378,9035637,1719023,2301615,45208850179,45208865588,15409,45209851018,45210151436,300418,45211073169,45211271792,198623,45208847052,45208876998,29946
+18,45212854993,45225893712,13038719,45221888939,45223593704,9033946,1704765,2300008,45221901732,45221924983,23251,45222908795,45223203590,294795,45224105803,45224313354,207551,45221899792,45221938802,39010
+19,45225893712,45238941242,13047530,45234926295,45236640454,9032583,1714159,2300788,45234938628,45234957237,18609,45235942710,45236239983,297273,45237159532,45237356140,196608,45234938330,45234976170,37840
+20,45238941242,45251979177,13037935,45247977674,45249678116,9036432,1700442,2301061,45247990919,45248013476,22557,45248991451,45249294742,303291,45250195733,45250395760,200027,45247988950,45248024969,36019
+21,45251979177,45265018752,13039575,45261005416,45262718472,9026239,1713056,2300280,0,0,0,0,0,0,0,0,0,0,0,0
+22,45265018752,45278062782,13044030,45274047185,45275762095,9028433,1714910,2300687,0,0,0,0,0,0,0,0,0,0,0,0
+23,45278062782,45291105708,13042926,45287094000,45288805223,9031218,1711223,2300485,0,0,0,0,0,0,0,0,0,0,0,0
+24,45291105708,45304155918,13050210,45300150844,45301854040,9045136,1703196,2301878,0,0,0,0,0,0,0,0,0,0,0,0
+25,45304155918,45317206695,13050777,45313191948,45314905714,9036030,1713766,2300981,0,0,0,0,0,0,0,0,0,0,0,0
+26,45317206695,45330265105,13058410,45326256021,45327964581,9049326,1708560,2300524,0,0,0,0,0,0,0,0,0,0,0,0
+27,45330265105,45343324012,13058907,45339305124,45341023739,9040019,1718615,2300273,0,0,0,0,0,0,0,0,0,0,0,0
+28,45343324012,45356374571,13050559,45352366211,45354073401,9042199,1707190,2301170,0,0,0,0,0,0,0,0,0,0,0,0
+29,45356374571,45369429514,13054943,45365417827,45367128283,9043256,1710456,2301231,0,0,0,0,0,0,0,0,0,0,0,0
+30,45369429514,45382479199,13049685,45378476397,45380177297,9046883,1700900,2301902,0,0,0,0,0,0,0,0,0,0,0,0
+31,45382479199,45395530376,13051177,45391510137,45393229377,9030938,1719240,2300999,0,0,0,0,0,0,0,0,0,0,0,0
+32,45395530376,45408571765,13041389,45404559082,45406270720,9028706,1711638,2301045,0,0,0,0,0,0,0,0,0,0,0,0
+33,45408571765,45421635175,13063410,45417619223,45419334221,9047458,1714998,2300954,0,0,0,0,0,0,0,0,0,0,0,0
+34,45421635175,45434672219,13037044,45430669445,45432371312,9034270,1701867,2300907,0,0,0,0,0,0,0,0,0,0,0,0
+35,45434672219,45447714036,13041817,45443704548,45445413852,9032329,1709304,2300184,0,0,0,0,0,0,0,0,0,0,0,0
+36,45447714036,45460765153,13051117,45456753675,45458463701,9039639,1710026,2301452,0,0,0,0,0,0,0,0,0,0,0,0
+37,45460765153,45473829105,13063952,45469808281,45471527400,9043128,1719119,2301705,0,0,0,0,0,0,0,0,0,0,0,0
+38,45473829105,45486884190,13055085,45482867237,45484583534,9038132,1716297,2300656,0,0,0,0,0,0,0,0,0,0,0,0
+39,45486884190,45499928571,13044381,45495917628,45497627921,9033438,1710293,2300650,0,0,0,0,0,0,0,0,0,0,0,0
+40,45499928571,45512973815,13045244,45508968990,45510673699,9040419,1704709,2300116,0,0,0,0,0,0,0,0,0,0,0,0
+-,45251983006,45265032725,13049720,45261020353,45262731761,9037347,1711408,2300964,21986676455,21986686280,9825,21987163213,21987310272,147058,21987754537,21987851587,97050,21986676441,21986691731,15290

tests/ut/python/profiler/__init__.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Unit test for profiler."""
+import os
+
+RAW_DATA_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../data/profiler_data'))
+RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB1'))
+RAW_DATA_JOB2 = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB2'))
+PROFILER_DIR = os.path.realpath(os.path.join(RAW_DATA_BASE, 'profiler'))

tests/ut/python/profiler/parser/__init__.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

tests/ut/python/profiler/parser/test_aicpu_parser.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Test the aicpu parser."""
+import os
+import tempfile
+import shutil
+
+from unittest import TestCase
+
+from mindspore.profiler.parser.aicpu_data_parser import DataPreProcessParser
+
+
+def get_result(file_path):
+    """
+    Get result from the aicpu file.
+
+    Args:
+        file_path (str): The aicpu file path.
+
+    Returns:
+        list[list], the parsed aicpu information.
+    """
+    result = []
+    try:
+        file = open(file_path, 'r')
+        result.append(file.read())
+        return result
+    finally:
+        if file:
+            file.close()
+
+
+class TestAicpuParser(TestCase):
+    """Test the class of Aicpu Parser."""
+
+    def setUp(self) -> None:
+        """Initialization before test case execution."""
+        self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
+                                                           '../../../data/profiler_data/'
+                                                           'JOB_AICPU/data'))
+        self.expect_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
+                                                        '../../../data/profiler_data/'
+                                                        'JOB_AICPU/expect'))
+        self.output_path = tempfile.mkdtemp(prefix='output_data_preprocess_aicpu_')
+        self.output_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt')
+        self.expect_file = os.path.join(self.expect_dir, 'output_data_preprocess_aicpu_0.txt')
+
+    def test_aicpu_parser(self):
+        """Test the class of Aicpu Parser."""
+        data = DataPreProcessParser(self.profiling_dir, self.output_file)
+        data.execute()
+        expect_result = get_result(self.expect_file)
+        result = get_result(self.output_file)
+        shutil.rmtree(self.output_path)
+        assert expect_result == result
+
+    def test_aicpu_parser_file_not_exist(self):
+        """Test the class of Aicpu Parser."""
+        profiling_dir = os.path.realpath(os.path.join(self.profiling_dir, 'data'))
+        data = DataPreProcessParser(profiling_dir, self.output_file)
+        data.execute()
+        shutil.rmtree(self.output_path)

tests/ut/python/profiler/parser/test_framework_parser.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Test the framework parser module."""
+import csv
+import os
+import shutil
+import tempfile
+from unittest import mock
+
+import pytest
+
+from mindspore.profiler.common.exceptions.exceptions import \
+    ProfilerFileNotFoundException
+from mindspore.profiler.parser.framework_parser import FrameworkParser
+from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA_BASE
+
+
+def get_framework_result(file_path):
+    """
+    Get framework result from the framework file.
+
+    Args:
+        file_path (str): The framework file path.
+
+    Returns:
+        list[list], the parsed framework information.
+    """
+    result = []
+    with open(file_path, 'r') as file:
+        csv_reader = csv.reader(file)
+        for row in csv_reader:
+            result.append(row)
+    return result
+
+
+class TestFrameworkParser:
+    """Test the class of `FrameworkParser`."""
+    def setup_method(self):
+        """Initialization before test case execution."""
+        with mock.patch.object(FrameworkParser, '_raw_data_dir', RAW_DATA_BASE):
+            self._output_path_1 = tempfile.mkdtemp(prefix='test_framework_parser_')
+            self._parser_1 = FrameworkParser('JOB1', '0', self._output_path_1)
+            self._output_path_2 = tempfile.mkdtemp(prefix='test_framework_parser_')
+            self._parser_2 = FrameworkParser('JOB2', '0', self._output_path_2)
+            self._output_path_4 = tempfile.mkdtemp(prefix='test_framework_parser_')
+            self._parser_4 = FrameworkParser('JOB4', '0', self._output_path_4)
+
+    def teardown_method(self) -> None:
+        """Clear up after test case execution."""
+        shutil.rmtree(self._output_path_1)
+        shutil.rmtree(self._output_path_2)
+        shutil.rmtree(self._output_path_4)
+
+    def test_save_path(self):
+        """Test the querying save path function."""
+        expect_result = os.path.join(self._output_path_1, 'framework_raw_0.csv')
+        assert expect_result == self._parser_1.save_path
+
+        expect_result = os.path.join(self._output_path_2, 'framework_raw_0.csv')
+        assert expect_result == self._parser_2.save_path
+
+    def test_point_info(self):
+        """Test the querying point info function."""
+        expect_result = {
+            1: 'Default/Cast-op6',
+            2: 'Default/TransData-op7'
+        }
+        assert expect_result == self._parser_4.point_info
+
+    def test_to_task_id_full_op_name_dict(self):
+        """Test the querying task id and full operator name dict function."""
+        expect_result = {
+            '51517': 'Default/Cast-op6',
+            '51518': 'Default/TransData-op7',
+            '51519': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
+            '51522': 'Default/network-WithLossCell/_backbone-ResNet/'
+                     'layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28'
+        }
+        assert expect_result == self._parser_1.to_task_id_full_op_name_dict()
+        assert expect_result == self._parser_2.to_task_id_full_op_name_dict()
+
+        expect_result = {
+            '0_1': 'Default/Cast-op6',
+            '0_2': 'Default/TransData-op7',
+            '0_3': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
+            '0_4': 'Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/'
+                   '0-ResidualBlock/conv1-Conv2d/Cast-op28'
+        }
+        assert expect_result == self._parser_4.to_task_id_full_op_name_dict()
+
+    def test_parse(self):
+        """Test the parse function."""
+        expect_framework_file = os.path.join(PROFILER_DIR, 'framework_raw_0.csv')
+        expect_framework_file = os.path.realpath(expect_framework_file)
+        expect_result = get_framework_result(expect_framework_file)
+
+        self._parser_1.parse()
+        framework_file = os.path.join(self._output_path_1, 'framework_raw_0.csv')
+        result = get_framework_result(framework_file)
+        assert expect_result == result
+
+        self._parser_2.parse()
+        framework_file = os.path.join(self._output_path_2, 'framework_raw_0.csv')
+        result = get_framework_result(framework_file)
+        assert expect_result == result
+
+    @mock.patch('os.listdir')
+    @mock.patch('os.path.isdir')
+    def test_create_framework_parser_fail_1(self, *args):
+        """Test the function of fail to create framework parser."""
+        args[0].return_value = True
+        args[1].return_value = []
+        with pytest.raises(ProfilerFileNotFoundException) as exc_info:
+            FrameworkParser('JOB1', '0')
+        assert exc_info.value.error_code == '50546084'
+        assert exc_info.value.message == 'The file <Framework> not found.'

tests/ut/python/profiler/parser/test_minddata_pipeline_parser.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Test the minddata pipeline parser module."""
+import csv
+import os
+import shutil
+import tempfile
+
+from mindspore.profiler.parser.minddata_pipeline_parser import \
+    MinddataPipelineParser
+from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA, RAW_DATA_JOB2
+
+
+def get_minddata_pipeline_result(file_path):
+    """
+    Get minddata pipeline result from the minddata pipeline file.
+
+    Args:
+        file_path (str): The minddata pipeline file path.
+
+    Returns:
+        list[list], the parsed minddata pipeline information.
+    """
+    result = []
+    with open(file_path, 'r') as file:
+        csv_reader = csv.reader(file)
+        for row in csv_reader:
+            result.append(row)
+    return result
+
+
+class TestMinddataPipelineParser:
+    """Test the class of `MinddataPipelineParser`."""
+    def setup_method(self):
+        """Initialization before test case execution."""
+        self._output_path_1 = tempfile.mkdtemp(
+            prefix='test_minddata_pipeline_parser_'
+        )
+        self._parser_1 = MinddataPipelineParser(
+            RAW_DATA, '0', self._output_path_1
+        )
+
+        self._output_path_2 = tempfile.mkdtemp(
+            prefix='test_minddata_pipeline_parser_'
+        )
+        self._parser_2 = MinddataPipelineParser(
+            RAW_DATA_JOB2, '0', self._output_path_2
+        )
+
+    def teardown_method(self) -> None:
+        """Clear up after test case execution."""
+        shutil.rmtree(self._output_path_1)
+        shutil.rmtree(self._output_path_2)
+
+    def test_save_path(self):
+        """Test the querying save path function."""
+        expect_result = os.path.join(
+            self._output_path_1, 'minddata_pipeline_raw_0.csv'
+        )
+        assert expect_result == self._parser_1.save_path
+
+    def test_parse(self):
+        """Test the parse function."""
+        expect_pipeline_file = os.path.join(
+            PROFILER_DIR, 'minddata_pipeline_raw_0.csv'
+        )
+        expect_result = get_minddata_pipeline_result(expect_pipeline_file)
+
+        self._parser_1.parse()
+        pipeline_file = os.path.join(
+            self._output_path_1, 'minddata_pipeline_raw_0.csv'
+        )
+        result = get_minddata_pipeline_result(pipeline_file)
+        assert expect_result == result
+
+        self._parser_2.parse()
+        pipeline_file = os.path.join(
+            self._output_path_2, 'minddata_pipeline_raw_0.csv'
+        )
+        result = get_minddata_pipeline_result(pipeline_file)
+        assert expect_result == result