diff --git a/e3d_data_iter/datasets.py b/e3d_data_iter/datasets.py index f9296a598073bebe7e15253225e24fd2d25b4a31..bcd6dbb836d6fdabd27ef82a9fc0d32dd714244c 100644 --- a/e3d_data_iter/datasets.py +++ b/e3d_data_iter/datasets.py @@ -156,7 +156,8 @@ def draw_msra_gaussian(heatmap, center, sigma): return heatmap def get_heatmap(img_fix_size,x1y1x2y2,handpose_2d,ratio, dw, dh,offset_x1=0,offset_y1=0,radius=20,vis = False): num_objs = 21 - hm = np.zeros((256,256,num_objs), dtype=np.float32) + + hm = np.zeros((img_fix_size.shape[0],img_fix_size.shape[1],num_objs), dtype=np.float32) draw_gaussian = draw_msra_gaussian if False else draw_umich_gaussian for k in range(num_objs): @@ -449,12 +450,12 @@ class LoadImagesAndLabels(Dataset): - hand_w = int((x2_-x1_)/8) - hand_h = int((y2_-y1_)/8) - offset_x1 = random.randint(-hand_w,int(hand_w/2)) - offset_y1 = random.randint(-hand_h,int(hand_h/2)) - offset_x2 = random.randint(-int(hand_w/2),hand_w) - offset_y2 = random.randint(-int(hand_h/2),hand_h) + hand_w = int((x2_-x1_)/2) + hand_h = int((y2_-y1_)/2) + offset_x1 = random.randint(-hand_w,int(hand_w/6)) + offset_y1 = random.randint(-hand_h,int(hand_h/6)) + offset_x2 = random.randint(-int(hand_w/6),hand_w) + offset_y2 = random.randint(-int(hand_h/6),hand_h) # print(" A : x1_,y1_,x2_,y2_ ： ",x1_,y1_,x2_,y2_) x1_new = x1_+offset_x1 y1_new = y1_+offset_y1 diff --git a/yolo_inference.py b/yolo_inference.py index d96092eff18f3e5a1e8816bf975fe6194d902d12..78e6929e0b467414a768a1568013120128996329 100644 --- a/yolo_inference.py +++ b/yolo_inference.py @@ -36,8 +36,12 @@ from manopth import manolayer if __name__ == "__main__": parser = argparse.ArgumentParser(description=' Project Hand Pose 3D Inference') - parser.add_argument('--model_path', type=str, default = './model_exp/2021-06-16_10-53-50/resnet_50-size-256-loss-wing_loss-model_epoch-225.pth', - help = 'model_path') # 模型路径 + parser.add_argument('--model_path', type=str, default = '././model_exp/2021-06-19_14-20-56/resnet_50-size-128-loss-wing_loss-model_epoch-196.pth', + help = 'model_path') # e3d handpose 模型路径 + parser.add_argument('--detect_model_path', type=str, default = './if_package/hand_detect_416-20210606.pt', + help = 'model_path') # detect 模型路径 + parser.add_argument('--handpose_x2d_model_path', type=str, default = './if_package/handposex_2d_resnet_50-size-256-wingloss102-0.119.pth', + help = 'model_path') # 手2维关键点 模型路径 parser.add_argument('--model', type=str, default = 'resnet_50', help = '''model : resnet_18,resnet_34,resnet_50,resnet_101''') # 模型类型 parser.add_argument('--num_classes', type=int , default = 63, @@ -46,7 +50,7 @@ if __name__ == "__main__": help = 'GPUS') # GPU选择 parser.add_argument('--test_path', type=str, default = './image/', help = 'test_path') # 测试图片路径 - parser.add_argument('--img_size', type=tuple , default = (256,256), + parser.add_argument('--img_size', type=tuple , default = (128,128), help = 'img_size') # 输入模型图片尺寸 parser.add_argument('--vis', type=bool , default = True, help = 'vis') # 是否可视化图片 @@ -90,16 +94,17 @@ if __name__ == "__main__": print('load test model : {}'.format(ops.model_path)) #----------------- 构建 handpose_x 2D关键点检测模型 - handpose_2d_model = handpose_x_model() + handpose_2d_model = handpose_x_model(model_path = ops.handpose_x2d_model_path) #----------------- 构建 yolo 检测模型 - hand_detect_model = yolo_v3_hand_model() + hand_detect_model = yolo_v3_hand_model(model_path = ops.detect_model_path,model_arch = "yolo",conf_thres = 0.3) + # hand_detect_model = yolo_v3_hand_model() #----------------- 构建 manopth g_side = "right" print('load model finished') pose, shape = func.initiate("zero") pre_useful_bone_len = np.zeros((1, 15)) # 骨架点信息 - solver = LM_Solver(num_Iter=666, th_beta=shape.cpu(), th_pose=pose.cpu(), lb_target=pre_useful_bone_len, + solver = LM_Solver(num_Iter=99, th_beta=shape.cpu(), th_pose=pose.cpu(), lb_target=pre_useful_bone_len, weight=1e-5) pose0 = torch.eye(3).repeat(1, 16, 1, 1) @@ -110,8 +115,8 @@ if __name__ == "__main__": root_rot_mode='rotmat', joint_rot_mode='rotmat') print('start ~') - point_fliter = smoother.OneEuroFilter(4.0, 0.0) - mesh_fliter = smoother.OneEuroFilter(4.0, 0.0) + point_fliter = smoother.OneEuroFilter(23.0, 0.0) + mesh_fliter = smoother.OneEuroFilter(23.0, 0.0) shape_fliter = smoother.OneEuroFilter(1.5, 0.0) #--------------------------- 配置点云 view_mat = np.array([[1.0, 0.0, 0.0], @@ -136,7 +141,6 @@ if __name__ == "__main__": test_pcd = open3d.geometry.PointCloud() # 定义点云 viewer.add_geometry(test_pcd) - print('start pose estimate') pre_uv = None @@ -145,7 +149,6 @@ if __name__ == "__main__": shape_flag = True #---------------------------------------------------------------- 预测图片 - with torch.no_grad(): idx = 0 cap = cv2.VideoCapture(0) #一般usb默认相机号为 0，如果没有相机无法启动，如果相机不为0需要自行确定其编号。 @@ -159,7 +162,7 @@ if __name__ == "__main__": x_min,y_min,x_max,y_max,_ = hand_bbox[0] w_ = max(abs(x_max-x_min),abs(y_max-y_min)) - w_ = w_*1.5 + w_ = w_*1.6 x_mid = (x_max+x_min)/2 y_mid = (y_max+y_min)/2 # @@ -195,9 +198,9 @@ if __name__ == "__main__": #-------------------------------- img_lbox,ratio, dw, dh = letterbox(img.copy(), height=ops.img_size[0], color=(0,0,0)) - if ops.vis: - cv2.namedWindow("letterbox",0) - cv2.imshow("letterbox",img_lbox) + # if ops.vis: + # cv2.namedWindow("letterbox",0) + # cv2.imshow("letterbox",img_lbox) #-------------------------------- get heatmap x1y1x2y2 = 0,0,0,0 @@ -219,24 +222,23 @@ if __name__ == "__main__": image_fusion = image_fusion.unsqueeze_(0) if use_cuda: image_fusion = image_fusion.cuda() # (bs, channel, h, w) - print("image_fusion size : {}".format(image_fusion.size())) - + # print("image_fusion size : {}".format(image_fusion.size())) #-------------------------------- # handpose_3d predict pre_ = model_(image_fusion.float()) # 模型推理 output = pre_.cpu().detach().numpy() output = np.squeeze(output) - print("handpose_3d output shape : {}".format(output.shape)) + # print("handpose_3d output shape : {}".format(output.shape)) pre_3d_joints = output.reshape((21,3)) - print("pre_3d_joints shape : {}".format(pre_3d_joints.shape)) + # print("pre_3d_joints shape : {}".format(pre_3d_joints.shape)) if g_side == "left": print("------------------->>. left") pre_3d_joints[:,0] *=(-1.) pre_3d_joints = torch.tensor(pre_3d_joints).squeeze(0) pre_3d_joints= pre_3d_joints.cuda() - print(pre_3d_joints.size()) + # print(pre_3d_joints.size()) #-------------------------------------------------------------------- # now_uv = result['uv'].clone().detach().cpu().numpy()[0, 0] # now_uv = now_uv.astype(np.float) @@ -251,6 +253,8 @@ if __name__ == "__main__": # fliter_ax.cla() # # filted_ax = vis.plot3d(flited_joints + new_tran, fliter_ax) + # pre_useful_bone_len = bone.caculate_length(pre_joints, label="useful") + pre_useful_bone_len = bone.caculate_length(pre_joints, label="useful") NGEN = 0 # PSO 迭代次数 @@ -274,7 +278,7 @@ if __name__ == "__main__": pose_R = torch.from_numpy(pose_R).float() # reconstruction hand_verts, j3d_recon = mano(pose_R, opt_tensor_shape.float()) - hand_verts[:,:,:] = hand_verts[:,:,:]*(0.55) + hand_verts[:,:,:] = hand_verts[:,:,:]*(0.573) # print(j3d_recon.size()) mesh.triangles = open3d.utility.Vector3iVector(mano.th_faces) @@ -312,7 +316,7 @@ if __name__ == "__main__": pre_joints = pre_joints*70 pre_joints[:,1] -= 40 pre_joints[:,0] -= 0 - print("pre_joints",pre_joints.shape) + # print("pre_joints",pre_joints.shape) test_pcd.points = open3d.utility.Vector3dVector(pre_joints) # test_pcd.points = open3d.utility.Vector3dVector(pre_joints[1,:].reshape(1,3)) # rgb = np.asarray([250,0,250]) @@ -342,8 +346,8 @@ if __name__ == "__main__": mask_2 = np.where(image_3d[:,:,2]!=120,1,0) img_mask = np.logical_or(mask_0,mask_1) img_mask = np.logical_or(img_mask,mask_2) - cv2.namedWindow("img_mask",0) - cv2.imshow("img_mask",img_mask.astype(np.float)) + # cv2.namedWindow("img_mask",0) + # cv2.imshow("img_mask",img_mask.astype(np.float)) locs = np.where(img_mask != 0) xx1 = np.min(locs[1]) @@ -356,7 +360,7 @@ if __name__ == "__main__": #---------- cv2.namedWindow("image_3d",0) cv2.imshow("image_3d",image_3d) - scale_ = ((x_max-x_min)/(xx2-xx1) + (y_max-y_min)/(yy2-yy1))/2. + scale_ = ((x_max-x_min)/(xx2-xx1) + (y_max-y_min)/(yy2-yy1))/2.*1.01 w_3d_ = (xx2-xx1)*scale_ h_3d_ = (yy2-yy1)*scale_ x_mid_3d = (xx1+xx2)/2. @@ -372,7 +376,6 @@ if __name__ == "__main__": crop_mask = np.where(crop_mask[:,:]>0.,1.,0.) crop_mask = np.expand_dims(crop_mask, 2) - # print("-----------------------------ppppppppppppppwwww : ",crop_.shape,crop_mask.shape,h_r,w_r) try: img_ff = img_yolo_x[int(y_mid - h_r ):int(y_mid + h_r ),int(x_mid - w_r ):int(x_mid + w_r ),:]*(1.-crop_mask) + crop_*crop_mask img_yolo_x[int(y_mid - h_r ):int(y_mid + h_r ),int(x_mid - w_r ):int(x_mid + w_r ),:] = img_ff.astype(np.uint8) @@ -383,15 +386,6 @@ if __name__ == "__main__": real_hand_h = h_r*2 depth_z = (model_hand_h/real_hand_h + model_hand_w/real_hand_w)/2.# 相对深度 z # - # ff_img = (img_yolo_x[int(y_mid - h_r ):int(y_mid + h_r ),int(x_mid - w_r ):int(x_mid + w_r ),:]*(1.-crop_mask) - # + crop_mask * crop_) - # img_yolo_x[int(y_mid - h_r ):int(y_mid + h_r ),int(x_mid - w_r ):int(x_mid + w_r ),:] - # cv2.namedWindow("crop_mask",0) - # cv2.imshow("crop_mask",crop_mask) - - # cv2.rectangle(img_yolo_x, (x1,y1),(x2,y2), (0,255,0), 5) # 绘制空心矩形 - # print("scale : {}".format(scale_)) - # cv2.rectangle(image_3d, (xx1,yy1), (xx2,yy2), (255,0,255), 5) # 绘制空心矩形 cv2.putText(img_yolo_x, " Relative Depth_Z :{:.3f} ".format(depth_z), (4,42),cv2.FONT_HERSHEY_DUPLEX, 1.1, (55, 0, 220),7) cv2.putText(img_yolo_x, " Relative Depth_Z :{:.3f} ".format(depth_z), (4,42),cv2.FONT_HERSHEY_DUPLEX, 1.1, (25, 180, 250),2)