diff --git a/ppdet/modeling/backbones/lite_hrnet.py b/ppdet/modeling/backbones/lite_hrnet.py
index fbf1dfe21635b380acb4fbb4c5138028ee0caa0d..da98be3a4b3323bc9f4b77ff31202009f43b2b1a 100644
--- a/ppdet/modeling/backbones/lite_hrnet.py
+++ b/ppdet/modeling/backbones/lite_hrnet.py
@@ -651,7 +651,9 @@ class LiteHRNetModule(nn.Layer):
             for i in range(len(self.fuse_layers)):
                 y = out[0] if i == 0 else self.fuse_layers[i][0](out[0])
                 for j in range(self.num_branches):
-                    if i == j:
+                    if j == 0:
+                        y += y
+                    elif i == j:
                         y += out[j]
                     else:
                         y += self.fuse_layers[i][j](out[j])