diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 617030727ca85b4fd67dfac2a0c051027b2152f2..9507be0f4083ee565ebb08952b645b7ea4f04391 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -272,7 +272,6 @@ struct radeon_fence {
 	uint64_t			seq;
 	/* RB, DMA, etc. */
 	unsigned			ring;
-	struct radeon_semaphore		*semaphore;
 };
 
 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring);
@@ -624,13 +623,14 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
  */
 
 struct radeon_ib {
-	struct radeon_sa_bo	*sa_bo;
-	uint32_t		length_dw;
-	uint64_t		gpu_addr;
-	uint32_t		*ptr;
-	struct radeon_fence	*fence;
-	unsigned		vm_id;
-	bool			is_const_ib;
+	struct radeon_sa_bo		*sa_bo;
+	uint32_t			length_dw;
+	uint64_t			gpu_addr;
+	uint32_t			*ptr;
+	struct radeon_fence		*fence;
+	unsigned			vm_id;
+	bool				is_const_ib;
+	struct radeon_semaphore		*semaphore;
 };
 
 struct radeon_ring {
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 5c065bf2d1621ea3c49f1f94acb6faf875f3e321..dcfe2a0bcdc0f50aae36bcc99a96e3a58af31747 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -138,12 +138,12 @@ static int radeon_cs_sync_rings(struct radeon_cs_parser *p)
 		return 0;
 	}
 
-	r = radeon_semaphore_create(p->rdev, &p->ib->fence->semaphore);
+	r = radeon_semaphore_create(p->rdev, &p->ib->semaphore);
 	if (r) {
 		return r;
 	}
 
-	return radeon_semaphore_sync_rings(p->rdev, p->ib->fence->semaphore,
+	return radeon_semaphore_sync_rings(p->rdev, p->ib->semaphore,
 					   sync_to_ring, p->ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 3a49311fc711ce77d43408f4f964a08f77230d74..48ec5e343844b9b1046c8a90e7d930eeda79de22 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -139,8 +139,6 @@ static void radeon_fence_destroy(struct kref *kref)
 
 	fence = container_of(kref, struct radeon_fence, kref);
 	fence->seq = RADEON_FENCE_NOTEMITED_SEQ;
-	if (fence->semaphore)
-		radeon_semaphore_free(fence->rdev, fence->semaphore, NULL);
 	kfree(fence);
 }
 
@@ -156,7 +154,6 @@ int radeon_fence_create(struct radeon_device *rdev,
 	(*fence)->rdev = rdev;
 	(*fence)->seq = RADEON_FENCE_NOTEMITED_SEQ;
 	(*fence)->ring = ring;
-	(*fence)->semaphore = NULL;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index b3d6942a2be9912ed4bd08c461afcb9be40e31f5..af8e1ee1dc0156d0d02c24462405e9593ad2022c 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -93,6 +93,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring,
 	(*ib)->gpu_addr = radeon_sa_bo_gpu_addr((*ib)->sa_bo);
 	(*ib)->vm_id = 0;
 	(*ib)->is_const_ib = false;
+	(*ib)->semaphore = NULL;
 
 	return 0;
 }
@@ -105,6 +106,7 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib)
 	if (tmp == NULL) {
 		return;
 	}
+	radeon_semaphore_free(rdev, tmp->semaphore, tmp->fence);
 	radeon_sa_bo_free(rdev, &tmp->sa_bo, tmp->fence);
 	radeon_fence_unref(&tmp->fence);
 	kfree(tmp);