diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index 67f0b91a99de057933583d8aa786f23c73cb1af8..c9bbb3b560a5683cef6cc9c6b81454f9d537ca94 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -176,8 +176,6 @@ static void sun4i_rgb_encoder_mode_set(struct drm_encoder *encoder,
 
 	sun4i_tcon0_mode_set(tcon, mode);
 
-	clk_set_rate(tcon->dclk, mode->crtc_clock * 1000);
-
 	/* FIXME: This seems to be board specific */
 	clk_set_phase(tcon->dclk, 120);
 }
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index 5d7866fce78b83a76e268917d505d72a346e79ec..8b6aaa60037d00b96a5bdbad49bcfe9cbbce8a36 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -130,6 +130,9 @@ void sun4i_tcon0_mode_set(struct sun4i_tcon *tcon,
 	u8 clk_delay;
 	u32 val = 0;
 
+	/* Configure the dot clock */
+	clk_set_rate(tcon->dclk, mode->crtc_clock * 1000);
+
 	/* Adjust clock delay */
 	clk_delay = sun4i_tcon_get_clk_delay(mode, 0);
 	regmap_update_bits(tcon->regs, SUN4I_TCON0_CTL_REG,
@@ -205,6 +208,9 @@ void sun4i_tcon1_mode_set(struct sun4i_tcon *tcon,
 
 	WARN_ON(!tcon->quirks->has_channel_1);
 
+	/* Configure the dot clock */
+	clk_set_rate(tcon->sclk1, mode->crtc_clock * 1000);
+
 	/* Adjust clock delay */
 	clk_delay = sun4i_tcon_get_clk_delay(mode, 1);
 	regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index 49c49431a053e34a3ebb9cb34a7490adda1c2860..542da220818ba1a222c03871957cd0f9c3333841 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -486,8 +486,6 @@ static void sun4i_tv_mode_set(struct drm_encoder *encoder,
 		      SUN4I_TVE_RESYNC_FIELD : 0));
 
 	regmap_write(tv->regs, SUN4I_TVE_SLAVE_REG, 0);
-
-	clk_set_rate(tcon->sclk1, mode->crtc_clock * 1000);
 }
 
 static struct drm_encoder_helper_funcs sun4i_tv_helper_funcs = {