diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
index 3d803f3cd27292e0f98f2c50672a376cb6dae4f7..448d4a7c09dd3f8d3c4a8398de23302e29ce8a45 100644
--- a/sound/soc/soc-core.c
+++ b/sound/soc/soc-core.c
@@ -983,7 +983,9 @@ static void soc_remove_dai_link(struct snd_soc_card *card, int num, int order)
 		}
 		cpu_dai->probed = 0;
 		list_del(&cpu_dai->card_list);
-		module_put(cpu_dai->dev->driver->owner);
+
+		if (!cpu_dai->codec)
+			module_put(cpu_dai->dev->driver->owner);
 	}
 }
 
@@ -1257,11 +1259,13 @@ static int soc_probe_dai_link(struct snd_soc_card *card, int num, int order)
 	/* probe the cpu_dai */
 	if (!cpu_dai->probed &&
 			cpu_dai->driver->probe_order == order) {
-		cpu_dai->dapm.card = card;
-		if (!try_module_get(cpu_dai->dev->driver->owner))
-			return -ENODEV;
+		if (!cpu_dai->codec) {
+			cpu_dai->dapm.card = card;
+			if (!try_module_get(cpu_dai->dev->driver->owner))
+				return -ENODEV;
 
-		snd_soc_dapm_new_dai_widgets(&cpu_dai->dapm, cpu_dai);
+			snd_soc_dapm_new_dai_widgets(&cpu_dai->dapm, cpu_dai);
+		}
 
 		if (cpu_dai->driver->probe) {
 			ret = cpu_dai->driver->probe(cpu_dai);