diff --git a/arch/arm/mach-at91/Kconfig b/arch/arm/mach-at91/Kconfig
index c3f9ebc5f9f52fd9956682535a96a11c5404488f..474c855fa83c25a65593764ff1d4986864df3728 100644
--- a/arch/arm/mach-at91/Kconfig
+++ b/arch/arm/mach-at91/Kconfig
@@ -33,16 +33,11 @@ config OLD_IRQ_AT91
 	select MULTI_IRQ_HANDLER
 	select SPARSE_IRQ
 
-config AT91_SAM9_TIME
-	select CLKSRC_OF if OF
-	bool
-
 config HAVE_AT91_SMD
 	bool
 
 config SOC_AT91SAM9
 	bool
-	select AT91_SAM9_TIME
 	select ATMEL_AIC_IRQ if !OLD_IRQ_AT91
 	select CPU_ARM926T
 	select GENERIC_CLOCKEVENTS
@@ -51,7 +46,6 @@ config SOC_AT91SAM9
 
 config SOC_SAMA5
 	bool
-	select AT91_SAM9_TIME
 	select ATMEL_AIC5_IRQ
 	select CPU_V7
 	select GENERIC_CLOCKEVENTS
diff --git a/arch/arm/mach-at91/Makefile b/arch/arm/mach-at91/Makefile
index 306c82b3d45c038ca50036227ba0d581e9b7df8c..61d04f9314e744ac587d679821d4aaf7638c8188 100644
--- a/arch/arm/mach-at91/Makefile
+++ b/arch/arm/mach-at91/Makefile
@@ -9,7 +9,6 @@ obj-		:=
 
 obj-$(CONFIG_OLD_IRQ_AT91)	+= irq.o
 obj-$(CONFIG_OLD_CLK_AT91)	+= clock.o
-obj-$(CONFIG_AT91_SAM9_TIME)	+= at91sam926x_time.o
 obj-$(CONFIG_SOC_AT91SAM9)	+= sam9_smc.o
 
 # CPU-specific support
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index cfd6519df6615fc1a9268fcaf092859adf4323c9..82a2ebe41e27706213a1670a2f3cdee3488b0a94 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -120,6 +120,10 @@ config CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK
 	help
 	 Use ARM global timer clock source as sched_clock
 
+config ATMEL_PIT
+	select CLKSRC_OF if OF
+	def_bool SOC_AT91SAM9 || SOC_SAMA5
+
 config CLKSRC_METAG_GENERIC
 	def_bool y if METAG
 	help
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 7fd9fd1dff42f29aebec34bfa684bc950cb97ab8..e566f6c7ded4925c2904132f376f5380ce4bb6d1 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -1,4 +1,5 @@
 obj-$(CONFIG_CLKSRC_OF)	+= clksrc-of.o
+obj-$(CONFIG_ATMEL_PIT)		+= timer-atmel-pit.o
 obj-$(CONFIG_ATMEL_TCB_CLKSRC)	+= tcb_clksrc.o
 obj-$(CONFIG_X86_PM_TIMER)	+= acpi_pm.o
 obj-$(CONFIG_SCx200HR_TIMER)	+= scx200_hrt.o
diff --git a/arch/arm/mach-at91/at91sam926x_time.c b/drivers/clocksource/timer-atmel-pit.c
similarity index 100%
rename from arch/arm/mach-at91/at91sam926x_time.c
rename to drivers/clocksource/timer-atmel-pit.c