memcpy_64.S 3.5 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0-only */
L
Linus Torvalds 已提交
2
/* Copyright 2002 Andi Kleen */
3

4
#include <linux/linkage.h>
5
#include <asm/errno.h>
6
#include <asm/cpufeatures.h>
7
#include <asm/alternative.h>
8
#include <asm/export.h>
9

10 11
.pushsection .noinstr.text, "ax"

12 13 14 15 16 17 18
/*
 * We build a jump to memcpy_orig by default which gets NOPped out on
 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
 */

L
Linus Torvalds 已提交
19 20 21
/*
 * memcpy - Copy a memory block.
 *
I
Ingo Molnar 已提交
22 23 24 25 26
 * Input:
 *  rdi destination
 *  rsi source
 *  rdx count
 *
L
Linus Torvalds 已提交
27 28
 * Output:
 * rax original destination
I
Ingo Molnar 已提交
29
 */
J
Jiri Slaby 已提交
30
SYM_FUNC_START_ALIAS(__memcpy)
31
SYM_FUNC_START_WEAK(memcpy)
32 33
	ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
		      "jmp memcpy_erms", X86_FEATURE_ERMS
L
Linus Torvalds 已提交
34

I
Ingo Molnar 已提交
35
	movq %rdi, %rax
36 37
	movq %rdx, %rcx
	shrq $3, %rcx
I
Ingo Molnar 已提交
38
	andl $7, %edx
39
	rep movsq
I
Ingo Molnar 已提交
40
	movl %edx, %ecx
41 42
	rep movsb
	ret
43
SYM_FUNC_END(memcpy)
J
Jiri Slaby 已提交
44
SYM_FUNC_END_ALIAS(__memcpy)
45 46
EXPORT_SYMBOL(memcpy)
EXPORT_SYMBOL(__memcpy)
47

48
/*
49 50
 * memcpy_erms() - enhanced fast string memcpy. This is faster and
 * simpler than memcpy. Use memcpy_erms when possible.
51
 */
52
SYM_FUNC_START_LOCAL(memcpy_erms)
53
	movq %rdi, %rax
54
	movq %rdx, %rcx
55 56
	rep movsb
	ret
57
SYM_FUNC_END(memcpy_erms)
58

59
SYM_FUNC_START_LOCAL(memcpy_orig)
60
	movq %rdi, %rax
61

62
	cmpq $0x20, %rdx
63
	jb .Lhandle_tail
64

I
Ingo Molnar 已提交
65
	/*
66
	 * We check whether memory false dependence could occur,
67
	 * then jump to corresponding copy mode.
I
Ingo Molnar 已提交
68
	 */
69 70
	cmp  %dil, %sil
	jl .Lcopy_backward
71
	subq $0x20, %rdx
72 73
.Lcopy_forward_loop:
	subq $0x20,	%rdx
74

I
Ingo Molnar 已提交
75
	/*
76
	 * Move in blocks of 4x8 bytes:
I
Ingo Molnar 已提交
77
	 */
78 79 80 81 82 83 84 85 86 87 88 89
	movq 0*8(%rsi),	%r8
	movq 1*8(%rsi),	%r9
	movq 2*8(%rsi),	%r10
	movq 3*8(%rsi),	%r11
	leaq 4*8(%rsi),	%rsi

	movq %r8,	0*8(%rdi)
	movq %r9,	1*8(%rdi)
	movq %r10,	2*8(%rdi)
	movq %r11,	3*8(%rdi)
	leaq 4*8(%rdi),	%rdi
	jae  .Lcopy_forward_loop
90
	addl $0x20,	%edx
91 92 93 94 95 96 97 98 99 100 101
	jmp  .Lhandle_tail

.Lcopy_backward:
	/*
	 * Calculate copy position to tail.
	 */
	addq %rdx,	%rsi
	addq %rdx,	%rdi
	subq $0x20,	%rdx
	/*
	 * At most 3 ALU operations in one cycle,
102
	 * so append NOPS in the same 16 bytes trunk.
103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
	 */
	.p2align 4
.Lcopy_backward_loop:
	subq $0x20,	%rdx
	movq -1*8(%rsi),	%r8
	movq -2*8(%rsi),	%r9
	movq -3*8(%rsi),	%r10
	movq -4*8(%rsi),	%r11
	leaq -4*8(%rsi),	%rsi
	movq %r8,		-1*8(%rdi)
	movq %r9,		-2*8(%rdi)
	movq %r10,		-3*8(%rdi)
	movq %r11,		-4*8(%rdi)
	leaq -4*8(%rdi),	%rdi
	jae  .Lcopy_backward_loop
118

119 120 121
	/*
	 * Calculate copy position to head.
	 */
122
	addl $0x20,	%edx
123 124
	subq %rdx,	%rsi
	subq %rdx,	%rdi
125
.Lhandle_tail:
126
	cmpl $16,	%edx
127
	jb   .Lless_16bytes
I
Ingo Molnar 已提交
128

129 130 131 132 133 134 135 136 137 138 139 140
	/*
	 * Move data from 16 bytes to 31 bytes.
	 */
	movq 0*8(%rsi), %r8
	movq 1*8(%rsi),	%r9
	movq -2*8(%rsi, %rdx),	%r10
	movq -1*8(%rsi, %rdx),	%r11
	movq %r8,	0*8(%rdi)
	movq %r9,	1*8(%rdi)
	movq %r10,	-2*8(%rdi, %rdx)
	movq %r11,	-1*8(%rdi, %rdx)
	retq
141
	.p2align 4
142
.Lless_16bytes:
143
	cmpl $8,	%edx
144 145 146 147 148 149 150 151 152 153 154
	jb   .Lless_8bytes
	/*
	 * Move data from 8 bytes to 15 bytes.
	 */
	movq 0*8(%rsi),	%r8
	movq -1*8(%rsi, %rdx),	%r9
	movq %r8,	0*8(%rdi)
	movq %r9,	-1*8(%rdi, %rdx)
	retq
	.p2align 4
.Lless_8bytes:
155
	cmpl $4,	%edx
156
	jb   .Lless_3bytes
I
Ingo Molnar 已提交
157

158 159 160 161 162 163 164 165
	/*
	 * Move data from 4 bytes to 7 bytes.
	 */
	movl (%rsi), %ecx
	movl -4(%rsi, %rdx), %r8d
	movl %ecx, (%rdi)
	movl %r8d, -4(%rdi, %rdx)
	retq
166
	.p2align 4
167
.Lless_3bytes:
168 169
	subl $1, %edx
	jb .Lend
170 171 172
	/*
	 * Move data from 1 bytes to 3 bytes.
	 */
173 174 175 176 177 178 179 180
	movzbl (%rsi), %ecx
	jz .Lstore_1byte
	movzbq 1(%rsi), %r8
	movzbq (%rsi, %rdx), %r9
	movb %r8b, 1(%rdi)
	movb %r9b, (%rdi, %rdx)
.Lstore_1byte:
	movb %cl, (%rdi)
181

I
Ingo Molnar 已提交
182
.Lend:
183
	retq
184
SYM_FUNC_END(memcpy_orig)
185

186
.popsection
新手
引导
客服 返回
顶部