6730276: JDI_REGRESSION tests fail with "Error: count must be non-zero" error on x86

Summary: Modify assembler code to check for 0 count for all copy routines.
Reviewed-by: never, ysr, jcoomes

src/os_cpu/linux_x86/vm/copy_linux_x86.inline.hpp

@@ -26,7 +26,7 @@ static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
 #ifdef AMD64
   (void)memmove(to, from, count * HeapWordSize);
 #else
-  // Same as pd_aligned_conjoint_words, except includes a zero-count check.
+  // Includes a zero-count check.
   intx temp;
   __asm__ volatile("        testl   %6,%6         ;"
                    "        jz      7f            ;"
@@ -84,7 +84,7 @@ static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
     break;
   }
 #else
-  // Same as pd_aligned_disjoint_words, except includes a zero-count check.
+  // Includes a zero-count check.
   intx temp;
   __asm__ volatile("        testl   %6,%6       ;"
                    "        jz      3f          ;"
@@ -130,75 +130,18 @@ static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count)
 }
 
 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
-#ifdef AMD64
-  (void)memmove(to, from, count * HeapWordSize);
-#else
-  // Same as pd_conjoint_words, except no zero-count check.
-  intx temp;
-  __asm__ volatile("        cmpl    %4,%5         ;"
-                   "        leal    -4(%4,%6,4),%3;"
-                   "        jbe     1f            ;"
-                   "        cmpl    %7,%5         ;"
-                   "        jbe     4f            ;"
-                   "1:      cmpl    $32,%6        ;"
-                   "        ja      3f            ;"
-                   "        subl    %4,%1         ;"
-                   "2:      movl    (%4),%3       ;"
-                   "        movl    %7,(%5,%4,1)  ;"
-                   "        addl    $4,%0         ;"
-                   "        subl    $1,%2          ;"
-                   "        jnz     2b            ;"
-                   "        jmp     7f            ;"
-                   "3:      rep;    smovl         ;"
-                   "        jmp     7f            ;"
-                   "4:      cmpl    $32,%2        ;"
-                   "        movl    %7,%0         ;"
-                   "        leal    -4(%5,%6,4),%1;"
-                   "        ja      6f            ;"
-                   "        subl    %4,%1         ;"
-                   "5:      movl    (%4),%3       ;"
-                   "        movl    %7,(%5,%4,1)  ;"
-                   "        subl    $4,%0         ;"
-                   "        subl    $1,%2          ;"
-                   "        jnz     5b            ;"
-                   "        jmp     7f            ;"
-                   "6:      std                   ;"
-                   "        rep;    smovl         ;"
-                   "        cld                   ;"
-                   "7:      nop                    "
-                   : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
-                   : "0"  (from), "1"  (to), "2"  (count), "3"  (temp)
-                   : "memory", "flags");
-#endif // AMD64
+  pd_conjoint_words(from, to, count);
 }
 
 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
-#ifdef AMD64
   pd_disjoint_words(from, to, count);
-#else
-  // Same as pd_disjoint_words, except no zero-count check.
-  intx temp;
-  __asm__ volatile("        cmpl    $32,%6      ;"
-                   "        ja      2f          ;"
-                   "        subl    %4,%1       ;"
-                   "1:      movl    (%4),%3     ;"
-                   "        movl    %7,(%5,%4,1);"
-                   "        addl    $4,%0       ;"
-                   "        subl    $1,%2        ;"
-                   "        jnz     1b          ;"
-                   "        jmp     3f          ;"
-                   "2:      rep;    smovl       ;"
-                   "3:      nop                  "
-                   : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
-                   : "0"  (from), "1"  (to), "2"  (count), "3"  (temp)
-                   : "memory", "cc");
-#endif // AMD64
 }
 
 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
 #ifdef AMD64
   (void)memmove(to, from, count);
 #else
+  // Includes a zero-count check.
   intx temp;
   __asm__ volatile("        testl   %6,%6          ;"
                    "        jz      13f            ;"

src/os_cpu/linux_x86/vm/linux_x86_32.s

@@ -121,10 +121,10 @@ cb_CopyRight:
         jnz      3b
         addl     %esi,%edi
 4:      movl     %eax,%ecx            # byte count less prefix
-        andl     $3,%ecx              # suffix byte count
+5:      andl     $3,%ecx              # suffix byte count
         jz       7f                   # no suffix
         # copy suffix
-5:      xorl     %eax,%eax
+        xorl     %eax,%eax
 6:      movb     (%esi,%eax,1),%dl
         movb     %dl,(%edi,%eax,1)
         addl     $1,%eax
@@ -159,10 +159,10 @@ cb_CopyLeft:
         # copy dwords, aligned or not
 3:      rep;     smovl
 4:      movl     %eax,%ecx            # byte count
-        andl     $3,%ecx              # suffix byte count
+5:      andl     $3,%ecx              # suffix byte count
         jz       7f                   # no suffix
         # copy suffix
-5:      subl     %esi,%edi
+        subl     %esi,%edi
         addl     $3,%esi
 6:      movb     (%esi),%dl
         movb     %dl,(%edi,%esi,1)
@@ -214,10 +214,10 @@ acb_CopyRight:
         # copy aligned dwords
 3:      rep;     smovl
 4:      movl     %eax,%ecx
-        andl     $3,%ecx
+5:      andl     $3,%ecx
         jz       7f
         # copy suffix
-5:      xorl     %eax,%eax
+        xorl     %eax,%eax
 6:      movb     (%esi,%eax,1),%dl
         movb     %dl,(%edi,%eax,1)
         addl     $1,%eax
@@ -250,9 +250,9 @@ acb_CopyLeft:
         jnz      3b
         addl     %esi,%edi
 4:      movl     %eax,%ecx
-        andl     $3,%ecx
+5:      andl     $3,%ecx
         jz       7f
-5:      subl     %esi,%edi
+        subl     %esi,%edi
         addl     $3,%esi
 6:      movb     (%esi),%dl
         movb     %dl,(%edi,%esi,1)
@@ -287,11 +287,12 @@ cs_CopyRight:
         andl     $3,%eax              # either 0 or 2
         jz       1f                   # no prefix
         # copy prefix
+        subl     $1,%ecx
+        jl       5f                   # zero count
         movw     (%esi),%dx
         movw     %dx,(%edi)
         addl     %eax,%esi            # %eax == 2
         addl     %eax,%edi
-        subl     $1,%ecx
 1:      movl     %ecx,%eax            # word count less prefix
         sarl     %ecx                 # dword count
         jz       4f                   # no dwords to move
@@ -454,12 +455,13 @@ ci_CopyRight:
         ret
         .=.+10
 2:      subl     %esi,%edi
+        jmp      4f
         .p2align 4,,15
 3:      movl     (%esi),%edx
         movl     %edx,(%edi,%esi,1)
         addl     $4,%esi
-        subl     $1,%ecx
-        jnz      3b
+4:      subl     $1,%ecx
+        jge      3b
         popl     %edi
         popl     %esi
         ret
@@ -467,19 +469,20 @@ ci_CopyLeft:
         std
         leal     -4(%edi,%ecx,4),%edi # to + count*4 - 4
         cmpl     $32,%ecx
-        ja       3f                   # > 32 dwords
+        ja       4f                   # > 32 dwords
         subl     %eax,%edi            # eax == from + count*4 - 4
+        jmp      3f
         .p2align 4,,15
 2:      movl     (%eax),%edx
         movl     %edx,(%edi,%eax,1)
         subl     $4,%eax
-        subl     $1,%ecx
-        jnz      2b
+3:      subl     $1,%ecx
+        jge      2b
         cld
         popl     %edi
         popl     %esi
         ret
-3:      movl     %eax,%esi            # from + count*4 - 4
+4:      movl     %eax,%esi            # from + count*4 - 4
         rep;     smovl
         cld
         popl     %edi

src/os_cpu/solaris_x86/vm/solaris_x86_32.s

@@ -154,10 +154,10 @@ cb_CopyRight:
         jnz      3b
         addl     %esi,%edi
 4:      movl     %eax,%ecx            / byte count less prefix
-        andl     $3,%ecx              / suffix byte count
+5:      andl     $3,%ecx              / suffix byte count
         jz       7f                   / no suffix
         / copy suffix
-5:      xorl     %eax,%eax
+        xorl     %eax,%eax
 6:      movb     (%esi,%eax,1),%dl
         movb     %dl,(%edi,%eax,1)
         addl     $1,%eax
@@ -192,10 +192,10 @@ cb_CopyLeft:
         / copy dwords, aligned or not
 3:      rep;     smovl
 4:      movl     %eax,%ecx            / byte count
-        andl     $3,%ecx              / suffix byte count
+5:      andl     $3,%ecx              / suffix byte count
         jz       7f                   / no suffix
         / copy suffix
-5:      subl     %esi,%edi
+        subl     %esi,%edi
         addl     $3,%esi
 6:      movb     (%esi),%dl
         movb     %dl,(%edi,%esi,1)
@@ -246,10 +246,10 @@ acb_CopyRight:
         / copy aligned dwords
 3:      rep;     smovl
 4:      movl     %eax,%ecx
-        andl     $3,%ecx
+5:      andl     $3,%ecx
         jz       7f
         / copy suffix
-5:      xorl     %eax,%eax
+        xorl     %eax,%eax
 6:      movb     (%esi,%eax,1),%dl
         movb     %dl,(%edi,%eax,1)
         addl     $1,%eax
@@ -282,9 +282,9 @@ acb_CopyLeft:
         jnz      3b
         addl     %esi,%edi
 4:      movl     %eax,%ecx
-        andl     $3,%ecx
+5:      andl     $3,%ecx
         jz       7f
-5:      subl     %esi,%edi
+        subl     %esi,%edi
         addl     $3,%esi
 6:      movb     (%esi),%dl
         movb     %dl,(%edi,%esi,1)
@@ -318,11 +318,12 @@ cs_CopyRight:
         andl     $3,%eax              / either 0 or 2
         jz       1f                   / no prefix
         / copy prefix
+        subl     $1,%ecx
+        jl       5f                   / zero count
         movw     (%esi),%dx
         movw     %dx,(%edi)
         addl     %eax,%esi            / %eax == 2
         addl     %eax,%edi
-        subl     $1,%ecx
 1:      movl     %ecx,%eax            / word count less prefix
         sarl     %ecx                 / dword count
         jz       4f                   / no dwords to move
@@ -482,12 +483,13 @@ ci_CopyRight:
         ret
         .=.+10
 2:      subl     %esi,%edi
+        jmp      4f
         .align   16
 3:      movl     (%esi),%edx
         movl     %edx,(%edi,%esi,1)
         addl     $4,%esi
-        subl     $1,%ecx
-        jnz      3b
+4:      subl     $1,%ecx
+        jge      3b
         popl     %edi
         popl     %esi
         ret
@@ -495,19 +497,20 @@ ci_CopyLeft:
         std
         leal     -4(%edi,%ecx,4),%edi / to + count*4 - 4
         cmpl     $32,%ecx
-        ja       3f                   / > 32 dwords
+        ja       4f                   / > 32 dwords
         subl     %eax,%edi            / eax == from + count*4 - 4
+        jmp      3f
         .align   16
 2:      movl     (%eax),%edx
         movl     %edx,(%edi,%eax,1)
         subl     $4,%eax
-        subl     $1,%ecx
-        jnz      2b
+3:      subl     $1,%ecx
+        jge      2b
         cld
         popl     %edi
         popl     %esi
         ret
-3:      movl     %eax,%esi            / from + count*4 - 4
+4:      movl     %eax,%esi            / from + count*4 - 4
         rep;     smovl
         cld
         popl     %edi

src/share/vm/asm/codeBuffer.cpp

@@ -404,7 +404,7 @@ void CodeSection::expand_locs(int new_capacity) {
       locs_start = REALLOC_RESOURCE_ARRAY(relocInfo, _locs_start, old_capacity, new_capacity);
     } else {
       locs_start = NEW_RESOURCE_ARRAY(relocInfo, new_capacity);
-      Copy::conjoint_bytes(_locs_start, locs_start, old_capacity * sizeof(relocInfo));
+      Copy::conjoint_jbytes(_locs_start, locs_start, old_capacity * sizeof(relocInfo));
       _locs_own = true;
     }
     _locs_start    = locs_start;
@@ -581,7 +581,7 @@ csize_t CodeBuffer::copy_relocations_to(CodeBlob* dest) const {
                              (HeapWord*)(buf+buf_offset),
                              (lsize + HeapWordSize-1) / HeapWordSize);
       } else {
-        Copy::conjoint_bytes(lstart, buf+buf_offset, lsize);
+        Copy::conjoint_jbytes(lstart, buf+buf_offset, lsize);
       }
     }
     buf_offset += lsize;

src/share/vm/c1/c1_Runtime1.cpp

@@ -1122,7 +1122,7 @@ JRT_LEAF(void, Runtime1::primitive_arraycopy(HeapWord* src, HeapWord* dst, int l
   if (length == 0) return;
   // Not guaranteed to be word atomic, but that doesn't matter
   // for anything but an oop array, which is covered by oop_arraycopy.
-  Copy::conjoint_bytes(src, dst, length);
+  Copy::conjoint_jbytes(src, dst, length);
 JRT_END
 
 JRT_LEAF(void, Runtime1::oop_arraycopy(HeapWord* src, HeapWord* dst, int num))

src/share/vm/interpreter/interpreterRuntime.cpp

@@ -1268,7 +1268,7 @@ IRT_LEAF(void, InterpreterRuntime::popframe_move_outgoing_args(JavaThread* threa
   Bytecode_invoke* invoke = Bytecode_invoke_at(mh, bci);
   ArgumentSizeComputer asc(invoke->signature());
   int size_of_arguments = (asc.size() + (invoke->has_receiver() ? 1 : 0)); // receiver
-  Copy::conjoint_bytes(src_address, dest_address,
+  Copy::conjoint_jbytes(src_address, dest_address,
                        size_of_arguments * Interpreter::stackElementSize);
 IRT_END
 #endif

src/share/vm/runtime/stubRoutines.cpp

@@ -135,28 +135,32 @@ typedef void (*arraycopy_fn)(address src, address dst, int count);
 static void test_arraycopy_func(address func, int alignment) {
   int v = 0xcc;
   int v2 = 0x11;
-  jlong lbuffer[2];
-  jlong lbuffer2[2];
-  address buffer  = (address) lbuffer;
-  address buffer2 = (address) lbuffer2;
+  jlong lbuffer[8];
+  jlong lbuffer2[8];
+  address fbuffer  = (address) lbuffer;
+  address fbuffer2 = (address) lbuffer2;
   unsigned int i;
   for (i = 0; i < sizeof(lbuffer); i++) {
-    buffer[i] = v; buffer2[i] = v2;
+    fbuffer[i] = v; fbuffer2[i] = v2;
   }
+  // C++ does not guarantee jlong[] array alignment to 8 bytes.
+  // Use middle of array to check that memory before it is not modified.
+  address buffer  = (address) round_to((intptr_t)&lbuffer[4], BytesPerLong);
+  address buffer2 = (address) round_to((intptr_t)&lbuffer2[4], BytesPerLong);
   // do an aligned copy
   ((arraycopy_fn)func)(buffer, buffer2, 0);
   for (i = 0; i < sizeof(lbuffer); i++) {
-    assert(buffer[i] == v && buffer2[i] == v2, "shouldn't have copied anything");
+    assert(fbuffer[i] == v && fbuffer2[i] == v2, "shouldn't have copied anything");
   }
   // adjust destination alignment
   ((arraycopy_fn)func)(buffer, buffer2 + alignment, 0);
   for (i = 0; i < sizeof(lbuffer); i++) {
-    assert(buffer[i] == v && buffer2[i] == v2, "shouldn't have copied anything");
+    assert(fbuffer[i] == v && fbuffer2[i] == v2, "shouldn't have copied anything");
   }
   // adjust source alignment
   ((arraycopy_fn)func)(buffer + alignment, buffer2, 0);
   for (i = 0; i < sizeof(lbuffer); i++) {
-    assert(buffer[i] == v && buffer2[i] == v2, "shouldn't have copied anything");
+    assert(fbuffer[i] == v && fbuffer2[i] == v2, "shouldn't have copied anything");
   }
 }
 #endif
@@ -183,7 +187,7 @@ void StubRoutines::initialize2() {
   test_arraycopy_func(arrayof_##type##_arraycopy(),          sizeof(HeapWord)); \
   test_arraycopy_func(arrayof_##type##_disjoint_arraycopy(), sizeof(HeapWord))
 
-  // Make sure all the arraycopy stubs properly handle zeros
+  // Make sure all the arraycopy stubs properly handle zero count
   TEST_ARRAYCOPY(jbyte);
   TEST_ARRAYCOPY(jshort);
   TEST_ARRAYCOPY(jint);
@@ -191,6 +195,25 @@ void StubRoutines::initialize2() {
 
 #undef TEST_ARRAYCOPY
 
+#define TEST_COPYRTN(type) \
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::conjoint_##type##s_atomic),  sizeof(type)); \
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::arrayof_conjoint_##type##s), (int)MAX2(sizeof(HeapWord), sizeof(type)))
+
+  // Make sure all the copy runtime routines properly handle zero count
+  TEST_COPYRTN(jbyte);
+  TEST_COPYRTN(jshort);
+  TEST_COPYRTN(jint);
+  TEST_COPYRTN(jlong);
+
+#undef TEST_COPYRTN
+
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::conjoint_words), sizeof(HeapWord));
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::disjoint_words), sizeof(HeapWord));
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::disjoint_words_atomic), sizeof(HeapWord));
+  // Aligned to BytesPerLong
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::aligned_conjoint_words), sizeof(jlong));
+  test_arraycopy_func(CAST_FROM_FN_PTR(address, Copy::aligned_disjoint_words), sizeof(jlong));
+
 #endif
 }
 
@@ -221,15 +244,13 @@ JRT_LEAF(void, StubRoutines::jbyte_copy(jbyte* src, jbyte* dest, size_t count))
 #ifndef PRODUCT
   SharedRuntime::_jbyte_array_copy_ctr++;      // Slow-path byte array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
-  Copy::conjoint_bytes_atomic(src, dest, count);
+  Copy::conjoint_jbytes_atomic(src, dest, count);
 JRT_END
 
 JRT_LEAF(void, StubRoutines::jshort_copy(jshort* src, jshort* dest, size_t count))
 #ifndef PRODUCT
   SharedRuntime::_jshort_array_copy_ctr++;     // Slow-path short/char array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
   Copy::conjoint_jshorts_atomic(src, dest, count);
 JRT_END
 
@@ -237,7 +258,6 @@ JRT_LEAF(void, StubRoutines::jint_copy(jint* src, jint* dest, size_t count))
 #ifndef PRODUCT
   SharedRuntime::_jint_array_copy_ctr++;       // Slow-path int/float array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
   Copy::conjoint_jints_atomic(src, dest, count);
 JRT_END
 
@@ -245,7 +265,6 @@ JRT_LEAF(void, StubRoutines::jlong_copy(jlong* src, jlong* dest, size_t count))
 #ifndef PRODUCT
   SharedRuntime::_jlong_array_copy_ctr++;      // Slow-path long/double array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
   Copy::conjoint_jlongs_atomic(src, dest, count);
 JRT_END
 
@@ -263,15 +282,13 @@ JRT_LEAF(void, StubRoutines::arrayof_jbyte_copy(HeapWord* src, HeapWord* dest, s
 #ifndef PRODUCT
   SharedRuntime::_jbyte_array_copy_ctr++;      // Slow-path byte array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
-  Copy::arrayof_conjoint_bytes(src, dest, count);
+  Copy::arrayof_conjoint_jbytes(src, dest, count);
 JRT_END
 
 JRT_LEAF(void, StubRoutines::arrayof_jshort_copy(HeapWord* src, HeapWord* dest, size_t count))
 #ifndef PRODUCT
   SharedRuntime::_jshort_array_copy_ctr++;     // Slow-path short/char array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
   Copy::arrayof_conjoint_jshorts(src, dest, count);
 JRT_END
 
@@ -279,7 +296,6 @@ JRT_LEAF(void, StubRoutines::arrayof_jint_copy(HeapWord* src, HeapWord* dest, si
 #ifndef PRODUCT
   SharedRuntime::_jint_array_copy_ctr++;       // Slow-path int/float array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
   Copy::arrayof_conjoint_jints(src, dest, count);
 JRT_END
 
@@ -287,7 +303,6 @@ JRT_LEAF(void, StubRoutines::arrayof_jlong_copy(HeapWord* src, HeapWord* dest, s
 #ifndef PRODUCT
   SharedRuntime::_jlong_array_copy_ctr++;       // Slow-path int/float array copy
 #endif // !PRODUCT
-  assert(count != 0, "count should be non-zero");
   Copy::arrayof_conjoint_jlongs(src, dest, count);
 JRT_END
 

src/share/vm/runtime/thread.cpp

@@ -2698,7 +2698,7 @@ void JavaThread::popframe_preserve_args(ByteSize size_in_bytes, void* start) {
   if (in_bytes(size_in_bytes) != 0) {
     _popframe_preserved_args = NEW_C_HEAP_ARRAY(char, in_bytes(size_in_bytes));
     _popframe_preserved_args_size = in_bytes(size_in_bytes);
-    Copy::conjoint_bytes(start, _popframe_preserved_args, _popframe_preserved_args_size);
+    Copy::conjoint_jbytes(start, _popframe_preserved_args, _popframe_preserved_args_size);
   }
 }
 

src/share/vm/runtime/vframeArray.cpp

@@ -355,9 +355,9 @@ void vframeArrayElement::unpack_on_stack(int callee_parameters,
       } else {
         base = iframe()->interpreter_frame_expression_stack();
       }
-      Copy::conjoint_bytes(saved_args,
-                           base,
-                           popframe_preserved_args_size_in_bytes);
+      Copy::conjoint_jbytes(saved_args,
+                            base,
+                            popframe_preserved_args_size_in_bytes);
       thread->popframe_free_preserved_args();
     }
   }

src/share/vm/utilities/copy.cpp

@@ -48,7 +48,7 @@ void Copy::conjoint_memory_atomic(void* from, void* to, size_t size) {
     Copy::conjoint_jshorts_atomic((jshort*) src, (jshort*) dst, size / sizeof(jshort));
   } else {
     // Not aligned, so no need to be atomic.
-    Copy::conjoint_bytes((void*) src, (void*) dst, size);
+    Copy::conjoint_jbytes((void*) src, (void*) dst, size);
   }
 }
 

src/share/vm/utilities/copy.hpp

@@ -73,6 +73,9 @@ class Copy : AllStatic {
   // whole alignment units.  E.g., if BytesPerLong is 2x word alignment, an odd
   // count may copy an extra word.  In the arrayof case, we are allowed to copy
   // only the number of copy units specified.
+  //
+  // All callees check count for 0.
+  //
 
   // HeapWords
 
@@ -99,7 +102,6 @@ class Copy : AllStatic {
   // Object-aligned words,  conjoint, not atomic on each word
   static void aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
     assert_params_aligned(from, to);
-    assert_non_zero(count);
     pd_aligned_conjoint_words(from, to, count);
   }
 
@@ -107,49 +109,42 @@ class Copy : AllStatic {
   static void aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
     assert_params_aligned(from, to);
     assert_disjoint(from, to, count);
-    assert_non_zero(count);
     pd_aligned_disjoint_words(from, to, count);
   }
 
   // bytes, jshorts, jints, jlongs, oops
 
   // bytes,                 conjoint, not atomic on each byte (not that it matters)
-  static void conjoint_bytes(void* from, void* to, size_t count) {
-    assert_non_zero(count);
+  static void conjoint_jbytes(void* from, void* to, size_t count) {
     pd_conjoint_bytes(from, to, count);
   }
 
   // bytes,                 conjoint, atomic on each byte (not that it matters)
-  static void conjoint_bytes_atomic(void* from, void* to, size_t count) {
-    assert_non_zero(count);
+  static void conjoint_jbytes_atomic(void* from, void* to, size_t count) {
     pd_conjoint_bytes(from, to, count);
   }
 
   // jshorts,               conjoint, atomic on each jshort
   static void conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerShort);
-    assert_non_zero(count);
     pd_conjoint_jshorts_atomic(from, to, count);
   }
 
   // jints,                 conjoint, atomic on each jint
   static void conjoint_jints_atomic(jint* from, jint* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerInt);
-    assert_non_zero(count);
     pd_conjoint_jints_atomic(from, to, count);
   }
 
   // jlongs,                conjoint, atomic on each jlong
   static void conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerLong);
-    assert_non_zero(count);
     pd_conjoint_jlongs_atomic(from, to, count);
   }
 
   // oops,                  conjoint, atomic on each oop
   static void conjoint_oops_atomic(oop* from, oop* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerHeapOop);
-    assert_non_zero(count);
     pd_conjoint_oops_atomic(from, to, count);
   }
 
@@ -157,7 +152,6 @@ class Copy : AllStatic {
   static void conjoint_oops_atomic(narrowOop* from, narrowOop* to, size_t count) {
     assert(sizeof(narrowOop) == sizeof(jint), "this cast is wrong");
     assert_params_ok(from, to, LogBytesPerInt);
-    assert_non_zero(count);
     pd_conjoint_jints_atomic((jint*)from, (jint*)to, count);
   }
 
@@ -168,36 +162,31 @@ class Copy : AllStatic {
   static void conjoint_memory_atomic(void* from, void* to, size_t size);
 
   // bytes,                 conjoint array, atomic on each byte (not that it matters)
-  static void arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) {
-    assert_non_zero(count);
+  static void arrayof_conjoint_jbytes(HeapWord* from, HeapWord* to, size_t count) {
     pd_arrayof_conjoint_bytes(from, to, count);
   }
 
   // jshorts,               conjoint array, atomic on each jshort
   static void arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerShort);
-    assert_non_zero(count);
     pd_arrayof_conjoint_jshorts(from, to, count);
   }
 
   // jints,                 conjoint array, atomic on each jint
   static void arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerInt);
-    assert_non_zero(count);
     pd_arrayof_conjoint_jints(from, to, count);
   }
 
   // jlongs,                conjoint array, atomic on each jlong
   static void arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerLong);
-    assert_non_zero(count);
     pd_arrayof_conjoint_jlongs(from, to, count);
   }
 
   // oops,                  conjoint array, atomic on each oop
   static void arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) {
     assert_params_ok(from, to, LogBytesPerHeapOop);
-    assert_non_zero(count);
     pd_arrayof_conjoint_oops(from, to, count);
   }
 
@@ -319,14 +308,6 @@ class Copy : AllStatic {
 #endif
   }
 
-  static void assert_non_zero(size_t count) {
-#ifdef ASSERT
-    if (count == 0) {
-      basic_fatal("count must be non-zero");
-    }
-#endif
-  }
-
   static void assert_byte_count_ok(size_t byte_count, size_t unit_size) {
 #ifdef ASSERT
     if ((size_t)round_to(byte_count, unit_size) != byte_count) {