From c906b754aaf2289c6c39d28a56b9fa4729ee2318 Mon Sep 17 00:00:00 2001 From: Jonathan Chambers Date: Sat, 11 May 2013 16:30:26 -0400 Subject: [PATCH] Initial work for concurrent Boehm with memory copy. --- libgc/allchblk.c | 13 ++++++- libgc/alloc.c | 32 +++++++++++++++- libgc/gcj_mlc.c | 6 +-- libgc/headers.c | 1 + libgc/include/private/gc_pmark.h | 8 +++- libgc/include/private/gc_priv.h | 37 ++++++++++++++++-- libgc/include/private/gcconfig.h | 3 ++ libgc/malloc.c | 18 ++++----- libgc/mark.c | 64 ++++++++++++++++++++++++++++++++ libgc/mark_rts.c | 12 ++++++ libgc/reclaim.c | 4 +- libgc/typd_mlc.c | 2 +- libgc/win32_threads.c | 64 ++++++++++++++++++++++++++++++++ 13 files changed, 243 insertions(+), 21 deletions(-) diff --git a/libgc/allchblk.c b/libgc/allchblk.c index 8f9b8957bd2..4ff98d0633e 100644 --- a/libgc/allchblk.c +++ b/libgc/allchblk.c @@ -581,9 +581,13 @@ int n; register hdr * thishdr; /* Header corr. to hbp */ size_t size_needed; /* number of bytes in requested objects */ size_t size_avail; /* bytes available in this block */ - size_needed = HBLKSIZE * OBJ_SZ_TO_BLOCKS(sz); +#ifdef DOPPELGANGER + if(kind != PTRFREE) + size_needed *= 2; +#endif + /* search for a big enough block in free list */ hbp = GC_hblkfreelist[n]; for(; 0 != hbp; hbp = hhdr -> hb_next) { @@ -764,8 +768,9 @@ int n; } GC_large_free_bytes -= size_needed; - + GC_ASSERT(IS_MAPPED(hhdr)); + return( hbp ); } @@ -791,6 +796,10 @@ signed_word size; size = hhdr->hb_sz; size = HBLKSIZE * OBJ_SZ_TO_BLOCKS(size); GC_remove_counts(hbp, (word)size); +#ifdef DOPPELGANGER + if(hhdr->hb_obj_kind != PTRFREE) + size *= 2; +#endif hhdr->hb_sz = size; # ifdef USE_MUNMAP hhdr -> hb_last_reclaimed = GC_gc_no; diff --git a/libgc/alloc.c b/libgc/alloc.c index db05da27676..0f63d9e99ab 100644 --- a/libgc/alloc.c +++ b/libgc/alloc.c @@ -391,14 +391,24 @@ static int set_realtime (int nanoseconds) GC_bool GC_try_to_collect_inner(stop_func) GC_stop_func stop_func; { + char buf[1024]; + CLOCK_TYPE time1; + CLOCK_TYPE time2; + CLOCK_TYPE time3; + CLOCK_TYPE time4; # ifdef CONDPRINT CLOCK_TYPE start_time, current_time; # endif if (GC_dont_gc) return FALSE; - + GET_TIME(time1); if (GC_notify_event) GC_notify_event (GC_EVENT_START); +#ifdef DOPPELGANGER_CONCURRENT + GC_complete_last(); +#endif + + GET_TIME(time2); if (GC_incremental && GC_collection_in_progress()) { # ifdef CONDPRINT if (GC_print_stats) { @@ -439,6 +449,7 @@ GC_stop_func stop_func; } GC_invalidate_mark_state(); /* Flush mark stack. */ GC_clear_marks(); + GET_TIME(time3); # ifdef SAVE_CALL_CHAIN_IN_GC GC_save_callers(GC_last_stack); # endif @@ -454,7 +465,9 @@ GC_stop_func stop_func; /* finish incrementally. */ return(FALSE); } +#ifndef DOPPELGANGER_CONCURRENT GC_finish_collection(); +#endif # if defined(CONDPRINT) if (GC_print_stats) { GET_TIME(current_time); @@ -464,6 +477,12 @@ GC_stop_func stop_func; # endif if (GC_notify_event) GC_notify_event (GC_EVENT_END); + GET_TIME(time4); + sprintf(buf, "Collection complete %lu us clear %lu us finish %lu us\n", + US_TIME_DIFF(time2,time1), + US_TIME_DIFF(time3,time2), + US_TIME_DIFF(time4,time3)); + OutputDebugStringA(buf); return(TRUE); } @@ -539,6 +558,10 @@ int GC_collect_a_little GC_PROTO(()) return(result); } +#ifdef DOPPELGANGER_CONCURRENT +extern int GC_mark_state; +#endif + /* * Assumes lock is held, signals are disabled. * We stop the world. @@ -605,6 +628,13 @@ GC_stop_func stop_func; return(FALSE); } if (GC_mark_some((ptr_t)(&dummy))) break; +#ifdef DOPPELGANGER_CONCURRENT + if (GC_mark_state == 3) + { + GC_doppelgang_mark(); + break; + } +#endif } GC_gc_no++; diff --git a/libgc/gcj_mlc.c b/libgc/gcj_mlc.c index 8b1da826c7a..77f415cc209 100644 --- a/libgc/gcj_mlc.c +++ b/libgc/gcj_mlc.c @@ -180,7 +180,7 @@ DCL_LOCK_STATE; *(void **)op = ptr_to_struct_containing_descr; UNLOCK(); } - return((GC_PTR) op); + return(GC_premark((GC_PTR) op)); } /* Similar to GC_gcj_malloc, but add debug info. This is allocated */ @@ -239,7 +239,7 @@ DCL_LOCK_STATE; } *(void **)op = ptr_to_struct_containing_descr; UNLOCK(); - return((GC_PTR) op); + return(GC_premark((GC_PTR) op)); } /* And a debugging version of the above: */ @@ -311,7 +311,7 @@ DCL_LOCK_STATE; } UNLOCK(); } - return((GC_PTR) op); + return(GC_premark((GC_PTR) op)); } #else diff --git a/libgc/headers.c b/libgc/headers.c index b7be1d84930..eb7e969449f 100644 --- a/libgc/headers.c +++ b/libgc/headers.c @@ -212,6 +212,7 @@ register struct hblk * h; # ifdef USE_MUNMAP result -> hb_last_reclaimed = GC_gc_no; # endif + result -> hb_obj_kind = PTRFREE; return(result); } diff --git a/libgc/include/private/gc_pmark.h b/libgc/include/private/gc_pmark.h index 51981914dc5..d5164a2e7b9 100644 --- a/libgc/include/private/gc_pmark.h +++ b/libgc/include/private/gc_pmark.h @@ -150,6 +150,12 @@ mse * GC_signal_mark_stack_overflow GC_PROTO((mse *msp)); # define ADD_TO_COMPOSITE(sz) # endif +#ifdef DOPPELGANGER +word * GC_doppelganger_for(word *obj, hdr * hhdr); +#else +#define GC_doppelganger_for(obj,hhdr) (obj) +#endif + /* Push the object obj with corresponding heap block header hhdr onto */ /* the mark stack. */ # define PUSH_OBJ(obj, hhdr, mark_stack_top, mark_stack_limit) \ @@ -164,7 +170,7 @@ mse * GC_signal_mark_stack_overflow GC_PROTO((mse *msp)); if (mark_stack_top >= mark_stack_limit) { \ mark_stack_top = GC_signal_mark_stack_overflow(mark_stack_top); \ } \ - mark_stack_top -> mse_start = (obj); \ + mark_stack_top -> mse_start = GC_doppelganger_for(obj, hhdr); \ mark_stack_top -> mse_descr = _descr; \ } \ } diff --git a/libgc/include/private/gc_priv.h b/libgc/include/private/gc_priv.h index ebba43f24d6..6641aaba992 100644 --- a/libgc/include/private/gc_priv.h +++ b/libgc/include/private/gc_priv.h @@ -309,9 +309,30 @@ long GC_time_diff_ms(uint64_t a, uint64_t b); # if defined(MSWIN32) || defined(MSWINCE) # include # include -# define CLOCK_TYPE DWORD -# define GET_TIME(x) x = GetTickCount() -# define MS_TIME_DIFF(a,b) ((long)((a)-(b))) +# define CLOCK_TYPE LONGLONG +# define GET_TIME(x) QueryPerformanceCounter((LARGE_INTEGER*)&x) +static long time_diff_ms(LONGLONG a, LONGLONG b) +{ + static LARGE_INTEGER frequency = {0}; + static double freq; + if (!frequency.QuadPart) { + QueryPerformanceFrequency (&frequency); + freq = frequency.QuadPart / 1000.0; + } + return (long)((((LARGE_INTEGER*)&a)->QuadPart - ((LARGE_INTEGER*)&b)->QuadPart) / freq); +} +static long time_diff_us(LONGLONG a, LONGLONG b) +{ + static LARGE_INTEGER frequency = {0}; + static double freq; + if (!frequency.QuadPart) { + QueryPerformanceFrequency (&frequency); + freq = frequency.QuadPart / 1000000.0; + } + return (long)((((LARGE_INTEGER*)&a)->QuadPart - ((LARGE_INTEGER*)&b)->QuadPart) / freq); +} +# define MS_TIME_DIFF(a,b) time_diff_ms(a,b) +# define US_TIME_DIFF(a,b) time_diff_us(a,b) # else /* !MSWIN32, !MSWINCE, !BSD_TIME */ # include # if !defined(__STDC__) && defined(SPARC) && defined(SUNOS4) @@ -1299,6 +1320,10 @@ struct hblk * GC_prev_block GC_PROTO((struct hblk * h)); /* use. */ void GC_mark_init GC_PROTO((void)); void GC_clear_marks GC_PROTO((void)); /* Clear mark bits for all heap objects. */ +#ifdef DOPPELGANGER_CONCURRENT +void GC_doppelgang_mark GC_PROTO((void)); +void GC_doppelganger_clear_roots GC_PROTO((void)); +#endif void GC_invalidate_mark_state GC_PROTO((void)); /* Tell the marker that marked */ /* objects may point to unmarked */ @@ -1811,6 +1836,12 @@ void GC_dirty_init GC_PROTO((void)); GC_API GC_bool GC_is_marked GC_PROTO((ptr_t p)); void GC_clear_mark_bit GC_PROTO((ptr_t p)); void GC_set_mark_bit GC_PROTO((ptr_t p)); + +#ifdef DOPPELGANGER_CONCURRENT +GC_PTR GC_premark(GC_PTR val); +#else +#define GC_premark(val) (val) +#endif /* Stubborn objects: */ void GC_read_changed GC_PROTO((void)); /* Analogous to GC_read_dirty */ diff --git a/libgc/include/private/gcconfig.h b/libgc/include/private/gcconfig.h index eb8fbb053c6..06a2094f57c 100644 --- a/libgc/include/private/gcconfig.h +++ b/libgc/include/private/gcconfig.h @@ -32,6 +32,9 @@ typedef struct GC_undefined_struct * ptr_t; # endif +#define DOPPELGANGER 1 +#define DOPPELGANGER_CONCURRENT 1 + /* Machine dependent parameters. Some tuning parameters can be found */ /* near the top of gc_private.h. */ diff --git a/libgc/malloc.c b/libgc/malloc.c index 1430f8c9476..374bb845daa 100644 --- a/libgc/malloc.c +++ b/libgc/malloc.c @@ -91,7 +91,7 @@ unsigned flags; /* Clear the whole block, in case of GC_realloc call. */ BZERO(result, n_blocks * HBLKSIZE); } - return result; + return GC_premark(result); } /* allocate lb bytes for an object of kind k. */ @@ -156,7 +156,7 @@ register ptr_t *opp; GC_words_allocd += lw; out: - return op; + return GC_premark(op); } /* Allocate a composite object of size n bytes. The caller guarantees */ @@ -226,7 +226,7 @@ register int k; if (0 == result) { return((*GC_oom_fn)(lb)); } else { - return(result); + return(GC_premark(result)); } } @@ -259,15 +259,15 @@ DCL_LOCK_STATE; FASTLOCK(); if( EXPECT(!FASTLOCK_SUCCEEDED() || (op = *opp) == 0, 0) ) { FASTUNLOCK(); - return(GENERAL_MALLOC((word)lb, PTRFREE)); + return(GC_premark(GENERAL_MALLOC((word)lb, PTRFREE))); } /* See above comment on signals. */ *opp = obj_link(op); GC_words_allocd += lw; FASTUNLOCK(); - return((GC_PTR) op); + return(GC_premark((GC_PTR) op)); } else { - return(GENERAL_MALLOC((word)lb, PTRFREE)); + return(GC_premark(GENERAL_MALLOC((word)lb, PTRFREE))); } } @@ -294,7 +294,7 @@ DCL_LOCK_STATE; FASTLOCK(); if( EXPECT(!FASTLOCK_SUCCEEDED() || (op = *opp) == 0, 0) ) { FASTUNLOCK(); - return(GENERAL_MALLOC((word)lb, NORMAL)); + return(GC_premark(GENERAL_MALLOC((word)lb, NORMAL))); } /* See above comment on signals. */ GC_ASSERT(0 == obj_link(op) @@ -306,9 +306,9 @@ DCL_LOCK_STATE; obj_link(op) = 0; GC_words_allocd += lw; FASTUNLOCK(); - return((GC_PTR) op); + return(GC_premark((GC_PTR) op)); } else { - return(GENERAL_MALLOC((word)lb, NORMAL)); + return(GC_premark(GENERAL_MALLOC((word)lb, NORMAL))); } } diff --git a/libgc/mark.c b/libgc/mark.c index a19d29e0d65..086831168e1 100644 --- a/libgc/mark.c +++ b/libgc/mark.c @@ -170,6 +170,13 @@ register hdr * hhdr; # endif { register hdr * hhdr = HDR(h); + +#ifdef DOPPELGANGER + if(hhdr->hb_obj_kind != PTRFREE) { + size_t size_needed = HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); + memcpy((((char*)h)+size_needed), h, size_needed); + } +#endif if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) return; /* Mark bit for these is cleared only once the object is */ @@ -209,6 +216,13 @@ ptr_t p; return(mark_bit_from_hdr(hhdr, word_no)); } +#ifdef DOPPELGANGER_CONCURRENT +GC_PTR GC_premark(GC_PTR val) +{ + GC_set_mark_bit(val); + return val; +} +#endif /* * Clear mark bits in all allocated heap blocks. This invalidates @@ -595,6 +609,19 @@ mse * msp; return(msp - GC_MARK_STACK_DISCARDS); } +#ifdef DOPPELGANGER +word * GC_doppelganger_for(word *obj, hdr * hhdr) +{ + word * my_current = obj; + if (hhdr->hb_obj_kind != PTRFREE) { + size_t offset = OBJ_SZ_TO_BLOCKS(hhdr->hb_sz) * HBLKSIZE; + my_current = (word*)(((ptr_t)obj) + offset); + } + + return my_current; +} +#endif + /* * Mark objects pointed to by the regions described by * mark stack entries between GC_mark_stack and GC_mark_stack_top, @@ -669,6 +696,7 @@ mse * mark_stack_limit; # endif /* PARALLEL_MARK */ mark_stack_top -> mse_start = limit = current_p + SPLIT_RANGE_WORDS-1; + // TODO, do we need doppelganger lookup here? mark_stack_top -> mse_descr = descr - WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1); /* Make sure that pointers overlapping the two ranges are */ @@ -1184,6 +1212,28 @@ void GC_mark_init() alloc_mark_stack(INITIAL_MARK_STACK_SIZE); } +#ifdef DOPPELGANGER +static HANDLE doppelganger_handle = 0; +static char* doppelganger_buffer = 0; +static int doppelganger_buffer_length = 1024*1024*10; +static int doppelganger_buffer_offset = 0; +static int doppelganger_roots_index = 0; +static char* doppelganger_roots[1024]; + +void GC_doppelganger_clear_roots() +{ + int i; + for (i = 0; i < doppelganger_roots_index; i++) + { + //HeapFree(doppelganger_handle, 0, doppelganger_roots[i]); + //free(doppelganger_roots[i]); + doppelganger_roots[i] = NULL; + } + doppelganger_roots_index = 0; + doppelganger_buffer_offset = 0; +} +#endif + /* * Push all locations between b and t onto the mark stack. * b is the first location to be checked. t is one past the last @@ -1209,8 +1259,22 @@ ptr_t top; length += GC_DS_TAGS; length &= ~GC_DS_TAGS; # endif +#ifdef DOPPELGANGER + if (!doppelganger_handle) { + doppelganger_handle = HeapCreate(0, GC_page_size*10, 0); + doppelganger_buffer = HeapAlloc(doppelganger_handle, 0, doppelganger_buffer_length); + } + doppelganger_roots[doppelganger_roots_index] = doppelganger_buffer + doppelganger_buffer_offset; + memcpy(doppelganger_roots[doppelganger_roots_index], bottom, length); + GC_mark_stack_top -> mse_start = (word *)doppelganger_roots[doppelganger_roots_index]; + GC_mark_stack_top -> mse_descr = length; + doppelganger_roots_index++; + doppelganger_buffer_offset += length; + GC_ASSERT(doppelganger_buffer_offset < doppelganger_buffer_length); +#else GC_mark_stack_top -> mse_start = (word *)bottom; GC_mark_stack_top -> mse_descr = length; +#endif } /* diff --git a/libgc/mark_rts.c b/libgc/mark_rts.c index d763c3dfea6..bd957ba2547 100644 --- a/libgc/mark_rts.c +++ b/libgc/mark_rts.c @@ -576,6 +576,14 @@ ptr_t cold_gc_frame; { int i; int kind; + CLOCK_TYPE time1; + CLOCK_TYPE time2; + char buf[1024]; + GET_TIME(time1); + +#ifdef DOPPELGANGER + GC_doppelganger_clear_roots(); +#endif /* * Next push static data. This must happen early on, since it's @@ -650,5 +658,9 @@ ptr_t cold_gc_frame; /* Note that without interior pointer recognition lots */ /* of stuff may have been pushed already, and this */ /* should be careful about mark stack overflows. */ + GET_TIME(time2); + + //sprintf(buf, "push_roots %lu us\n", US_TIME_DIFF(time2,time1)); + //OutputDebugStringA(buf); } diff --git a/libgc/reclaim.c b/libgc/reclaim.c index 864c0cad8af..5208298eed1 100644 --- a/libgc/reclaim.c +++ b/libgc/reclaim.c @@ -967,7 +967,9 @@ int report_if_found; /* Abort if a GC_reclaimable object is found */ /* Go through all heap blocks (in hblklist) and reclaim unmarked objects */ /* or enqueue the block for later processing. */ GC_apply_to_all_blocks(GC_reclaim_block, (word)report_if_found); - +#ifdef DOPPELGANGER_CONCURRENT +#define EAGER_SWEEP 1 +#endif # ifdef EAGER_SWEEP /* This is a very stupid thing to do. We make it possible anyway, */ /* so that you can convince yourself that it really is very stupid. */ diff --git a/libgc/typd_mlc.c b/libgc/typd_mlc.c index 373257cd260..f93eb2edf96 100644 --- a/libgc/typd_mlc.c +++ b/libgc/typd_mlc.c @@ -668,7 +668,7 @@ DCL_LOCK_STATE; } if (op != NULL) ((word *)op)[lw - 1] = d; - return((GC_PTR) op); + return(GC_premark((GC_PTR) op)); } #if defined(__STDC__) || defined(__cplusplus) diff --git a/libgc/win32_threads.c b/libgc/win32_threads.c index 7837cd0abd7..42ae8f2882f 100644 --- a/libgc/win32_threads.c +++ b/libgc/win32_threads.c @@ -36,6 +36,9 @@ typedef LONG * IE_t; GC_bool GC_thr_initialized = FALSE; DWORD GC_main_thread = 0; +#ifdef DOPPELGANGER_CONCURRENT +DWORD GC_background_thread = 0; +#endif extern GC_bool GC_use_dll_main; @@ -495,6 +498,11 @@ void GC_push_all_stacks() GC_push_all_stack(stack_min, thread->stack_base); } } +#ifdef DOPPELGANGER_CONCURRENT + else if (thread -> id == thread_id && thread -> id == GC_background_thread) { + found_me = TRUE; + } +#endif } if (!found_me) ABORT("Collecting from unknown thread."); } @@ -664,8 +672,57 @@ DWORD WINAPI main_thread_start(LPVOID arg) # else /* !MSWINCE */ +#ifdef DOPPELGANGER_CONCURRENT +HANDLE start_mark_event; +HANDLE start_clean_event; +HANDLE finish_mark_event; + +DWORD WINAPI BackgroundThread(LPVOID arg) +{ + GC_thread me; + me = GC_new_thread(); + GC_background_thread = GetCurrentThreadId(); + me->should_scan = FALSE; + while(TRUE) + { + WaitForSingleObject(start_mark_event, INFINITE); + while(TRUE) { + int dummy = 0; + if (GC_mark_some((ptr_t)(&dummy))) break; + } + //if (!GC_stopped_mark(GC_never_stop_func)) { + // abort(); + //} + WaitForSingleObject(start_clean_event, INFINITE); + GC_finish_collection(); + SetEvent(finish_mark_event); + + } +} + +void GC_complete_last (void) +{ + static int intialized = 0; + if (intialized) { + SetEvent(start_clean_event); + WaitForSingleObject(finish_mark_event, INFINITE); + } + intialized = 1; +} + +void GC_doppelgang_mark (void) +{ + //WaitForSingleObject(finish_mark_event, INFINITE); + SetEvent(start_mark_event); +} + +#endif + /* Called by GC_init() - we hold the allocation lock. */ void GC_thr_init() { +#ifdef DOPPELGANGER_CONCURRENT + DWORD thread_id = 0; +#endif GC_thread me; if (GC_thr_initialized) return; GC_main_thread = GetCurrentThreadId(); @@ -674,6 +731,13 @@ void GC_thr_init() { /* Add the initial thread, so we can stop it. */ me = GC_new_thread(); me->should_scan = TRUE; + +#ifdef DOPPELGANGER_CONCURRENT + start_mark_event = CreateEvent(NULL, FALSE, FALSE, NULL); + start_clean_event = CreateEvent(NULL, FALSE, FALSE, NULL); + finish_mark_event = CreateEvent(NULL, FALSE, FALSE, NULL); + CreateThread(NULL, 0, &BackgroundThread, NULL, 0, &thread_id); +#endif } #ifdef CYGWIN32 -- GitLab