Skip to content

Add heap sized based full gc heuristic #60376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gbaraldi wants to merge 1 commit into
base: master
Choose a base branch
from
+35 −26
Open

Add heap sized based full gc heuristic #60376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion base/timing.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ end
@static if Base.USING_STOCK_GC
# must be kept in sync with `src/gc-stock.h``
const FULL_SWEEP_REASONS = [:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL, :FULL_SWEEP_REASON_FORCED_FULL_SWEEP,
:FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE]
:FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE, :FULL_SWEEP_REASON_LARGE_HEAP_GROWTH]
end

"""
Expand Down
56 changes: 32 additions & 24 deletions src/gc-stock.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ static int mark_reset_age = 0;

static int64_t scanned_bytes; // young bytes scanned while marking
static int64_t perm_scanned_bytes; // old bytes scanned while marking
static int64_t heap_size_after_last_full_gc = 0;
int prev_sweep_full = 1;
int current_sweep_full = 0;
int next_sweep_full = 0;
Expand Down Expand Up @@ -2256,13 +2257,13 @@ JL_DLLEXPORT void jl_gc_mark_queue_objarray(jl_ptls_t ptls, jl_value_t *parent,
// `_new_obj` has its lowest bit tagged if it's in the remset (in which case we shouldn't update page metadata)
FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_new_obj) JL_NOTSAFEPOINT
{
int meta_updated = (uintptr_t)_new_obj & GC_REMSET_PTR_TAG;
int remset_object = (uintptr_t)_new_obj & GC_REMSET_PTR_TAG;
jl_value_t *new_obj = (jl_value_t *)((uintptr_t)_new_obj & ~(uintptr_t)GC_REMSET_PTR_TAG);
mark_obj: {
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
uintptr_t vtag = o->header & ~(uintptr_t)0xf;
uint8_t bits = (gc_old(o->header) && !mark_reset_age) ? GC_OLD_MARKED : GC_MARKED;
int update_meta = __likely(!meta_updated && !gc_verifying);
int update_meta = __likely(!remset_object && !gc_verifying);
int foreign_alloc = 0;
if (update_meta && o->bits.in_image) {
foreign_alloc = 1;
Expand Down Expand Up @@ -2362,7 +2363,7 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_
uintptr_t nptr = (npointers << 2) | 1 | bits;
new_obj = gc_mark_obj8(ptls, obj8_parent, obj8_begin, obj8_end, nptr);
if (new_obj != NULL) {
if (!meta_updated)
if (!remset_object)
goto mark_obj;
else
gc_ptr_queue_push(mq, new_obj);
Expand Down Expand Up @@ -2402,13 +2403,13 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_
}
else if (how == 1) {
if (update_meta || foreign_alloc) {
size_t nb = jl_genericmemory_nbytes(m);
gc_heap_snapshot_record_hidden_edge(new_obj, m->ptr, nb, 0);
if (bits == GC_OLD_MARKED) {
ptls->gc_tls.gc_cache.perm_scanned_bytes += nb;
}
else {
ptls->gc_tls.gc_cache.scanned_bytes += nb;
size_t nb = jl_genericmemory_nbytes(m);
gc_heap_snapshot_record_hidden_edge(new_obj, m->ptr, nb, 0);
if (bits == GC_OLD_MARKED) {
ptls->gc_tls.gc_cache.perm_scanned_bytes += nb;
}
else {
ptls->gc_tls.gc_cache.scanned_bytes += nb;
}
}
}
Expand Down Expand Up @@ -2484,7 +2485,7 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_
assert(obj8_begin < obj8_end);
new_obj = gc_mark_obj8(ptls, obj8_parent, obj8_begin, obj8_end, nptr);
if (new_obj != NULL) {
if (!meta_updated)
if (!remset_object)
goto mark_obj;
else
gc_ptr_queue_push(mq, new_obj);
Expand All @@ -2497,7 +2498,7 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_
assert(obj16_begin < obj16_end);
new_obj = gc_mark_obj16(ptls, obj16_parent, obj16_begin, obj16_end, nptr);
if (new_obj != NULL) {
if (!meta_updated)
if (!remset_object)
goto mark_obj;
else
gc_ptr_queue_push(mq, new_obj);
Expand All @@ -2512,7 +2513,7 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_
assert(obj32_begin < obj32_end);
new_obj = gc_mark_obj32(ptls, obj32_parent, obj32_begin, obj32_end, nptr);
if (new_obj != NULL) {
if (!meta_updated)
if (!remset_object)
goto mark_obj;
else
gc_ptr_queue_push(mq, new_obj);
Expand Down Expand Up @@ -3049,7 +3050,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS
uint64_t gc_start_time = jl_hrtime();
uint64_t mutator_time = gc_end_time == 0 ? old_mut_time : gc_start_time - gc_end_time;
uint64_t before_free_heap_size = jl_atomic_load_relaxed(&gc_heap_stats.heap_size);
int64_t last_perm_scanned_bytes = perm_scanned_bytes;
uint64_t start_mark_time = jl_hrtime();
JL_PROBE_GC_MARK_BEGIN();
{
Expand Down Expand Up @@ -3149,8 +3149,9 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS
gc_stats_all_pool();
gc_stats_big_obj();
gc_num.total_allocd += gc_num.allocd;
if (!prev_sweep_full)
promoted_bytes += perm_scanned_bytes - last_perm_scanned_bytes;
// promoted_bytes are all the new bytes scanned that got promoted to old but that have never seen a full GC as old
promoted_bytes += scanned_bytes;
scanned_bytes = 0;
// 4. next collection decision
int remset_nptr = 0;
int sweep_full = next_sweep_full;
Expand All @@ -3176,13 +3177,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS
recollect = 1;
gc_record_full_sweep_reason(FULL_SWEEP_REASON_FORCED_FULL_SWEEP);
}
if (sweep_full) {
// these are the difference between the number of gc-perm bytes scanned
// on the first collection after sweep_full, and the current scan
perm_scanned_bytes = 0;
promoted_bytes = 0;
}
scanned_bytes = 0;
// 5. start sweeping
uint64_t start_sweep_time = jl_hrtime();
JL_PROBE_GC_SWEEP_BEGIN(sweep_full);
Expand Down Expand Up @@ -3334,8 +3328,18 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS
target_heap = heap_size + jl_options.heap_target_increment;
jl_atomic_store_relaxed(&gc_heap_stats.heap_target, target_heap);
}

if (sweep_full) {
// these are the difference between the number of gc-perm bytes scanned
// on the first collection after sweep_full, and the current scan
perm_scanned_bytes = 0;
promoted_bytes = 0;
heap_size_after_last_full_gc = jl_atomic_load_relaxed(&gc_heap_stats.heap_size);
}
// We want to trigger full GCs either if the heap size has grown a lot since the last full GC.
// For this we use the overallocation function to see what a reasonable rate of growth is,
// or if there is too much memory that has not seen a full GC after being promoted to old.
double old_ratio = (double)promoted_bytes/(double)heap_size;
double last_full_gc_heap_ratio = (double)heap_size/(double)overallocation(heap_size_after_last_full_gc, 0, UINT64_MAX);
if (heap_size > user_max) {
next_sweep_full = 1;
gc_record_full_sweep_reason(FULL_SWEEP_REASON_USER_MAX_EXCEEDED);
Expand All @@ -3344,6 +3348,10 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS
next_sweep_full = 1;
gc_record_full_sweep_reason(FULL_SWEEP_REASON_LARGE_PROMOTION_RATE);
}
else if (last_full_gc_heap_ratio > 1) {
next_sweep_full = 1;
gc_record_full_sweep_reason(FULL_SWEEP_REASON_LARGE_HEAP_GROWTH);
}
else {
next_sweep_full = 0;
}
Expand Down
3 changes: 2 additions & 1 deletion src/gc-stock.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,8 @@ FORCE_INLINE void gc_big_object_link(bigval_t *sentinel_node, bigval_t *node) JL
#define FULL_SWEEP_REASON_FORCED_FULL_SWEEP (1)
#define FULL_SWEEP_REASON_USER_MAX_EXCEEDED (2)
#define FULL_SWEEP_REASON_LARGE_PROMOTION_RATE (3)
#define FULL_SWEEP_NUM_REASONS (4)
#define FULL_SWEEP_REASON_LARGE_HEAP_GROWTH (4)
#define FULL_SWEEP_NUM_REASONS (5)

extern JL_DLLEXPORT uint64_t jl_full_sweep_reasons[FULL_SWEEP_NUM_REASONS];
STATIC_INLINE void gc_record_full_sweep_reason(int reason) JL_NOTSAFEPOINT
Expand Down