Changes to support moving immix

src/builtins.c

@@ -344,6 +344,9 @@ static uintptr_t type_object_id_(jl_value_t *v, jl_varidx_t *env) JL_NOTSAFEPOIN
             i++;
             pe = pe->prev;
         }
+        if(mmtk_object_is_managed_by_mmtk(v)) {
+            mmtk_pin_object(v);
+        }
         return inthash((uintptr_t)v);
     }
     if (tv == jl_uniontype_type) {
@@ -392,6 +395,9 @@ static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOT
         return ~h;
     size_t f, nf = jl_datatype_nfields(dt);
     if (nf == 0 || (!dt->layout->haspadding && dt->layout->npointers == 0)) {
+        if(mmtk_object_is_managed_by_mmtk(v)) {
+            mmtk_pin_object(v);
+        }
         // operate element-wise if there are unused bits inside,
         // otherwise just take the whole data block at once
         // a few select pointers (notably symbol) also have special hash values
@@ -452,8 +458,12 @@ static uintptr_t NOINLINE jl_object_id__cold(jl_datatype_t *dt, jl_value_t *v) J
         jl_module_t *m = (jl_module_t*)v;
         return m->hash;
     }
-    if (dt->name->mutabl)
+    if (dt->name->mutabl) {
+        if(mmtk_object_is_managed_by_mmtk(v)) {
+            mmtk_pin_object(v);
+        }
         return inthash((uintptr_t)v);
+    }
     return immut_id_(dt, v, dt->hash);
 }
 

src/datatype.c

@@ -65,6 +65,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu
     jl_typename_t *tn =
         (jl_typename_t*)jl_gc_alloc(ct->ptls, sizeof(jl_typename_t),
                                     jl_typename_type);
+    mmtk_pin_object(tn);
     tn->name = name;
     tn->module = module;
     tn->wrapper = NULL;
@@ -96,6 +97,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void)
 {
     jl_task_t *ct = jl_current_task;
     jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type);
+    mmtk_pin_object(t);
     jl_set_typetagof(t, jl_datatype_tag, 0);
     t->hash = 0;
     t->hasfreetypevars = 0;

src/gc-stacks.c

@@ -66,7 +66,7 @@ static void *malloc_stack(size_t bufsz) JL_NOTSAFEPOINT
     return stk;
 }
 
-static void free_stack(void *stkbuf, size_t bufsz)
+void free_stack(void *stkbuf, size_t bufsz)
 {
     munmap(stkbuf, bufsz);
     jl_atomic_fetch_add(&num_stack_mappings, -1);
@@ -104,7 +104,7 @@ static unsigned select_pool(size_t nb) JL_NOTSAFEPOINT
 }
 
 
-static void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz)
+void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz)
 {
 #ifdef _COMPILER_ASAN_ENABLED_
     __asan_unpoison_stack_memory((uintptr_t)stkbuf, bufsz);

src/interpreter.c

@@ -762,6 +762,7 @@ jl_value_t *NOINLINE jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t
     unsigned nroots = jl_source_nslots(src) + jl_source_nssavalues(src);
     JL_GC_PUSHFRAME(s, s->locals, nroots);
     jl_array_t *stmts = src->code;
+    JL_GC_PUSH1(&stmts);
     assert(jl_typetagis(stmts, jl_array_any_type));
     s->src = src;
     s->module = m;
@@ -774,6 +775,7 @@ jl_value_t *NOINLINE jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t
     jl_value_t *r = eval_body(stmts, s, 0, 1);
     ct->world_age = last_age;
     JL_GC_POP();
+    JL_GC_POP();
     return r;
 }
 

src/ircode.c

@@ -1164,12 +1164,15 @@ void jl_init_serializer(void)
     assert(LAST_TAG+1+i < 256);
 
     for (i = 2; i < 256; i++) {
-        if (deser_tag[i])
+        if (deser_tag[i]) {
+            PTRHASH_PIN(deser_tag[i])
             ptrhash_put(&ser_tag, deser_tag[i], (void*)i);
+        }
     }
 
     i = 2;
     while (common_symbols[i-2] != NULL) {
+        PTRHASH_PIN(common_symbols[i-2])
         ptrhash_put(&common_symbol_tag, common_symbols[i-2], (void*)i);
         deser_symbols[i] = (jl_value_t*)common_symbols[i-2];
         i += 1;

src/julia.h

@@ -3,6 +3,19 @@
 #ifndef JULIA_H
 #define JULIA_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int mmtk_object_is_managed_by_mmtk(void* addr);
+extern unsigned char mmtk_pin_object(void* obj);
+#define PTRHASH_PIN(key)                        \
+        mmtk_pin_object(key);                   \
+
+#ifdef __cplusplus
+}
+#endif
+
 #if defined(JL_LIBRARY_EXPORTS_INTERNAL) || defined(JL_LIBRARY_EXPORTS_CODEGEN)
 #define JL_LIBRARY_EXPORTS
 #endif

src/julia_internal.h

@@ -534,7 +534,9 @@ JL_DLLEXPORT uintptr_t jl_get_buff_tag(void);
 typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer
 STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz)
 {
-    return jl_gc_alloc(ptls, sz, (void*)jl_buff_tag);
+    jl_gc_tracked_buffer_t *buf = jl_gc_alloc(ptls, sz, (void*)jl_buff_tag);
+    mmtk_pin_object(buf);
+    return buf;
 }
 
 STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT

src/mmtk-gc.c

@@ -431,6 +431,9 @@ jl_value_t *jl_gc_realloc_string(jl_value_t *s, size_t sz)
     size_t len = jl_string_len(s);
     jl_value_t *snew = jl_alloc_string(sz);
     memcpy(jl_string_data(snew), jl_string_data(s), sz <= len ? sz : len);
+    if(mmtk_is_pinned(s)) {
+        mmtk_pin_object(snew);
+    }
     return snew;
 }
 

src/runtime_ccall.cpp

@@ -352,6 +352,8 @@ jl_value_t *jl_get_cfunction_trampoline(
     tramp = trampoline_alloc();
     ((void**)result)[0] = tramp;
     tramp = init_trampoline(tramp, nval);
+    PTRHASH_PIN((void*)fobj)
+    PTRHASH_PIN(result)
     ptrhash_put(cache, (void*)fobj, result);
     uv_mutex_unlock(&trampoline_lock);
     return result;

src/staticdata.c

@@ -656,6 +656,8 @@ static int needs_uniquing(jl_value_t *v) JL_NOTSAFEPOINT
 
 static void record_field_change(jl_value_t **addr, jl_value_t *newval) JL_NOTSAFEPOINT
 {
+    PTRHASH_PIN((void*)addr)
+    PTRHASH_PIN((void*)newval)
     ptrhash_put(&field_replace, (void*)addr, newval);
 }
 
@@ -2138,6 +2140,8 @@ static jl_svec_t *jl_prune_type_cache_hash(jl_svec_t *cache) JL_GC_DISABLED
     assert(serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] == cache);
     cache = cache_rehash_set(cache, l);
     // redirect all references to the old cache to relocate to the new cache object
+    PTRHASH_PIN((void*)cache)
+    PTRHASH_PIN((void*)idx)
     ptrhash_put(&serialization_order, cache, idx);
     serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] = cache;
     return cache;
@@ -2388,6 +2392,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array,
     htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs));
     uintptr_t i;
     for (i = 0; id_to_fptrs[i] != NULL; i++) {
+        PTRHASH_PIN((void*)(uintptr_t)id_to_fptrs[i])
         ptrhash_put(&fptr_to_id, (void*)(uintptr_t)id_to_fptrs[i], (void*)(i + 2));
     }
     htable_new(&serialization_order, 25000);
@@ -2474,6 +2479,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array,
             htable_new(&external_objects, NUM_TAGS);
             for (size_t i = 0; tags[i] != NULL; i++) {
                 jl_value_t *tag = *tags[i];
+                PTRHASH_PIN(tag)
                 ptrhash_put(&external_objects, tag, tag);
             }
             // Queue the worklist itself as the first item we serialize
@@ -3045,6 +3051,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl
                 assert(tag == 0);
                 arraylist_push(&delay_list, pfld);
                 arraylist_push(&delay_list, obj);
+                PTRHASH_PIN(obj)
                 ptrhash_put(&new_dt_objs, (void*)obj, obj); // mark obj as invalid
                 *pfld = (uintptr_t)NULL;
                 continue;
@@ -3078,6 +3085,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl
                     }
                     static_assert(offsetof(jl_datatype_t, name) == 0, "");
                     newdt->name = dt->name;
+                    PTRHASH_PIN(newdt)
+                    PTRHASH_PIN(dt)
                     ptrhash_put(&new_dt_objs, (void*)newdt, dt);
                 }
                 else {

src/staticdata_utils.c

@@ -272,6 +272,7 @@ static void jl_collect_new_roots(jl_array_t *roots, jl_array_t *new_specializati
         assert(jl_is_code_instance(ci));
         jl_method_t *m = ci->def->def.method;
         assert(jl_is_method(m));
+        PTRHASH_PIN(m)
         ptrhash_put(&mset, (void*)m, (void*)m);
     }
     int nwithkey;
@@ -434,6 +435,7 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra
         for (size_t i = 0; i < jl_array_len(external_cis); i++) {
             jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(external_cis, i);
             jl_method_instance_t *mi = ci->def;
+            PTRHASH_PIN(mi)
             ptrhash_put(&external_mis, (void*)mi, (void*)mi);
         }
     }
@@ -469,6 +471,8 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra
     for (size_t i = 0; i < l / 2; i++) {
         jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, i * 2);
         void *target = (void*)((char*)HT_NOTFOUND + i + 1);
+        PTRHASH_PIN(caller)
+        PTRHASH_PIN(target)
         ptrhash_put(&edges_ids, (void*)caller, target);
     }
     // process target list to turn it into a memoized validity table
@@ -545,6 +549,8 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra
                 jl_array_ptr_1d_push(ext_targets, callee);
                 jl_array_ptr_1d_push(ext_targets, matches);
                 target = (void*)((char*)HT_NOTFOUND + jl_array_len(ext_targets) / 3);
+                PTRHASH_PIN(callee)
+                PTRHASH_PIN(target)
                 ptrhash_put(&edges_map2, (void*)callee, target);
             }
             idxs[++nt] = (char*)target - (char*)HT_NOTFOUND - 1;
@@ -1090,6 +1096,8 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a
         jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(ci_list, i);
         assert(ci->min_world == minworld);
         if (ci->max_world == 1) { // sentinel value: has edges to external callables
+            PTRHASH_PIN((void*)ci->def)
+            PTRHASH_PIN((void*)ci)
             ptrhash_put(&visited, (void*)ci->def, (void*)ci);
         }
         else {
@@ -1155,6 +1163,7 @@ static void classify_callers(htable_t *callers_with_edges, jl_array_t *edges)
     size_t l = edges ? jl_array_len(edges) / 2 : 0;
     for (size_t i = 0; i < l; i++) {
         jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i);
+        PTRHASH_PIN((void*)caller)
         ptrhash_put(callers_with_edges, (void*)caller, (void*)caller);
     }
 }

src/task.c

@@ -1207,6 +1207,7 @@ CFI_NORETURN
     jl_task_t *ct = jl_get_current_task();
 #else
     jl_task_t *ct = jl_current_task;
+    JL_GC_PUSH1(&ct);
 #endif
     jl_ptls_t ptls = ct->ptls;
     jl_value_t *res;
@@ -1247,6 +1248,7 @@ skip_pop_exception:;
     ct->result = res;
     jl_gc_wb(ct, ct->result);
     jl_finish_task(ct);
+    JL_GC_POP();
     jl_gc_debug_critical_error();
     abort();
 }

src/threading.c

@@ -694,6 +694,12 @@ void jl_init_threading(void)
     }
     int16_t ngcthreads = jl_n_markthreads + jl_n_sweepthreads;
 
+#ifdef MMTK_GC
+    // MMTk gets the number of GC threads from jl_options.ngcthreads, and spawn its GC threads.
+    // So we just set ngcthreads to 0 here to avoid spawning any GC threads in Julia.
+    ngcthreads = 0;
+#endif
+
 #ifdef MMTK_GC
     // MMTk gets the number of GC threads from jl_options.ngcthreads, and spawn its GC threads.
     // So we just set ngcthreads to 0 here to avoid spawning any GC threads in Julia.

src/toplevel.c

@@ -140,6 +140,8 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
     jl_value_t *form = (jl_value_t*)newm;
     JL_GC_PUSH1(&form);
     JL_LOCK(&jl_modules_mutex);
+    PTRHASH_PIN(newm)
+    PTRHASH_PIN((void*)((uintptr_t)HT_NOTFOUND + 1))
     ptrhash_put(&jl_current_modules, (void*)newm, (void*)((uintptr_t)HT_NOTFOUND + 1));
     JL_UNLOCK(&jl_modules_mutex);