julia_locks.h

// This file is a part of Julia. License is MIT: https://julialang.org/license

#ifndef JL_LOCKS_H
#define JL_LOCKS_H

#include "julia_assert.h"

#ifdef __cplusplus
extern "C" {
#endif

// Lock acquire and release primitives

// JL_LOCK and jl_mutex_lock are GC safe points, use uv_mutex_t if that is not desired.
// Always use JL_LOCK unless no one holding the lock can trigger a GC or GC
// safepoint. uv_mutex_t should only be needed for GC internal locks.
// The JL_LOCK* and JL_UNLOCK* macros are no-op for non-threading build
// while the jl_mutex_* functions are always locking and unlocking the locks.

static inline void jl_mutex_wait(jl_mutex_t *lock, int safepoint)
{
    jl_task_t *self = jl_current_task;
    jl_task_t *owner = jl_atomic_load_relaxed(&lock->owner);
    if (owner == self) {
        lock->count++;
        return;
    }
    while (1) {
        if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) {
            lock->count = 1;
            return;
        }
        if (safepoint) {
            jl_gc_safepoint_(self->ptls);
        }
        jl_cpu_pause();
        owner = jl_atomic_load_relaxed(&lock->owner);
    }
}

static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT
{
#ifndef __clang_gcanalyzer__
    // Hide this body from the analyzer, otherwise it complains that we're calling
    // a non-safepoint from this function. The 0 arguments guarantees that we do
    // not reach the safepoint, but the analyzer can't figure that out
    jl_mutex_wait(lock, 0);
#endif
}

static inline void jl_lock_frame_push(jl_mutex_t *lock)
{
    jl_ptls_t ptls = jl_current_task->ptls;
    small_arraylist_t *locks = &ptls->locks;
    uint32_t len = locks->len;
    if (__unlikely(len >= locks->max)) {
        small_arraylist_grow(locks, 1);
    }
    else {
        locks->len = len + 1;
    }
    locks->items[len] = (void*)lock;
}
static inline void jl_lock_frame_pop(void)
{
    jl_ptls_t ptls = jl_current_task->ptls;
    assert(ptls->locks.len > 0);
    ptls->locks.len--;
}

#define JL_SIGATOMIC_BEGIN() do {               \
        jl_current_task->ptls->defer_signal++;  \
        jl_signal_fence();                      \
    } while (0)
#define JL_SIGATOMIC_END() do {                                 \
        jl_signal_fence();                                      \
        if (--jl_current_task->ptls->defer_signal == 0) {       \
            jl_sigint_safepoint(jl_current_task->ptls);         \
        }                                                       \
    } while (0)

static inline void jl_mutex_lock(jl_mutex_t *lock)
{
    JL_SIGATOMIC_BEGIN();
    jl_mutex_wait(lock, 1);
    jl_lock_frame_push(lock);
}

static inline int jl_mutex_trylock_nogc(jl_mutex_t *lock)
{
    jl_task_t *self = jl_current_task;
    jl_task_t *owner = jl_atomic_load_acquire(&lock->owner);
    if (owner == self) {
        lock->count++;
        return 1;
    }
    if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) {
        lock->count = 1;
        return 1;
    }
    return 0;
}

static inline int jl_mutex_trylock(jl_mutex_t *lock)
{
    int got = jl_mutex_trylock_nogc(lock);
    if (got) {
        JL_SIGATOMIC_BEGIN();
        jl_lock_frame_push(lock);
    }
    return got;
}
static inline void jl_mutex_unlock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT
{
#ifndef __clang_gcanalyzer__
    assert(jl_atomic_load_relaxed(&lock->owner) == jl_current_task &&
           "Unlocking a lock in a different thread.");
    if (--lock->count == 0) {
        jl_atomic_store_release(&lock->owner, (jl_task_t*)NULL);
        jl_cpu_wake();
    }
#endif
}

static inline void jl_mutex_unlock(jl_mutex_t *lock)
{
    jl_mutex_unlock_nogc(lock);
    jl_lock_frame_pop();
    JL_SIGATOMIC_END();
    if (jl_atomic_load_relaxed(&jl_gc_have_pending_finalizers)) {
        jl_gc_run_pending_finalizers(jl_current_task); // may GC
    }
}

static inline void jl_mutex_init(jl_mutex_t *lock) JL_NOTSAFEPOINT
{
    jl_atomic_store_relaxed(&lock->owner, (jl_task_t*)NULL);
    lock->count = 0;
}

#define JL_MUTEX_INIT(m) jl_mutex_init(m)
#define JL_LOCK(m) jl_mutex_lock(m)
#define JL_UNLOCK(m) jl_mutex_unlock(m)
#define JL_LOCK_NOGC(m) jl_mutex_lock_nogc(m)
#define JL_UNLOCK_NOGC(m) jl_mutex_unlock_nogc(m)

#ifdef __cplusplus
}
#endif

#endif