Skip to content

Commit

Permalink
vulkan: switch to VK_KHR_synchronization2
Browse files Browse the repository at this point in the history 
Largely boilerplate changes, but more inherently future-compatible.
  • Loading branch information
@haasn
haasn committed Apr 21, 2023
1 parent 44fdc77 commit e5fb4e6
Show file tree
Hide file tree
Showing 12 changed files with 206 additions and 198 deletions.
2 changes: 1 addition & 1 deletion src/tests/vulkan.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ static void vulkan_interop_tests(pl_vulkan pl_vk,
struct vk_cmd *cmd = vk_cmd_begin(vk, vk->pool_graphics, NULL);
REQUIRE(cmd);
struct pl_sync_vk *sync_vk = PL_PRIV(sync);
vk_cmd_sig(cmd, (pl_vulkan_sem){ sync_vk->signal });
vk_cmd_sig(cmd, VK_PIPELINE_STAGE_2_NONE, (pl_vulkan_sem){ sync_vk->signal });
REQUIRE(vk_cmd_submit(vk, &cmd));

// Do something with the image again to "import" it
Expand Down
77 changes: 37 additions & 40 deletions src/vulkan/command.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,7 @@ static void vk_cmd_reset(struct vk_ctx *vk, struct vk_cmd *cmd)

cmd->callbacks.num = 0;
cmd->deps.num = 0;
cmd->depstages.num = 0;
cmd->depvalues.num = 0;
cmd->sigs.num = 0;
cmd->sigvalues.num = 0;
}

static void vk_cmd_destroy(struct vk_ctx *vk, struct vk_cmd *cmd)
Expand Down Expand Up @@ -130,33 +127,40 @@ void vk_cmd_callback(struct vk_cmd *cmd, vk_cb callback,
});
}

void vk_cmd_dep(struct vk_cmd *cmd, VkPipelineStageFlags stage, pl_vulkan_sem dep)
void vk_cmd_dep(struct vk_cmd *cmd, VkPipelineStageFlags2 stage, pl_vulkan_sem dep)
{
assert(cmd->deps.num == cmd->depstages.num);
assert(cmd->deps.num == cmd->depvalues.num);
PL_ARRAY_APPEND(cmd, cmd->deps, dep.sem);
PL_ARRAY_APPEND(cmd, cmd->depvalues, dep.value);
PL_ARRAY_APPEND(cmd, cmd->depstages, stage);
PL_ARRAY_APPEND(cmd, cmd->deps, (VkSemaphoreSubmitInfo) {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO,
.semaphore = dep.sem,
.value = dep.value,
.stageMask = stage,
});
}

void vk_cmd_sig(struct vk_cmd *cmd, pl_vulkan_sem sig)
void vk_cmd_sig(struct vk_cmd *cmd, VkPipelineStageFlags2 stage, pl_vulkan_sem sig)
{
VkSemaphoreSubmitInfo sinfo = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO,
.semaphore = sig.sem,
.value = sig.value,
.stageMask = stage,
};

// Try updating existing semaphore signal operations in-place
for (int i = 0; i < cmd->sigs.num; i++) {
if (cmd->sigs.elem[i] == sig.sem) {
cmd->sigvalues.elem[i] = PL_MAX(cmd->sigvalues.elem[i], sig.value);
if (cmd->sigs.elem[i].semaphore == sig.sem) {
pl_assert(sig.value > cmd->sigs.elem[i].value);
cmd->sigs.elem[i] = sinfo;
return;
}
}

assert(cmd->sigs.num == cmd->sigvalues.num);
PL_ARRAY_APPEND(cmd, cmd->sigs, sig.sem);
PL_ARRAY_APPEND(cmd, cmd->sigvalues, sig.value);
PL_ARRAY_APPEND(cmd, cmd->sigs, sinfo);
}

struct vk_sync_scope vk_sem_barrier(struct vk_ctx *vk, struct vk_cmd *cmd,
struct vk_sem *sem, VkPipelineStageFlags stage,
VkAccessFlags access, bool is_trans)
struct vk_sem *sem, VkPipelineStageFlags2 stage,
VkAccessFlags2 access, bool is_trans)
{
bool is_write = (access & vk_access_write) || is_trans;

Expand Down Expand Up @@ -299,7 +303,7 @@ struct vk_cmd *vk_cmd_begin(struct vk_ctx *vk, struct vk_cmdpool *pool,
PL_VK_NAME(SEMAPHORE, cmd->sync.sem, debug_tag);

cmd->sync.value++;
vk_cmd_sig(cmd, cmd->sync);
vk_cmd_sig(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, cmd->sync);
return cmd;

error:
Expand All @@ -320,45 +324,38 @@ bool vk_cmd_submit(struct vk_ctx *vk, struct vk_cmd **pcmd)

VK(vk->EndCommandBuffer(cmd->buf));

VkTimelineSemaphoreSubmitInfo tinfo = {
.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO,
.waitSemaphoreValueCount = cmd->depvalues.num,
.pWaitSemaphoreValues = cmd->depvalues.elem,
.signalSemaphoreValueCount = cmd->sigvalues.num,
.pSignalSemaphoreValues = cmd->sigvalues.elem,
};

VkSubmitInfo sinfo = {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.pNext = &tinfo,
.commandBufferCount = 1,
.pCommandBuffers = &cmd->buf,
.waitSemaphoreCount = cmd->deps.num,
.pWaitSemaphores = cmd->deps.elem,
.pWaitDstStageMask = cmd->depstages.elem,
.signalSemaphoreCount = cmd->sigs.num,
.pSignalSemaphores = cmd->sigs.elem,
VkSubmitInfo2 sinfo = {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.waitSemaphoreInfoCount = cmd->deps.num,
.pWaitSemaphoreInfos = cmd->deps.elem,
.signalSemaphoreInfoCount = cmd->sigs.num,
.pSignalSemaphoreInfos = cmd->sigs.elem,
.commandBufferInfoCount = 1,
.pCommandBufferInfos = &(VkCommandBufferSubmitInfo) {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
.commandBuffer = cmd->buf,
},
};

if (pl_msg_test(vk->log, PL_LOG_TRACE)) {
PL_TRACE(vk, "Submitting command %p on queue %p (QF %d):",
(void *) cmd->buf, (void *) cmd->queue, pool->qf);
for (int n = 0; n < cmd->deps.num; n++) {
PL_TRACE(vk, " waits on semaphore 0x%"PRIx64" = %"PRIu64,
(uint64_t) cmd->deps.elem[n], cmd->depvalues.elem[n]);
(uint64_t) cmd->deps.elem[n].semaphore, cmd->deps.elem[n].value);
}
for (int n = 0; n < cmd->sigs.num; n++) {
PL_TRACE(vk, " signals semaphore 0x%"PRIx64" = %"PRIu64,
(uint64_t) cmd->sigs.elem[n], cmd->sigvalues.elem[n]);
(uint64_t) cmd->sigs.elem[n].semaphore, cmd->sigs.elem[n].value);
}
if (cmd->callbacks.num)
PL_TRACE(vk, " signals %d callbacks", cmd->callbacks.num);
}

vk->lock_queue(vk->queue_ctx, pool->qf, cmd->qindex);
VkResult res = vk->QueueSubmit(cmd->queue, 1, &sinfo, VK_NULL_HANDLE);
VkResult res = vk->QueueSubmit2(cmd->queue, 1, &sinfo, VK_NULL_HANDLE);
vk->unlock_queue(vk->queue_ctx, pool->qf, cmd->qindex);
PL_VK_ASSERT(res, "vkQueueSubmit");
PL_VK_ASSERT(res, "vkQueueSubmit2");

pl_mutex_lock(&vk->lock);
PL_ARRAY_APPEND(vk->alloc, vk->cmds_pending, cmd);
Expand Down
34 changes: 10 additions & 24 deletions src/vulkan/command.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,9 @@ struct vk_cmd {
VkQueue queue; // the submission queue (for recording/pending)
int qindex; // the index of `queue` in `pool`
VkCommandBuffer buf; // the command buffer itself
// The semaphores represent dependencies that need to complete before
// this command can be executed. These are *not* owned by the vk_cmd
PL_ARRAY(VkSemaphore) deps;
PL_ARRAY(VkPipelineStageFlags) depstages;
PL_ARRAY(uint64_t) depvalues;
// The signals represent semaphores that fire once the command finishes
// executing. These are also not owned by the vk_cmd
PL_ARRAY(VkSemaphore) sigs;
PL_ARRAY(uint64_t) sigvalues;
// Command dependencies and signals. Not owned by the vk_cmd.
PL_ARRAY(VkSemaphoreSubmitInfo) deps;
PL_ARRAY(VkSemaphoreSubmitInfo) sigs;
// "Callbacks" to fire once a command completes. These are used for
// multiple purposes, ranging from resource deallocation to fencing.
PL_ARRAY(struct vk_callback) callbacks;
Expand All @@ -66,41 +60,33 @@ void vk_cmd_callback(struct vk_cmd *cmd, vk_cb callback,

// Associate a raw dependency for the current command. This semaphore must
// signal by the corresponding stage before the command may execute.
void vk_cmd_dep(struct vk_cmd *cmd, VkPipelineStageFlags stage, pl_vulkan_sem dep);
void vk_cmd_dep(struct vk_cmd *cmd, VkPipelineStageFlags2 stage, pl_vulkan_sem dep);

// Associate a raw signal with the current command. This semaphore will signal
// after the command completes.
void vk_cmd_sig(struct vk_cmd *cmd, pl_vulkan_sem sig);
// after the given stage completes.
void vk_cmd_sig(struct vk_cmd *cmd, VkPipelineStageFlags2 stage, pl_vulkan_sem sig);

// Synchronization scope
struct vk_sync_scope {
pl_vulkan_sem sync; // semaphore of last access
VkQueue queue; // source queue of last access
VkPipelineStageFlags stage; // stage bitmask of last access
VkAccessFlags access; // access type bitmask
VkPipelineStageFlags2 stage;// stage bitmask of last access
VkAccessFlags2 access; // access type bitmask
};

// Synchronization primitive
struct vk_sem {
struct vk_sync_scope read, write;
};

static inline void vk_sem_init(struct vk_sem *sem)
{
*sem = (struct vk_sem) {
.write.stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
.read.stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
};
};

// Updates the `vk_sem` state for a given access. If `is_trans` is set, this
// access is treated as a write (since it alters the resource's state).
//
// Returns a struct describing the previous access to a resource. A pipeline
// barrier is only required if the previous access scope is nonzero.
struct vk_sync_scope vk_sem_barrier(struct vk_ctx *vk, struct vk_cmd *cmd,
struct vk_sem *sem, VkPipelineStageFlags stage,
VkAccessFlags access, bool is_trans);
struct vk_sem *sem, VkPipelineStageFlags2 stage,
VkAccessFlags2 access, bool is_trans);

// Command pool / queue family hybrid abstraction
struct vk_cmdpool {
Expand Down
8 changes: 3 additions & 5 deletions src/vulkan/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,16 +139,14 @@ struct vk_ctx {
PL_VK_FUN(CmdDrawIndexed);
PL_VK_FUN(CmdEndDebugUtilsLabelEXT);
PL_VK_FUN(CmdEndRenderPass);
PL_VK_FUN(CmdPipelineBarrier);
PL_VK_FUN(CmdPipelineBarrier2);
PL_VK_FUN(CmdPushConstants);
PL_VK_FUN(CmdPushDescriptorSetKHR);
PL_VK_FUN(CmdResetQueryPool);
PL_VK_FUN(CmdSetEvent);
PL_VK_FUN(CmdSetScissor);
PL_VK_FUN(CmdSetViewport);
PL_VK_FUN(CmdUpdateBuffer);
PL_VK_FUN(CmdWaitEvents);
PL_VK_FUN(CmdWriteTimestamp);
PL_VK_FUN(CmdWriteTimestamp2);
PL_VK_FUN(CreateBuffer);
PL_VK_FUN(CreateBufferView);
PL_VK_FUN(CreateCommandPool);
Expand Down Expand Up @@ -212,7 +210,7 @@ struct vk_ctx {
PL_VK_FUN(InvalidateMappedMemoryRanges);
PL_VK_FUN(MapMemory);
PL_VK_FUN(QueuePresentKHR);
PL_VK_FUN(QueueSubmit);
PL_VK_FUN(QueueSubmit2);
PL_VK_FUN(QueueWaitIdle);
PL_VK_FUN(ResetEvent);
PL_VK_FUN(ResetFences);
Expand Down
11 changes: 5 additions & 6 deletions src/vulkan/context.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,8 @@ const VkPhysicalDeviceFeatures2 pl_vulkan_recommended_features = {
// Required features
static const VkPhysicalDeviceVulkan13Features required_vk13 = {
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES,
.maintenance4 = true
.maintenance4 = true,
.synchronization2 = true,
};

static const VkPhysicalDeviceVulkan12Features required_vk12 = {
Expand Down Expand Up @@ -283,15 +284,13 @@ static const struct vk_fun vk_dev_funs[] = {
PL_VK_DEV_FUN(CmdDrawIndexed),
PL_VK_DEV_FUN(CmdEndDebugUtilsLabelEXT),
PL_VK_DEV_FUN(CmdEndRenderPass),
PL_VK_DEV_FUN(CmdPipelineBarrier),
PL_VK_DEV_FUN(CmdPipelineBarrier2),
PL_VK_DEV_FUN(CmdPushConstants),
PL_VK_DEV_FUN(CmdResetQueryPool),
PL_VK_DEV_FUN(CmdSetEvent),
PL_VK_DEV_FUN(CmdSetScissor),
PL_VK_DEV_FUN(CmdSetViewport),
PL_VK_DEV_FUN(CmdUpdateBuffer),
PL_VK_DEV_FUN(CmdWaitEvents),
PL_VK_DEV_FUN(CmdWriteTimestamp),
PL_VK_DEV_FUN(CmdWriteTimestamp2),
PL_VK_DEV_FUN(CreateBuffer),
PL_VK_DEV_FUN(CreateBufferView),
PL_VK_DEV_FUN(CreateCommandPool),
Expand Down Expand Up @@ -344,7 +343,7 @@ static const struct vk_fun vk_dev_funs[] = {
PL_VK_DEV_FUN(GetQueryPoolResults),
PL_VK_DEV_FUN(InvalidateMappedMemoryRanges),
PL_VK_DEV_FUN(MapMemory),
PL_VK_DEV_FUN(QueueSubmit),
PL_VK_DEV_FUN(QueueSubmit2),
PL_VK_DEV_FUN(QueueWaitIdle),
PL_VK_DEV_FUN(ResetEvent),
PL_VK_DEV_FUN(ResetFences),
Expand Down
8 changes: 4 additions & 4 deletions src/vulkan/gpu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,8 @@ static void timer_begin(pl_gpu gpu, struct vk_cmd *cmd, pl_timer timer)
vk->ResetQueryPool(vk->dev, timer->qpool, timer->index_write, 2);
}

vk->CmdWriteTimestamp(cmd->buf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
timer->qpool, timer->index_write);
vk->CmdWriteTimestamp2(cmd->buf, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
timer->qpool, timer->index_write);

p->cmd_timer = timer;
}
Expand Down Expand Up @@ -207,8 +207,8 @@ bool _end_cmd(pl_gpu gpu, struct vk_cmd **pcmd, bool submit)

if (p->cmd_timer) {
pl_timer timer = p->cmd_timer;
vk->CmdWriteTimestamp(cmd->buf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
timer->qpool, timer->index_write + 1);
vk->CmdWriteTimestamp2(cmd->buf, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
timer->qpool, timer->index_write + 1);

timer->pending |= timer_bit(timer->index_write);
vk_cmd_callback(cmd, (vk_cb) timer_end_cb, timer,
Expand Down
8 changes: 4 additions & 4 deletions src/vulkan/gpu.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,8 @@ bool vk_tex_upload(pl_gpu, const struct pl_tex_transfer_params *);
bool vk_tex_download(pl_gpu, const struct pl_tex_transfer_params *);
bool vk_tex_poll(pl_gpu, pl_tex, uint64_t timeout);
bool vk_tex_export(pl_gpu, pl_tex, pl_sync);
void vk_tex_barrier(pl_gpu, struct vk_cmd *, pl_tex, VkPipelineStageFlags,
VkAccessFlags, VkImageLayout, uint32_t qf);
void vk_tex_barrier(pl_gpu, struct vk_cmd *, pl_tex, VkPipelineStageFlags2,
VkAccessFlags2, VkImageLayout, uint32_t qf);

struct pl_buf_vk {
pl_rc_t rc;
Expand All @@ -152,8 +152,8 @@ bool vk_buf_export(pl_gpu, pl_buf);
bool vk_buf_poll(pl_gpu, pl_buf, uint64_t timeout);

// Helper to ease buffer barrier creation. (`offset` is relative to pl_buf)
void vk_buf_barrier(pl_gpu, struct vk_cmd *, pl_buf, VkPipelineStageFlags,
VkAccessFlags, size_t offset, size_t size, bool export);
void vk_buf_barrier(pl_gpu, struct vk_cmd *, pl_buf, VkPipelineStageFlags2,
VkAccessFlags2, size_t offset, size_t size, bool export);

// Flush visible writes to a buffer made by the API
void vk_buf_flush(pl_gpu, struct vk_cmd *, pl_buf, size_t offset, size_t size);
Expand Down
Loading

0 comments on commit e5fb4e6

Please sign in to comment.