diff --git a/Makefile b/Makefile
index f1622a017340..77e173474b45 100644
--- a/Makefile
+++ b/Makefile
@@ -404,6 +404,7 @@ install: githooks
 ifneq ($(E2E_EXECUTOR),emissary)
 	# only change the executor from the default it we need to
 	kubectl patch cm/workflow-controller-configmap -p "{\"data\": {\"containerRuntimeExecutor\": \"$(E2E_EXECUTOR)\"}}"
+	kubectl apply -f manifests/quick-start/base/executor/$(E2E_EXECUTOR)
 endif
 ifeq ($(PROFILE),stress)
 	kubectl -n $(KUBE_NAMESPACE) apply -f test/stress/massive-workflow.yaml
diff --git a/docs/security.md b/docs/security.md
index e93edad45dee..9a9571ace27b 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -38,12 +38,7 @@ Workflow pods run using either:
 
 There is no restriction on which service account in a namespace may be used.
 
-This service account typically needs the following permissions:
-
-* Get/watch/patch pods.
-* Get/watch pod logs.
-
-See [workflow-role.yaml](https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start/base/workflow-role.yaml).
+This service account typically needs [permissions](workflow-rbac.md).
 
 Different service accounts should be used if a workflow pod needs to have elevated permissions, e.g. to create other resources.
 
diff --git a/docs/workflow-rbac.md b/docs/workflow-rbac.md
index 04df0eb91b42..d48634b3ee18 100644
--- a/docs/workflow-rbac.md
+++ b/docs/workflow-rbac.md
@@ -5,22 +5,26 @@ the `default` service account of the workflow's namespace. The amount of access
 what the workflow needs to do. For example, if your workflow needs to deploy a resource, then the workflow's service
 account will require 'create' privileges on that resource.
 
-The bare minimum for a workflow running using the Emissary executor to function is outlined below:
+**Warning**: We do not recommend using the `default` service account in production. It is a shared account so may have
+permissions added to it you do not want. Instead, create a service account only for your workflow.
+
+The minimum for the executor to function:
 
 ```yaml
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: workflow-role
+  name: executor
 rules:
   - apiGroups:
       - ""
     resources:
       - pods
     verbs:
+      - get
       - patch
 ```
 
-If you are using another executor, or using resource template, you'll need additional permissions,
-see [workflow-role](https://github.com/argoproj/argo-workflows/blob/master/manifests/quick-start/base/workflow-role.yaml)
-.
+If you are not using the emissary, you'll need additional permissions.
+See [executor](https://github.com/argoproj/argo-workflows/tree/master/manifests/quick-start/base/executor) for suitable
+permissions.
diff --git a/hack/test-examples.sh b/hack/test-examples.sh
index 9cc95faecbe1..62cb4acad5b5 100755
--- a/hack/test-examples.sh
+++ b/hack/test-examples.sh
@@ -3,6 +3,8 @@ set -eu -o pipefail
 
 # Load the configmaps that contains the parameter values used for certain examples.
 kubectl apply -f examples/configmaps/simple-parameters-configmap.yaml
+# Needed for examples/selected-executor-workflow.yaml.
+kubectl apply -f manifests/quick-start/base/executor/pns/executor-role.yaml
 
 echo "Checking for banned images..."
 grep -lR 'workflows.argoproj.io/test' examples/*  | while read f ; do
diff --git a/manifests/quick-start-minimal.yaml b/manifests/quick-start-minimal.yaml
index 807155af6736..baa5ce4e723c 100644
--- a/manifests/quick-start-minimal.yaml
+++ b/manifests/quick-start-minimal.yaml
@@ -259,6 +259,30 @@ metadata:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
+metadata:
+  annotations:
+    workflows.argoproj.io/description: |
+      This is the minimum recommended permissions needed if you want to use the agent, e.g. for HTTP or plugin templates.
+
+      If <= v3.2 you must replace `workflowtasksets/status` with `patch workflowtasksets`.
+  name: agent
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtasksets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtasksets/status
+  verbs:
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
 metadata:
   name: argo-role
 rules:
@@ -437,86 +461,75 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: submit-workflow-template
+  annotations:
+    workflows.argoproj.io/description: |
+      Recomended minimum permissions for the `emissary` executor.
+  name: executor
 rules:
 - apiGroups:
-  - argoproj.io
-  resources:
-  - workfloweventbindings
-  verbs:
-  - list
-- apiGroups:
-  - argoproj.io
-  resources:
-  - workflowtemplates
-  verbs:
-  - get
-- apiGroups:
-  - argoproj.io
+  - ""
   resources:
-  - workflows
+  - pods
   verbs:
-  - create
+  - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: workflow-role
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other pods. The same pattern would be suitable for other resurces, e.g. a service
+  name: pod-manager
 rules:
 - apiGroups:
   - ""
   resources:
   - pods
   verbs:
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/log
-  verbs:
+  - create
   - get
-  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: submit-workflow-template
+rules:
 - apiGroups:
-  - ""
+  - argoproj.io
   resources:
-  - pods/exec
+  - workfloweventbindings
   verbs:
-  - create
+  - list
 - apiGroups:
-  - ""
+  - argoproj.io
   resources:
-  - configmaps
+  - workflowtemplates
   verbs:
-  - create
   - get
-  - update
 - apiGroups:
   - argoproj.io
   resources:
   - workflows
   verbs:
   - create
-  - get
-- apiGroups:
-  - argoproj.io
-  resources:
-  - workflowtasksets
-  verbs:
-  - list
-  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other workflows. The same pattern would be suitable for other resurces, e.g. a service
+  name: workflow-manager
+rules:
 - apiGroups:
   - argoproj.io
   resources:
-  - workflowtasksets/status
+  - workflows
   verbs:
-  - patch
+  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -552,16 +565,16 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: RoleBinding
 metadata:
-  name: kubelet-executor
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes/proxy
-  verbs:
-  - get
+  name: agent-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: agent
+subjects:
+- kind: ServiceAccount
+  name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -589,6 +602,18 @@ subjects:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
+metadata:
+  name: executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: executor
+subjects:
+- kind: ServiceAccount
+  name: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
 metadata:
   name: github.com
 roleRef:
@@ -603,52 +628,51 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: workflow-default-binding
+  name: pod-manager-default
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: workflow-role
+  name: pod-manager
 subjects:
 - kind: ServiceAccount
   name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
-  name: argo-clusterworkflowtemplate-role-binding
+  name: workflow-manager-default
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: argo-clusterworkflowtemplate-role
+  kind: Role
+  name: workflow-manager
 subjects:
 - kind: ServiceAccount
-  name: argo
-  namespace: argo
+  name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: argo-server-clusterworkflowtemplate-role-binding
+  name: argo-clusterworkflowtemplate-role-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: argo-server-clusterworkflowtemplate-role
+  name: argo-clusterworkflowtemplate-role
 subjects:
 - kind: ServiceAccount
-  name: argo-server
+  name: argo
   namespace: argo
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: kubelet-executor-default
+  name: argo-server-clusterworkflowtemplate-role-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: kubelet-executor
+  name: argo-server-clusterworkflowtemplate-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: argo-server
   namespace: argo
 ---
 apiVersion: v1
diff --git a/manifests/quick-start-mysql.yaml b/manifests/quick-start-mysql.yaml
index 599865b1276f..b94b85b0f868 100644
--- a/manifests/quick-start-mysql.yaml
+++ b/manifests/quick-start-mysql.yaml
@@ -259,6 +259,30 @@ metadata:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
+metadata:
+  annotations:
+    workflows.argoproj.io/description: |
+      This is the minimum recommended permissions needed if you want to use the agent, e.g. for HTTP or plugin templates.
+
+      If <= v3.2 you must replace `workflowtasksets/status` with `patch workflowtasksets`.
+  name: agent
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtasksets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtasksets/status
+  verbs:
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
 metadata:
   name: argo-role
 rules:
@@ -437,86 +461,75 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: submit-workflow-template
+  annotations:
+    workflows.argoproj.io/description: |
+      Recomended minimum permissions for the `emissary` executor.
+  name: executor
 rules:
 - apiGroups:
-  - argoproj.io
-  resources:
-  - workfloweventbindings
-  verbs:
-  - list
-- apiGroups:
-  - argoproj.io
-  resources:
-  - workflowtemplates
-  verbs:
-  - get
-- apiGroups:
-  - argoproj.io
+  - ""
   resources:
-  - workflows
+  - pods
   verbs:
-  - create
+  - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: workflow-role
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other pods. The same pattern would be suitable for other resurces, e.g. a service
+  name: pod-manager
 rules:
 - apiGroups:
   - ""
   resources:
   - pods
   verbs:
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/log
-  verbs:
+  - create
   - get
-  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: submit-workflow-template
+rules:
 - apiGroups:
-  - ""
+  - argoproj.io
   resources:
-  - pods/exec
+  - workfloweventbindings
   verbs:
-  - create
+  - list
 - apiGroups:
-  - ""
+  - argoproj.io
   resources:
-  - configmaps
+  - workflowtemplates
   verbs:
-  - create
   - get
-  - update
 - apiGroups:
   - argoproj.io
   resources:
   - workflows
   verbs:
   - create
-  - get
-- apiGroups:
-  - argoproj.io
-  resources:
-  - workflowtasksets
-  verbs:
-  - list
-  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other workflows. The same pattern would be suitable for other resurces, e.g. a service
+  name: workflow-manager
+rules:
 - apiGroups:
   - argoproj.io
   resources:
-  - workflowtasksets/status
+  - workflows
   verbs:
-  - patch
+  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -552,16 +565,16 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: RoleBinding
 metadata:
-  name: kubelet-executor
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes/proxy
-  verbs:
-  - get
+  name: agent-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: agent
+subjects:
+- kind: ServiceAccount
+  name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -589,6 +602,18 @@ subjects:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
+metadata:
+  name: executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: executor
+subjects:
+- kind: ServiceAccount
+  name: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
 metadata:
   name: github.com
 roleRef:
@@ -603,52 +628,51 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: workflow-default-binding
+  name: pod-manager-default
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: workflow-role
+  name: pod-manager
 subjects:
 - kind: ServiceAccount
   name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
-  name: argo-clusterworkflowtemplate-role-binding
+  name: workflow-manager-default
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: argo-clusterworkflowtemplate-role
+  kind: Role
+  name: workflow-manager
 subjects:
 - kind: ServiceAccount
-  name: argo
-  namespace: argo
+  name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: argo-server-clusterworkflowtemplate-role-binding
+  name: argo-clusterworkflowtemplate-role-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: argo-server-clusterworkflowtemplate-role
+  name: argo-clusterworkflowtemplate-role
 subjects:
 - kind: ServiceAccount
-  name: argo-server
+  name: argo
   namespace: argo
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: kubelet-executor-default
+  name: argo-server-clusterworkflowtemplate-role-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: kubelet-executor
+  name: argo-server-clusterworkflowtemplate-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: argo-server
   namespace: argo
 ---
 apiVersion: v1
diff --git a/manifests/quick-start-postgres.yaml b/manifests/quick-start-postgres.yaml
index d8c5f6e67a56..288ad57727d0 100644
--- a/manifests/quick-start-postgres.yaml
+++ b/manifests/quick-start-postgres.yaml
@@ -259,6 +259,30 @@ metadata:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
+metadata:
+  annotations:
+    workflows.argoproj.io/description: |
+      This is the minimum recommended permissions needed if you want to use the agent, e.g. for HTTP or plugin templates.
+
+      If <= v3.2 you must replace `workflowtasksets/status` with `patch workflowtasksets`.
+  name: agent
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtasksets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtasksets/status
+  verbs:
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
 metadata:
   name: argo-role
 rules:
@@ -437,86 +461,75 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: submit-workflow-template
+  annotations:
+    workflows.argoproj.io/description: |
+      Recomended minimum permissions for the `emissary` executor.
+  name: executor
 rules:
 - apiGroups:
-  - argoproj.io
-  resources:
-  - workfloweventbindings
-  verbs:
-  - list
-- apiGroups:
-  - argoproj.io
-  resources:
-  - workflowtemplates
-  verbs:
-  - get
-- apiGroups:
-  - argoproj.io
+  - ""
   resources:
-  - workflows
+  - pods
   verbs:
-  - create
+  - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: workflow-role
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other pods. The same pattern would be suitable for other resurces, e.g. a service
+  name: pod-manager
 rules:
 - apiGroups:
   - ""
   resources:
   - pods
   verbs:
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/log
-  verbs:
+  - create
   - get
-  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: submit-workflow-template
+rules:
 - apiGroups:
-  - ""
+  - argoproj.io
   resources:
-  - pods/exec
+  - workfloweventbindings
   verbs:
-  - create
+  - list
 - apiGroups:
-  - ""
+  - argoproj.io
   resources:
-  - configmaps
+  - workflowtemplates
   verbs:
-  - create
   - get
-  - update
 - apiGroups:
   - argoproj.io
   resources:
   - workflows
   verbs:
   - create
-  - get
-- apiGroups:
-  - argoproj.io
-  resources:
-  - workflowtasksets
-  verbs:
-  - list
-  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other workflows. The same pattern would be suitable for other resurces, e.g. a service
+  name: workflow-manager
+rules:
 - apiGroups:
   - argoproj.io
   resources:
-  - workflowtasksets/status
+  - workflows
   verbs:
-  - patch
+  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -552,16 +565,16 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: RoleBinding
 metadata:
-  name: kubelet-executor
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes/proxy
-  verbs:
-  - get
+  name: agent-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: agent
+subjects:
+- kind: ServiceAccount
+  name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -589,6 +602,18 @@ subjects:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
+metadata:
+  name: executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: executor
+subjects:
+- kind: ServiceAccount
+  name: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
 metadata:
   name: github.com
 roleRef:
@@ -603,52 +628,51 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: workflow-default-binding
+  name: pod-manager-default
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: workflow-role
+  name: pod-manager
 subjects:
 - kind: ServiceAccount
   name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
-  name: argo-clusterworkflowtemplate-role-binding
+  name: workflow-manager-default
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: argo-clusterworkflowtemplate-role
+  kind: Role
+  name: workflow-manager
 subjects:
 - kind: ServiceAccount
-  name: argo
-  namespace: argo
+  name: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: argo-server-clusterworkflowtemplate-role-binding
+  name: argo-clusterworkflowtemplate-role-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: argo-server-clusterworkflowtemplate-role
+  name: argo-clusterworkflowtemplate-role
 subjects:
 - kind: ServiceAccount
-  name: argo-server
+  name: argo
   namespace: argo
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: kubelet-executor-default
+  name: argo-server-clusterworkflowtemplate-role-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: kubelet-executor
+  name: argo-server-clusterworkflowtemplate-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: argo-server
   namespace: argo
 ---
 apiVersion: v1
diff --git a/manifests/quick-start/base/agent-default-rolebinding.yaml b/manifests/quick-start/base/agent-default-rolebinding.yaml
new file mode 100644
index 000000000000..f22f07123d31
--- /dev/null
+++ b/manifests/quick-start/base/agent-default-rolebinding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: agent
+subjects:
+  - kind: ServiceAccount
+    name: default
diff --git a/manifests/quick-start/base/agent-role.yaml b/manifests/quick-start/base/agent-role.yaml
new file mode 100644
index 000000000000..6f7f52ed121b
--- /dev/null
+++ b/manifests/quick-start/base/agent-role.yaml
@@ -0,0 +1,24 @@
+# https://argoproj.github.io/argo-workflows/workflow-rbac/
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: agent
+  annotations:
+    workflows.argoproj.io/description: |
+      This is the minimum recommended permissions needed if you want to use the agent, e.g. for HTTP or plugin templates.
+
+      If <= v3.2 you must replace `workflowtasksets/status` with `patch workflowtasksets`.
+rules:
+  - apiGroups:
+      - argoproj.io
+    resources:
+      - workflowtasksets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - argoproj.io
+    resources:
+      - workflowtasksets/status
+    verbs:
+      - patch
diff --git a/manifests/quick-start/base/executor-default-rolebinding.yaml b/manifests/quick-start/base/executor-default-rolebinding.yaml
new file mode 100644
index 000000000000..124c7ca315a7
--- /dev/null
+++ b/manifests/quick-start/base/executor-default-rolebinding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: executor
+subjects:
+  - kind: ServiceAccount
+    name: default
diff --git a/manifests/quick-start/base/executor/docker/executor-role.yaml b/manifests/quick-start/base/executor/docker/executor-role.yaml
new file mode 100644
index 000000000000..77706a6cce0e
--- /dev/null
+++ b/manifests/quick-start/base/executor/docker/executor-role.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: executor
+  annotations:
+    workflows.argoproj.io/description: |
+      Recommended minimum permissions for the `docker` executor.
+
+      This executor is superseded by  the `emmisary` executor, so we do not recommend you use it anymore.
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - patch
diff --git a/manifests/quick-start/base/executor/emissary/executor-role.yaml b/manifests/quick-start/base/executor/emissary/executor-role.yaml
new file mode 100644
index 000000000000..8f93206cf310
--- /dev/null
+++ b/manifests/quick-start/base/executor/emissary/executor-role.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: executor
+  annotations:
+    workflows.argoproj.io/description: |
+      Recomended minimum permissions for the `emissary` executor.
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - patch
diff --git a/manifests/quick-start/base/executor/k8sapi/executor-role.yaml b/manifests/quick-start/base/executor/k8sapi/executor-role.yaml
new file mode 100644
index 000000000000..88c113b2194a
--- /dev/null
+++ b/manifests/quick-start/base/executor/k8sapi/executor-role.yaml
@@ -0,0 +1,30 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: executor
+  annotations:
+    workflows.argoproj.io/description: |
+      Recommended minimum permissions for `k8siapi` executor.
+
+      This executor is superseded by  the `emmisary` executor, so we do not recommend you use it anymore.
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - patch
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - pods/log
+    verbs:
+      - get
diff --git a/manifests/quick-start/base/executor/kubelet/executor-role.yaml b/manifests/quick-start/base/executor/kubelet/executor-role.yaml
new file mode 100644
index 000000000000..d3d2e11d3a8f
--- /dev/null
+++ b/manifests/quick-start/base/executor/kubelet/executor-role.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: executor
+  annotations:
+    workflows.argoproj.io/description: |
+      Recommended minimum permissions for `kubelet` executor.
+
+      This executor is superseded by  the `emmisary` executor, so we do not recommend you use it anymore.
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - patch
\ No newline at end of file
diff --git a/manifests/quick-start/base/kubelet-executor-clusterrole.yaml b/manifests/quick-start/base/executor/kubelet/kubelet-executor-clusterrole.yaml
similarity index 100%
rename from manifests/quick-start/base/kubelet-executor-clusterrole.yaml
rename to manifests/quick-start/base/executor/kubelet/kubelet-executor-clusterrole.yaml
diff --git a/manifests/quick-start/base/kubelet-executor-default-clusterrolebinding.yaml b/manifests/quick-start/base/executor/kubelet/kubelet-executor-default-clusterrolebinding.yaml
similarity index 100%
rename from manifests/quick-start/base/kubelet-executor-default-clusterrolebinding.yaml
rename to manifests/quick-start/base/executor/kubelet/kubelet-executor-default-clusterrolebinding.yaml
diff --git a/manifests/quick-start/base/executor/pns/executor-role.yaml b/manifests/quick-start/base/executor/pns/executor-role.yaml
new file mode 100644
index 000000000000..392999f5855a
--- /dev/null
+++ b/manifests/quick-start/base/executor/pns/executor-role.yaml
@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: executor
+  annotations:
+    workflows.argoproj.io/description: |
+      Recomended minimum permissions for `pns` executor.
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - patch
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/log
+    verbs:
+      - get
\ No newline at end of file
diff --git a/manifests/quick-start/base/kustomization.yaml b/manifests/quick-start/base/kustomization.yaml
index b809453642d7..f8d0da97f921 100644
--- a/manifests/quick-start/base/kustomization.yaml
+++ b/manifests/quick-start/base/kustomization.yaml
@@ -6,10 +6,14 @@ resources:
   - minio
   - webhooks
   - argo-server-sso-secret.yaml
-  - workflow-role.yaml
-  - kubelet-executor-clusterrole.yaml
-  - kubelet-executor-default-clusterrolebinding.yaml
-  - workflow-default-rolebinding.yaml
+  - executor/emissary/executor-role.yaml
+  - executor-default-rolebinding.yaml
+  - pod-manager-role.yaml
+  - pod-manager-default-rolebinding.yaml
+  - workflow-manager-role.yaml
+  - workflow-manager-default-rolebinding.yaml
+  - agent-role.yaml
+  - agent-default-rolebinding.yaml
   - cluster-workflow-template-rbac.yaml
   - artifact-repositories-configmap.yaml
 
diff --git a/manifests/quick-start/base/memoizer-default-rolebinding.yaml b/manifests/quick-start/base/memoizer-default-rolebinding.yaml
new file mode 100644
index 000000000000..1f415115b36a
--- /dev/null
+++ b/manifests/quick-start/base/memoizer-default-rolebinding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: memoizer-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: memoizer
+subjects:
+  - kind: ServiceAccount
+    name: default
diff --git a/manifests/quick-start/base/memoizer-role.yaml b/manifests/quick-start/base/memoizer-role.yaml
new file mode 100644
index 000000000000..d5cb71c74671
--- /dev/null
+++ b/manifests/quick-start/base/memoizer-role.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: memoizer
+  annotations:
+    workflows.argoproj.io/description: |
+      Only needed if you are using ConfigMap-based cache for memoization.
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+      - get
+      - update
diff --git a/manifests/quick-start/base/pod-manager-default-rolebinding.yaml b/manifests/quick-start/base/pod-manager-default-rolebinding.yaml
new file mode 100644
index 000000000000..42587b643a02
--- /dev/null
+++ b/manifests/quick-start/base/pod-manager-default-rolebinding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pod-manager-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-manager
+subjects:
+  - kind: ServiceAccount
+    name: default
diff --git a/manifests/quick-start/base/pod-manager-role.yaml b/manifests/quick-start/base/pod-manager-role.yaml
new file mode 100644
index 000000000000..e9196fffad25
--- /dev/null
+++ b/manifests/quick-start/base/pod-manager-role.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pod-manager
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other pods. The same pattern would be suitable for other resurces, e.g. a service
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - create
+      - get
\ No newline at end of file
diff --git a/manifests/quick-start/base/workflow-default-rolebinding.yaml b/manifests/quick-start/base/workflow-default-rolebinding.yaml
index 9cca04007764..cb930cfb27a7 100644
--- a/manifests/quick-start/base/workflow-default-rolebinding.yaml
+++ b/manifests/quick-start/base/workflow-default-rolebinding.yaml
@@ -5,7 +5,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: workflow-role
+  name: workflow
 subjects:
   - kind: ServiceAccount
     name: default
diff --git a/manifests/quick-start/base/workflow-manager-default-rolebinding.yaml b/manifests/quick-start/base/workflow-manager-default-rolebinding.yaml
new file mode 100644
index 000000000000..b2f8783f6827
--- /dev/null
+++ b/manifests/quick-start/base/workflow-manager-default-rolebinding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: workflow-manager-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: workflow-manager
+subjects:
+  - kind: ServiceAccount
+    name: default
diff --git a/manifests/quick-start/base/workflow-manager-role.yaml b/manifests/quick-start/base/workflow-manager-role.yaml
new file mode 100644
index 000000000000..d4e6606f52c3
--- /dev/null
+++ b/manifests/quick-start/base/workflow-manager-role.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: workflow-manager
+  annotations:
+    workflows.argoproj.io/description: |
+      This is an example of the permissions you would need if you wanted to use a resource template to create and manage
+      other workflows. The same pattern would be suitable for other resurces, e.g. a service
+rules:
+  - apiGroups:
+      - argoproj.io
+    resources:
+      - workflows
+    verbs:
+      - create
+      - get
\ No newline at end of file
diff --git a/manifests/quick-start/base/workflow-role.yaml b/manifests/quick-start/base/workflow-role.yaml
deleted file mode 100644
index 9d026666297a..000000000000
--- a/manifests/quick-start/base/workflow-role.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-# https://argoproj.github.io/argo-workflows/workflow-rbac/
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: workflow-role
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - patch
-  # pod get/watch is used to identify the container IDs of the current pod.
-  # This is not needed for the emissary executor.
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-      - watch
-  # logs get/watch are used to get the pods logs for script outputs, and for log archival
-  # This is not needed for the emissary executor.
-  - apiGroups:
-      - ""
-    resources:
-      - pods/log
-    verbs:
-      - get
-      - watch
-  # Only needed if you are running the `k8sapi` executor.
-  - apiGroups:
-      - ""
-    resources:
-      - pods/exec
-    verbs:
-      - create
-  # Only needed if you are using ConfigMap-based cache for memoization.
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    verbs:
-      - create
-      - get
-      - update
-  # This allows one workflow to create another.
-  # This is only needed for resource templates.
-  - apiGroups:
-      - argoproj.io
-    resources:
-      - workflows
-    verbs:
-      - create
-      - get
-  # This allows executor to patch the taskset.
-  - apiGroups:
-      - argoproj.io
-    resources:
-      - workflowtasksets
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - argoproj.io
-    resources:
-      - workflowtasksets/status
-    verbs:
-      - patch
\ No newline at end of file
diff --git a/workflow/executor/agent.go b/workflow/executor/agent.go
index 0443738c0764..0ab463954b43 100644
--- a/workflow/executor/agent.go
+++ b/workflow/executor/agent.go
@@ -187,7 +187,7 @@ func (ae *AgentExecutor) patchWorker(ctx context.Context, taskSetInterface v1alp
 			}, errors.IsTransientErr, func() error {
 				_, err := taskSetInterface.Patch(ctx, ae.WorkflowName, types.MergePatchType, patch, metav1.PatchOptions{}, "status")
 				if apierr.IsForbidden(err) {
-					ae.log.Warn("forbidden to patch workflowtaskset/status,  falling back to less secure patching workflowtaskset, please updated your agent's RBAC")
+					ae.log.WithError(err).Warn("falling back to less secure patching workflowtaskset, please updated your agent's RBAC")
 					_, err = taskSetInterface.Patch(ctx, ae.WorkflowName, types.MergePatchType, patch, metav1.PatchOptions{})
 				}
 				return err