Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 154: BK Operator should support multiple ledger PVCs, multiple journal PVCs and multiple index PVCs for 1 bookie #155

Closed
wants to merge 2 commits into from
Closed

Issue 154: BK Operator should support multiple ledger PVCs, multiple journal PVCs and multiple index PVCs for 1 bookie #155

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
90fbf8a
support multi PVCs
CraneShiEMC Jun 22, 2021
79fef46
Add flag whether to use multi-PVC support
CraneShiEMC Jul 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 147 additions & 34 deletions pkg/controller/bookkeepercluster/bookie.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ func makeBookiePodSpec(bk *v1alpha1.BookkeeperCluster) *corev1.PodSpec {
var hostPathVolumeMounts []string
var emptyDirVolumeMounts []string
var configMapVolumeMounts []string
var multiVolumesSetPerBookieEnabled bool
var ok bool

if _, ok = bk.Spec.Options["ledgerDirectories"]; ok {
Expand Down Expand Up @@ -170,6 +171,13 @@ func makeBookiePodSpec(bk *v1alpha1.BookkeeperCluster) *corev1.PodSpec {
configMapVolumeMounts = strings.Split(bk.Spec.Options["configMapVolumeMounts"], ",")
}

if _, ok = bk.Spec.Options["multiVolumesSetPerBookieEnabled"]; ok {
multiVolumesSetPerBookieEnabled, _ = strconv.ParseBool(bk.Spec.Options["multiVolumesSetPerBookieEnabled"])
} else {
// default value if user did not set multiVolumesSetPerBookieEnabled in options
multiVolumesSetPerBookieEnabled = false
}

var volumes []corev1.Volume
var cmVolumeMounts []corev1.VolumeMount

Expand Down Expand Up @@ -225,7 +233,8 @@ func makeBookiePodSpec(bk *v1alpha1.BookkeeperCluster) *corev1.PodSpec {
}

volumeMounts := createVolumeMount(ledgerDirs, journalDirs, indexDirs,
ledgerSubPath, journalSubPath, indexSubPath, hostPathVolumeMounts, emptyDirVolumeMounts)
ledgerSubPath, journalSubPath, indexSubPath, hostPathVolumeMounts,
emptyDirVolumeMounts, multiVolumesSetPerBookieEnabled)

if len(cmVolumeMounts) > 0 {
volumeMounts = append(volumeMounts, cmVolumeMounts...)
Expand Down Expand Up @@ -288,17 +297,26 @@ func makeBookiePodSpec(bk *v1alpha1.BookkeeperCluster) *corev1.PodSpec {
return podSpec
}

func createVolumeMount(ledgerDirs []string, journalDirs []string, indexDirs []string, ledgerSubPath string, journalSubPath string, indexSubPath string, hostPathVolumeMounts []string, emptyDirVolumeMounts []string) []corev1.VolumeMount {
func createVolumeMount(ledgerDirs []string, journalDirs []string, indexDirs []string, ledgerSubPath string, journalSubPath string, indexSubPath string, hostPathVolumeMounts []string, emptyDirVolumeMounts []string, multiVolumesSetPerBookieEnabled bool) []corev1.VolumeMount {
var volumeMounts []corev1.VolumeMount
if len(ledgerDirs) > 1 {
for i, ledger := range ledgerDirs {
name := ledgerSubPath + strconv.Itoa(i)
v := corev1.VolumeMount{
Name: LedgerDiskName,
MountPath: ledger,
SubPath: name,
if multiVolumesSetPerBookieEnabled {
for i, ledger := range ledgerDirs {
v := corev1.VolumeMount{
Name: LedgerDiskName + strconv.Itoa(i),
MountPath: ledger,
}
volumeMounts = append(volumeMounts, v)
}
} else {
for i, ledger := range ledgerDirs {
v := corev1.VolumeMount{
Name: LedgerDiskName,
MountPath: ledger,
SubPath: ledgerSubPath + strconv.Itoa(i),
}
volumeMounts = append(volumeMounts, v)
}
volumeMounts = append(volumeMounts, v)
}
} else {
v := corev1.VolumeMount{
Expand All @@ -308,14 +326,23 @@ func createVolumeMount(ledgerDirs []string, journalDirs []string, indexDirs []st
volumeMounts = append(volumeMounts, v)
}
if len(journalDirs) > 1 {
for i, journal := range journalDirs {
name := journalSubPath + strconv.Itoa(i)
v := corev1.VolumeMount{
Name: JournalDiskName,
MountPath: journal,
SubPath: name,
if multiVolumesSetPerBookieEnabled {
for i, journal := range journalDirs {
v := corev1.VolumeMount{
Name: JournalDiskName + strconv.Itoa(i),
MountPath: journal,
}
volumeMounts = append(volumeMounts, v)
}
} else {
for i, journal := range journalDirs {
v := corev1.VolumeMount{
Name: JournalDiskName,
MountPath: journal,
SubPath: journalSubPath + strconv.Itoa(i),
}
volumeMounts = append(volumeMounts, v)
}
volumeMounts = append(volumeMounts, v)
}
} else {
v := corev1.VolumeMount{
Expand All @@ -325,14 +352,23 @@ func createVolumeMount(ledgerDirs []string, journalDirs []string, indexDirs []st
volumeMounts = append(volumeMounts, v)
}
if len(indexDirs) > 1 {
for i, index := range indexDirs {
name := indexSubPath + strconv.Itoa(i)
v := corev1.VolumeMount{
Name: IndexDiskName,
MountPath: index,
SubPath: name,
if multiVolumesSetPerBookieEnabled {
for i, index := range indexDirs {
v := corev1.VolumeMount{
Name: IndexDiskName + strconv.Itoa(i),
MountPath: index,
}
volumeMounts = append(volumeMounts, v)
}
} else {
for i, index := range indexDirs {
v := corev1.VolumeMount{
Name: IndexDiskName,
MountPath: index,
SubPath: indexSubPath + strconv.Itoa(i),
}
volumeMounts = append(volumeMounts, v)
}
volumeMounts = append(volumeMounts, v)
}
} else {
v := corev1.VolumeMount{
Expand Down Expand Up @@ -365,29 +401,106 @@ func createVolumeMount(ledgerDirs []string, journalDirs []string, indexDirs []st
}

func makeBookieVolumeClaimTemplates(bk *v1alpha1.BookkeeperCluster) []corev1.PersistentVolumeClaim {
return []corev1.PersistentVolumeClaim{
{
var ledgerDirs, journalDirs, indexDirs []string
var multiVolumesSetPerBookieEnabled bool
var ok bool

if _, ok = bk.Spec.Options["ledgerDirectories"]; ok {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lines 407-428 might not be required, if we are making multivolume support to false . Could you please verify that?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lines 407-428 might not be required, if we are making multivolume support to false . Could you please verify that?

OK. I will optimize this part according to your suggestion.

ledgerDirs = strings.Split(bk.Spec.Options["ledgerDirectories"], ",")
} else {
// default value if user did not set ledgerDirectories in options
ledgerDirs = append(ledgerDirs, "/bk/ledgers")
}

if _, ok = bk.Spec.Options["journalDirectories"]; ok {
journalDirs = strings.Split(bk.Spec.Options["journalDirectories"], ",")
} else {
// default value if user did not set journalDirectories in options
journalDirs = append(journalDirs, "/bk/journal")
}

if _, ok = bk.Spec.Options["indexDirectories"]; ok {
indexDirs = strings.Split(bk.Spec.Options["indexDirectories"], ",")
} else {
// default value if user did not set indexDirectories in options
indexDirs = append(indexDirs, "/bk/index")
}

if _, ok = bk.Spec.Options["multiVolumesSetPerBookieEnabled"]; ok {
multiVolumesSetPerBookieEnabled, _ = strconv.ParseBool(bk.Spec.Options["multiVolumesSetPerBookieEnabled"])
} else {
// default value if user did not set multiVolumesSetPerBookieEnabled in options
multiVolumesSetPerBookieEnabled = false
}

var claims []corev1.PersistentVolumeClaim
if multiVolumesSetPerBookieEnabled && len(ledgerDirs) > 1 {
for i := 0; i < len(ledgerDirs); i++ {
pvc := corev1.PersistentVolumeClaim{
CraneShiEMC marked this conversation as resolved.
Show resolved Hide resolved
ObjectMeta: metav1.ObjectMeta{
Name: LedgerDiskName + strconv.Itoa(i),
Namespace: bk.Namespace,
},
Spec: *bk.Spec.Storage.LedgerVolumeClaimTemplate,
}
claims = append(claims, pvc)
}
} else {
pvc := corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: JournalDiskName,
Name: LedgerDiskName,
Namespace: bk.Namespace,
},
Spec: *bk.Spec.Storage.JournalVolumeClaimTemplate,
},
{
Spec: *bk.Spec.Storage.LedgerVolumeClaimTemplate,
}
claims = append(claims, pvc)
}

if multiVolumesSetPerBookieEnabled && len(journalDirs) > 1 {
for i := 0; i < len(journalDirs); i++ {
pvc := corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: JournalDiskName + strconv.Itoa(i),
Namespace: bk.Namespace,
},
Spec: *bk.Spec.Storage.JournalVolumeClaimTemplate,
}
claims = append(claims, pvc)
}
} else {
pvc := corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: LedgerDiskName,
Name: JournalDiskName,
Namespace: bk.Namespace,
},
Spec: *bk.Spec.Storage.LedgerVolumeClaimTemplate,
},
{
Spec: *bk.Spec.Storage.JournalVolumeClaimTemplate,
}
claims = append(claims, pvc)
}

if multiVolumesSetPerBookieEnabled && len(indexDirs) > 1 {
for i := 0; i < len(indexDirs); i++ {
pvc := corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: IndexDiskName + strconv.Itoa(i),
Namespace: bk.Namespace,
},
Spec: *bk.Spec.Storage.IndexVolumeClaimTemplate,
}
claims = append(claims, pvc)
}
} else {
pvc := corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: IndexDiskName,
Namespace: bk.Namespace,
},
Spec: *bk.Spec.Storage.IndexVolumeClaimTemplate,
},
}
claims = append(claims, pvc)
}

return claims
}

func MakeBookieConfigMap(bk *v1alpha1.BookkeeperCluster) *corev1.ConfigMap {
Expand Down