diff --git a/conf/broker.conf b/conf/broker.conf index e6b3aef8811b5..44cf26d21707b 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1014,6 +1014,8 @@ bookkeeperClientMinAvailableBookiesInIsolationGroups= # Enable/disable having read operations for a ledger to be sticky to a single bookie. # If this flag is enabled, the client will use one single bookie (by preference) to read # all entries for a ledger. +# Please notice that enabling sticky reads also requires that the ensemble size equals write quorum size for the ledger's +# persistence policies. bookkeeperEnableStickyReads=true # Set the client security provider factory class name. @@ -1059,13 +1061,15 @@ bookkeeperExplicitLacIntervalInMills=0 ### --- Managed Ledger --- ### -# Number of bookies to use when creating a ledger +# Ensemble size (E), Number of bookies to use for storing entries in a ledger. +# Please notice that sticky reads enabled by bookkeeperEnableStickyReads=true aren’t used +# unless ensemble size (E) equals write quorum (Qw) size. managedLedgerDefaultEnsembleSize=2 -# Number of copies to store for each message +# Write quorum (Qw) size, Replication factor for storing entries (messages) in a ledger. managedLedgerDefaultWriteQuorum=2 -# Number of guaranteed copies (acks to wait before write is complete) +# Ack quorum (Qa) size, Number of guaranteed copies (acks to wait for before a write is considered completed) managedLedgerDefaultAckQuorum=2 # with OpportunisticStriping=true the ensembleSize is adapted automatically to writeQuorum diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 6683d36c36e06..6e327fd8b2609 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1785,19 +1785,22 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se @FieldContext( minValue = 1, category = CATEGORY_STORAGE_ML, - doc = "Number of bookies to use when creating a ledger" + doc = "Ensemble (E) size, Number of bookies to use for storing entries in a ledger.\n" + + "Please notice that sticky reads enabled by bookkeeperEnableStickyReads=true aren’t used " + + " unless ensemble size (E) equals write quorum (Qw) size." ) private int managedLedgerDefaultEnsembleSize = 2; @FieldContext( minValue = 1, category = CATEGORY_STORAGE_ML, - doc = "Number of copies to store for each message" + doc = "Write quorum (Qw) size, Replication factor for storing entries (messages) in a ledger." ) private int managedLedgerDefaultWriteQuorum = 2; @FieldContext( minValue = 1, category = CATEGORY_STORAGE_ML, - doc = "Number of guaranteed copies (acks to wait before write is complete)" + doc = "Ack quorum (Qa) size, Number of guaranteed copies " + + "(acks to wait for before a write is considered completed)" ) private int managedLedgerDefaultAckQuorum = 2; diff --git a/site2/docs/administration-zk-bk.md b/site2/docs/administration-zk-bk.md index 56555ccb673da..03cc889d676d7 100644 --- a/site2/docs/administration-zk-bk.md +++ b/site2/docs/administration-zk-bk.md @@ -259,9 +259,9 @@ You can run the following command to check if the bookie you have decommissioned In Pulsar, you can set *persistence policies* at the namespace level, which determines how BookKeeper handles persistent storage of messages. Policies determine four things: -* The number of acks (guaranteed copies) to wait for each ledger entry. -* The number of bookies to use for a topic. -* The number of writes to make for each ledger entry. +* Ensemble (E) size, Number of [bookies](reference-terminology.md#bookie) to use for storing entries in a ledger. +* Write quorum (Qw) size, Replication factor for storing entries (messages) in a ledger. +* Ack quorum (Qa) size, Number of guaranteed copies (acks to wait for before a write is considered completed). * The throttling rate for mark-delete operations. ### Set persistence policies @@ -272,21 +272,39 @@ You can set persistence policies for BookKeeper at the [namespace](reference-ter Use the [`set-persistence`](/tools/pulsar-admin/) subcommand and specify a namespace as well as any policies that you want to apply. The available flags are: -Flag | Description | Default -:----|:------------|:------- -`-a`, `--bookkeeper-ack-quorum` | The number of acks (guaranteed copies) to wait on for each entry | 0 -`-e`, `--bookkeeper-ensemble` | The number of [bookies](reference-terminology.md#bookie) to use for topics in the namespace | 0 -`-w`, `--bookkeeper-write-quorum` | The number of writes to make for each entry | 0 -`-r`, `--ml-mark-delete-max-rate` | Throttling rate for mark-delete operations (0 means no throttle) | 0 +Flag | Description | Default +:----|:---------------------------------------------------------------------------------------------------------------------------|:------- +`-e`, `--bookkeeper-ensemble` | Ensemble (E) size, Number of [bookies](reference-terminology.md#bookie) to use for storing entries in a ledger.| 0 +`-w`, `--bookkeeper-write-quorum` | Write quorum (Qw) size, Replication factor for storing entries (messages) in a ledger. | 0 +`-a`, `--bookkeeper-ack-quorum` | Ack quorum (Qa) size, Number of guaranteed copies (acks to wait for before a write is considered completed) | 0 +`-r`, `--ml-mark-delete-max-rate` | Throttling rate for mark-delete operations (0 means no throttle) | 0 + +Please notice that sticky reads enabled by `bookkeeperEnableStickyReads=true` aren’t used unless ensemble size (E) equals write quorum (Qw) size. Sticky reads improve the efficiency of the Bookkeeper read ahead cache when all reads for a single ledger are sent to a single bookie. + +Some rules for choosing the values: + +Rule | Description | +:----|:------------| +E >= Qw >= Qa|Ensemble size must be larger or equal than write quorum size, write quorum size must be larger or equal than ack quorum size. +Max bookie failures = Qa-1, |This rule must be fulfilled if data durability is desired in case of bookie failures. To safely tolerate at least one bookie failure at a time in the ensemble, Qa must be set to a value at least 2. +E == Qw | Sticky reads enabled by `bookkeeperEnableStickyReads=true` aren't used unless ensemble size (E) equals write quorum (Qw) size. The following is an example: ```shell pulsar-admin namespaces set-persistence my-tenant/my-ns \ ---bookkeeper-ack-quorum 2 \ ---bookkeeper-ensemble 3 +--bookkeeper-ensemble 3 \ +--bookkeeper-write-quorum 3 \ +--bookkeeper-ack-quorum 3 +``` + +Short example: + +```shell +pulsar-admin namespaces set-persistence my-tenant/my-ns -e 3 -w 3 -a 3 ``` + #### REST API {@inject: endpoint|POST|/admin/v2/namespaces/:tenant/:namespace/persistence|operation/setPersistence?version=@pulsar:version_number@} @@ -294,13 +312,14 @@ pulsar-admin namespaces set-persistence my-tenant/my-ns \ #### Java ```java -// The following must be true: bkEnsemble >= bkQuorum >= bkAckQuorum +// The following must be true: bkEnsemble >= bkWriteQuorum >= bkAckQuorum +// Please notice that sticky reads cannot be used unless bkEnsemble == bkWriteQuorum. int bkEnsemble = 3; -int bkQuorum = 2; -int bkAckQuorum = 2; +int bkWriteQuorum = 3; +int bkAckQuorum = 3; double markDeleteRate = 0.7; PersistencePolicies policies = - new PersistencePolicies(ensemble, quorum, ackQuorum, markDeleteRate); + new PersistencePolicies(bkEnsemble, bkWriteQuorum, bkAckQuorum, markDeleteRate); admin.namespaces().setPersistence(namespace, policies); ```