Add admin failover for managed domains

tools/cli/admin.go

@@ -811,6 +811,20 @@ func newAdminClusterCommands() []cli.Command {
 				AdminDescribeCluster(c)
 			},
 		},
+		{
+			Name:    "failover",
+			Aliases: []string{"fo"},
+			Usage:   "Failover domains with domain data IsManagedByCadence=true to target cluster",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  FlagActiveClusterNameWithAlias,
+					Usage: "Target active cluster name",
+				},
+			},
+			Action: func(c *cli.Context) {
+				newDomainCLI(c, false).FailoverDomains(c)
+			},
+		},
 	}
 }
 

tools/cli/app_test.go

@@ -525,6 +525,38 @@ func (s *cliAppSuite) TestAdminAddSearchAttribute() {
 	s.Nil(err)
 }
 
+func (s *cliAppSuite) TestAdminFailover() {
+	var listDomainsResponse = &serverShared.ListDomainsResponse{
+		Domains: []*serverShared.DescribeDomainResponse{
+			{
+				DomainInfo: &serverShared.DomainInfo{
+					Name:        common.StringPtr("test-domain"),
+					Description: common.StringPtr("a test domain"),
+					OwnerEmail:  common.StringPtr("[email protected]"),
+					Data: map[string]string{
+						managedFailoverDomainData: "true",
+					},
+				},
+				ReplicationConfiguration: &serverShared.DomainReplicationConfiguration{
+					ActiveClusterName: common.StringPtr("active"),
+					Clusters: []*serverShared.ClusterReplicationConfiguration{
+						{
+							ClusterName: common.StringPtr("active"),
+						},
+						{
+							ClusterName: common.StringPtr("standby"),
+						},
+					},
+				},
+			},
+		},
+	}
+	s.serverFrontendClient.EXPECT().ListDomains(gomock.Any(), gomock.Any()).Return(listDomainsResponse, nil).Times(1)
+	s.serverFrontendClient.EXPECT().UpdateDomain(gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
+	err := s.app.Run([]string{"", "admin", "cl", "fo", "--ac", "standby"})
+	s.Nil(err)
+}
+
 func (s *cliAppSuite) TestDescribeTaskList() {
 	resp := describeTaskListResponse
 	s.clientFrontendClient.EXPECT().DescribeTaskList(gomock.Any(), gomock.Any(), callOptions...).Return(resp, nil)

tools/cli/defs.go

@@ -65,6 +65,8 @@ const (
 	showErrorStackEnv    = `CADENCE_CLI_SHOW_STACKS`
 
 	searchAttrInputSeparator = "|"
+
+	managedFailoverDomainData = "IsManagedByCadence"
 )
 
 var envKeysForUserName = []string{

tools/cli/domainCommands.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/olekukonko/tablewriter"
@@ -282,6 +283,72 @@ func (d *domainCLIImpl) UpdateDomain(c *cli.Context) {
 	}
 }
 
+// FailoverDomains is used for managed failover all domains with domain data IsManagedByCadence=true
+func (d *domainCLIImpl) FailoverDomains(c *cli.Context) {
+	targetCluster := getRequiredOption(c, FlagActiveClusterName)
+
+	domains := d.getAllDomains()
+	shouldFailover := func(domain *shared.DescribeDomainResponse) bool {
+		isDomainNotActiveInTargetCluster := domain.ReplicationConfiguration.GetActiveClusterName() != targetCluster
+		return isDomainNotActiveInTargetCluster && isDomainFailoverManagedByCadence(domain)
+	}
+	var succeedDomains []string
+	var failedDomains []string
+	for _, domain := range domains {
+		if shouldFailover(domain) {
+			domainName := domain.GetDomainInfo().GetName()
+			err := d.failover(c, domainName, targetCluster)
+			if err != nil {
+				printError(fmt.Sprintf("Failed failover domain: %s\n", domainName), err)
+				failedDomains = append(failedDomains, domainName)
+			} else {
+				fmt.Printf("Success failover domain: %s\n", domainName)
+				succeedDomains = append(succeedDomains, domainName)
+			}
+		}
+	}
+	fmt.Printf("Succeed %d: %v\n", len(succeedDomains), succeedDomains)
+	fmt.Printf("Failed  %d: %v\n", len(failedDomains), failedDomains)
+}
+
+func (d *domainCLIImpl) getAllDomains() []*shared.DescribeDomainResponse {
+	var res []*shared.DescribeDomainResponse
+	pagesize := int32(200)
+	var token []byte
+	for more := true; more; more = len(token) > 0 {
+		listRequest := &shared.ListDomainsRequest{
+			PageSize:      common.Int32Ptr(pagesize),
+			NextPageToken: token,
+		}
+		listResp, err := d.listDomains(context.Background(), listRequest)
+		if err != nil {
+			ErrorAndExit("Error when list domains info", err)
+		}
+		token = listResp.GetNextPageToken()
+		res = append(res, listResp.GetDomains()...)
+	}
+	return res
+}
+
+func isDomainFailoverManagedByCadence(domain *shared.DescribeDomainResponse) bool {
+	domainData := domain.DomainInfo.GetData()
+	return strings.ToLower(strings.TrimSpace(domainData[managedFailoverDomainData])) == "true"
+}
+
+func (d *domainCLIImpl) failover(c *cli.Context, domainName string, targetCluster string) error {
+	replicationConfig := &shared.DomainReplicationConfiguration{
+		ActiveClusterName: common.StringPtr(targetCluster),
+	}
+	updateRequest := &shared.UpdateDomainRequest{
+		Name:                     common.StringPtr(domainName),
+		ReplicationConfiguration: replicationConfig,
+	}
+	ctx, cancel := newContext(c)
+	defer cancel()
+	_, err := d.updateDomain(ctx, updateRequest)
+	return err
+}
+
 // DescribeDomain updates a domain
 func (d *domainCLIImpl) DescribeDomain(c *cli.Context) {
 	domainName := c.GlobalString(FlagDomain)
@@ -349,6 +416,18 @@ func (d *domainCLIImpl) DescribeDomain(c *cli.Context) {
 	}
 }
 
+func (d *domainCLIImpl) listDomains(
+	ctx context.Context,
+	request *shared.ListDomainsRequest,
+) (*shared.ListDomainsResponse, error) {
+
+	if d.frontendClient != nil {
+		return d.frontendClient.ListDomains(ctx, request)
+	}
+
+	return d.domainHandler.ListDomains(ctx, request)
+}
+
 func (d *domainCLIImpl) registerDomain(
 	ctx context.Context,
 	request *shared.RegisterDomainRequest,

tools/cli/util.go

@@ -509,8 +509,7 @@ func mapKeysToArray(m map[string]string) []string {
 	return out
 }
 
-// ErrorAndExit print easy to understand error msg first then error detail in a new line
-func ErrorAndExit(msg string, err error) {
+func printError(msg string, err error) {
 	if err != nil {
 		fmt.Printf("%s %s\n%s %+v\n", colorRed("Error:"), msg, colorMagenta("Error Details:"), err)
 		if os.Getenv(showErrorStackEnv) != `` {
@@ -522,6 +521,11 @@ func ErrorAndExit(msg string, err error) {
 	} else {
 		fmt.Printf("%s %s\n", colorRed("Error:"), msg)
 	}
+}
+
+// ErrorAndExit print easy to understand error msg first then error detail in a new line
+func ErrorAndExit(msg string, err error) {
+	printError(msg, err)
 	osExit(1)
 }