From 7e8a5b04e0de32d9eb58543aee35407c7c86f86a Mon Sep 17 00:00:00 2001 From: srinandan Date: Mon, 30 Aug 2021 13:09:25 -0700 Subject: [PATCH] support exporting all revisions --- client/apis/apis.go | 70 +++++++++++++++++++++++++++++++++++++++------ cmd/apis/expapis.go | 22 +++++++++++++- cmd/org/export.go | 6 +++- 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/client/apis/apis.go b/client/apis/apis.go index f0623deb..e5f67cc4 100644 --- a/client/apis/apis.go +++ b/client/apis/apis.go @@ -259,7 +259,7 @@ func CleanProxy(name string, reportOnly bool, keepList []string) (err error) { } //ExportProxies -func ExportProxies(conn int, folder string) (err error) { +func ExportProxies(conn int, folder string, allRevisions bool) (err error) { //parent workgroup var pwg sync.WaitGroup const entityType = "apis" @@ -299,7 +299,7 @@ func ExportProxies(conn int, folder string) (err error) { pwg.Add(1) end = (i * conn) + conn clilog.Info.Printf("Exporting batch %d of proxies\n", (i + 1)) - go batchExport(entities.Proxies[start:end], entityType, folder, &pwg) + go batchExport(entities.Proxies[start:end], entityType, folder, allRevisions, &pwg) start = end pwg.Wait() } @@ -307,7 +307,7 @@ func ExportProxies(conn int, folder string) (err error) { if remaining > 0 { pwg.Add(1) clilog.Info.Printf("Exporting remaining %d proxies\n", remaining) - go batchExport(entities.Proxies[start:numEntities], entityType, folder, &pwg) + go batchExport(entities.Proxies[start:numEntities], entityType, folder, allRevisions, &pwg) pwg.Wait() } @@ -367,23 +367,75 @@ func ImportProxies(conn int, folder string) error { return nil } -func batchExport(entities []proxy, entityType string, folder string, pwg *sync.WaitGroup) { +func batchExport(entities []proxy, entityType string, folder string, allRevisions bool, pwg *sync.WaitGroup) { defer pwg.Done() //batch workgroup var bwg sync.WaitGroup + //proxy revision wait group + var prwg sync.WaitGroup + //number of revisions to download concurrently + conn := 2 - bwg.Add(len(entities)) + numProxies := len(entities) + bwg.Add(numProxies) for _, entity := range entities { - //download only the last revision - lastRevision := maxRevision(entity.Revision) - clilog.Info.Printf("Downloading revision %s of proxy %s\n", lastRevision, entity.Name) - go apiclient.FetchAsyncBundle(entityType, folder, entity.Name, lastRevision, &bwg) + if !allRevisions { + //download only the last revision + lastRevision := maxRevision(entity.Revision) + clilog.Info.Printf("Downloading revision %s of proxy %s\n", lastRevision, entity.Name) + go apiclient.FetchAsyncBundle(entityType, folder, entity.Name, lastRevision, &bwg) + } else { + //download all revisions + if len(entity.Revision) == 1 { + lastRevision := maxRevision(entity.Revision) + clilog.Info.Printf("Downloading revision %s of proxy %s\n", lastRevision, entity.Name) + apiclient.FetchAsyncBundle(entityType, folder, entity.Name, lastRevision, &bwg) + } else { + + numRevisions := len(entity.Revision) + numOfLoops, remaining := numRevisions/conn, numRevisions%conn + + start := 0 + + for i, end := 0, 0; i < numOfLoops; i++ { + prwg.Add(1) + end = (i * conn) + conn + clilog.Info.Printf("Exporting batch %d of proxy revisions\n", (i + 1)) + go batchExportRevisions(entityType, folder, entity.Name, entity.Revision[start:end], &prwg) + start = end + prwg.Wait() + } + + if remaining > 0 { + prwg.Add(1) + clilog.Info.Printf("Exporting remaining %d proxy revisions\n", remaining) + go batchExportRevisions(entityType, folder, entity.Name, entity.Revision[start:numRevisions], &prwg) + prwg.Wait() + } + bwg.Done() + } + } } + bwg.Wait() } +func batchExportRevisions(entityType string, folder string, name string, revisions []string, prwg *sync.WaitGroup) { + defer prwg.Done() + //batch workgroup + var brwg sync.WaitGroup + + brwg.Add(len(revisions)) + + for _, revision := range revisions { + clilog.Info.Printf("Exporting proxy %s revision %s\n", name, revision) + go apiclient.FetchAsyncBundle(entityType, folder, name, revision, &brwg) + } + brwg.Wait() +} + //batch creates a batch of proxies to import func batchImport(entities []string, pwg *sync.WaitGroup) { diff --git a/cmd/apis/expapis.go b/cmd/apis/expapis.go index 69b6591a..46636aee 100644 --- a/cmd/apis/expapis.go +++ b/cmd/apis/expapis.go @@ -15,6 +15,9 @@ package apis import ( + "fmt" + "os" + "github.com/spf13/cobra" "github.com/srinandan/apigeecli/apiclient" "github.com/srinandan/apigeecli/client/apis" @@ -29,15 +32,32 @@ var ExpCmd = &cobra.Command{ return apiclient.SetApigeeOrg(org) }, RunE: func(cmd *cobra.Command, args []string) (err error) { - return apis.ExportProxies(conn, folder) + if err = folderExists(folder); err != nil { + return err + } + return apis.ExportProxies(conn, folder, allRevisions) }, } +var allRevisions bool + func init() { ExpCmd.Flags().IntVarP(&conn, "conn", "c", 4, "Number of connections") ExpCmd.Flags().StringVarP(&folder, "folder", "f", "", "folder to export API proxy bundles") + ExpCmd.Flags().BoolVarP(&allRevisions, "all", "", + false, "Export all proxy revisions") +} +func folderExists(folder string) (err error) { + if folder == "" { + return nil + } + _, err = os.Stat(folder) + if err != nil { + return fmt.Errorf("folder not found or write permission denied") + } + return nil } diff --git a/cmd/org/export.go b/cmd/org/export.go index 34ba1700..def43013 100644 --- a/cmd/org/export.go +++ b/cmd/org/export.go @@ -60,7 +60,7 @@ var ExportCmd = &cobra.Command{ clilog.Warning.Println("Calls to Apigee APIs have a quota of 6000 per min. Running this tool against large list of entities can exhaust that quota and impact the usage of the platform.") fmt.Println("Exporting API Proxies...") - if err = apis.ExportProxies(conn, proxiesFolderName); err != nil { + if err = apis.ExportProxies(conn, proxiesFolderName, allRevisions); err != nil { return err } @@ -171,12 +171,16 @@ var ExportCmd = &cobra.Command{ }, } +var allRevisions bool + func init() { ExportCmd.Flags().StringVarP(&org, "org", "o", "", "Apigee organization name") ExportCmd.Flags().IntVarP(&conn, "conn", "c", 4, "Number of connections") + ExportCmd.Flags().BoolVarP(&allRevisions, "all", "", + false, "Expose all revisions") } func createFolders() (err error) {