Skip to content
This repository has been archived by the owner on May 6, 2022. It is now read-only.

Commit

Permalink
fix #18: dont append revision
Browse files Browse the repository at this point in the history
  • Loading branch information
srinandan committed Jan 15, 2022
1 parent 7eef8f5 commit 6e6c8d1
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 26 deletions.
12 changes: 12 additions & 0 deletions apiclient/bundles.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"archive/zip"
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
Expand Down Expand Up @@ -274,3 +275,14 @@ func ImportBundle(entityType string, name string, bundlePath string) error {
clilog.Info.Printf("Completed entity: %s", u.String())
return nil
}

func FolderExists(folder string) (err error) {
if folder == "" {
return nil
}
_, err = os.Stat(folder)
if err != nil {
return fmt.Errorf("folder not found or write permission denied")
}
return nil
}
70 changes: 61 additions & 9 deletions client/sharedflows/sharedflows.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ func Fetch(name string, revision int) (err error) {
}

//Export
func Export(conn int, folder string) (err error) {
func Export(conn int, folder string, allRevisions bool) (err error) {
//parent workgroup
var pwg sync.WaitGroup
const entityType = "sharedflows"
Expand Down Expand Up @@ -298,15 +298,15 @@ func Export(conn int, folder string) (err error) {
pwg.Add(1)
end = (i * conn) + conn
clilog.Info.Printf("Exporting batch %d of proxies\n", (i + 1))
go batchExport(entities.Proxies[start:end], entityType, folder, &pwg)
go batchExport(entities.Proxies[start:end], entityType, folder, allRevisions, &pwg)
start = end
pwg.Wait()
}

if remaining > 0 {
pwg.Add(1)
clilog.Info.Printf("Exporting remaining %d proxies\n", remaining)
go batchExport(entities.Proxies[start:numEntities], entityType, folder, &pwg)
go batchExport(entities.Proxies[start:numEntities], entityType, folder, allRevisions, &pwg)
pwg.Wait()
}

Expand Down Expand Up @@ -365,22 +365,74 @@ func Import(conn int, folder string) error {
return nil
}

func batchExport(entities []sharedflow, entityType string, folder string, pwg *sync.WaitGroup) {
func batchExport(entities []sharedflow, entityType string, folder string, allRevisions bool, pwg *sync.WaitGroup) {
defer pwg.Done()
//batch workgroup
var bwg sync.WaitGroup
//proxy revision wait group
var prwg sync.WaitGroup
//number of revisions to download concurrently
conn := 2

bwg.Add(len(entities))
numSharedFlows := len(entities)
bwg.Add(numSharedFlows)

for _, entity := range entities {
//download only the last revision
lastRevision := maxRevision(entity.Revision)
clilog.Info.Printf("Downloading revision %s of sharedflow %s\n", lastRevision, entity.Name)
go apiclient.FetchAsyncBundle(entityType, folder, entity.Name, lastRevision, true, &bwg)
if !allRevisions {
//download only the last revision
lastRevision := maxRevision(entity.Revision)
clilog.Info.Printf("Downloading revision %s of sharedflow %s\n", lastRevision, entity.Name)
go apiclient.FetchAsyncBundle(entityType, folder, entity.Name, lastRevision, allRevisions, &bwg)
} else {
//download all revisions
if len(entity.Revision) == 1 {
lastRevision := maxRevision(entity.Revision)
clilog.Info.Printf("Downloading revision %s of sharedflow %s\n", lastRevision, entity.Name)
go apiclient.FetchAsyncBundle(entityType, folder, entity.Name, lastRevision, allRevisions, &bwg)
} else {

numRevisions := len(entity.Revision)
numOfLoops, remaining := numRevisions/conn, numRevisions%conn

start := 0

for i, end := 0, 0; i < numOfLoops; i++ {
prwg.Add(1)
end = (i * conn) + conn
clilog.Info.Printf("Exporting batch %d of sharedflow revisions\n", (i + 1))
go batchExportRevisions(entityType, folder, entity.Name, entity.Revision[start:end], &prwg)
start = end
prwg.Wait()
}

if remaining > 0 {
prwg.Add(1)
clilog.Info.Printf("Exporting remaining %d sharedflow revisions\n", remaining)
go batchExportRevisions(entityType, folder, entity.Name, entity.Revision[start:numRevisions], &prwg)
prwg.Wait()
}
bwg.Done()
}
}
}
bwg.Wait()
}

//batchExportRevisions
func batchExportRevisions(entityType string, folder string, name string, revisions []string, prwg *sync.WaitGroup) {
defer prwg.Done()
//batch workgroup
var brwg sync.WaitGroup

brwg.Add(len(revisions))

for _, revision := range revisions {
clilog.Info.Printf("Exporting sharedflow %s revision %s\n", name, revision)
go apiclient.FetchAsyncBundle(entityType, folder, name, revision, true, &brwg)
}
brwg.Wait()
}

//batch creates a batch of sharedflow to import
func batchImport(entities []string, pwg *sync.WaitGroup) {

Expand Down
16 changes: 1 addition & 15 deletions cmd/apis/expapis.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@
package apis

import (
"fmt"
"os"

"github.com/spf13/cobra"
"github.com/srinandan/apigeecli/apiclient"
"github.com/srinandan/apigeecli/client/apis"
Expand All @@ -32,7 +29,7 @@ var ExpCmd = &cobra.Command{
return apiclient.SetApigeeOrg(org)
},
RunE: func(cmd *cobra.Command, args []string) (err error) {
if err = folderExists(folder); err != nil {
if err = apiclient.FolderExists(folder); err != nil {
return err
}
return apis.ExportProxies(conn, folder, allRevisions)
Expand All @@ -50,14 +47,3 @@ func init() {
ExpCmd.Flags().BoolVarP(&allRevisions, "all", "",
false, "Export all proxy revisions")
}

func folderExists(folder string) (err error) {
if folder == "" {
return nil
}
_, err = os.Stat(folder)
if err != nil {
return fmt.Errorf("folder not found or write permission denied")
}
return nil
}
2 changes: 1 addition & 1 deletion cmd/org/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ var ExportCmd = &cobra.Command{
}

fmt.Println("Exporting Sharedflows...")
if err = sharedflows.Export(conn, sharedFlowsFolderName); err != nil {
if err = sharedflows.Export(conn, sharedFlowsFolderName, allRevisions); err != nil {
return err
}

Expand Down
9 changes: 8 additions & 1 deletion cmd/sharedflows/expsf.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,21 @@ var ExpCmd = &cobra.Command{
return apiclient.SetApigeeOrg(org)
},
RunE: func(cmd *cobra.Command, args []string) (err error) {
return sharedflows.Export(conn, folder)
if err = apiclient.FolderExists(folder); err != nil {
return err
}
return sharedflows.Export(conn, folder, allRevisions)
},
}

var allRevisions bool

func init() {

ExpCmd.Flags().IntVarP(&conn, "conn", "c",
4, "Number of connections")
ExpCmd.Flags().StringVarP(&folder, "folder", "f",
"", "folder to export sharedflow bundles")
ExpCmd.Flags().BoolVarP(&allRevisions, "all", "",
false, "Export all proxy revisions")
}

0 comments on commit 6e6c8d1

Please sign in to comment.