From 3e678f2abc6818814adb0602f6efe3db87e01145 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 16 Feb 2023 10:33:47 -0700 Subject: [PATCH 1/7] support network name overrides in pack to build releases --- Makefile | 2 +- build/actors/pack.sh | 2 +- gen/bundle/bundle.go | 32 +++++++++++++++++++++++++------- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 023ca61f540..385ea3482d1 100644 --- a/Makefile +++ b/Makefile @@ -298,7 +298,7 @@ actors-gen: actors-code-gen fiximports .PHONY: actors-gen bundle-gen: - $(GOCC) run ./gen/bundle $(RELEASE) + $(GOCC) run ./gen/bundle $(RELEASE) $(RELEASE_OVERRIDES) $(GOCC) fmt ./build/... .PHONY: bundle-gen diff --git a/build/actors/pack.sh b/build/actors/pack.sh index 863a3c5c70f..b8d2bdb3dec 100755 --- a/build/actors/pack.sh +++ b/build/actors/pack.sh @@ -52,4 +52,4 @@ popd echo "Generating metadata..." -make -C ../../ RELEASE="$RELEASE" bundle-gen +make -C ../../ RELEASE="$RELEASE" RELEASE_OVERRIDES="${RELEASE_OVERRIDES[*]}" bundle-gen diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index d953b99c99a..da38f192f0c 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -2,6 +2,7 @@ package main import ( "os" + "strings" "text/template" "github.com/filecoin-project/lotus/build" @@ -32,19 +33,36 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet } `)) +func splitOverride(override string) (string, string) { + x := strings.Split(override, "=") + return x[0], x[1] +} + func main() { metadata, err := build.ReadEmbeddedBuiltinActorsMetadata() if err != nil { panic(err) } - // TODO: Re-enable this when we can set the tag for ONLY the appropriate version - // https://github.com/filecoin-project/lotus/issues/10185#issuecomment-1422864836 - //if len(os.Args) > 1 { - // for _, m := range metadata { - // m.BundleGitTag = os.Args[1] - // } - //} + // see ./build/actors/pack.sh + // expected args are git bundle tag then number of per network overrides + // overrides are in the format network_name=override + overrides := map[string]string{} + for _, override := range os.Args[2:] { + network, version := splitOverride(override) + overrides[network] = version + } + + if len(os.Args) > 1 { + for _, m := range metadata { + override, ok := overrides[m.Network] + if ok { + m.BundleGitTag = override + } else { + m.BundleGitTag = os.Args[1] + } + } + } fi, err := os.Create("./build/builtin_actors_gen.go") if err != nil { From 16b7d4525fe7cd7ba938658bb70746b365c89f0d Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 16 Feb 2023 10:40:16 -0700 Subject: [PATCH 2/7] lintfix --- gen/bundle/bundle.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index da38f192f0c..e369fd3dc26 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -44,9 +44,9 @@ func main() { panic(err) } - // see ./build/actors/pack.sh - // expected args are git bundle tag then number of per network overrides - // overrides are in the format network_name=override + // see ./build/actors/pack.sh + // expected args are git bundle tag then number of per network overrides + // overrides are in the format network_name=override overrides := map[string]string{} for _, override := range os.Args[2:] { network, version := splitOverride(override) From ea03e1b6e70969e3c4bc7194bdf12b4b2aac7a45 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 16 Feb 2023 13:10:25 -0700 Subject: [PATCH 3/7] only override the correct version for bundle gen --- gen/bundle/bundle.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index e369fd3dc26..908ef57e8a7 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "strings" "text/template" @@ -56,7 +57,7 @@ func main() { if len(os.Args) > 1 { for _, m := range metadata { override, ok := overrides[m.Network] - if ok { + if ok && strings.HasPrefix(override, fmt.Sprintf("v%d", m.Version)) { m.BundleGitTag = override } else { m.BundleGitTag = os.Args[1] From 0f6cbf1be25aa2d255051fcdc0b2e1d280910174 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 16 Feb 2023 14:01:45 -0700 Subject: [PATCH 4/7] include the version that pack is running on for filtering the build gen metadata --- Makefile | 2 +- build/actors/pack.sh | 2 +- gen/bundle/bundle.go | 30 ++++++++++++++++++++---------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 385ea3482d1..d1e7d159a11 100644 --- a/Makefile +++ b/Makefile @@ -298,7 +298,7 @@ actors-gen: actors-code-gen fiximports .PHONY: actors-gen bundle-gen: - $(GOCC) run ./gen/bundle $(RELEASE) $(RELEASE_OVERRIDES) + $(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES) $(GOCC) fmt ./build/... .PHONY: bundle-gen diff --git a/build/actors/pack.sh b/build/actors/pack.sh index b8d2bdb3dec..e594bb2daf6 100755 --- a/build/actors/pack.sh +++ b/build/actors/pack.sh @@ -52,4 +52,4 @@ popd echo "Generating metadata..." -make -C ../../ RELEASE="$RELEASE" RELEASE_OVERRIDES="${RELEASE_OVERRIDES[*]}" bundle-gen +make -C ../../ VERSION="$VERSION" RELEASE="$RELEASE" RELEASE_OVERRIDES="${RELEASE_OVERRIDES[*]}" bundle-gen diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index 908ef57e8a7..a1eab4e0466 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -45,24 +45,34 @@ func main() { panic(err) } + var metadataSpecificVersion []*build.BuiltinActorsMetadata + // see ./build/actors/pack.sh // expected args are git bundle tag then number of per network overrides // overrides are in the format network_name=override - overrides := map[string]string{} - for _, override := range os.Args[2:] { - network, version := splitOverride(override) - overrides[network] = version - } if len(os.Args) > 1 { + version := os.Args[1] + overrides := map[string]string{} + for _, override := range os.Args[3:] { + k, v := splitOverride(override) + overrides[k] = v + } for _, m := range metadata { - override, ok := overrides[m.Network] - if ok && strings.HasPrefix(override, fmt.Sprintf("v%d", m.Version)) { - m.BundleGitTag = override - } else { - m.BundleGitTag = os.Args[1] + if strings.HasPrefix(version, fmt.Sprintf("v%d", m.Version)) { + // correct version + override, ok := overrides[m.Network] + if ok { + m.BundleGitTag = override + } else { + m.BundleGitTag = os.Args[2] + } + fmt.Println("hi") + metadataSpecificVersion = append(metadataSpecificVersion, m) } } + metadata = metadataSpecificVersion + } fi, err := os.Create("./build/builtin_actors_gen.go") From dbcf30470f8ddb1e916ab230dee4bd7a97a9f39e Mon Sep 17 00:00:00 2001 From: mike seiler Date: Fri, 17 Feb 2023 11:16:40 -0700 Subject: [PATCH 5/7] clean up comment remove print --- gen/bundle/bundle.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index a1eab4e0466..bc038569df7 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -46,9 +46,9 @@ func main() { } var metadataSpecificVersion []*build.BuiltinActorsMetadata - // see ./build/actors/pack.sh - // expected args are git bundle tag then number of per network overrides + // expected args are: + // $(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES) // overrides are in the format network_name=override if len(os.Args) > 1 { @@ -67,7 +67,6 @@ func main() { } else { m.BundleGitTag = os.Args[2] } - fmt.Println("hi") metadataSpecificVersion = append(metadataSpecificVersion, m) } } From 4dedced1752e7fefebb3196b98ed6619bb447c99 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Fri, 17 Feb 2023 11:52:40 -0700 Subject: [PATCH 6/7] lintfix --- gen/bundle/bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index bc038569df7..d3e5ebf6bee 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -47,7 +47,7 @@ func main() { var metadataSpecificVersion []*build.BuiltinActorsMetadata // see ./build/actors/pack.sh - // expected args are: + // expected args are: // $(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES) // overrides are in the format network_name=override From f41435d2858a00bf465252aa7f003c74b731bd0e Mon Sep 17 00:00:00 2001 From: Aayush Date: Fri, 17 Feb 2023 14:17:30 -0500 Subject: [PATCH 7/7] feat: bundle: only write git tags for newly packed bundles --- gen/bundle/bundle.go | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go index d3e5ebf6bee..f57ced15b43 100644 --- a/gen/bundle/bundle.go +++ b/gen/bundle/bundle.go @@ -1,8 +1,8 @@ package main import ( - "fmt" "os" + "strconv" "strings" "text/template" @@ -40,38 +40,42 @@ func splitOverride(override string) (string, string) { } func main() { + // read metadata from the embedded bundle, includes all info except git tags metadata, err := build.ReadEmbeddedBuiltinActorsMetadata() if err != nil { panic(err) } - var metadataSpecificVersion []*build.BuiltinActorsMetadata - // see ./build/actors/pack.sh - // expected args are: - // $(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES) - // overrides are in the format network_name=override - + // IF args have been provided, extract git tag info from them, otherwise + // rely on previously embedded metadata for git tags. if len(os.Args) > 1 { - version := os.Args[1] + // see ./build/actors/pack.sh + // (optional) expected args are: + // $(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES) + // overrides are in the format network_name=override + gitTag := os.Args[2] + packedActorsVersion, err := strconv.Atoi(os.Args[1][1:]) + if err != nil { + panic(err) + } + overrides := map[string]string{} for _, override := range os.Args[3:] { k, v := splitOverride(override) overrides[k] = v } for _, m := range metadata { - if strings.HasPrefix(version, fmt.Sprintf("v%d", m.Version)) { - // correct version + if int(m.Version) == packedActorsVersion { override, ok := overrides[m.Network] if ok { m.BundleGitTag = override } else { - m.BundleGitTag = os.Args[2] + m.BundleGitTag = gitTag } - metadataSpecificVersion = append(metadataSpecificVersion, m) + } else { + m.BundleGitTag = getOldGitTagFromEmbeddedMetadata(m) } } - metadata = metadataSpecificVersion - } fi, err := os.Create("./build/builtin_actors_gen.go") @@ -85,3 +89,14 @@ func main() { panic(err) } } + +func getOldGitTagFromEmbeddedMetadata(m *build.BuiltinActorsMetadata) string { + for _, v := range build.EmbeddedBuiltinActorsMetadata { + // if we agree on the manifestCid for the previously embedded metadata, use the previously set tag + if m.Version == v.Version && m.Network == v.Network && m.ManifestCid == v.ManifestCid { + return m.BundleGitTag + } + } + + return "" +}