Go: revert changes to make and CI to postpone them in a separate PR

github · May 2, 2024 · 12b9b80 · 12b9b80
1 parent a8d3226
commit 12b9b80
Show file tree

Hide file tree

Showing 3 changed files with 191 additions and 14 deletions.
diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml
@@ -7,6 +7,8 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
+env:
+  GO_VERSION: '~1.22.0'
 
 permissions:
   contents: read
@@ -16,17 +18,72 @@ jobs:
     name: Test MacOS
     runs-on: macos-latest
     steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+        id: go
+
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Run tests
-        uses: ./go/actions/test
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
 
   test-win:
     if: github.repository_owner == 'github'
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+        id: go
+
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Run tests
-        uses: ./go/actions/test
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
@@ -16,6 +16,9 @@ on:
       - .github/actions/**
       - codeql-workspace.yml
 
+env:
+  GO_VERSION: '~1.22.0'
+
 permissions:
   contents: read
 
@@ -25,9 +28,51 @@ jobs:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+        id: go
+
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Run tests
-        uses: ./go/actions/test
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Check that all Go code is autoformatted
+        run: |
+          cd go
+          make check-formatting
+
+      - name: Compile qhelp files to markdown
+        run: |
+          cd go
+          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
+
+      - name: Upload qhelp markdown
+        uses: actions/upload-artifact@v3
         with:
-          run-code-checks: true
+          name: qhelp-markdown
+          path: go/qhelp-out/**/*.md
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
diff --git a/go/Makefile b/go/Makefile
@@ -1,10 +1,30 @@
-all: gen extractor
+all: extractor ql/lib/go.dbscheme
+
+ifeq ($(OS),Windows_NT)
+EXE = .exe
+CODEQL_PLATFORM = win64
+else
+EXE =
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+CODEQL_PLATFORM = linux64
+endif
+ifeq ($(UNAME_S),Darwin)
+CODEQL_PLATFORM = osx64
+endif
+endif
+
+CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh baseline-config-empty.json baseline-config-vendor.json configure-baseline.cmd configure-baseline.sh identify-environment.cmd identify-environment.sh index.cmd index.sh pre-finalize.cmd pre-finalize.sh tracing-config.lua)
 
 EXTRACTOR_PACK_OUT = build/codeql-extractor-go
 
-.PHONY: extractor gen clean autoformat check-formatting
+BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme
+
+.PHONY: tools tools-codeql tools-codeql-full clean autoformat \
+	tools-linux64 tools-osx64 tools-win64 check-formatting
 
 clean:
+	rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv
 	rm -rf $(EXTRACTOR_PACK_OUT) build/stats build/testdb
 
 autoformat:
@@ -27,11 +47,66 @@ endif
 qhelp-to-markdown:
 	scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)"
 
-extractor:
-	bazel run :create-extractor-pack
+tools: tools-codeql tools/tokenizer.jar
+
+.PHONY: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES)))
+$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))):
+	go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F))
+
+tools-codeql: tools-$(CODEQL_PLATFORM)
+
+tools-codeql-full: tools-linux64 tools-osx64 tools-win64
+
+tools-linux64: $(addprefix tools/linux64/,$(BINARIES))
+
+.PHONY: $(addprefix tools/linux64/,$(BINARIES))
+$(addprefix tools/linux64/,$(BINARIES)):
+	GOOS=linux GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(@F)
+
+tools-osx64: $(addprefix tools/osx64/,$(BINARIES))
+
+.PHONY: $(addprefix tools/osx64/,$(BINARIES))
+$(addprefix tools/osx64/,$(BINARIES)):
+	GOOS=darwin GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@.amd64 ./cli/$(@F)
+	GOOS=darwin GOARCH=arm64 go build -C extractor -mod=vendor -o ../$@.arm64 ./cli/$(@F)
+	lipo -create $@.amd64 $@.arm64 -output $@
+	rm $@.amd64 $@.arm64
+
+tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES)))
+
+.PHONY: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES)))
+$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))):
+	env GOOS=windows GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F))
+
+.PHONY: extractor-common extractor extractor-full
+extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \
+	tools/tokenizer.jar $(CODEQL_TOOLS)
+	rm -rf $(EXTRACTOR_PACK_OUT)
+	mkdir -p $(EXTRACTOR_PACK_OUT)
+	cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
+	mkdir $(EXTRACTOR_PACK_OUT)/tools
+	cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools
+	cp -r downgrades $(EXTRACTOR_PACK_OUT)
+
+extractor: extractor-common tools-codeql
+	cp -r tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools
+
+extractor-full: extractor-common tools-codeql-full
+	cp -r $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools
+
+tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class
+	jar cf $@ -C tools net
+	jar uf $@ -C tools opencsv
+
+tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cpd/GoLanguage.java
+	javac -cp extractor -d tools $<
+	rm tools/net/sourceforge/pmd/cpd/AbstractLanguage.class
+	rm tools/net/sourceforge/pmd/cpd/SourceCode.class
+	rm tools/net/sourceforge/pmd/cpd/TokenEntry.class
+	rm tools/net/sourceforge/pmd/cpd/Tokenizer.class
 
-gen:
-	bazel run :gen
+ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE)
+	$< $@
 
 build/stats/src.stamp:
 	mkdir -p $(@D)/src
@@ -48,7 +123,7 @@ test: all build/testdb/check-upgrade-path
 	codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
 #	use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported
 	env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
-	cd extractor; bazel test ...
+	cd extractor; go test -mod=vendor ./...
 	bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1)
 
 .PHONY: build/testdb/check-upgrade-path