diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index 9915b0869db7..ded53f868b70 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,6 +7,8 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' permissions: contents: read @@ -16,17 +18,72 @@ jobs: name: Test MacOS runs-on: macos-latest steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" test-win: if: github.repository_owner == 'github' name: Test Windows runs-on: windows-latest-xl steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 63e2b7c49740..6d9cac5dae91 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,6 +16,9 @@ on: - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' + permissions: contents: read @@ -25,9 +28,51 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Check that all Go code is autoformatted + run: | + cd go + make check-formatting + + - name: Compile qhelp files to markdown + run: | + cd go + env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown + + - name: Upload qhelp markdown + uses: actions/upload-artifact@v3 with: - run-code-checks: true + name: qhelp-markdown + path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" diff --git a/go/Makefile b/go/Makefile index 3ebeb69fcaf1..d0289a093a52 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,10 +1,30 @@ -all: gen extractor +all: extractor ql/lib/go.dbscheme + +ifeq ($(OS),Windows_NT) +EXE = .exe +CODEQL_PLATFORM = win64 +else +EXE = +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +CODEQL_PLATFORM = linux64 +endif +ifeq ($(UNAME_S),Darwin) +CODEQL_PLATFORM = osx64 +endif +endif + +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh baseline-config-empty.json baseline-config-vendor.json configure-baseline.cmd configure-baseline.sh identify-environment.cmd identify-environment.sh index.cmd index.sh pre-finalize.cmd pre-finalize.sh tracing-config.lua) EXTRACTOR_PACK_OUT = build/codeql-extractor-go -.PHONY: extractor gen clean autoformat check-formatting +BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme + +.PHONY: tools tools-codeql tools-codeql-full clean autoformat \ + tools-linux64 tools-osx64 tools-win64 check-formatting clean: + rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv rm -rf $(EXTRACTOR_PACK_OUT) build/stats build/testdb autoformat: @@ -27,11 +47,66 @@ endif qhelp-to-markdown: scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)" -extractor: - bazel run :create-extractor-pack +tools: tools-codeql tools/tokenizer.jar + +.PHONY: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) +$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): + go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) + +tools-codeql: tools-$(CODEQL_PLATFORM) + +tools-codeql-full: tools-linux64 tools-osx64 tools-win64 + +tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) + +.PHONY: $(addprefix tools/linux64/,$(BINARIES)) +$(addprefix tools/linux64/,$(BINARIES)): + GOOS=linux GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(@F) + +tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) + +.PHONY: $(addprefix tools/osx64/,$(BINARIES)) +$(addprefix tools/osx64/,$(BINARIES)): + GOOS=darwin GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@.amd64 ./cli/$(@F) + GOOS=darwin GOARCH=arm64 go build -C extractor -mod=vendor -o ../$@.arm64 ./cli/$(@F) + lipo -create $@.amd64 $@.arm64 -output $@ + rm $@.amd64 $@.arm64 + +tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) + +.PHONY: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) +$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): + env GOOS=windows GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) + +.PHONY: extractor-common extractor extractor-full +extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \ + tools/tokenizer.jar $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r downgrades $(EXTRACTOR_PACK_OUT) + +extractor: extractor-common tools-codeql + cp -r tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools + +extractor-full: extractor-common tools-codeql-full + cp -r $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools + +tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class + jar cf $@ -C tools net + jar uf $@ -C tools opencsv + +tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cpd/GoLanguage.java + javac -cp extractor -d tools $< + rm tools/net/sourceforge/pmd/cpd/AbstractLanguage.class + rm tools/net/sourceforge/pmd/cpd/SourceCode.class + rm tools/net/sourceforge/pmd/cpd/TokenEntry.class + rm tools/net/sourceforge/pmd/cpd/Tokenizer.class -gen: - bazel run :gen +ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE) + $< $@ build/stats/src.stamp: mkdir -p $(@D)/src @@ -48,7 +123,7 @@ test: all build/testdb/check-upgrade-path codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) # use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) - cd extractor; bazel test ... + cd extractor; go test -mod=vendor ./... bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1) .PHONY: build/testdb/check-upgrade-path