From 647945c54e71abafb9c01cac95b8dd4e21d11afc Mon Sep 17 00:00:00 2001 From: Felipe Martin <812088+fmartingr@users.noreply.github.com> Date: Tue, 14 May 2024 08:01:52 +0200 Subject: [PATCH] ci: unify local and ci docker workflows (#907) * deps: go to 1.22.3 * removed ci dockerfile * add make buildx * updated dockerfile * ci uses make buildx command * commented upx for the future * disable openbsd/arm tests * wip * put dist file in dist path * removed unused make command * build-local to speed up local tests * don't clean when buildx * podman workaround * manually define source files for tests --- .github/workflows/_buildx.yml | 14 ++------- .github/workflows/_test.yml | 3 -- .github/workflows/docker/Dockerfile.ci | 11 ------- .github/workflows/docker/etc/group | 2 -- .github/workflows/docker/etc/passwd | 2 -- .gitignore | 3 ++ .goreleaser.yaml | 9 ++++++ Dockerfile | 41 ++++++++++++++++--------- Makefile | 42 ++++++++++++++++++++++++-- go.mod | 2 +- scripts/buildx.sh | 30 ++++++++++++++++++ scripts/test.sh | 4 +-- 12 files changed, 113 insertions(+), 50 deletions(-) delete mode 100644 .github/workflows/docker/Dockerfile.ci delete mode 100644 .github/workflows/docker/etc/group delete mode 100644 .github/workflows/docker/etc/passwd create mode 100755 scripts/buildx.sh diff --git a/.github/workflows/_buildx.yml b/.github/workflows/_buildx.yml index 8dbbfd987..a9415258b 100644 --- a/.github/workflows/_buildx.yml +++ b/.github/workflows/_buildx.yml @@ -14,15 +14,7 @@ jobs: - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # 4.1.7 with: name: dist - - name: Prep binaries - working-directory: .github/workflows/docker - run: | - mkdir binaries - cp -r ../../../shiori_linux_* binaries/ - mv binaries/shiori_linux_arm_7 binaries/shiori_linux_arm - mv binaries/shiori_linux_amd64_v1 binaries/shiori_linux_amd64 - gzip -d -S binaries/.gz__ -r . - chmod 755 binaries/shiori_linux_*/shiori + path: ./dist # Every pull request that goes into master - name: Prepare master push tags @@ -55,9 +47,7 @@ jobs: echo "tag_flags=--tag $REPO:pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV - name: Buildx - working-directory: .github/workflows/docker run: | set -x echo "${{ secrets.GITHUB_TOKEN }}" | docker login -u "${{ github.repository_owner }}" --password-stdin ghcr.io - docker buildx create --use --name builder - docker buildx build -f Dockerfile.ci --platform=linux/amd64,arm64,linux/arm/v7 --push ${{ env.tag_flags }} . + make buildx CONTAINER_BUILDX_OPTIONS="--push ${{ env.tag_flags }}" diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 0b0a4dc76..46ea8f576 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -107,9 +107,6 @@ jobs: architecture: x86-64 version: '7.5' - - name: openbsd - architecture: arm64 - version: '7.5' steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 diff --git a/.github/workflows/docker/Dockerfile.ci b/.github/workflows/docker/Dockerfile.ci deleted file mode 100644 index 679f32afc..000000000 --- a/.github/workflows/docker/Dockerfile.ci +++ /dev/null @@ -1,11 +0,0 @@ -FROM docker.io/alpine:3.19 -ARG TARGETARCH -LABEL org.opencontainers.image.source https://github.com/go-shiori/shiori -COPY etc /etc -COPY binaries/shiori_linux_$TARGETARCH/shiori /usr/bin/shiori -USER shiori -WORKDIR /shiori -EXPOSE 8080 -ENV SHIORI_DIR=/shiori -ENTRYPOINT ["/usr/bin/shiori"] -CMD ["server"] diff --git a/.github/workflows/docker/etc/group b/.github/workflows/docker/etc/group deleted file mode 100644 index 1aae0e48c..000000000 --- a/.github/workflows/docker/etc/group +++ /dev/null @@ -1,2 +0,0 @@ -root:x:0:root -shiori:x:1000:shiori \ No newline at end of file diff --git a/.github/workflows/docker/etc/passwd b/.github/workflows/docker/etc/passwd deleted file mode 100644 index 4c33c088e..000000000 --- a/.github/workflows/docker/etc/passwd +++ /dev/null @@ -1,2 +0,0 @@ -root:x:0:0::/root:/bin/ash -shiori:x:1000:1000::/shiori:/bin/ash \ No newline at end of file diff --git a/.gitignore b/.gitignore index bb7f726cb..e096d70e6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ dist/ # frontend node_modules + +# workaround for buildx using podman +type=docker diff --git a/.goreleaser.yaml b/.goreleaser.yaml index bdb8a14de..a1787a81f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -36,6 +36,15 @@ archives: - goos: windows format: zip +# TODO: +# upx: +# - enabled: true +# ids: +# - shiori +# goos: [linux, darwin] +# goarch: [amd64, arm, arm64] +# goarm: ["7"] + checksum: name_template: 'checksums.txt' snapshot: diff --git a/Dockerfile b/Dockerfile index 2153baf6d..8a6f3894c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,30 @@ -# build stage -FROM ghcr.io/ghcri/golang:1.21-alpine3.19 AS builder -WORKDIR /src -COPY . . -RUN go build -ldflags '-s -w' +# Build stage +ARG ALPINE_VERSION +ARG GOLANG_VERSION -# server image +FROM docker.io/library/alpine:${ALPINE_VERSION} AS builder +ARG TARGETARCH +ARG TARGETOS +ARG TARGETVARIANT +COPY dist/shiori_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}/shiori /usr/bin/shiori +RUN apk add --no-cache ca-certificates tzdata && \ + chmod +x /usr/bin/shiori + +# Server image +FROM scratch + +ENV PORT 8080 +ENV SHIORI_DIR=/shiori +WORKDIR ${SHIORI_DIR} + +LABEL org.opencontainers.image.source="https://github.com/go-shiori/shiori" +LABEL maintainer="Felipe Martin " + +COPY --from=builder /usr/bin/shiori /usr/bin/shiori +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +EXPOSE ${PORT} -FROM docker.io/alpine:3.19 -LABEL org.opencontainers.image.source https://github.com/go-shiori/shiori -COPY --from=builder /src/shiori /usr/bin/ -RUN addgroup -g 1000 shiori \ - && adduser -D -h /shiori -g '' -G shiori -u 1000 shiori -USER shiori -WORKDIR /shiori -EXPOSE 8080 -ENV SHIORI_DIR /shiori/ ENTRYPOINT ["/usr/bin/shiori"] CMD ["server"] diff --git a/Makefile b/Makefile index efd8e36a9..37dcbb1ea 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ GO ?= $(shell command -v go 2> /dev/null) BASH ?= $(shell command -v bash 2> /dev/null) +GOLANG_VERSION := $(shell head -n 4 go.mod | tail -n 1 | cut -d " " -f 2) # Development SHIORI_DIR ?= dev-data +SOURCE_FILES ?=./internal/... # Build CGO_ENABLED ?= 0 @@ -11,6 +13,16 @@ BUILD_HASH := $(shell git describe --tags) BUILD_TAGS ?= osusergo,netgo,fts5 LDFLAGS += -s -w -X main.version=$(BUILD_HASH) -X main.date=$(BUILD_TIME) +# Build (container) +CONTAINER_RUNTIME := docker +CONTAINERFILE_NAME := Dockerfile +CONTAINER_ALPINE_VERSION := 3.19 +BUILDX_PLATFORMS := linux/amd64,linux/arm64,linux/arm/v7 + +# This is used for local development only, forcing linux to create linux only images but with the arch +# of the running machine. Far from perfect but works. +LOCAL_BUILD_PLATFORM = linux/$(shell go env GOARCH) + # Testing GO_TEST_FLAGS ?= -v -race -count=1 -tags $(BUILD_TAGS) -covermode=atomic -coverprofile=coverage.out GOTESTFMT_FLAGS ?= @@ -26,6 +38,15 @@ SWAGGER_DOCS_PATH ?= ./docs/swagger # Frontend CLEANCSS_OPTS ?= --with-rebase +# Common exports +export GOLANG_VERSION +export CONTAINER_RUNTIME +export CONTAINERFILE_NAME +export CONTAINER_ALPINE_VERSION +export BUILDX_PLATFORMS + +export SOURCE_FILES + # Help documentatin à la https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html .PHONY: help help: @@ -87,12 +108,29 @@ styles-check: ## Build binary .PHONY: build build: clean - GIN_MODE=$(GIN_MODE) goreleaser build --rm-dist --snapshot + GIN_MODE=$(GIN_MODE) goreleaser build --clean --snapshot + +## Build binary for current targer +build-local: clean + GIN_MODE=$(GIN_MODE) goreleaser build --clean --snapshot --single-target + +## Build docker image using Buildx. +# used for multi-arch builds suing mainly the CI, that's why the task does not +# build the binaries using a dependency task. +.PHONY: buildx +buildx: + $(info: Make: Buildx) + @bash scripts/buildx.sh + +## Build docker image for local development +buildx-local: build-local + $(info: Make: Build image locally) + CONTAINER_BUILDX_OPTIONS="-t shiori:localdev --output type=docker" BUILDX_PLATFORMS=$(LOCAL_BUILD_PLATFORM) scripts/buildx.sh ## Creates a coverage report .PHONY: coverage coverage: - $(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt ./... + $(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt $(SOURCE_FILES) $(GO) tool cover -html=coverage.txt ## Run generate accross the project diff --git a/go.mod b/go.mod index cbf2f89b8..e4d5fe679 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/go-shiori/shiori // +heroku goVersion go1.22 -go 1.22.2 +go 1.22.3 require ( git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc diff --git a/scripts/buildx.sh b/scripts/buildx.sh new file mode 100755 index 000000000..7690a2925 --- /dev/null +++ b/scripts/buildx.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -ex + +# Check if the shiori_builder builder exists +if [ "$CONTAINER_RUNTIME" == "docker" ]; then + if [ -z "$($CONTAINER_RUNTIME buildx ls | grep shiori_builder)" ]; then + echo "Creating shiori_builder builder" + $CONTAINER_RUNTIME buildx create --use --name shiori_builder + fi +fi + +if [[ -d "dist/shiori_linux_arm_7" ]]; then + cp -r dist/shiori_linux_arm_7 dist/shiori_linux_armv7 +fi + +if [[ -d "dist/shiori_linux_amd64_v1" ]]; then + cp -r dist/shiori_linux_amd64_v1 dist/shiori_linux_amd64 +fi + +$CONTAINER_RUNTIME buildx build \ + -f ${CONTAINERFILE_NAME} \ + --platform=${BUILDX_PLATFORMS} \ + --build-arg "ALPINE_VERSION=${CONTAINER_ALPINE_VERSION}" \ + --build-arg "GOLANG_VERSION=${GOLANG_VERSION}" \ + ${CONTAINER_BUILDX_OPTIONS} \ + . + +if [ "$CONTAINER_RUNTIME" == "docker" ]; then + $CONTAINER_RUNTIME buildx rm shiori_builder +fi diff --git a/scripts/test.sh b/scripts/test.sh index 1893616da..08b65a958 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -8,7 +8,7 @@ fi # if gotestfmt is installed, run with it if [ -x "$(command -v gotestfmt)" ]; then set -o pipefail - go test ./... ${GO_TEST_FLAGS} -json | gotestfmt ${GOTESTFMT_FLAGS} + go test ${SOURCE_FILES} ${GO_TEST_FLAGS} -json | gotestfmt ${GOTESTFMT_FLAGS} else - go test ./... ${GO_TEST_FLAGS} + go test ${SOURCE_FILES} ${GO_TEST_FLAGS} fi