From ba2d7b31eda3b7d7a059c8bc313a07631de8199f Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 16:02:00 +0200 Subject: [PATCH 01/13] #138: Upgrade dependencies --- .github/workflows/broken_links_checker.yml | 5 + .github/workflows/ci-build-next-java.yml | 11 +- .github/workflows/ci-build.yml | 114 +++++++-- .github/workflows/dependencies_check.yml | 78 ++++--- .github/workflows/dependencies_update.yml | 143 ++++++------ .github/workflows/release.yml | 219 ++++++++++++++++++ ...elease_droid_prepare_original_checksum.yml | 39 ---- .../release_droid_print_quick_checksum.yml | 26 --- ...release_droid_release_on_maven_central.yml | 35 --- ...ase_droid_upload_github_release_assets.yml | 47 ---- dependencies.md | 177 +++++++------- doc/changes/changelog.md | 1 + doc/changes/changes_3.5.5.md | 47 ++++ pk_generated_parent.pom | 31 ++- pom.xml | 39 ++-- release_config.yml | 4 - 16 files changed, 617 insertions(+), 399 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/release_droid_prepare_original_checksum.yml delete mode 100644 .github/workflows/release_droid_print_quick_checksum.yml delete mode 100644 .github/workflows/release_droid_release_on_maven_central.yml delete mode 100644 .github/workflows/release_droid_upload_github_release_assets.yml create mode 100644 doc/changes/changes_3.5.5.md delete mode 100644 release_config.yml diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml index 0fbcad5..39612b7 100644 --- a/.github/workflows/broken_links_checker.yml +++ b/.github/workflows/broken_links_checker.yml @@ -13,6 +13,11 @@ on: jobs: linkChecker: runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + shell: "bash" concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/ci-build-next-java.yml b/.github/workflows/ci-build-next-java.yml index e3acdb7..e8302fe 100644 --- a/.github/workflows/ci-build-next-java.yml +++ b/.github/workflows/ci-build-next-java.yml @@ -10,6 +10,11 @@ on: jobs: java-17-compatibility: runs-on: ubuntu-latest + defaults: + run: + shell: "bash" + permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -29,9 +34,3 @@ jobs: mvn --batch-mode --update-snapshots clean package -DtrimStackTrace=false \ -Djava.version=17 \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - name: Publish Test Report for Java 17 - uses: scacap/action-surefire-report@v1 - if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - fail_if_no_tests: false diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 8852bbe..4c4b328 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -1,61 +1,127 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml +# This file was generated by Project Keeper. name: CI Build on: push: - branches: - - main - pull_request: - + branches: [ + main + ] + + pull_request: null + workflow_dispatch: null jobs: build: runs-on: ubuntu-latest - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + defaults: + run: { + shell: bash + } + permissions: { + contents: read, + issues: read + } + concurrency: { + group: '${{ github.workflow }}-${{ github.ref }}', cancel-in-progress: true + } + outputs: { + release-required: '${{ steps.check-release.outputs.release-required }}' + } steps: - name: Free Disk Space + id: free-disk-space if: ${{ false }} run: | sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/share/dotnet - name: Checkout the repository + id: checkout uses: actions/checkout@v4 - with: + with: { fetch-depth: 0 + } - name: Set up JDKs + id: setup-java uses: actions/setup-java@v4 with: - distribution: "temurin" + distribution: temurin java-version: | 11 17 - cache: "maven" + cache: maven - name: Cache SonarCloud packages + id: cache-sonar uses: actions/cache@v4 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Enable testcontainer reuse + with: { + path: ~/.sonar/cache, + key: '${{ runner.os }}-sonar', + restore-keys: '${{ runner.os }}-sonar' + } + - { + name: Enable testcontainer reuse, + id: enable-testcontainer-reuse, run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" + } - name: Run tests and build with Maven + id: build-pk-verify run: | mvn --batch-mode clean verify \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -DtrimStackTrace=false - - name: Publish Test Report - uses: scacap/action-surefire-report@v1 - if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - name: Sonar analysis + id: sonar-analysis if: ${{ env.SONAR_TOKEN != null }} run: | mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -DtrimStackTrace=false \ -Dsonar.token=$SONAR_TOKEN - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + env: { + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}', + SONAR_TOKEN: '${{ secrets.SONAR_TOKEN }}' + } + - name: Verify Release Artifacts + id: verify-release-artifacts + run: "print_message() {\n local -r message=$1\n echo \"$message\"\n echo \"$message\" >> \"$GITHUB_STEP_SUMMARY\"\n}\n\nprint_message \"### Release Artifacts\"\n\nIFS=$'\\n' artifacts_array=($ARTIFACTS)\nmissing_files=()\nfor file in \"${artifacts_array[@]}\";\ndo \n echo \"Checking if file $file exists...\"\n if ! [[ -f \"$file\" ]]; then\n print_message \"* ⚠️ \\`$file\\` does not exist ⚠️\"\n echo \"Content of directory $(dirname \"$file\"):\"\n ls \"$(dirname \"$file\")\"\n missing_files+=(\"$file\")\n else\n print_message \"* \\`$file\\` ✅\" \n fi\ndone\nprint_message \"\"\nnumber_of_missing_files=${#missing_files[@]}\nif [[ $number_of_missing_files -gt 0 ]]; then\n print_message \"⚠️ $number_of_missing_files release artifact(s) missing ⚠️\"\n exit 1\nfi\n" + env: { + ARTIFACTS: '${{ steps.build-pk-verify.outputs.release-artifacts }}' + } + - name: Upload artifacts + id: upload-artifacts + uses: actions/upload-artifact@v4 + with: { + name: artifacts, + path: '${{ steps.build-pk-verify.outputs.release-artifacts }}', + retention-days: 5 + } + - name: Check if release is needed + id: check-release + if: ${{ github.ref == 'refs/heads/main' }} + run: | + if mvn --batch-mode com.exasol:project-keeper-maven-plugin:verify-release --projects .; then + echo "### ✅ Release preconditions met, start release" >> "$GITHUB_STEP_SUMMARY" + echo "release-required=true" >> "$GITHUB_OUTPUT" + else + echo "### 🛑 Release precondition not met, skipping release" >> "$GITHUB_STEP_SUMMARY" + echo "See log output for details." >> "$GITHUB_STEP_SUMMARY" + echo "release-required=false" >> "$GITHUB_OUTPUT" + fi + env: { + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + } + start_release: + needs: build + if: ${{ github.ref == 'refs/heads/main' && needs.build.outputs.release-required == 'true' }} + concurrency: { + cancel-in-progress: false, + group: release + } + secrets: inherit + permissions: { + contents: write, + actions: read, + issues: read + } + uses: ./.github/workflows/release.yml + with: { + started-from-ci: true + } diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index 6926e55..9c2365c 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -1,64 +1,80 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/dependencies_check.yml +# This file was generated by Project Keeper. name: Report Security Issues on: - workflow_dispatch: + workflow_dispatch: null schedule: - - cron: "0 2 * * *" - + - { + cron: 0 2 * * * + } jobs: report_security_issues: runs-on: ubuntu-latest - permissions: - contents: read + defaults: + run: { + shell: bash + } + permissions: { + contents: read, issues: write - outputs: - created-issues: ${{ steps.security-issues.outputs.created-issues }} + } + outputs: { + created-issues: '${{ steps.security-issues.outputs.created-issues }}' + } + concurrency: { + group: '${{ github.workflow }}-report_security_issues', + cancel-in-progress: true + } steps: - - uses: actions/checkout@v4 - + - { + name: Checkout, + id: checkout, + uses: actions/checkout@v4 + } - name: Set up JDKs + id: setup-jdks uses: actions/setup-java@v4 with: - distribution: "temurin" + distribution: temurin java-version: | 11 17 - cache: "maven" - + cache: maven - name: Generate ossindex report + id: ossindex-report run: | mvn --batch-mode org.sonatype.ossindex.maven:ossindex-maven-plugin:audit \ org.sonatype.ossindex.maven:ossindex-maven-plugin:audit-aggregate \ -Dossindex.reportFile=$(pwd)/ossindex-report.json \ -Dossindex.fail=false - - name: Report Security Issues id: security-issues uses: exasol/python-toolbox/.github/actions/security-issues@main - with: - format: "maven" - command: "cat ossindex-report.json" - github-token: ${{ secrets.GITHUB_TOKEN }} - + with: { + format: maven, + command: cat ossindex-report.json, + github-token: '${{ secrets.GITHUB_TOKEN }}' + } - name: Output security issues (Debugging) + id: debug-print-security-issues run: | echo "$CREATED_ISSUES" > test.jsonl cat test.jsonl - env: - CREATED_ISSUES: ${{ steps.security-issues.outputs.created-issues }} - + env: { + CREATED_ISSUES: '${{ steps.security-issues.outputs.created-issues }}' + } start_dependency_udpate: needs: report_security_issues if: ${{ needs.report_security_issues.outputs.created-issues }} - concurrency: - cancel-in-progress: true - group: "dependency_update" - # Workflow needs secret INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK + concurrency: { + group: '${{ github.workflow }}-start_dependency_update', + cancel-in-progress: false + } secrets: inherit - permissions: - contents: write + permissions: { + contents: write, pull-requests: write + } uses: ./.github/workflows/dependencies_update.yml - with: - vulnerability_issues: ${{ needs.report_security_issues.outputs.created-issues }} + with: { + vulnerability_issues: '${{ needs.report_security_issues.outputs.created-issues }}' + } diff --git a/.github/workflows/dependencies_update.yml b/.github/workflows/dependencies_update.yml index 58222ba..0fa7180 100644 --- a/.github/workflows/dependencies_update.yml +++ b/.github/workflows/dependencies_update.yml @@ -1,70 +1,68 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/dependencies_update.yml +# This file was generated by Project Keeper. name: Update dependencies on: workflow_call: inputs: - vulnerability_issues: - description: "GitHub issues for vulnerable dependencies as JSONL" - required: true + vulnerability_issues: { + description: GitHub issues for vulnerable dependencies as JSONL, + required: true, type: string - workflow_dispatch: - + } + workflow_dispatch: null jobs: update_dependencies: runs-on: ubuntu-latest - permissions: - contents: write + defaults: + run: { + shell: bash + } + permissions: { + contents: write, pull-requests: write - + } + concurrency: { + group: '${{ github.workflow }}', + cancel-in-progress: false + } steps: - uses: actions/checkout@v4 - with: + id: checkout + with: { fetch-depth: 0 - + } - name: Set up JDKs + id: setup-jdks uses: actions/setup-java@v4 with: - distribution: "temurin" + distribution: temurin java-version: | 11 17 - cache: "maven" - + cache: maven - name: Print issues + id: debug-print-issues run: | echo "Issues from Action input: $ISSUES" - env: - ISSUES: ${{ inputs.vulnerability_issues }} - + env: { + ISSUES: '${{ inputs.vulnerability_issues }}' + } - name: Fail if not running on a branch + id: check-branch if: ${{ !startsWith(github.ref, 'refs/heads/') }} uses: actions/github-script@v7 with: script: | core.setFailed('Not running on a branch, github.ref is ${{ github.ref }}. Please start this workflow only on main or a branch') - - name: Update dependencies + id: update-dependencies run: | mvn --batch-mode com.exasol:project-keeper-maven-plugin:update-dependencies --projects . \ -Dproject-keeper:vulnerabilities="$CREATED_ISSUES" - env: - CREATED_ISSUES: ${{ inputs.vulnerability_issues }} - - - name: Project Keeper Fix - run: | - mvn --batch-mode com.exasol:project-keeper-maven-plugin:fix --projects . - - - name: Project Keeper Fix for updated Project Keeper version - # Calling PK fix a second time is necessary because the first invocation potentially updated PK itself. - # So we need to run PK fix again with the latest PK version. - # [impl->dsn~dependency-updater.workflow.start-pk-fix~1] - run: | - mvn --batch-mode com.exasol:project-keeper-maven-plugin:fix --projects . - + env: { + CREATED_ISSUES: '${{ inputs.vulnerability_issues }}' + } - name: Generate Pull Request comment id: pr-comment - # [impl->dsn~dependency-updater.workflow.pull-request-trigger-ci-build~1] run: | echo 'comment<> "$GITHUB_OUTPUT" echo 'This Pull Request was created by [`dependencies_update.yml`](https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/dependencies_update.yml) workflow.' >> "$GITHUB_OUTPUT" @@ -75,15 +73,19 @@ jobs: echo 'It updates dependencies.' >> "$GITHUB_OUTPUT" fi echo >> "$GITHUB_OUTPUT" - echo '# ⚠️ This PR does not trigger CI workflows by default ⚠️' >> "$GITHUB_OUTPUT" + echo '# ⚠️ Notes ⚠️' >> "$GITHUB_OUTPUT" + echo '## Run PK fix manually' >> "$GITHUB_OUTPUT" + echo 'Due to restrictions workflow `dependencies_update.yml` cannot update other workflows, see https://github.com/exasol/project-keeper/issues/578 for details.' >> "$GITHUB_OUTPUT" + echo 'Please checkout this PR locally and run `mvn com.exasol:project-keeper-maven-plugin:fix --projects .`' >> "$GITHUB_OUTPUT" + echo '## This PR does not trigger CI workflows' >> "$GITHUB_OUTPUT" echo 'Please click the **Close pull request** button and then **Reopen pull request** to trigger running checks.' >> "$GITHUB_OUTPUT" echo 'See https://github.com/exasol/project-keeper/issues/534 for details.' >> "$GITHUB_OUTPUT" echo 'EOF' >> "$GITHUB_OUTPUT" cat "$GITHUB_OUTPUT" - env: - CREATED_ISSUES: ${{ inputs.vulnerability_issues }} - + env: { + CREATED_ISSUES: '${{ inputs.vulnerability_issues }}' + } - name: Generate Pull Request Title id: pr-title run: | @@ -96,22 +98,23 @@ jobs: fi cat "$GITHUB_OUTPUT" - env: - CREATED_ISSUES: ${{ inputs.vulnerability_issues }} - + env: { + CREATED_ISSUES: '${{ inputs.vulnerability_issues }}' + } - name: Configure git + id: configure-git run: | git config --global user.email "opensource@exasol.com" git config --global user.name "Automatic Dependency Updater" - - name: Create branch + id: create-branch if: ${{ github.ref == 'refs/heads/main' }} run: | branch_name="dependency-update/$(date "+%Y%m%d%H%M%S")" echo "Creating branch $branch_name" git checkout -b "$branch_name" - - name: Commit changes & push + id: publish-branch if: ${{ startsWith(github.ref, 'refs/heads/' ) }} run: | branch_name=$(git rev-parse --abbrev-ref HEAD) @@ -129,9 +132,9 @@ jobs: echo "Pushing branch $branch_name..." git push --set-upstream origin "$branch_name" echo "Done." - env: - TITLE: ${{ steps.pr-title.outputs.title }} - + env: { + TITLE: '${{ steps.pr-title.outputs.title }}' + } - name: Create pull request id: create-pr if: ${{ github.ref == 'refs/heads/main' }} @@ -139,31 +142,35 @@ jobs: pr_url=$(gh pr create --base main --title "$TITLE" --body "$COMMENT") echo "Created Pull Request: $pr_url" echo "pr_url=$pr_url" >> "$GITHUB_OUTPUT" - env: - COMMENT: ${{ steps.pr-comment.outputs.comment }} - TITLE: ${{ steps.pr-title.outputs.title }} - GH_TOKEN: ${{ github.token }} - + env: { + COMMENT: '${{ steps.pr-comment.outputs.comment }}', + TITLE: '${{ steps.pr-title.outputs.title }}', + GH_TOKEN: '${{ github.token }}' + } - name: Report failure Status to Slack channel - # Also run this step in case of failures + id: report-failure-slack if: ${{ always() }} uses: ravsamhq/notify-slack-action@v2 - with: - status: ${{ job.status }} - token: ${{ secrets.GITHUB_TOKEN }} - notification_title: "Dependency check in {repo} has {status_message}" - message_format: "{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>" - notify_when: "failure,cancelled,warnings" - env: - SLACK_WEBHOOK_URL: ${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }} - + with: { + status: '${{ job.status }}', + token: '${{ secrets.GITHUB_TOKEN }}', + notification_title: 'Dependency check in {repo} has {status_message}', + message_format: '{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>', + notify_when: 'failure,cancelled,warnings' + } + env: { + SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }}' + } - name: Report new Pull Request to Slack channel + id: report-pr-slack if: ${{ steps.create-pr.outputs.pr_url }} uses: ravsamhq/notify-slack-action@v2 - with: - status: ${{ job.status }} - token: ${{ secrets.GITHUB_TOKEN }} - notification_title: "Dependency update for {repo} created a Pull Request" - message_format: "{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}" - env: - SLACK_WEBHOOK_URL: ${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }} + with: { + status: '${{ job.status }}', + token: '${{ secrets.GITHUB_TOKEN }}', + notification_title: 'Dependency update for {repo} created a Pull Request', + message_format: '{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}' + } + env: { + SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }}' + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5be64c8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,219 @@ +# This file was generated by Project Keeper. +name: Release +on: + workflow_call: + inputs: + started-from-ci: { + description: 'Marks this release as started from CI, skipping precondition check', + type: boolean, + required: true, + default: false + } + workflow_dispatch: + inputs: + skip-maven-central: { + description: Skip deployment to Maven Central, + required: true, + type: boolean, + default: false + } + skip-github-release: { + description: Skip creating the GitHub release, + required: true, + type: boolean, + default: false + } +jobs: + release: + runs-on: ubuntu-latest + defaults: + run: { + shell: bash + } + concurrency: { + group: '${{ github.workflow }}', + cancel-in-progress: false + } + permissions: { + contents: write, + actions: read, + issues: read + } + steps: + - name: Checkout the repository + id: checkout + uses: actions/checkout@v4 + with: { + fetch-depth: 0 + } + - name: Set up Maven Central Repository + id: configure-maven-central-credentials + if: ${{ true }} + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: | + 11 + 17 + cache: maven + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Set up JDKs + id: setup-jdks + if: ${{ ! true }} + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: | + 11 + 17 + cache: maven + - name: Fail if not running on main branch + id: check-main-branch + if: ${{ github.ref != 'refs/heads/main' }} + uses: actions/github-script@v7 + with: + script: | + core.setFailed('Not running on main branch, github.ref is ${{ github.ref }}. Please start this workflow only on main') + - name: Check CI build of this commit succeeded + id: check-ci-build-status + if: ${{ ! inputs.started-from-ci }} + run: | + echo "Commit SHA: $COMMIT_SHA" + gh run list --workflow ci-build.yml --branch main --event push --commit $COMMIT_SHA + ci_build_status=$(gh run list --workflow ci-build.yml --branch main --event push --commit $COMMIT_SHA --json conclusion --template '{{range .}}{{.conclusion}}{{"\n"}}{{end}}') + echo "CI build status at commit $COMMIT_SHA was '$ci_build_status'" + if [[ "$ci_build_status" != "success" ]]; then + gh run list --workflow ci-build.yml --commit $COMMIT_SHA >> $GITHUB_STEP_SUMMARY + echo "Status of CI build for commit $COMMIT_SHA was '$ci_build_status', expected 'success'" >> $GITHUB_STEP_SUMMARY + cat $GITHUB_STEP_SUMMARY + exit 1 + fi + env: { + COMMIT_SHA: '${{ github.sha }}', + GH_TOKEN: '${{ github.token }}' + } + - name: Verify release preconditions + id: verify-release + run: | + mvn --batch-mode com.exasol:project-keeper-maven-plugin:verify-release --projects . + echo "$GITHUB_OUTPUT" + env: { + GITHUB_TOKEN: '${{ github.token }}' + } + - { + name: Build project, + id: build, + run: mvn --batch-mode -DskipTests clean verify + } + - { + name: List secret GPG keys, + id: list-secret-gpg-keys, + if: '${{ true && (! inputs.skip-maven-central) }}', + run: gpg --list-secret-keys + } + - name: Publish to Central Repository + id: deploy-maven-central + if: ${{ true && (! inputs.skip-maven-central) }} + run: | + echo "#### Maven Central Release" >> "$GITHUB_STEP_SUMMARY" + mvn --batch-mode -Dgpg.skip=false -DskipTests deploy + echo "Published to Maven Central ✅" >> "$GITHUB_STEP_SUMMARY" + env: { + MAVEN_USERNAME: '${{ secrets.OSSRH_USERNAME }}', + MAVEN_PASSWORD: '${{ secrets.OSSRH_PASSWORD }}', + MAVEN_GPG_PASSPHRASE: '${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }}' + } + - name: Calculate Artifact Checksums + id: artifact-checksum + if: ${{ ! inputs.skip-github-release }} + run: | + echo "Calculating sha256 checksum for artifact files" + echo "artifacts<> "$GITHUB_OUTPUT" + IFS=$'\n' artifacts_array=($ARTIFACTS) + for file in "${artifacts_array[@]}"; + do + full_path=$(realpath "$file") + echo "Calculate sha256sum for file '$full_path'" + file_dir="$(dirname "$full_path")" + file_name=$(basename "$full_path") + pushd "$file_dir" + checksum_file_name="${file_name}.sha256" + sha256sum "$file_name" > "$checksum_file_name" + echo "$full_path" >> "$GITHUB_OUTPUT" + echo "${file_dir}/$checksum_file_name" >> "$GITHUB_OUTPUT" + popd + done + echo "EOF" >> "$GITHUB_OUTPUT" + echo "Full artifact file list" + cat "$GITHUB_OUTPUT" + env: { + ARTIFACTS: '${{ steps.verify-release.outputs.release-artifacts }}' + } + - name: Create GitHub Release + id: create-github-release + if: ${{ ! inputs.skip-github-release }} + run: | + echo "### GitHub Release" >> "$GITHUB_STEP_SUMMARY" + IFS=$'\n' artifacts_array=($ARTIFACTS) + echo "#### Attaching Release Artifacts" >> "$GITHUB_STEP_SUMMARY" + for file in "${artifacts_array[@]}"; + do + echo "Attaching artifact '$file'" + echo "* \`$file\`" >> "$GITHUB_STEP_SUMMARY" + done + echo "" >> "$GITHUB_STEP_SUMMARY" + release_url=$(gh release create --latest --title "$TITLE" --notes "$NOTES" --target main $TAG "${artifacts_array[@]}") + echo "Created release $TAG with title '$TITLE' at $release_url ✅" >> "$GITHUB_STEP_SUMMARY" + echo "release-url=$release_url" >> "$GITHUB_OUTPUT" + + # [impl->dsn~release-workflow.create-golang-tags~1] + echo "#### Creating Additional Tags" >> "$GITHUB_STEP_SUMMARY" + IFS=$'\n' tags_array=($ADDITIONAL_TAGS) + for tag in "${tags_array[@]}"; + do + echo "Creating tag '$tag'" + git tag "$tag" + git push origin "$tag" + echo "* \`$tag\`" >> "$GITHUB_STEP_SUMMARY" + done + + git fetch --tags origin + env: { + GH_TOKEN: '${{ github.token }}', + TAG: '${{ steps.verify-release.outputs.release-tag }}', + ADDITIONAL_TAGS: '${{ steps.verify-release.outputs.additional-release-tags }}', + NOTES: '${{ steps.verify-release.outputs.release-notes }}', + TITLE: '${{ steps.verify-release.outputs.release-title }}', + ARTIFACTS: '${{ steps.artifact-checksum.outputs.artifacts }}' + } + - name: Report failure Status to Slack channel + id: report-failure-status-slack + if: ${{ always() }} + uses: ravsamhq/notify-slack-action@v2 + with: { + status: '${{ job.status }}', + token: '${{ github.token }}', + notification_title: 'Release build in {repo} has {status_message}', + message_format: '{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>', + notify_when: 'failure,cancelled,warnings,skipped' + } + env: { + SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }}' + } + - name: Report new release to Slack channel + id: report-new-release-slack + if: ${{ steps.create-github-release.outputs.release-url }} + uses: ravsamhq/notify-slack-action@v2 + with: { + status: '${{ job.status }}', + token: '${{ github.token }}', + notification_title: 'Release build for {repo} created a new release', + message_format: '{workflow} created release ${{ steps.create-github-release.outputs.release-url }}' + } + env: { + SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }}' + } diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml deleted file mode 100644 index 9801107..0000000 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_prepare_original_checksum.yml -name: Release Droid - Prepare Original Checksum -on: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Free Disk Space - if: ${{ false }} - run: | - sudo rm -rf /usr/local/lib/android - sudo rm -rf /usr/share/dotnet - - name: Checkout the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up JDKs - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: | - 11 - 17 - cache: "maven" - - name: Enable testcontainer reuse - run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" - - name: Run tests and build with Maven - run: mvn --batch-mode clean verify --file pom.xml - - name: Prepare checksum - run: find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + > original_checksum - - name: Upload checksum to the artifactory - uses: actions/upload-artifact@v4 - with: - name: original_checksum - retention-days: 5 - path: original_checksum diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml deleted file mode 100644 index 86979cd..0000000 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_print_quick_checksum.yml -name: Release Droid - Print Quick Checksum -on: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up JDKs - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: | - 11 - 17 - cache: "maven" - - name: Build with Maven skipping tests - run: mvn --batch-mode clean verify -DskipTests - - name: Print checksum - run: echo 'checksum_start==';find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + | xargs;echo '==checksum_end' diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml deleted file mode 100644 index 0a5ee04..0000000 --- a/.github/workflows/release_droid_release_on_maven_central.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_release_on_maven_central.yml -name: Release Droid - Release On Maven Central -on: - workflow_dispatch: - -jobs: - publish: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Maven Central Repository - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: | - 11 - 17 - cache: "maven" - server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: List secret GPG keys - run: gpg --list-secret-keys - - name: Publish to Central Repository - run: mvn --batch-mode -Dgpg.skip=false -DskipTests clean deploy - env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml deleted file mode 100644 index b19f7cf..0000000 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_upload_github_release_assets.yml -name: Release Droid - Upload GitHub Release Assets -on: - workflow_dispatch: - inputs: - upload_url: - description: "Assets upload URL" - required: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up JDKs - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: | - 11 - 17 - cache: "maven" - - name: Build with Maven skipping tests - run: mvn --batch-mode clean verify -DskipTests - - name: Generate sha256sum files - run: | - cd target - find . -maxdepth 1 -name \*.jar -exec bash -c 'sha256sum {} > {}.sha256' \; - - name: Upload assets to the GitHub release draft - uses: shogo82148/actions-upload-release-asset@v1 - with: - upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/*.jar - - name: Upload sha256sum files - uses: shogo82148/actions-upload-release-asset@v1 - with: - upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/*.sha256 - - name: Upload error-code-report - uses: shogo82148/actions-upload-release-asset@v1 - with: - upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/error_code_report.json diff --git a/dependencies.md b/dependencies.md index bb1e88d..4816e46 100644 --- a/dependencies.md +++ b/dependencies.md @@ -12,102 +12,105 @@ | Dependency | License | | ----------------------------------------------- | ---------------------------------------------------------------------- | -| [EXASolution JDBC Driver][4] | [EXAClient License][5] | +| [Exasol JDBC Driver][4] | [EXAClient License][5] | | [MySQL Connector/J][6] | The GNU General Public License, v2 with Universal FOSS Exception, v1.0 | -| [PostgreSQL JDBC Driver][7] | [BSD-2-Clause][8] | -| [ojdbc11][9] | [Oracle Free Use Terms and Conditions (FUTC)][10] | -| [junit-pioneer][11] | [Eclipse Public License v2.0][12] | -| [Test containers for Exasol on Docker][13] | [MIT License][14] | -| [Testcontainers :: JUnit Jupiter Extension][15] | [MIT][16] | -| [Testcontainers :: JDBC :: MySQL][15] | [MIT][16] | -| [Testcontainers :: JDBC :: PostgreSQL][15] | [MIT][16] | -| [Testcontainers :: JDBC :: Oracle XE][15] | [MIT][16] | -| [Matcher for SQL Result Sets][17] | [MIT License][18] | -| [Hamcrest][19] | [BSD License 3][20] | -| [JUnit Jupiter API][21] | [Eclipse Public License v2.0][12] | -| [JUnit Jupiter Engine][21] | [Eclipse Public License v2.0][12] | -| [mockito-junit-jupiter][22] | [MIT][23] | -| [EqualsVerifier \| release normal jar][24] | [Apache License, Version 2.0][25] | -| [SLF4J JDK14 Provider][26] | [MIT License][27] | +| [Protocol Buffers [Core]][7] | [BSD-3-Clause][8] | +| [PostgreSQL JDBC Driver][9] | [BSD-2-Clause][10] | +| [ojdbc11][11] | [Oracle Free Use Terms and Conditions (FUTC)][12] | +| [junit-pioneer][13] | [Eclipse Public License v2.0][14] | +| [Test containers for Exasol on Docker][15] | [MIT License][16] | +| [Testcontainers :: JUnit Jupiter Extension][17] | [MIT][18] | +| [Testcontainers :: JDBC :: MySQL][17] | [MIT][18] | +| [Testcontainers :: JDBC :: PostgreSQL][17] | [MIT][18] | +| [Testcontainers :: JDBC :: Oracle XE][17] | [MIT][18] | +| [Matcher for SQL Result Sets][19] | [MIT License][20] | +| [Hamcrest][21] | [BSD-3-Clause][22] | +| [JUnit Jupiter API][23] | [Eclipse Public License v2.0][14] | +| [JUnit Jupiter Engine][23] | [Eclipse Public License v2.0][14] | +| [mockito-junit-jupiter][24] | [MIT][25] | +| [EqualsVerifier \| release normal jar][26] | [Apache License, Version 2.0][27] | +| [SLF4J JDK14 Provider][28] | [MIT License][29] | ## Plugin Dependencies | Dependency | License | | ------------------------------------------------------- | ------------------------------------- | -| [SonarQube Scanner for Maven][28] | [GNU LGPL 3][29] | -| [Apache Maven Toolchains Plugin][30] | [Apache License, Version 2.0][25] | -| [Apache Maven Compiler Plugin][31] | [Apache-2.0][25] | -| [Apache Maven Enforcer Plugin][32] | [Apache-2.0][25] | -| [Maven Flatten Plugin][33] | [Apache Software Licenese][25] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][34] | [ASL2][35] | -| [Maven Surefire Plugin][36] | [Apache-2.0][25] | -| [Versions Maven Plugin][37] | [Apache License, Version 2.0][25] | -| [duplicate-finder-maven-plugin Maven Mojo][38] | [Apache License 2.0][39] | -| [Apache Maven Deploy Plugin][40] | [Apache-2.0][25] | -| [Apache Maven GPG Plugin][41] | [Apache-2.0][25] | -| [Apache Maven Source Plugin][42] | [Apache License, Version 2.0][25] | -| [Apache Maven Javadoc Plugin][43] | [Apache-2.0][25] | -| [Nexus Staging Maven Plugin][44] | [Eclipse Public License][45] | -| [Maven Failsafe Plugin][46] | [Apache-2.0][25] | -| [JaCoCo :: Maven Plugin][47] | [Eclipse Public License 2.0][48] | -| [error-code-crawler-maven-plugin][49] | [MIT License][50] | -| [Reproducible Build Maven Plugin][51] | [Apache 2.0][35] | -| [OpenFastTrace Maven Plugin][52] | [GNU General Public License v3.0][53] | -| [Project Keeper Maven plugin][54] | [The MIT License][55] | +| [SonarQube Scanner for Maven][30] | [GNU LGPL 3][31] | +| [Apache Maven Toolchains Plugin][32] | [Apache-2.0][27] | +| [Apache Maven Compiler Plugin][33] | [Apache-2.0][27] | +| [Apache Maven Enforcer Plugin][34] | [Apache-2.0][27] | +| [Maven Flatten Plugin][35] | [Apache Software Licenese][27] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][36] | [ASL2][37] | +| [Maven Surefire Plugin][38] | [Apache-2.0][27] | +| [Versions Maven Plugin][39] | [Apache License, Version 2.0][27] | +| [duplicate-finder-maven-plugin Maven Mojo][40] | [Apache License 2.0][41] | +| [Apache Maven Deploy Plugin][42] | [Apache-2.0][27] | +| [Apache Maven GPG Plugin][43] | [Apache-2.0][27] | +| [Apache Maven Source Plugin][44] | [Apache License, Version 2.0][27] | +| [Apache Maven Javadoc Plugin][45] | [Apache-2.0][27] | +| [Nexus Staging Maven Plugin][46] | [Eclipse Public License][47] | +| [Maven Failsafe Plugin][48] | [Apache-2.0][27] | +| [JaCoCo :: Maven Plugin][49] | [EPL-2.0][50] | +| [error-code-crawler-maven-plugin][51] | [MIT License][52] | +| [Reproducible Build Maven Plugin][53] | [Apache 2.0][37] | +| [OpenFastTrace Maven Plugin][54] | [GNU General Public License v3.0][55] | +| [Project Keeper Maven plugin][56] | [The MIT License][57] | [0]: https://github.com/exasol/db-fundamentals-java/ [1]: https://github.com/exasol/db-fundamentals-java/blob/main/LICENSE [2]: https://github.com/exasol/error-reporting-java/ [3]: https://github.com/exasol/error-reporting-java/blob/main/LICENSE -[4]: http://www.exasol.com -[5]: https://repo1.maven.org/maven2/com/exasol/exasol-jdbc/7.1.20/exasol-jdbc-7.1.20-license.txt +[4]: http://www.exasol.com/ +[5]: https://repo1.maven.org/maven2/com/exasol/exasol-jdbc/24.1.2/exasol-jdbc-24.1.2-license.txt [6]: http://dev.mysql.com/doc/connector-j/en/ -[7]: https://jdbc.postgresql.org -[8]: https://jdbc.postgresql.org/license/ -[9]: https://www.oracle.com/database/technologies/maven-central-guide.html -[10]: https://www.oracle.com/downloads/licenses/oracle-free-license.html -[11]: https://junit-pioneer.org/ -[12]: https://www.eclipse.org/legal/epl-v20.html -[13]: https://github.com/exasol/exasol-testcontainers/ -[14]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE -[15]: https://java.testcontainers.org -[16]: http://opensource.org/licenses/MIT -[17]: https://github.com/exasol/hamcrest-resultset-matcher/ -[18]: https://github.com/exasol/hamcrest-resultset-matcher/blob/main/LICENSE -[19]: http://hamcrest.org/JavaHamcrest/ -[20]: http://opensource.org/licenses/BSD-3-Clause -[21]: https://junit.org/junit5/ -[22]: https://github.com/mockito/mockito -[23]: https://opensource.org/licenses/MIT -[24]: https://www.jqno.nl/equalsverifier -[25]: https://www.apache.org/licenses/LICENSE-2.0.txt -[26]: http://www.slf4j.org -[27]: http://www.opensource.org/licenses/mit-license.php -[28]: http://sonarsource.github.io/sonar-scanner-maven/ -[29]: http://www.gnu.org/licenses/lgpl.txt -[30]: https://maven.apache.org/plugins/maven-toolchains-plugin/ -[31]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[32]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[33]: https://www.mojohaus.org/flatten-maven-plugin/ -[34]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[35]: http://www.apache.org/licenses/LICENSE-2.0.txt -[36]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[37]: https://www.mojohaus.org/versions/versions-maven-plugin/ -[38]: https://basepom.github.io/duplicate-finder-maven-plugin -[39]: http://www.apache.org/licenses/LICENSE-2.0.html -[40]: https://maven.apache.org/plugins/maven-deploy-plugin/ -[41]: https://maven.apache.org/plugins/maven-gpg-plugin/ -[42]: https://maven.apache.org/plugins/maven-source-plugin/ -[43]: https://maven.apache.org/plugins/maven-javadoc-plugin/ -[44]: http://www.sonatype.com/public-parent/nexus-maven-plugins/nexus-staging/nexus-staging-maven-plugin/ -[45]: http://www.eclipse.org/legal/epl-v10.html -[46]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[47]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[48]: https://www.eclipse.org/legal/epl-2.0/ -[49]: https://github.com/exasol/error-code-crawler-maven-plugin/ -[50]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE -[51]: http://zlika.github.io/reproducible-build-maven-plugin -[52]: https://github.com/itsallcode/openfasttrace-maven-plugin -[53]: https://www.gnu.org/licenses/gpl-3.0.html -[54]: https://github.com/exasol/project-keeper/ -[55]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[7]: https://developers.google.com/protocol-buffers/protobuf-java/ +[8]: https://opensource.org/licenses/BSD-3-Clause +[9]: https://jdbc.postgresql.org +[10]: https://jdbc.postgresql.org/license/ +[11]: https://www.oracle.com/database/technologies/maven-central-guide.html +[12]: https://www.oracle.com/downloads/licenses/oracle-free-license.html +[13]: https://junit-pioneer.org/ +[14]: https://www.eclipse.org/legal/epl-v20.html +[15]: https://github.com/exasol/exasol-testcontainers/ +[16]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE +[17]: https://java.testcontainers.org +[18]: http://opensource.org/licenses/MIT +[19]: https://github.com/exasol/hamcrest-resultset-matcher/ +[20]: https://github.com/exasol/hamcrest-resultset-matcher/blob/main/LICENSE +[21]: http://hamcrest.org/JavaHamcrest/ +[22]: https://raw.githubusercontent.com/hamcrest/JavaHamcrest/master/LICENSE +[23]: https://junit.org/junit5/ +[24]: https://github.com/mockito/mockito +[25]: https://opensource.org/licenses/MIT +[26]: https://www.jqno.nl/equalsverifier +[27]: https://www.apache.org/licenses/LICENSE-2.0.txt +[28]: http://www.slf4j.org +[29]: http://www.opensource.org/licenses/mit-license.php +[30]: http://sonarsource.github.io/sonar-scanner-maven/ +[31]: http://www.gnu.org/licenses/lgpl.txt +[32]: https://maven.apache.org/plugins/maven-toolchains-plugin/ +[33]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[34]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[35]: https://www.mojohaus.org/flatten-maven-plugin/ +[36]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[37]: http://www.apache.org/licenses/LICENSE-2.0.txt +[38]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[39]: https://www.mojohaus.org/versions/versions-maven-plugin/ +[40]: https://basepom.github.io/duplicate-finder-maven-plugin +[41]: http://www.apache.org/licenses/LICENSE-2.0.html +[42]: https://maven.apache.org/plugins/maven-deploy-plugin/ +[43]: https://maven.apache.org/plugins/maven-gpg-plugin/ +[44]: https://maven.apache.org/plugins/maven-source-plugin/ +[45]: https://maven.apache.org/plugins/maven-javadoc-plugin/ +[46]: http://www.sonatype.com/public-parent/nexus-maven-plugins/nexus-staging/nexus-staging-maven-plugin/ +[47]: http://www.eclipse.org/legal/epl-v10.html +[48]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[49]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[50]: https://www.eclipse.org/legal/epl-2.0/ +[51]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[52]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[53]: http://zlika.github.io/reproducible-build-maven-plugin +[54]: https://github.com/itsallcode/openfasttrace-maven-plugin +[55]: https://www.gnu.org/licenses/gpl-3.0.html +[56]: https://github.com/exasol/project-keeper/ +[57]: https://github.com/exasol/project-keeper/blob/main/LICENSE diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 75a1118..09d3b3a 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [3.5.5](changes_3.5.5.md) * [3.5.4](changes_3.5.4.md) * [3.5.3](changes_3.5.3.md) * [3.5.2](changes_3.5.2.md) diff --git a/doc/changes/changes_3.5.5.md b/doc/changes/changes_3.5.5.md new file mode 100644 index 0000000..b95bcab --- /dev/null +++ b/doc/changes/changes_3.5.5.md @@ -0,0 +1,47 @@ +# Test Database Builder for Java 3.5.5, released 2024-??-?? + +Code name: Fix CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1` + +## Summary + +This release fixes CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1`. + +## Security + +* #138: Fixed CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1` + +## Dependency Updates + +### Test Dependency Updates + +* Updated `com.exasol:exasol-jdbc:7.1.20` to `24.1.2` +* Updated `com.exasol:exasol-testcontainers:7.0.1` to `7.1.1` +* Updated `com.exasol:hamcrest-resultset-matcher:1.6.5` to `1.7.0` +* Added `com.google.protobuf:protobuf-java:4.28.2` +* Updated `com.mysql:mysql-connector-j:8.3.0` to `9.0.0` +* Updated `com.oracle.database.jdbc:ojdbc11:23.3.0.23.09` to `23.5.0.24.07` +* Updated `nl.jqno.equalsverifier:equalsverifier:3.15.8` to `3.16.2` +* Updated `org.hamcrest:hamcrest:2.2` to `3.0` +* Updated `org.junit.jupiter:junit-jupiter-api:5.10.2` to `5.11.0` +* Updated `org.junit.jupiter:junit-jupiter-engine:5.10.2` to `5.11.0` +* Updated `org.mockito:mockito-junit-jupiter:5.11.0` to `5.13.0` +* Updated `org.postgresql:postgresql:42.7.2` to `42.7.4` +* Updated `org.slf4j:slf4j-jdk14:2.0.12` to `2.0.16` +* Updated `org.testcontainers:junit-jupiter:1.19.7` to `1.20.1` +* Updated `org.testcontainers:mysql:1.19.7` to `1.20.1` +* Updated `org.testcontainers:oracle-xe:1.19.7` to `1.20.1` +* Updated `org.testcontainers:postgresql:1.19.7` to `1.20.1` + +### Plugin Dependency Updates + +* Updated `com.exasol:error-code-crawler-maven-plugin:2.0.0` to `2.0.3` +* Updated `com.exasol:project-keeper-maven-plugin:4.1.0` to `4.3.3` +* Updated `org.apache.maven.plugins:maven-compiler-plugin:3.12.1` to `3.13.0` +* Updated `org.apache.maven.plugins:maven-deploy-plugin:3.1.1` to `3.1.2` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.1` to `3.5.0` +* Updated `org.apache.maven.plugins:maven-gpg-plugin:3.1.0` to `3.2.4` +* Updated `org.apache.maven.plugins:maven-javadoc-plugin:3.6.3` to `3.7.0` +* Updated `org.apache.maven.plugins:maven-toolchains-plugin:3.1.0` to `3.2.0` +* Updated `org.jacoco:jacoco-maven-plugin:0.8.11` to `0.8.12` +* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.10.0.2594` to `4.0.0.4121` +* Updated `org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13` to `1.7.0` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 132adab..d1c0769 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol test-db-builder-java-generated-parent - 3.5.4 + 3.5.5 pom UTF-8 @@ -50,12 +50,12 @@ org.sonarsource.scanner.maven sonar-maven-plugin - 3.10.0.2594 + 4.0.0.4121 org.apache.maven.plugins maven-toolchains-plugin - 3.1.0 + 3.2.0 @@ -74,22 +74,21 @@ org.apache.maven.plugins maven-compiler-plugin - 3.12.1 + 3.13.0 ${java.version} ${java.version} true - - -Xlint:all,-processing - + -Xlint:all + -Werror org.apache.maven.plugins maven-enforcer-plugin - 3.4.1 + 3.5.0 enforce-maven @@ -216,7 +215,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2 true @@ -224,7 +223,7 @@ org.apache.maven.plugins maven-gpg-plugin - 3.1.0 + 3.2.4 sign-artifacts @@ -245,8 +244,8 @@ org.apache.maven.plugins maven-source-plugin + Failed to execute goal org.apache.maven.plugins:maven-source-plugin:3.3.0:jar-no-fork (attach-sources) on project project-keeper-shared-model-classes: Presumably you have configured maven-source-plugn to execute twice times in your build. You have to configure a classifier for at least on of them. + Using goal "jar-no-fork" didn't help. See https://stackoverflow.com/questions/76305897/maven-build-fails-after-upgrading-to-maven-source-plugin-from-3-2-1-to-3-3-0 --> 3.2.1 @@ -260,7 +259,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.6.3 + 3.7.0 attach-javadocs @@ -282,7 +281,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.13 + 1.7.0 true ossrh @@ -323,7 +322,7 @@ org.jacoco jacoco-maven-plugin - 0.8.11 + 0.8.12 prepare-agent @@ -364,7 +363,7 @@ com.exasol error-code-crawler-maven-plugin - 2.0.0 + 2.0.3 verify diff --git a/pom.xml b/pom.xml index a003825..c872270 100644 --- a/pom.xml +++ b/pom.xml @@ -2,36 +2,43 @@ 4.0.0 test-db-builder-java - 3.5.4 + 3.5.5 Test Database Builder for Java pom.xml https://github.com/exasol/test-db-builder-java/ - 1.19.7 + 1.20.1 com.exasol exasol-jdbc - 7.1.20 + 24.1.2 test com.mysql mysql-connector-j - 8.3.0 + 9.0.0 + test + + + + com.google.protobuf + protobuf-java + 4.28.2 test org.postgresql postgresql - 42.7.2 + 42.7.4 test com.oracle.database.jdbc ojdbc11 - 23.3.0.23.09 + 23.5.0.24.07 test @@ -48,7 +55,7 @@ com.exasol exasol-testcontainers - 7.0.1 + 7.1.1 test @@ -78,37 +85,37 @@ com.exasol hamcrest-resultset-matcher - 1.6.5 + 1.7.0 test org.hamcrest hamcrest - 2.2 + 3.0 test org.junit.jupiter junit-jupiter-api - 5.10.2 + 5.11.0 test org.junit.jupiter junit-jupiter-engine - 5.10.2 + 5.11.0 test org.mockito mockito-junit-jupiter - 5.11.0 + 5.13.0 test nl.jqno.equalsverifier equalsverifier - 3.15.8 + 3.16.2 test @@ -120,7 +127,7 @@ org.slf4j slf4j-jdk14 - 2.0.12 + 2.0.16 test @@ -145,7 +152,7 @@ com.exasol project-keeper-maven-plugin - 4.1.0 + 4.3.3 @@ -159,7 +166,7 @@ test-db-builder-java-generated-parent com.exasol - 3.5.4 + 3.5.5 pk_generated_parent.pom diff --git a/release_config.yml b/release_config.yml deleted file mode 100644 index 473c219..0000000 --- a/release_config.yml +++ /dev/null @@ -1,4 +0,0 @@ -release-platforms: - - GitHub - - Maven -language: Java From e18d0a3b25ae5fff2269785b558eb896189b53ed Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 16:07:21 +0200 Subject: [PATCH 02/13] Fix compiler warnings, upgrade databases --- .../exasol/AttachToExistingObjectIT.java | 3 +- ...olDatabaseObjectCreationAndDeletionIT.java | 4 +-- ...asolImmediateDatabaseObjectWriterTest.java | 28 ++++++++++--------- ...QLDatabaseObjectCreationAndDeletionIT.java | 7 +++-- .../dialects/mysql/MySqlSchemaTest.java | 9 +++--- ...qlDatabaseObjectCreationAndDeletionIT.java | 2 +- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/test/java/com/exasol/dbbuilder/dialects/exasol/AttachToExistingObjectIT.java b/src/test/java/com/exasol/dbbuilder/dialects/exasol/AttachToExistingObjectIT.java index d6aa450..272713f 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/exasol/AttachToExistingObjectIT.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/exasol/AttachToExistingObjectIT.java @@ -22,6 +22,7 @@ @Tag("integration") class AttachToExistingObjectIT { @Container + @SuppressWarnings("resource") // Will be closed by JUnit rule private static final ExasolContainer> container = new ExasolContainer<>() .withReuse(true); private static ExasolObjectFactory factory; @@ -63,4 +64,4 @@ void testAttachToScriptFromSqlFile(@TempDir final Path tempDir) throws IOExcepti void testAttachToScriptThrowsExceptionOnNonExistingFile() { assertThrows(DatabaseObjectException.class, () -> factory.executeSqlFile(Path.of("non/existent/file.sql"))); } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolDatabaseObjectCreationAndDeletionIT.java b/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolDatabaseObjectCreationAndDeletionIT.java index f2fa83b..d186a72 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolDatabaseObjectCreationAndDeletionIT.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolDatabaseObjectCreationAndDeletionIT.java @@ -39,6 +39,7 @@ // [itest->dsn~exasol-object-factory~1] class ExasolDatabaseObjectCreationAndDeletionIT extends AbstractDatabaseObjectCreationAndDeletionIT { @Container + @SuppressWarnings("resource") // Will be closed by JUnit rule private static final ExasolContainer> container = new ExasolContainer<>() .withReuse(true); private static final String ADAPTER_SCRIPT_CONTENT = "def adapter_call(request):\n" + // @@ -359,8 +360,7 @@ void testCreateReturnsUdf() throws SQLException { private ResultSet getScriptDescription(final ExasolSchema exasolSchema) throws SQLException { final String sql = "SELECT SCRIPT_TEXT FROM EXA_ALL_SCRIPTS WHERE SCRIPT_SCHEMA = '" + exasolSchema.getName() + "'"; - final ResultSet result = getAdminConnection().createStatement().executeQuery(sql); - return result; + return getAdminConnection().createStatement().executeQuery(sql); } @Test diff --git a/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolImmediateDatabaseObjectWriterTest.java b/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolImmediateDatabaseObjectWriterTest.java index 129ea55..bc7961f 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolImmediateDatabaseObjectWriterTest.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/exasol/ExasolImmediateDatabaseObjectWriterTest.java @@ -25,12 +25,13 @@ protected DatabaseObjectWriter getDatabaseObjectWriter() { void testWriteAdapterScript() { final ExasolImmediateDatabaseObjectWriterStub objectWriter = spy(new ExasolImmediateDatabaseObjectWriterStub( this.connectionMock, ExasolObjectConfiguration.builder().build())); - final ExasolSchema schema = new ExasolSchema(objectWriter, ExasolIdentifier.of("TEST_SCHEMA")); - final AdapterScript adapterScript = schema.createAdapterScriptBuilder("MY_ADAPTER").content("content") - .language(AdapterScript.Language.JAVA).build(); - objectWriter.write(adapterScript); - assertThat(objectWriter.getLastQuery(), - equalTo("CREATE JAVA ADAPTER SCRIPT \"TEST_SCHEMA\".\"MY_ADAPTER\" AS\ncontent\n/")); + try (final ExasolSchema schema = new ExasolSchema(objectWriter, ExasolIdentifier.of("TEST_SCHEMA"))) { + final AdapterScript adapterScript = schema.createAdapterScriptBuilder("MY_ADAPTER").content("content") + .language(AdapterScript.Language.JAVA).build(); + objectWriter.write(adapterScript); + assertThat(objectWriter.getLastQuery(), + equalTo("CREATE JAVA ADAPTER SCRIPT \"TEST_SCHEMA\".\"MY_ADAPTER\" AS\ncontent\n/")); + } } @Test @@ -38,11 +39,12 @@ void testWriteAdapterScript() { void testWriteAdapterScriptWithJvmOption() { final ExasolImmediateDatabaseObjectWriterStub objectWriter = new ExasolImmediateDatabaseObjectWriterStub( this.connectionMock, ExasolObjectConfiguration.builder().withJvmOptions("-DmyProp=1").build()); - final ExasolSchema schema = new ExasolSchema(objectWriter, ExasolIdentifier.of("TEST_SCHEMA")); - final AdapterScript adapterScript = schema.createAdapterScriptBuilder("MY_ADAPTER").content("content") - .language(AdapterScript.Language.JAVA).build(); - objectWriter.write(adapterScript); - assertThat(objectWriter.getLastQuery(), equalTo( - "CREATE JAVA ADAPTER SCRIPT \"TEST_SCHEMA\".\"MY_ADAPTER\" AS\n%jvmoption -DmyProp=1;\ncontent\n/")); + try (final ExasolSchema schema = new ExasolSchema(objectWriter, ExasolIdentifier.of("TEST_SCHEMA"))) { + final AdapterScript adapterScript = schema.createAdapterScriptBuilder("MY_ADAPTER").content("content") + .language(AdapterScript.Language.JAVA).build(); + objectWriter.write(adapterScript); + assertThat(objectWriter.getLastQuery(), equalTo( + "CREATE JAVA ADAPTER SCRIPT \"TEST_SCHEMA\".\"MY_ADAPTER\" AS\n%jvmoption -DmyProp=1;\ncontent\n/")); + } } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java index f14c696..a6aaf1b 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java @@ -26,8 +26,9 @@ @Testcontainers // [itest->dsn~mysql-object-factory~1] class MySQLDatabaseObjectCreationAndDeletionIT extends AbstractDatabaseObjectCreationAndDeletionIT { - private static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:8.2.0"; + private static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:9.0.1"; @Container + @SuppressWarnings("resource") // Will be closed by JUnit rule private static final MySQLContainer container = new MySQLContainer<>(MYSQL_DOCKER_IMAGE_REFERENCE) .withUsername("root").withPassword(""); @@ -79,7 +80,7 @@ private void assertUserHasSchemaPrivilege(final String username, final String ob } @Test - void testGrantSchemaPrivilegeToUser() throws InterruptedException { + void testGrantSchemaPrivilegeToUser() { final Schema schema = this.factory.createSchema("OBJPRIVSCHEMA"); final User user = this.factory.createUser("OBJPRIVUSER").grant(schema, SELECT, DELETE); assertAll(() -> assertUserHasSchemaPrivilege(user.getName(), schema.getName(), "Select_priv"), @@ -87,7 +88,7 @@ void testGrantSchemaPrivilegeToUser() throws InterruptedException { } @Test - void testGrantTablePrivilegeToUser() throws InterruptedException { + void testGrantTablePrivilegeToUser() { final Schema schema = this.factory.createSchema("TABPRIVSCHEMA"); final Table table = schema.createTable("TABPRIVTABLE", "COL1", "DATE", "COL2", "INT"); final User user = this.factory.createUser("TABPRIVUSER").grant(table, SELECT, DELETE); diff --git a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchemaTest.java b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchemaTest.java index a96c0e3..0d9f512 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchemaTest.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchemaTest.java @@ -33,8 +33,9 @@ void testGetFullyQualifiedName() { @Test void testCreateTableBuilder() { - final MySqlSchema mySqlSchema = new MySqlSchema(this.writerMock, MySQLIdentifier.of("THE_SCHEMA")); - final Table table = mySqlSchema.createTableBuilder("TABLE_D").column("A", "DATE").build(); - assertThat(table.getName(), equalTo("TABLE_D")); + try (final MySqlSchema mySqlSchema = new MySqlSchema(this.writerMock, MySQLIdentifier.of("THE_SCHEMA"))) { + final Table table = mySqlSchema.createTableBuilder("TABLE_D").column("A", "DATE").build(); + assertThat(table.getName(), equalTo("TABLE_D")); + } } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/dbbuilder/dialects/postgres/PostgreSqlDatabaseObjectCreationAndDeletionIT.java b/src/test/java/com/exasol/dbbuilder/dialects/postgres/PostgreSqlDatabaseObjectCreationAndDeletionIT.java index e79df80..c28ab3a 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/postgres/PostgreSqlDatabaseObjectCreationAndDeletionIT.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/postgres/PostgreSqlDatabaseObjectCreationAndDeletionIT.java @@ -11,7 +11,7 @@ import com.exasol.errorreporting.ExaError; class PostgreSqlDatabaseObjectCreationAndDeletionIT extends AbstractDatabaseObjectCreationAndDeletionIT { - private static final String POSTGRES_DOCKER_IMAGE_REFERENCE = "postgres:16.1-bullseye"; + private static final String POSTGRES_DOCKER_IMAGE_REFERENCE = "postgres:16.4-bullseye"; @Container private static final PostgreSQLContainer> container = new PostgreSQLContainer<>( POSTGRES_DOCKER_IMAGE_REFERENCE); From 80c99af45e1bb2d452e57250e844d8b3202fcad4 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 16:09:47 +0200 Subject: [PATCH 03/13] Update gitattributes --- .gitattributes | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index 193b61e..57a4570 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,10 +6,7 @@ doc/changes/changelog.md linguist-genera .github/workflows/ci-build.yml linguist-generated=true .github/workflows/dependencies_check.yml linguist-generated=true .github/workflows/dependencies_update.yml linguist-generated=true -.github/workflows/release_droid_prepare_original_checksum.yml linguist-generated=true -.github/workflows/release_droid_release_on_maven_central.yml linguist-generated=true -.github/workflows/release_droid_print_quick_checksum.yml linguist-generated=true -.github/workflows/release_droid_upload_github_release_assets.yml linguist-generated=true +.github/workflows/release.yml linguist-generated=true .settings/org.eclipse.jdt.core.prefs linguist-generated=true .settings/org.eclipse.jdt.ui.prefs linguist-generated=true From 956d262798ed4f95908f59fac5404405e9277298 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 16:11:55 +0200 Subject: [PATCH 04/13] Fix compile errors --- pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index c872270..e166d72 100644 --- a/pom.xml +++ b/pom.xml @@ -161,6 +161,16 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:all,-path + -Werror + + + From 04013c4f9e1b71753b2c4f80a75fbb201ae68869 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 16:13:06 +0200 Subject: [PATCH 05/13] Run PK fix --- dependencies.md | 86 ++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/dependencies.md b/dependencies.md index 4816e46..fd1f7f8 100644 --- a/dependencies.md +++ b/dependencies.md @@ -37,24 +37,24 @@ | ------------------------------------------------------- | ------------------------------------- | | [SonarQube Scanner for Maven][30] | [GNU LGPL 3][31] | | [Apache Maven Toolchains Plugin][32] | [Apache-2.0][27] | -| [Apache Maven Compiler Plugin][33] | [Apache-2.0][27] | -| [Apache Maven Enforcer Plugin][34] | [Apache-2.0][27] | -| [Maven Flatten Plugin][35] | [Apache Software Licenese][27] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][36] | [ASL2][37] | -| [Maven Surefire Plugin][38] | [Apache-2.0][27] | -| [Versions Maven Plugin][39] | [Apache License, Version 2.0][27] | -| [duplicate-finder-maven-plugin Maven Mojo][40] | [Apache License 2.0][41] | -| [Apache Maven Deploy Plugin][42] | [Apache-2.0][27] | -| [Apache Maven GPG Plugin][43] | [Apache-2.0][27] | -| [Apache Maven Source Plugin][44] | [Apache License, Version 2.0][27] | -| [Apache Maven Javadoc Plugin][45] | [Apache-2.0][27] | -| [Nexus Staging Maven Plugin][46] | [Eclipse Public License][47] | -| [Maven Failsafe Plugin][48] | [Apache-2.0][27] | -| [JaCoCo :: Maven Plugin][49] | [EPL-2.0][50] | -| [error-code-crawler-maven-plugin][51] | [MIT License][52] | -| [Reproducible Build Maven Plugin][53] | [Apache 2.0][37] | -| [OpenFastTrace Maven Plugin][54] | [GNU General Public License v3.0][55] | -| [Project Keeper Maven plugin][56] | [The MIT License][57] | +| [OpenFastTrace Maven Plugin][33] | [GNU General Public License v3.0][34] | +| [Project Keeper Maven plugin][35] | [The MIT License][36] | +| [Apache Maven Compiler Plugin][37] | [Apache-2.0][27] | +| [Apache Maven Enforcer Plugin][38] | [Apache-2.0][27] | +| [Maven Flatten Plugin][39] | [Apache Software Licenese][27] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][40] | [ASL2][41] | +| [Maven Surefire Plugin][42] | [Apache-2.0][27] | +| [Versions Maven Plugin][43] | [Apache License, Version 2.0][27] | +| [duplicate-finder-maven-plugin Maven Mojo][44] | [Apache License 2.0][45] | +| [Apache Maven Deploy Plugin][46] | [Apache-2.0][27] | +| [Apache Maven GPG Plugin][47] | [Apache-2.0][27] | +| [Apache Maven Source Plugin][48] | [Apache License, Version 2.0][27] | +| [Apache Maven Javadoc Plugin][49] | [Apache-2.0][27] | +| [Nexus Staging Maven Plugin][50] | [Eclipse Public License][51] | +| [Maven Failsafe Plugin][52] | [Apache-2.0][27] | +| [JaCoCo :: Maven Plugin][53] | [EPL-2.0][54] | +| [error-code-crawler-maven-plugin][55] | [MIT License][56] | +| [Reproducible Build Maven Plugin][57] | [Apache 2.0][41] | [0]: https://github.com/exasol/db-fundamentals-java/ [1]: https://github.com/exasol/db-fundamentals-java/blob/main/LICENSE @@ -89,28 +89,28 @@ [30]: http://sonarsource.github.io/sonar-scanner-maven/ [31]: http://www.gnu.org/licenses/lgpl.txt [32]: https://maven.apache.org/plugins/maven-toolchains-plugin/ -[33]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[34]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[35]: https://www.mojohaus.org/flatten-maven-plugin/ -[36]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[37]: http://www.apache.org/licenses/LICENSE-2.0.txt -[38]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[39]: https://www.mojohaus.org/versions/versions-maven-plugin/ -[40]: https://basepom.github.io/duplicate-finder-maven-plugin -[41]: http://www.apache.org/licenses/LICENSE-2.0.html -[42]: https://maven.apache.org/plugins/maven-deploy-plugin/ -[43]: https://maven.apache.org/plugins/maven-gpg-plugin/ -[44]: https://maven.apache.org/plugins/maven-source-plugin/ -[45]: https://maven.apache.org/plugins/maven-javadoc-plugin/ -[46]: http://www.sonatype.com/public-parent/nexus-maven-plugins/nexus-staging/nexus-staging-maven-plugin/ -[47]: http://www.eclipse.org/legal/epl-v10.html -[48]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[49]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[50]: https://www.eclipse.org/legal/epl-2.0/ -[51]: https://github.com/exasol/error-code-crawler-maven-plugin/ -[52]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE -[53]: http://zlika.github.io/reproducible-build-maven-plugin -[54]: https://github.com/itsallcode/openfasttrace-maven-plugin -[55]: https://www.gnu.org/licenses/gpl-3.0.html -[56]: https://github.com/exasol/project-keeper/ -[57]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[33]: https://github.com/itsallcode/openfasttrace-maven-plugin +[34]: https://www.gnu.org/licenses/gpl-3.0.html +[35]: https://github.com/exasol/project-keeper/ +[36]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[37]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[38]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[39]: https://www.mojohaus.org/flatten-maven-plugin/ +[40]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[41]: http://www.apache.org/licenses/LICENSE-2.0.txt +[42]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[43]: https://www.mojohaus.org/versions/versions-maven-plugin/ +[44]: https://basepom.github.io/duplicate-finder-maven-plugin +[45]: http://www.apache.org/licenses/LICENSE-2.0.html +[46]: https://maven.apache.org/plugins/maven-deploy-plugin/ +[47]: https://maven.apache.org/plugins/maven-gpg-plugin/ +[48]: https://maven.apache.org/plugins/maven-source-plugin/ +[49]: https://maven.apache.org/plugins/maven-javadoc-plugin/ +[50]: http://www.sonatype.com/public-parent/nexus-maven-plugins/nexus-staging/nexus-staging-maven-plugin/ +[51]: http://www.eclipse.org/legal/epl-v10.html +[52]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[53]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[54]: https://www.eclipse.org/legal/epl-2.0/ +[55]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[56]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[57]: http://zlika.github.io/reproducible-build-maven-plugin From 728c3682e64959f40e40795ba5e06c514e9b0c26 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 16:29:35 +0200 Subject: [PATCH 06/13] #137: Use batch insert --- doc/changes/changes_3.5.5.md | 6 ++++++ error_code_config.yml | 2 +- .../AbstractImmediateDatabaseObjectWriter.java | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/doc/changes/changes_3.5.5.md b/doc/changes/changes_3.5.5.md index b95bcab..7ca03d9 100644 --- a/doc/changes/changes_3.5.5.md +++ b/doc/changes/changes_3.5.5.md @@ -6,10 +6,16 @@ Code name: Fix CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-ja This release fixes CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1`. +The release also speeds up inserting rows into a table by using batch insert. + ## Security * #138: Fixed CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1` +## Features + +* #137: Updated `AbstractImmediateDatabaseObjectWriter#write()` to use batching for inserting rows + ## Dependency Updates ### Test Dependency Updates diff --git a/error_code_config.yml b/error_code_config.yml index f0a5d0f..799dcf0 100644 --- a/error_code_config.yml +++ b/error_code_config.yml @@ -2,4 +2,4 @@ error-tags: TDBJ: packages: - com.exasol.dbbuilder - highest-index: 34 + highest-index: 35 diff --git a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java index 8e8ba48..2b9c69d 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java @@ -88,7 +88,8 @@ public void write(final Table table, final Stream> rows) { try (final PreparedStatement preparedStatement = this.connection.prepareStatement(sql)) { final boolean autoCommitOriginalState = this.connection.getAutoCommit(); this.connection.setAutoCommit(false); - rows.forEach(row -> writeRow(table, sql, preparedStatement, row)); + rows.forEach(row -> addBatch(table, preparedStatement, row)); + preparedStatement.executeBatch(); if (autoCommitOriginalState) { this.connection.commit(); this.connection.setAutoCommit(true); @@ -101,16 +102,15 @@ public void write(final Table table, final Stream> rows) { } } - private void writeRow(final Table table, final String sql, final PreparedStatement preparedStatement, - final List row) { + private void addBatch(final Table table, final PreparedStatement preparedStatement, final List row) { try { for (int i = 0; i < row.size(); ++i) { preparedStatement.setObject(i + 1, row.get(i)); } - preparedStatement.execute(); + preparedStatement.addBatch(); } catch (final SQLException exception) { - throw new DatabaseObjectException(table, ExaError.messageBuilder("E-TDBJ-1") - .message("Failed to execute insert query: {{statement}}", sql).toString(), exception); + throw new DatabaseObjectException(table, + ExaError.messageBuilder("E-TDBJ-35").message("Failed to row to batch").toString(), exception); } } From 2f4ab4435611b93b6c856b85a67b9c9535f60f92 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 23 Sep 2024 17:28:56 +0200 Subject: [PATCH 07/13] Increment minor version --- doc/changes/changelog.md | 2 +- doc/changes/{changes_3.5.5.md => changes_3.6.0.md} | 2 +- pk_generated_parent.pom | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename doc/changes/{changes_3.5.5.md => changes_3.6.0.md} (97%) diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 09d3b3a..8c1df46 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,6 @@ # Changes -* [3.5.5](changes_3.5.5.md) +* [3.6.0](changes_3.6.0.md) * [3.5.4](changes_3.5.4.md) * [3.5.3](changes_3.5.3.md) * [3.5.2](changes_3.5.2.md) diff --git a/doc/changes/changes_3.5.5.md b/doc/changes/changes_3.6.0.md similarity index 97% rename from doc/changes/changes_3.5.5.md rename to doc/changes/changes_3.6.0.md index 7ca03d9..78e2a75 100644 --- a/doc/changes/changes_3.5.5.md +++ b/doc/changes/changes_3.6.0.md @@ -1,4 +1,4 @@ -# Test Database Builder for Java 3.5.5, released 2024-??-?? +# Test Database Builder for Java 3.6.0, released 2024-??-?? Code name: Fix CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index d1c0769..239618b 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol test-db-builder-java-generated-parent - 3.5.5 + 3.6.0 pom UTF-8 diff --git a/pom.xml b/pom.xml index e166d72..2d9d761 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 test-db-builder-java - 3.5.5 + 3.6.0 Test Database Builder for Java pom.xml https://github.com/exasol/test-db-builder-java/ @@ -176,7 +176,7 @@ test-db-builder-java-generated-parent com.exasol - 3.5.5 + 3.6.0 pk_generated_parent.pom From c565bf92c23dbc256f84f874a3a21960ad42060f Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Tue, 24 Sep 2024 09:46:48 +0200 Subject: [PATCH 08/13] #134: Allowed specifying charset for MySQL tables --- doc/changes/changes_3.6.0.md | 3 +- doc/user_guide/user_guide.md | 15 ++- .../dbbuilder/dialects/AbstractSchema.java | 2 +- .../MySqlImmediateDatabaseObjectWriter.java | 24 +++- .../dbbuilder/dialects/mysql/MySqlSchema.java | 5 + .../dbbuilder/dialects/mysql/MySqlTable.java | 51 +++++++++ .../dialects/AbstractObjectFactoryTest.java | 4 +- ...QLDatabaseObjectCreationAndDeletionIT.java | 103 ++++++++++++++---- .../dialects/mysql/MySqlTableTest.java | 34 ++++++ 9 files changed, 214 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java create mode 100644 src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlTableTest.java diff --git a/doc/changes/changes_3.6.0.md b/doc/changes/changes_3.6.0.md index 78e2a75..11e73f9 100644 --- a/doc/changes/changes_3.6.0.md +++ b/doc/changes/changes_3.6.0.md @@ -6,7 +6,7 @@ Code name: Fix CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-ja This release fixes CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1`. -The release also speeds up inserting rows into a table by using batch insert. +The release also speeds up inserting rows into a table by using batch insert and allows specifying a charset when creating MySQL tables, see the [user guide](../user_guide/user_guide.md#mysql-specific-database-objects) for details. ## Security @@ -15,6 +15,7 @@ The release also speeds up inserting rows into a table by using batch insert. ## Features * #137: Updated `AbstractImmediateDatabaseObjectWriter#write()` to use batching for inserting rows +* #134: Allowed specifying charset for MySQL tables ## Dependency Updates diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 5b5c403..ea6f7fc 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -62,7 +62,7 @@ final Table table = schema.createTable("DAYS","DAY_NAME","VARCHAR(9), "SHORT_NAM In case you want to create more complex tables, you can also use a builder. ```java -final Table table=schema.createTableBuilder("DAYS") +final Table table = schema.createTableBuilder("DAYS") .column("DAY_NAME","VARCHAR(9)" .column("SHORT_NAME","VARCHAR(3)" .column("DAY_IN_WEEK","DECIMAL(1,0)" @@ -390,6 +390,17 @@ Given that a script of that name exists, you can then [execute the script](#exec ## MySQL-Specific Database Objects -So far there are no MySQL Specific Database Objects that are not described in [Dialect-Agnostic Database Objects](#dialect-agnostic-database-objects) section. +In addition to [Dialect-Agnostic Database Objects](#dialect-agnostic-database-objects), MySQL allows specifying a charset when creating a new table using the table builder of a `MySqlSchema`. When no charset is specified, MySql uses UTF8 as default. + +```java +final MySqlSchema schema = (MySqlSchema) factory.createSchema("TEST")); +final MySqlTable table = schema.createTableBuilder("ASCII_DAYS") + .charset("ASCII") + .column("DAY_NAME","VARCHAR(9)" + .column("SHORT_NAME","VARCHAR(3)" + .column("DAY_IN_WEEK","DECIMAL(1,0)" + // ... + .build() +``` Please keep in mind that Schema object represents a database in MySQL as a schema is a [synonym](https://dev.mysql.com/doc/refman/8.0/en/create-database.html) for a database in MySQL syntax. diff --git a/src/main/java/com/exasol/dbbuilder/dialects/AbstractSchema.java b/src/main/java/com/exasol/dbbuilder/dialects/AbstractSchema.java index 87c1224..b6b06d1 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/AbstractSchema.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/AbstractSchema.java @@ -63,7 +63,7 @@ public Table.Builder createTableBuilder(final String name) { public Table createTable(final String name, final List columnNames, final List columnTypes) { verifyNotDeleted(); if (columnNames.size() == columnTypes.size()) { - final Table.Builder builder = Table.builder(getWriter(), this, getIdentifier(name)); + final Table.Builder builder = createTableBuilder(name); passColumnsToTableBuilder(columnNames, columnTypes, builder); final Table table = builder.build(); this.tables.add(table); diff --git a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlImmediateDatabaseObjectWriter.java b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlImmediateDatabaseObjectWriter.java index 5767210..29ec636 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlImmediateDatabaseObjectWriter.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlImmediateDatabaseObjectWriter.java @@ -46,9 +46,31 @@ public void write(final User user, final DatabaseObject object, final ObjectPriv } } + @Override + public void write(final Table table) { + final MySqlTable mySqlTable = (MySqlTable) table; + final StringBuilder builder = new StringBuilder("CREATE TABLE "); + builder.append(mySqlTable.getFullyQualifiedName()).append(" ("); + int i = 0; + for (final Column column : mySqlTable.getColumns()) { + if (i++ > 0) { + builder.append(", "); + } + builder.append(getQuotedColumnName(column.getName())) // + .append(" ") // + .append(column.getType()); + } + builder.append(")"); + if (mySqlTable.getCharset() != null) { + builder.append(" CHARACTER SET ") // + .append(mySqlTable.getCharset()); + } + writeToObject(mySqlTable, builder.toString()); + } + @Override // [impl->dsn~dropping-schemas~2] public void drop(final Schema schema) { writeToObject(schema, "DROP SCHEMA " + schema.getFullyQualifiedName()); } -} \ No newline at end of file +} diff --git a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java index bb31781..8e7b933 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java @@ -31,4 +31,9 @@ public DatabaseObjectWriter getWriter() { protected Identifier getIdentifier(final String name) { return MySQLIdentifier.of(name); } + + @Override + public MySqlTable.MySqlTableBuilder createTableBuilder(final String name) { + return MySqlTable.builder(getWriter(), this, getIdentifier(name)); + } } diff --git a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java new file mode 100644 index 0000000..a9dba74 --- /dev/null +++ b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java @@ -0,0 +1,51 @@ +package com.exasol.dbbuilder.dialects.mysql; + +import com.exasol.db.Identifier; +import com.exasol.dbbuilder.dialects.*; + +public class MySqlTable extends Table { + + private final String charset; + + protected MySqlTable(final MySqlTableBuilder builder) { + super(builder); + this.charset = builder.charset; + } + + public String getCharset() { + return charset; + } + + public static MySqlTableBuilder builder(final DatabaseObjectWriter writer, final Schema parentSchema, + final Identifier tableName) { + return new MySqlTableBuilder(writer, parentSchema, tableName); + } + + public static class MySqlTableBuilder extends Table.Builder { + + private String charset; + + private MySqlTableBuilder(final DatabaseObjectWriter writer, final Schema parentSchema, + final Identifier tableName) { + super(writer, parentSchema, tableName); + } + + @Override + public MySqlTableBuilder column(final String columnName, final String columnType) { + super.column(columnName, columnType); + return this; + } + + public MySqlTableBuilder charset(final String charset) { + this.charset = charset; + return this; + } + + @Override + public MySqlTable build() { + final MySqlTable table = new MySqlTable(this); + this.writer.write(table); + return table; + } + } +} diff --git a/src/test/java/com/exasol/dbbuilder/dialects/AbstractObjectFactoryTest.java b/src/test/java/com/exasol/dbbuilder/dialects/AbstractObjectFactoryTest.java index 7e1c8bb..d5e93f0 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/AbstractObjectFactoryTest.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/AbstractObjectFactoryTest.java @@ -7,9 +7,9 @@ public abstract class AbstractObjectFactoryTest { - abstract protected AbstractImmediateDatabaseObjectWriter getWriterMock(); + protected abstract AbstractImmediateDatabaseObjectWriter getWriterMock(); - abstract protected DatabaseObjectFactory testee(); + protected abstract DatabaseObjectFactory testee(); @Test void createSchemaWritesObject() { diff --git a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java index a6aaf1b..730528c 100644 --- a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java +++ b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySQLDatabaseObjectCreationAndDeletionIT.java @@ -9,12 +9,14 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.sql.*; import org.hamcrest.Matcher; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.testcontainers.containers.JdbcDatabaseContainer.NoDriverFoundException; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -81,19 +83,21 @@ private void assertUserHasSchemaPrivilege(final String username, final String ob @Test void testGrantSchemaPrivilegeToUser() { - final Schema schema = this.factory.createSchema("OBJPRIVSCHEMA"); - final User user = this.factory.createUser("OBJPRIVUSER").grant(schema, SELECT, DELETE); - assertAll(() -> assertUserHasSchemaPrivilege(user.getName(), schema.getName(), "Select_priv"), - () -> assertUserHasSchemaPrivilege(user.getName(), schema.getName(), "Delete_priv")); + try (final Schema schema = this.factory.createSchema("OBJPRIVSCHEMA")) { + final User user = this.factory.createUser("OBJPRIVUSER").grant(schema, SELECT, DELETE); + assertAll(() -> assertUserHasSchemaPrivilege(user.getName(), schema.getName(), "Select_priv"), + () -> assertUserHasSchemaPrivilege(user.getName(), schema.getName(), "Delete_priv")); + } } @Test void testGrantTablePrivilegeToUser() { - final Schema schema = this.factory.createSchema("TABPRIVSCHEMA"); - final Table table = schema.createTable("TABPRIVTABLE", "COL1", "DATE", "COL2", "INT"); - final User user = this.factory.createUser("TABPRIVUSER").grant(table, SELECT, DELETE); - assertAll(() -> assertUserHasTablePrivilege(user.getName(), table.getName(), "Select"), - () -> assertUserHasTablePrivilege(user.getName(), table.getName(), "Delete")); + try (final Schema schema = this.factory.createSchema("TABPRIVSCHEMA")) { + final Table table = schema.createTable("TABPRIVTABLE", "COL1", "DATE", "COL2", "INT"); + final User user = this.factory.createUser("TABPRIVUSER").grant(table, SELECT, DELETE); + assertAll(() -> assertUserHasTablePrivilege(user.getName(), table.getName(), "Select"), + () -> assertUserHasTablePrivilege(user.getName(), table.getName(), "Delete")); + } } private void assertUserHasTablePrivilege(final String username, final String objectName, @@ -114,17 +118,76 @@ private void assertUserHasTablePrivilege(final String username, final String obj @Test void testInsertIntoTable() { - final Schema schema = this.factory.createSchema("INSERTSCHEMA"); - final Table table = schema.createTable("INSERTTABLE", "ID", "INT", "NAME", "VARCHAR(10)"); - table.insert(1, "FOO").insert(2, "BAR"); - try { - final ResultSet result = this.adminConnection.createStatement() - .executeQuery("SELECT ID, NAME FROM " + table.getFullyQualifiedName() + "ORDER BY ID ASC"); - assertThat(result, table().row(1, "FOO").row(2, "BAR").matches()); - } catch (final SQLException exception) { - throw new AssertionError(ExaError.messageBuilder("E-TDBJ-25") - .message("Unable to validate contents of table {{table}}", table.getFullyQualifiedName()) - .toString(), exception); + try (final Schema schema = this.factory.createSchema("INSERTSCHEMA")) { + final Table table = schema.createTable("INSERTTABLE", "ID", "INT", "NAME", "VARCHAR(10)"); + table.insert(1, "FOO").insert(2, "BAR"); + try { + final ResultSet result = this.adminConnection.createStatement() + .executeQuery("SELECT ID, NAME FROM " + table.getFullyQualifiedName() + "ORDER BY ID ASC"); + assertThat(result, table().row(1, "FOO").row(2, "BAR").matches()); + } catch (final SQLException exception) { + throw new AssertionError(ExaError.messageBuilder("E-TDBJ-25") + .message("Unable to validate contents of table {{table}}", table.getFullyQualifiedName()) + .toString(), exception); + } + } + } + + @Test + void testCreateTableWithDefaultCharsetUsesUtf8() { + try (final MySqlSchema schema = (MySqlSchema) this.factory.createSchema("CHARSET_SCHEMA_DEFAULT")) { + final MySqlTable table = schema.createTableBuilder("TABLE_WITH_CHARSET").column("ID", "INT") + .column("NAME", "VARCHAR(10)").build(); + assertAll( + () -> assertThat("column charset", + getColumnCharset("def", schema.getName(), table.getName(), "NAME"), equalTo("utf8mb4")), + () -> assertThat("table collation", getTableCollation("def", schema.getName(), table.getName()), + equalTo("utf8mb4_0900_ai_ci"))); + } + } + + @Test + void testCreateTableWithCharset() { + try (final MySqlSchema schema = (MySqlSchema) this.factory.createSchema("CHARSET_SCHEMA_ASCII")) { + final MySqlTable table = schema.createTableBuilder("TABLE_WITH_CHARSET").charset("ASCII") + .column("ID", "INT").column("NAME", "VARCHAR(10)").build(); + assertAll( + () -> assertThat("column charset", + getColumnCharset("def", schema.getName(), table.getName(), "NAME"), equalTo("ascii")), + () -> assertThat("table collation", getTableCollation("def", schema.getName(), table.getName()), + equalTo("ascii_general_ci"))); + } + } + + private String getColumnCharset(final String catalog, final String schema, final String table, + final String column) { + final String query = "select CHARACTER_SET_NAME from information_schema.COLUMNS " + + "where TABLE_CATALOG=? AND TABLE_SCHEMA=? AND TABLE_NAME=? AND COLUMN_NAME=?"; + try (Connection con = container.createConnection(""); PreparedStatement stmt = con.prepareStatement(query)) { + stmt.setString(1, catalog); + stmt.setString(2, schema); + stmt.setString(3, table); + stmt.setString(4, column); + final ResultSet rs = stmt.executeQuery(); + assertTrue(rs.next()); + return rs.getString("CHARACTER_SET_NAME"); + } catch (NoDriverFoundException | SQLException exception) { + throw new IllegalStateException("Query '" + query + "' failed: " + exception.getMessage(), exception); + } + } + + private String getTableCollation(final String catalog, final String schema, final String table) { + final String query = "select TABLE_COLLATION from information_schema.TABLES " + + "where TABLE_CATALOG=? AND TABLE_SCHEMA=? AND TABLE_NAME=?"; + try (Connection con = container.createConnection(""); PreparedStatement stmt = con.prepareStatement(query)) { + stmt.setString(1, catalog); + stmt.setString(2, schema); + stmt.setString(3, table); + final ResultSet rs = stmt.executeQuery(); + assertTrue(rs.next()); + return rs.getString("TABLE_COLLATION"); + } catch (NoDriverFoundException | SQLException exception) { + throw new IllegalStateException("Query '" + query + "' failed: " + exception.getMessage(), exception); } } diff --git a/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlTableTest.java b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlTableTest.java new file mode 100644 index 0000000..2e19481 --- /dev/null +++ b/src/test/java/com/exasol/dbbuilder/dialects/mysql/MySqlTableTest.java @@ -0,0 +1,34 @@ +package com.exasol.dbbuilder.dialects.mysql; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.exasol.dbbuilder.dialects.DatabaseObjectWriter; +import com.exasol.dbbuilder.dialects.Schema; + +@ExtendWith(MockitoExtension.class) +class MySqlTableTest { + + @Mock + DatabaseObjectWriter writerMock; + @Mock + Schema schemaMock; + + @Test + void createWithoutCharset() { + final MySqlTable table = MySqlTable.builder(writerMock, schemaMock, MySQLIdentifier.of("tableName")).build(); + assertThat(table.getCharset(), is(nullValue())); + } + + @Test + void createWithCharset() { + final MySqlTable table = MySqlTable.builder(writerMock, schemaMock, MySQLIdentifier.of("tableName")) + .charset("myCharset").build(); + assertThat(table.getCharset(), equalTo("myCharset")); + } +} From a28bf3e80832d6f8954feebf72dc45bfe69cdece Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Tue, 24 Sep 2024 10:32:01 +0200 Subject: [PATCH 09/13] #136: Add support for databases without transaction support --- doc/changes/changes_3.6.0.md | 3 +- error_code_config.yml | 2 +- ...AbstractImmediateDatabaseObjectWriter.java | 11 ++- .../exasol/dbbuilder/dialects/AutoCommit.java | 61 +++++++++++++++++ .../dbbuilder/dialects/AutoCommitTest.java | 68 +++++++++++++++++++ 5 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/exasol/dbbuilder/dialects/AutoCommit.java create mode 100644 src/test/java/com/exasol/dbbuilder/dialects/AutoCommitTest.java diff --git a/doc/changes/changes_3.6.0.md b/doc/changes/changes_3.6.0.md index 11e73f9..6514bf1 100644 --- a/doc/changes/changes_3.6.0.md +++ b/doc/changes/changes_3.6.0.md @@ -6,7 +6,7 @@ Code name: Fix CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-ja This release fixes CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1`. -The release also speeds up inserting rows into a table by using batch insert and allows specifying a charset when creating MySQL tables, see the [user guide](../user_guide/user_guide.md#mysql-specific-database-objects) for details. +The release also speeds up inserting rows into a table by using batch insert, allows specifying a charset when creating MySQL tables, see the [user guide](../user_guide/user_guide.md#mysql-specific-database-objects) for details and supports databases that don't support AutoCommit. ## Security @@ -16,6 +16,7 @@ The release also speeds up inserting rows into a table by using batch insert and * #137: Updated `AbstractImmediateDatabaseObjectWriter#write()` to use batching for inserting rows * #134: Allowed specifying charset for MySQL tables +* #136: Added support for databases without AutoCommit support ## Dependency Updates diff --git a/error_code_config.yml b/error_code_config.yml index 799dcf0..347a000 100644 --- a/error_code_config.yml +++ b/error_code_config.yml @@ -2,4 +2,4 @@ error-tags: TDBJ: packages: - com.exasol.dbbuilder - highest-index: 35 + highest-index: 37 diff --git a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java index 2b9c69d..e400614 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java @@ -85,15 +85,12 @@ public void truncate(final Table table) { public void write(final Table table, final Stream> rows) { final String valuePlaceholders = "?" + ", ?".repeat(table.getColumnCount() - 1); final String sql = "INSERT INTO " + table.getFullyQualifiedName() + " VALUES(" + valuePlaceholders + ")"; - try (final PreparedStatement preparedStatement = this.connection.prepareStatement(sql)) { - final boolean autoCommitOriginalState = this.connection.getAutoCommit(); - this.connection.setAutoCommit(false); + + try (@SuppressWarnings("try") // autoCommit never referenced in try block by intention + AutoCommit autoCommit = AutoCommit.tryDeactivate(connection); + final PreparedStatement preparedStatement = this.connection.prepareStatement(sql)) { rows.forEach(row -> addBatch(table, preparedStatement, row)); preparedStatement.executeBatch(); - if (autoCommitOriginalState) { - this.connection.commit(); - this.connection.setAutoCommit(true); - } } catch (final SQLException exception) { throw new DatabaseObjectException(table, ExaError.messageBuilder("E-TDBJ-2") diff --git a/src/main/java/com/exasol/dbbuilder/dialects/AutoCommit.java b/src/main/java/com/exasol/dbbuilder/dialects/AutoCommit.java new file mode 100644 index 0000000..ca0f71d --- /dev/null +++ b/src/main/java/com/exasol/dbbuilder/dialects/AutoCommit.java @@ -0,0 +1,61 @@ +package com.exasol.dbbuilder.dialects; + +import java.sql.*; +import java.util.logging.Logger; + +import com.exasol.errorreporting.ExaError; + +/** + * This class allows temporarily deactivating AutoCommit for a given {@link Connection} and restores the original state + * in {@link #close()}. If the database does not support deactivating AutoCommit (i.e. throws a + * {@link SQLFeatureNotSupportedException}), this class will silently ignore it. + */ +class AutoCommit implements AutoCloseable { + private static final Logger LOG = Logger.getLogger(AutoCommit.class.getName()); + private final Connection connection; + + private AutoCommit(final Connection connection) { + this.connection = connection; + } + + static AutoCommit tryDeactivate(final Connection connection) { + try { + final boolean originalState = connection.getAutoCommit(); + if (!originalState) { + return new AutoCommit(null); + } + if (deactivatingAutoCommitSuccessful(connection)) { + return new AutoCommit(connection); + } else { + return new AutoCommit(null); + } + } catch (final SQLException exception) { + throw new DatabaseObjectException( + ExaError.messageBuilder("E-TDBJ-36").message("Failed to check AutoCommit state").toString(), + exception); + } + } + + private static boolean deactivatingAutoCommitSuccessful(final Connection connection) throws SQLException { + try { + connection.setAutoCommit(false); + return true; + } catch (final SQLFeatureNotSupportedException exception) { + LOG.fine("Database does not support deactivating AutoCommit: " + exception.getMessage()); + return false; + } + } + + @Override + public void close() { + if (connection != null) { + try { + connection.setAutoCommit(true); + } catch (final SQLException exception) { + throw new DatabaseObjectException( + ExaError.messageBuilder("E-TDBJ-37").message("Failed to re-enable AutoCommit").toString(), + exception); + } + } + } +} diff --git a/src/test/java/com/exasol/dbbuilder/dialects/AutoCommitTest.java b/src/test/java/com/exasol/dbbuilder/dialects/AutoCommitTest.java new file mode 100644 index 0000000..279d3b1 --- /dev/null +++ b/src/test/java/com/exasol/dbbuilder/dialects/AutoCommitTest.java @@ -0,0 +1,68 @@ +package com.exasol.dbbuilder.dialects; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.*; + +import java.sql.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AutoCommitTest { + @Mock + Connection connectionMock; + + @Test + void autoCommitAlreadyDeactivated() throws SQLException { + when(connectionMock.getAutoCommit()).thenReturn(false); + AutoCommit.tryDeactivate(connectionMock).close(); + verify(connectionMock, never()).setAutoCommit(anyBoolean()); + verifyNoMoreInteractions(connectionMock); + } + + @Test + void autoCommitEnabledAndSupported() throws SQLException { + when(connectionMock.getAutoCommit()).thenReturn(true); + AutoCommit.tryDeactivate(connectionMock).close(); + final InOrder inOrder = inOrder(connectionMock); + inOrder.verify(connectionMock).setAutoCommit(false); + inOrder.verify(connectionMock).setAutoCommit(true); + inOrder.verifyNoMoreInteractions(); + } + + @Test + void autoCommitEnabledAndNotSupported() throws SQLException { + when(connectionMock.getAutoCommit()).thenReturn(true); + doThrow(new SQLFeatureNotSupportedException("unsupported")).when(connectionMock).setAutoCommit(false); + AutoCommit.tryDeactivate(connectionMock).close(); + verify(connectionMock).setAutoCommit(false); + verifyNoMoreInteractions(connectionMock); + } + + @Test + void settingAutoCommitFailsWithOtherException() throws SQLException { + when(connectionMock.getAutoCommit()).thenReturn(true); + doThrow(new SQLException("mock")).when(connectionMock).setAutoCommit(false); + final DatabaseObjectException exception = assertThrows(DatabaseObjectException.class, + () -> AutoCommit.tryDeactivate(connectionMock)); + assertThat(exception.getMessage(), equalTo("E-TDBJ-36: Failed to check AutoCommit state")); + assertThat(exception.getCause().getMessage(), equalTo("mock")); + } + + @Test + void reactivatingAutoCommitFails() throws SQLException { + when(connectionMock.getAutoCommit()).thenReturn(true); + final AutoCommit autoCommit = AutoCommit.tryDeactivate(connectionMock); + doThrow(new SQLException("mock")).when(connectionMock).setAutoCommit(true); + final DatabaseObjectException exception = assertThrows(DatabaseObjectException.class, autoCommit::close); + assertThat(exception.getMessage(), equalTo("E-TDBJ-37: Failed to re-enable AutoCommit")); + assertThat(exception.getCause().getMessage(), equalTo("mock")); + } +} From b249febb733a7c58ea4af78d4681917901e909f4 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Tue, 24 Sep 2024 10:32:55 +0200 Subject: [PATCH 10/13] Update changelog entry --- doc/changes/changes_3.6.0.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/changes/changes_3.6.0.md b/doc/changes/changes_3.6.0.md index 6514bf1..d5db58b 100644 --- a/doc/changes/changes_3.6.0.md +++ b/doc/changes/changes_3.6.0.md @@ -6,7 +6,7 @@ Code name: Fix CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-ja This release fixes CVE-2024-7254 in test dependency `com.google.protobuf:protobuf-java:3.25.1`. -The release also speeds up inserting rows into a table by using batch insert, allows specifying a charset when creating MySQL tables, see the [user guide](../user_guide/user_guide.md#mysql-specific-database-objects) for details and supports databases that don't support AutoCommit. +The release also speeds up inserting rows into a table by using batch insert, allows specifying a charset when creating MySQL tables, see the [user guide](../user_guide/user_guide.md#mysql-specific-database-objects) for details and supports databases that don't support transactions (i.e. setting autoCommit to `false`). ## Security @@ -16,7 +16,7 @@ The release also speeds up inserting rows into a table by using batch insert, al * #137: Updated `AbstractImmediateDatabaseObjectWriter#write()` to use batching for inserting rows * #134: Allowed specifying charset for MySQL tables -* #136: Added support for databases without AutoCommit support +* #136: Added support for databases without transaction support ## Dependency Updates From 7908f6d58af8412310716f51585ad379e9fe4368 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Tue, 24 Sep 2024 10:47:19 +0200 Subject: [PATCH 11/13] Fix compiler error --- .../dialects/AbstractImmediateDatabaseObjectWriter.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java index e400614..4d77e9f 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java @@ -82,12 +82,11 @@ public void truncate(final Table table) { protected abstract String getQuotedColumnName(String columnName); @Override + @SuppressWarnings("try") // autoCommit never referenced in try block by intention public void write(final Table table, final Stream> rows) { final String valuePlaceholders = "?" + ", ?".repeat(table.getColumnCount() - 1); final String sql = "INSERT INTO " + table.getFullyQualifiedName() + " VALUES(" + valuePlaceholders + ")"; - - try (@SuppressWarnings("try") // autoCommit never referenced in try block by intention - AutoCommit autoCommit = AutoCommit.tryDeactivate(connection); + try (AutoCommit autoCommit = AutoCommit.tryDeactivate(connection); final PreparedStatement preparedStatement = this.connection.prepareStatement(sql)) { rows.forEach(row -> addBatch(table, preparedStatement, row)); preparedStatement.executeBatch(); From 277c5bd3e494c80a6c971bbadbf6d9a146d58acd Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Tue, 24 Sep 2024 10:56:02 +0200 Subject: [PATCH 12/13] Add missing javadoc --- .../com/exasol/dbbuilder/dialects/Table.java | 4 +- .../dbbuilder/dialects/mysql/MySqlSchema.java | 2 +- .../dbbuilder/dialects/mysql/MySqlTable.java | 50 ++++++++++++++++--- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/exasol/dbbuilder/dialects/Table.java b/src/main/java/com/exasol/dbbuilder/dialects/Table.java index 0637511..41e0491 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/Table.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/Table.java @@ -30,7 +30,7 @@ protected Table(final Builder builder) { * @param writer database object writer * @param schema parent schema * @param tableName name of the database table - * @return new {@link Table} instance + * @return new {@link Builder} instance */ // [impl->dsn~creating-tables~1] public static Builder builder(final DatabaseObjectWriter writer, final Schema schema, final Identifier tableName) { @@ -151,4 +151,4 @@ public Table build() { return table; } } -} \ No newline at end of file +} diff --git a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java index 8e7b933..835bcd1 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlSchema.java @@ -33,7 +33,7 @@ protected Identifier getIdentifier(final String name) { } @Override - public MySqlTable.MySqlTableBuilder createTableBuilder(final String name) { + public MySqlTable.Builder createTableBuilder(final String name) { return MySqlTable.builder(getWriter(), this, getIdentifier(name)); } } diff --git a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java index a9dba74..eabb9c8 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/mysql/MySqlTable.java @@ -3,44 +3,78 @@ import com.exasol.db.Identifier; import com.exasol.dbbuilder.dialects.*; +/** + * A MySql table that allows specifying a charset. + */ public class MySqlTable extends Table { private final String charset; - protected MySqlTable(final MySqlTableBuilder builder) { + /** + * Create a new MySql table based on a given builder. + * + * @param builder the builder from which to copy the values + */ + protected MySqlTable(final Builder builder) { super(builder); this.charset = builder.charset; } + /** + * Get the table's charset. + * + * @return charset or {@code null} for the default charset + */ public String getCharset() { return charset; } - public static MySqlTableBuilder builder(final DatabaseObjectWriter writer, final Schema parentSchema, + /** + * Create a builder for a {@link MySqlTable}. + * + * @param writer database object writer + * @param parentSchema parent schema + * @param tableName name of the database table + * @return new {@link Builder} instance + */ + public static Builder builder(final DatabaseObjectWriter writer, final Schema parentSchema, final Identifier tableName) { - return new MySqlTableBuilder(writer, parentSchema, tableName); + return new Builder(writer, parentSchema, tableName); } - public static class MySqlTableBuilder extends Table.Builder { + /** + * Builder for {@link MySqlTable}s. + */ + public static class Builder extends Table.Builder { private String charset; - private MySqlTableBuilder(final DatabaseObjectWriter writer, final Schema parentSchema, - final Identifier tableName) { + private Builder(final DatabaseObjectWriter writer, final Schema parentSchema, final Identifier tableName) { super(writer, parentSchema, tableName); } @Override - public MySqlTableBuilder column(final String columnName, final String columnType) { + public Builder column(final String columnName, final String columnType) { super.column(columnName, columnType); return this; } - public MySqlTableBuilder charset(final String charset) { + /** + * Set a custom charset for the new table. Defaults to UTF-8. + * + * @param charset custom charset, e.g. {@code ascii} + * @return {@code this} for fluent programming + */ + public Builder charset(final String charset) { this.charset = charset; return this; } + /** + * Build a new {@link MySqlTable} instance. + * + * @return new {@link MySqlTable} instance + */ @Override public MySqlTable build() { final MySqlTable table = new MySqlTable(this); From 4716672dce610f39f99af1353728fcca9a3b7b6a Mon Sep 17 00:00:00 2001 From: Christoph Pirkl <4711730+kaklakariada@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:11:08 +0200 Subject: [PATCH 13/13] Update src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sebastian Bär --- .../dialects/AbstractImmediateDatabaseObjectWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java index 4d77e9f..a9b34a7 100644 --- a/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java +++ b/src/main/java/com/exasol/dbbuilder/dialects/AbstractImmediateDatabaseObjectWriter.java @@ -86,7 +86,7 @@ public void truncate(final Table table) { public void write(final Table table, final Stream> rows) { final String valuePlaceholders = "?" + ", ?".repeat(table.getColumnCount() - 1); final String sql = "INSERT INTO " + table.getFullyQualifiedName() + " VALUES(" + valuePlaceholders + ")"; - try (AutoCommit autoCommit = AutoCommit.tryDeactivate(connection); + try (final AutoCommit autoCommit = AutoCommit.tryDeactivate(connection); final PreparedStatement preparedStatement = this.connection.prepareStatement(sql)) { rows.forEach(row -> addBatch(table, preparedStatement, row)); preparedStatement.executeBatch();