Read Sources #598
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Read Sources | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 17 * * *' | |
| jobs: | |
| readsources: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| pip install poetry | |
| poetry install | |
| - name: Checkout remote repository (dnsReaper) | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: punk-security/dnsReaper | |
| path: dnsReaper | |
| - name: Checkout remote repository (nuclei-templates) | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: projectdiscovery/nuclei-templates | |
| path: nuclei-templates | |
| - name: Read Sources | |
| run: | | |
| poetry run python3 baddns/scripts/readsources.py | |
| - name: Stage, commit, and create PR for changed files | |
| env: | |
| GH_TOKEN: ${{ secrets.DNSBOT_TOKEN }} | |
| run: | | |
| echo "starting Read Sources run" >> readsources_action.log | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Action" | |
| echo "Current directory and contents:" >> readsources_action.log | |
| pwd >> readsources_action.log | |
| echo "Contents of signatures_to_test directory:" >> readsources_action.log | |
| ls -la signatures_to_test >> readsources_action.log | |
| # Check for existence of previous hashes file and read its content | |
| if [[ -f "baddns/signatures/signature_history.txt" ]]; then | |
| prev_hashes=$(cat baddns/signatures/signature_history.txt) | |
| echo "signature_history.txt was present and read into prev_hashes..." >> readsources_action.log | |
| else | |
| echo "signature_history.txt was NOT present" >> readsources_action.log | |
| prev_hashes="" | |
| fi | |
| # Get only the files in the signatures_to_test directory | |
| files=$(basename -a signatures_to_test/*.yml) | |
| for file in $files; do | |
| echo "attempting to parse signature file: $file" >> readsources_action.log | |
| if [[ ! -s "signatures_to_test/$file" ]]; then | |
| echo "File $file is empty or doesn't exist, skipping" >> readsources_action.log | |
| continue | |
| fi | |
| # Check if the file exists in the signatures directory and if it has been changed | |
| if [[ ! -f "baddns/signatures/$file" ]] || ! cmp -s "baddns/signatures/$file" "signatures_to_test/$file"; then | |
| # Calculate hash of the new file and check if we've seen it before | |
| new_hash=$(sha256sum "signatures_to_test/$file" | cut -d ' ' -f 1) | |
| if echo "$prev_hashes" | grep -q "$new_hash"; then | |
| echo "File $file has been seen before, skipping" >> readsources_action.log | |
| continue | |
| fi | |
| PR_BLOCKED=false | |
| # Check for existing PRs with the same title | |
| sleep 5 | |
| PR_TITLE="[SignatureBot] Add or update signature $file" | |
| SEARCH_QUERY="repo:${{ github.repository }} type:pr state:open in:title \"$PR_TITLE\"" | |
| # Encode the search query | |
| ENCODED_QUERY=$(echo "$SEARCH_QUERY" | jq -sRr @uri) | |
| EXISTING_OPEN_PRS=$(curl -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=$ENCODED_QUERY" | jq '.items') | |
| if [[ -n "$EXISTING_OPEN_PRS" && "$EXISTING_OPEN_PRS" != "[]" ]]; then | |
| echo "Found an open PR with title '$PR_TITLE', skipping this signature" >> readsources_action.log | |
| continue | |
| fi | |
| sleep 5 | |
| # Encode the PR title for use in a URL | |
| ENCODED_PR_TITLE=$(echo "$PR_TITLE" | jq -sRr @uri) | |
| # Check for existing closed PRs with the same title and label 'signature-blocked' | |
| SEARCH_RESULT=$(curl -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=repo:${{ github.repository }}+type:pr+state:closed+label:signature-blocked+in:title+$ENCODED_PR_TITLE") | |
| # Use jq to determine if any matching PRs are found. If so, skip processing this signature. | |
| MATCHING_CLOSED_BLOCKED_PR_COUNT=$(echo "$SEARCH_RESULT" | jq '.total_count') | |
| if [[ $MATCHING_CLOSED_BLOCKED_PR_COUNT -gt 0 ]]; then | |
| echo "Found a closed PR with title '$PR_TITLE' and label 'signature-blocked', skipping this signature" >> readsources_action.log | |
| continue | |
| fi | |
| # Add the new hash to the history file | |
| echo "adding hash $new_hash to signature file" >> readsources_action.log | |
| echo "$new_hash #$file" >> "baddns/signatures/signature_history.txt" | |
| # Copy the file to the signatures directory | |
| cp "signatures_to_test/$file" "baddns/signatures/$file" | |
| BRANCH_NAME="new-signature-$file" | |
| # Fetch the latest remote references | |
| git fetch | |
| # Check if the branch exists | |
| if git ls-remote --heads origin $BRANCH_NAME | grep $BRANCH_NAME; then | |
| # Check for existing PRs with the same title | |
| PR_TITLE="[SignatureBot] Add or update signature $file" | |
| # Construct the search query | |
| SEARCH_QUERY="repo:${{ github.repository }} type:pr state:open in:title \"$PR_TITLE\"" | |
| # Encode the search query for the URL | |
| ENCODED_QUERY=$(echo "$SEARCH_QUERY" | jq -sRr @uri) | |
| # Short delay, then check for existing PRs with the Search API | |
| sleep 5 | |
| EXISTING_OPEN_PRS=$(curl -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=$ENCODED_QUERY" | jq '.items') | |
| if [[ -n "$EXISTING_OPEN_PRS" && "$EXISTING_OPEN_PRS" != "[]" ]]; then | |
| echo "Found an open PR with title '$PR_TITLE', skipping branch deletion for safety" >> readsources_action.log | |
| continue | |
| fi | |
| # No open PR, so safe to delete the branch | |
| sleep 5 | |
| git push origin --delete $BRANCH_NAME | |
| echo "Found and deleted stale branch $BRANCH_NAME remotely" >> readsources_action.log | |
| # If needed, delete the branch locally too. This step might not be necessary if the action doesn't check out all branches. | |
| if git show-ref --verify --quiet refs/heads/$BRANCH_NAME; then | |
| git branch -D $BRANCH_NAME | |
| echo "Found and deleted stale branch $BRANCH_NAME locally" >> readsources_action.log | |
| fi | |
| else | |
| echo "No stale branch $BRANCH_NAME found remotely" >> readsources_action.log | |
| fi | |
| echo "checking out new branch $BRANCH_NAME" >> readsources_action.log | |
| git checkout -b $BRANCH_NAME | |
| git add "baddns/signatures/$file" | |
| git add "baddns/signatures/signature_history.txt" | |
| echo "about to commit with the following files changed:" >> readsources_action.log | |
| git status -s >> readsources_action.log | |
| echo "adding commit and pushing branch..." >> readsources_action.log | |
| # If the commit operation is successful, then push the changes and create a PR | |
| git commit -m "[SignatureBot] Add or update signature $file and update signature history" && { | |
| git push origin new-signature-$file | |
| echo "## Add or update signature: $file" > pr_message | |
| echo "This PR adds or updates the follow signature:" >> pr_message | |
| echo '```' >> pr_message | |
| echo "Attempting to read baddns/signatures/$file" >> readsources_action.log | |
| cat baddns/signatures/$file >> pr_message | |
| echo '```' >> pr_message | |
| echo "creating PR: '[SignatureBot] Add or update signature $file'" >> readsources_action.log | |
| sleep 5 | |
| gh pr create --title "[SignatureBot] Add or update signature $file" --body-file pr_message --head new-signature-$file --repo ${{ github.repository }} 2>&1 | tee -a readsources_action.log | |
| } | |
| git checkout main | |
| else | |
| echo "Skipping signature file: $file - no changes detected" >> readsources_action.log | |
| fi | |
| done | |
| echo "completed read sources" >> readsources_action.log | |
| - name: Upload readsources.py log | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: readsources.py-log | |
| path: readsources.log | |
| - name: Upload read-sources action log | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: read-sources-action-log | |
| path: readsources_action.log | |