fix: issue issues with snapshots in pipeline (#5243)

Author: nmerget

.github/workflows/00-scan-secrets.yml

@@ -8,6 +8,7 @@ on:
 
 jobs:
   TruffleHog:
+    if: ${{ github.event.pull_request != null }} # only scan on pull-requests
     runs-on: ubuntu-24.04 # Use Ubuntu 24.04 explicitly
     steps:
       - name: ⏬ Checkout repo
@@ -17,7 +18,6 @@ jobs:
 
       # https://github.com/marketplace/actions/trufflehog-oss#advanced-usage-scan-entire-branch
       - name: 🐷 TruffleHog OSS
-        if: ${{ github.event.pull_request != null }} # only scan on pull-requests
         uses: trufflesecurity/trufflehog@main
         with:
           # Setting base to an empty string scans the entire branch, per TruffleHog OSS advanced usage:

.github/workflows/01-get-playwright-version.yml

@@ -10,16 +10,23 @@ on:
       version:
         description: "Current playwright version"
         value: ${{ jobs.playwright-version.outputs.version }}
+      test-ally:
+        description: "If aria-snapshots are changed"
+        value: ${{ jobs.playwright-version.outputs.test-ally }}
 
 jobs:
   playwright-version:
     name: Get and save publish version
     runs-on: ubuntu-24.04 # Use Ubuntu 24.04 explicitly
     outputs:
       version: ${{ steps.version.outputs.version }}
+      test-ally: ${{ steps.ally.outputs.test-ally }}
     steps:
       - name: ⏬ Checkout repo
         uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          fetch-tags: false
 
       - name: 🔄 Init Cache
         uses: ./.github/actions/npm-cache
@@ -29,3 +36,11 @@ jobs:
         run: |
           OUTPUT=$(npx --no tsx scripts/github/get-playwright-version.ts)
           echo "version=$OUTPUT" >> $GITHUB_OUTPUT
+
+      - name: 👩 Should Test a11y
+        id: ally
+        shell: bash
+        run: |
+          ARIA_MAIN=$(npx --no tsx scripts/github/snapshot-changes/check-main-aria.ts)
+          echo "test-ally=$ARIA_MAIN"
+          echo "test-ally=$ARIA_MAIN" >> $GITHUB_OUTPUT

.github/workflows/02-e2e-showcases.yml

@@ -9,16 +9,9 @@ on:
       showcase:
         required: true
         type: string
-    outputs:
-      aria-snapshots-changed:
-        description: "If aria snapshots changed in current PR"
-        value: ${{ jobs.playwright-showcases.outputs.aria-snapshots-changed }}
-      aria-snapshots-changed-to-main:
-        description: "If aria snapshots changed to main branch"
-        value: ${{ jobs.playwright-showcases.outputs.aria-snapshots-changed-to-main }}
-      visual-snapshots-changed:
-        description: "If visual snapshots changed"
-        value: ${{ jobs.playwright-showcases.outputs.visual-snapshots-changed }}
+      test-ally:
+        required: false
+        type: string
 
 permissions:
   actions: write
@@ -32,20 +25,15 @@ jobs:
       image: mcr.microsoft.com/playwright:v${{ inputs.version }}
       options: --user 1001
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        shardIndex: [1, 2, 3, 4]
-        shardTotal: [4]
-    outputs:
-      aria-snapshots-changed: ${{ steps.aria-snapshots.outputs.aria-snapshots-changed }}
-      aria-snapshots-changed-to-main: ${{ steps.aria-snapshots.outputs.aria-snapshots-changed-to-main }}
-      visual-snapshots-changed: ${{ steps.visual-snapshots.outputs.visual-snapshots-changed }}
+        shardIndex: [1, 2, 3, 4, 5]
+        shardTotal: [5]
     steps:
       - name: ⏬ Checkout repo
         uses: actions/checkout@v5
         with:
-          fetch-depth: 0 # fetch all history for all branches
-          fetch-tags: false
+          fetch-depth: 2
 
       - name: 🔄 Init Cache
         uses: ./.github/actions/npm-cache
@@ -89,60 +77,37 @@ jobs:
         shell: bash
         if: github.event.pull_request != null
         run: |
-          npm run test:aria-snapshots --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --update-snapshots
-          OUTPUT=$(npx --no tsx scripts/github/snapshot-changes/check-aria.ts)
-          if [ "$OUTPUT" = "true" ]; then
-            echo "aria-snapshots-changed=$OUTPUT" >> $GITHUB_OUTPUT
-          fi
-          OUTPUT=$(npx --no tsx scripts/github/snapshot-changes/check-main-aria.ts)
-          if [ "$OUTPUT" = "true" ]; then
-            echo "aria-snapshots-changed-to-main=$OUTPUT" >> $GITHUB_OUTPUT
-          fi
+          npm run test:aria-snapshots --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
-      - name: 👩‍🔬 Test axe-core with Playwright 🎭
+      - name: 👩‍🔬 Test visual-snapshots with Playwright 🎭
+        id: visual-snapshots
         shell: bash
         if: github.event.pull_request != null
         run: |
-          npm run test:axe-core --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          npm run test:visual-snapshots --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
-      - name: 👩‍🔬 Test visual-snapshots with Playwright 🎭
-        id: visual-snapshots
+      - name: 👩‍🔬 Test axe-core with Playwright 🎭
         shell: bash
         if: github.event.pull_request != null
         run: |
-          npm run test:visual-snapshots --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --update-snapshots
-          OUTPUT=$(npx --no tsx scripts/github/snapshot-changes/check-visual.ts)
-          if [ "$OUTPUT" = "true" ]; then
-            echo "visual-snapshots-changed=$OUTPUT" >> $GITHUB_OUTPUT
-          fi
+          npm run test:axe-core --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
       - name: 👩‍🔬 Test a11y-checker with Playwright 🎭
         shell: bash
         if: |
           github.event.pull_request != null &&
-          steps.aria-snapshots.outputs.aria-snapshots-changed-to-main == 'true'
+          inputs.test-ally == 'true'
         run: |
           npm run test:a11y-checker --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
-      - name: 🌳 Log Snapshot Changes
-        if: github.event.pull_request != null
-        env:
-          ARIA: ${{ steps.aria-snapshots.outputs.aria-snapshots-changed}}
-          VISUAL: ${{ steps.visual-snapshots.outputs.visual-snapshots-changed}}
-          ARIA_MAIN: ${{ steps.aria-snapshots.outputs.aria-snapshots-changed-to-main }}
-        run: |
-          echo "ARIA: $ARIA"
-          echo "VISUAL: $VISUAL"
-          echo "ARIA_MAIN: $ARIA_MAIN"
-
       - name: 🔣 Print GitHub Report
         if: failure()
         shell: bash
         run: |
           npx playwright merge-reports --reporter github ./blob-report
 
       - name: 🆙 Upload test results
-        if: failure()
+        if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.showcase }}-playwright-results-${{ matrix.shardIndex }}

.github/workflows/default.yml

@@ -211,27 +211,31 @@ jobs:
     with:
       version: ${{ needs.get-playwright-version.outputs.version }}
       showcase: stencil-showcase
+      test-ally: ${{ needs.get-playwright-version.outputs.test-ally }}
 
   test-showcase-angular:
     uses: ./.github/workflows/02-e2e-showcases.yml
     needs: [build-showcase-angular, get-playwright-version]
     with:
       version: ${{ needs.get-playwright-version.outputs.version }}
       showcase: angular-showcase
+      test-ally: ${{ needs.get-playwright-version.outputs.test-ally }}
 
   test-showcase-react:
     uses: ./.github/workflows/02-e2e-showcases.yml
     needs: [build-showcase-react, get-playwright-version]
     with:
       version: ${{ needs.get-playwright-version.outputs.version }}
       showcase: react-showcase
+      test-ally: ${{ needs.get-playwright-version.outputs.test-ally }}
 
   test-showcase-vue:
     uses: ./.github/workflows/02-e2e-showcases.yml
     needs: [build-showcase-vue, get-playwright-version]
     with:
       version: ${{ needs.get-playwright-version.outputs.version }}
       showcase: vue-showcase
+      test-ally: ${{ needs.get-playwright-version.outputs.test-ally }}
 
   test-showcase-patternhub:
     uses: ./.github/workflows/02-e2e-patternhub.yml
@@ -273,68 +277,33 @@ jobs:
   # This job runs only if aria-snapshots were changed compared to main or if it runs on main branch
   test-screen-reader:
     if: |
-      needs.test-showcase-react.outputs.aria-snapshots-changed-to-main == 'true' ||
+      needs.get-playwright-version.outputs.test-ally == 'true' ||
       github.event.pull_request == null
     uses: ./.github/workflows/02-e2e-screen-reader.yml
     needs: [test-showcase-react, init-playwright, get-playwright-version]
     with:
       version: ${{ needs.get-playwright-version.outputs.version }}
 
-  should-regenerate-snapshots:
-    env:
-      ARIA_ANGULAR: ${{ needs.test-showcase-angular.outputs.aria-snapshots-changed }}
-      VISUAL_ANGULAR: ${{ needs.test-showcase-angular.outputs.visual-snapshots-changed }}
-      ARIA_REACT: ${{ needs.test-showcase-react.outputs.aria-snapshots-changed }}
-      VISUAL_REACT: ${{ needs.test-showcase-react.outputs.visual-snapshots-changed }}
-      ARIA_VUE: ${{ needs.test-showcase-vue.outputs.aria-snapshots-changed }}
-      VISUAL_VUE: ${{ needs.test-showcase-vue.outputs.visual-snapshots-changed }}
-      ARIA_STENCIL: ${{ needs.test-showcase-stencil.outputs.aria-snapshots-changed }}
-      VISUAL_STENCIL: ${{ needs.test-showcase-stencil.outputs.visual-snapshots-changed }}
+  regenerate-snapshots:
     if: |
-      !cancelled() && (
+      !cancelled() && !contains(github.actor,'[bot]') && (
         needs.test-showcase-angular.result == 'failure' ||
         needs.test-showcase-react.result == 'failure' ||
         needs.test-showcase-vue.result == 'failure' ||
-        needs.test-showcase-angular.outputs.aria-snapshots-changed == 'true' ||
-        needs.test-showcase-angular.outputs.visual-snapshots-changed == 'true' ||
-        needs.test-showcase-react.outputs.aria-snapshots-changed == 'true' ||
-        needs.test-showcase-react.outputs.visual-snapshots-changed == 'true' ||
-        needs.test-showcase-vue.outputs.aria-snapshots-changed == 'true' ||
-        needs.test-showcase-vue.outputs.visual-snapshots-changed == 'true' ||
-        needs.test-showcase-stencil.outputs.aria-snapshots-changed == 'true' ||
-        needs.test-showcase-stencil.outputs.visual-snapshots-changed == 'true'
+        needs.test-showcase-stencil.result == 'failure'
       )
+    uses: ./.github/workflows/02-e2e-regenerate.yml
+    with:
+      version: ${{ needs.get-playwright-version.outputs.version }}
+      type: showcases
     needs:
       [
         test-showcase-angular,
         test-showcase-react,
         test-showcase-vue,
-        test-showcase-stencil
+        test-showcase-stencil,
+        get-playwright-version
       ]
-    runs-on: ubuntu-24.04
-    steps:
-      - name: "Fail if actor is a bot and regeneration is triggered"
-        run: |
-          if [[ "${{ github.actor }}" == *"[bot]"* ]]; then
-              echo "The snapshots between frameworks are different." >> $GITHUB_STEP_SUMMARY
-              echo "| Framework | Aria Snapshots Changed | Visual Snapshots Changed |" >> $GITHUB_STEP_SUMMARY
-              echo "|-----------|------------------------|--------------------------|" >> $GITHUB_STEP_SUMMARY
-              echo "| Angular   | "$ARIA_ANGULAR"  | "$VISUAL_ANGULAR" |" >> $GITHUB_STEP_SUMMARY
-              echo "| React     | "$ARIA_REACT"    | "$VISUAL_REACT"   |" >> $GITHUB_STEP_SUMMARY
-              echo "| Vue       | "$ARIA_VUE"      | "$VISUAL_VUE"     |" >> $GITHUB_STEP_SUMMARY
-              echo "| Stencil   | "$ARIA_STENCIL"  | "$VISUAL_STENCIL" |" >> $GITHUB_STEP_SUMMARY
-              exit 1
-          fi
-        shell: bash
-
-  regenerate-snapshots:
-    if: |
-      !cancelled() && needs.should-regenerate-snapshots.result == 'success'
-    uses: ./.github/workflows/02-e2e-regenerate.yml
-    with:
-      version: ${{ needs.get-playwright-version.outputs.version }}
-      type: showcases
-    needs: [should-regenerate-snapshots, get-playwright-version]
 
   commit-regenerated-snapshots:
     uses: ./.github/workflows/02-e2e-regenerated-snapshots-commit.yml

__snapshots__/switch/showcase/chromium-highContrast/DBSwitch-should-match-screenshot-1/DBSwitch-should-match-screenshot.png

@@ -0,0 +1,99 @@
+- main:
+  - heading "DBSwitch" [level=1]
+  - link "Density"
+  - switch "functional"
+  - text: functional
+  - status
+  - switch "regular (Default)"
+  - text: regular (Default)
+  - status
+  - switch "expressive"
+  - text: expressive
+  - status
+  - link "Checked"
+  - switch "(Default) False"
+  - text: (Default) False
+  - status
+  - switch "True" [checked]
+  - text: "True"
+  - status
+  - link "Disabled"
+  - switch "(Default) False"
+  - text: (Default) False
+  - status
+  - switch "True" [disabled]
+  - text: "True"
+  - status
+  - link "Visual Aid"
+  - switch "(Default) False + Unchecked"
+  - text: (Default) False + Unchecked
+  - status
+  - switch "(Default) False + Checked" [checked]
+  - text: (Default) False + Checked
+  - status
+  - switch "True + Unchecked"
+  - text: True + Unchecked
+  - status
+  - switch "True + Checked" [checked]
+  - text: True + Checked
+  - status
+  - link "Size"
+  - switch "(Default) Medium"
+  - text: (Default) Medium
+  - status
+  - switch "Small"
+  - text: Small
+  - status
+  - link "Variant"
+  - switch "(Default) Trailing"
+  - text: (Default) Trailing
+  - status
+  - switch "Leading"
+  - text: Leading
+  - status
+  - link "Show Label"
+  - switch "(Default) True"
+  - text: (Default) True
+  - status
+  - switch "False"
+  - text: "False"
+  - status
+  - text: "False"
+  - link "Required"
+  - switch "(Default) False"
+  - text: (Default) False
+  - status
+  - switch "True"
+  - text: "True"
+  - status
+  - link "Show Required Asterisk"
+  - switch "(Default) True"
+  - text: (Default) True
+  - status
+  - switch "False"
+  - text: "False"
+  - status
+  - link "Validation"
+  - switch "(Default) No validation"
+  - text: (Default) No validation
+  - status
+  - switch "Invalid"
+  - text: Invalid Invalid Message
+  - status
+  - switch "Valid" [checked]
+  - text: Valid Valid message
+  - status
+  - link "Show Message"
+  - switch "(Default) False"
+  - text: (Default) False
+  - status
+  - switch "True"
+  - text: True Message
+  - status
+  - link "Examples"
+  - switch "Custom Icons"
+  - text: Custom Icons
+  - status
+  - switch "Required + Visual Aid"
+  - text: Required + Visual Aid
+  - status