IntelliJ-IdeaVim/.github/workflows/runUiTestsIJ.yml

name: Run UI Tests for IntelliJ IDEA
on:
  workflow_dispatch:
  schedule:
      - cron: '*/30 * * * *'
jobs:
  test-macos:
    if: false  # Temporarily disabled - change to: github.repository == 'JetBrains/ideavim'
    runs-on: macos-latest
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - name: Setup Java
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: 21
      - name: Setup FFmpeg
        run: brew install ffmpeg
      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v4
        with:
          cache-read-only: false
      - name: Build Plugin
        run: gradle :buildPlugin --configuration-cache
      - name: Run Idea
        run: |
          mkdir -p build/reports
          gradle runIdeForUiTests > build/reports/idea.log &
      - name: List available capture devices
        run: ffmpeg -f avfoundation -list_devices true -i "" 2>&1 || true
        continue-on-error: true
      - name: Start screen recording
        run: |
          mkdir -p build/reports/ci-screen-recording
          ffmpeg -f avfoundation -capture_cursor 1 -i "0:none" -r 30 -vcodec libx264 -pix_fmt yuv420p build/reports/ci-screen-recording/screen-recording.mp4 &
          echo $! > /tmp/ffmpeg_pid.txt
        continue-on-error: true
      - name: Auto-click Allow button for screen recording permission
        run: |
          sleep 3
          brew install cliclick || true

          for coords in "512:367" "960:540" "640:400" "800:450"; do
            x=$(echo $coords | cut -d: -f1)
            y=$(echo $coords | cut -d: -f2)
            echo "Trying coordinates: $x,$y"

            cliclick c:$x,$y 2>/dev/null && echo "cliclick succeeded" && break
            sleep 0.5

            osascript -e "tell application \"System Events\" to click at {$x, $y}" 2>/dev/null && echo "AppleScript succeeded" && break
            sleep 1
          done
        continue-on-error: true
      - name: Wait for Idea started
        uses: jtalk/url-health-check-action@v3
        with:
          url: http://127.0.0.1:8082
          max-attempts: 20
          retry-delay: 10s
      - name: Tests
        run: gradle :tests:ui-ij-tests:testUi
      - name: Stop screen recording
        if: always()
        run: |
          if [ -f /tmp/ffmpeg_pid.txt ]; then
            kill $(cat /tmp/ffmpeg_pid.txt) || true
            sleep 2
          fi
        continue-on-error: true
      - name: Move sandbox logs
        if: always()
        run: mv build/idea-sandbox/IU-*/log_runIdeForUiTests idea-sandbox-log
      - name: Upload macOS artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: macos-reports
          path: |
            build/reports
            tests/ui-ij-tests/build/reports
            idea-sandbox-log

  test-linux:
    if: github.repository == 'JetBrains/ideavim'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - name: Free up disk space
        run: |
          echo "Disk space before cleanup:"
          df -h
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          echo "Disk space after cleanup:"
          df -h
      - name: Setup Java
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: 21
      - name: Setup FFmpeg
        run: sudo apt-get update && sudo apt-get install -y ffmpeg
      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v4
        with:
          cache-read-only: false
      - name: Build Plugin
        run: gradle :buildPlugin --configuration-cache
      - name: Start Xvfb
        run: |
          export DISPLAY=:99.0
          Xvfb :99 -screen 0 1280x720x24 &
          echo "DISPLAY=:99.0" >> $GITHUB_ENV
      - name: Run Idea
        run: |
          mkdir -p build/reports
          gradle runIdeForUiTests > build/reports/idea.log 2>&1 &
      - name: Start screen recording
        run: |
          mkdir -p build/reports/ci-screen-recording
          ffmpeg -f x11grab -video_size 1280x720 -i :99.0 -r 15 -vcodec libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p build/reports/ci-screen-recording/screen-recording.mp4 &
          echo $! > /tmp/ffmpeg_pid.txt
        continue-on-error: true
      - name: Wait for Idea started
        uses: jtalk/url-health-check-action@v3
        with:
          url: http://127.0.0.1:8082
          max-attempts: 20
          retry-delay: 10s
      - name: Tests
        run: gradle :tests:ui-ij-tests:testUi
      - name: Stop screen recording
        if: always()
        run: |
          if [ -f /tmp/ffmpeg_pid.txt ]; then
            kill $(cat /tmp/ffmpeg_pid.txt) || true
            sleep 2
          fi
        continue-on-error: true
      - name: Move sandbox logs
        if: always()
        run: mv build/idea-sandbox/IU-*/log_runIdeForUiTests idea-sandbox-log
      - name: Upload Linux artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: linux-reports
          path: |
            build/reports
            tests/ui-ij-tests/build/reports
            idea-sandbox-log

  analyze-failures:
    needs: [test-macos, test-linux]
    if: always() && (needs.test-macos.result == 'failure' || needs.test-linux.result == 'failure')
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - name: Setup Java
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: 21
      - name: Setup FFmpeg
        run: sudo apt-get update && sudo apt-get install -y ffmpeg
      - name: Download macOS artifacts
        if: needs.test-macos.result == 'failure'
        uses: actions/download-artifact@v4
        with:
          name: macos-reports
          path: macos-reports
        continue-on-error: true
      - name: Download Linux artifacts
        if: needs.test-linux.result == 'failure'
        uses: actions/download-artifact@v4
        with:
          name: linux-reports
          path: linux-reports
        continue-on-error: true
      - name: AI Analysis of Test Failures
        uses: anthropics/claude-code-action@v1
        with:
          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          claude_args: '--allowed-tools "Read,Write,Edit,Glob,Grep,Bash(ffmpeg:*),Bash(ffprobe:*),Bash(mkdir:*),Bash(touch:*),Bash(echo:*),Bash(ls:*),Bash(cat:*),Bash(grep:*),Bash(find:*),Bash(cd:*),Bash(rm:*),Bash(git:*),Bash(gh:*),Bash(gradle:*),Bash(./gradlew:*),Bash(java:*),Bash(which:*)"'
          prompt: |
            ## Task: Analyze UI Test Failures Across Platforms

            Please analyze the UI test failures from both macOS and Linux platforms.

            Test results locations:
            - macOS reports: macos-reports/ (if macOS tests failed)
            - Linux reports: linux-reports/ (if Linux tests failed)

            Each platform's reports include:
            - CI screen recording at build/reports/ci-screen-recording/screen-recording.mp4
            - Screenshot at tests/ui-ij-tests/build/reports/ideaVimTest.png
            - IDE sandbox logs in idea-sandbox-log directory
            - UI hierarchy file at build/reports/hierarchy-ideaVimTest.html

            ffmpeg is available for video analysis:
              * Extract frame at specific time: `ffmpeg -i video.mp4 -ss 00:01:30 -vframes 1 output.png`
              * Extract multiple frames: `ffmpeg -i video.mp4 -vf fps=1/10 frame_%04d.png` (1 frame every 10 seconds)
              * Get video duration: `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 video.mp4`
              * Extract last N seconds: `ffmpeg -sseof -10 -i video.mp4 -update 1 last_frame.png` (last 10 seconds)
              * Create thumbnail grid: `ffmpeg -i video.mp4 -vf "select='not(mod(n\,300))',scale=160:120,tile=5x5" -frames:v 1 grid.png`

            Special troubleshooting for timeout failures:
            - If the test fails because it waited for something to appear, but according to the video and screenshot the element actually appeared on screen, check the UI hierarchy file at build/reports/hierarchy-ideaVimTest.html
            - This hierarchy file shows the actual structure of UI elements and their properties at the time of failure
            - The failure may be caused by a renamed property or changed class name in the UI element
            - If you find this is the case, suggest a new query or selector that matches the current element structure

            Analysis approach:
            1. Determine which platforms failed (macOS, Linux, or both)
            2. Compare failures across platforms to identify:
               - **Common issues**: Same root cause affecting both platforms (e.g., API changes, accessibility name changes)
               - **Platform-specific issues**: Problems unique to one platform (e.g., macOS permission dialogs, Linux display issues)
            3. For common issues, provide a single unified fix that works on both platforms
            4. For platform-specific issues, clearly identify the platform and provide targeted fixes

            Please provide:
            1. A detailed analysis of what went wrong on each platform
            2. Whether the issue is common across platforms or platform-specific
            3. The root cause of the failure(s)
            4. Potential fixes or suggestions

            Write your analysis to analysis-result.txt

            ## UI Test Best Practices

            When fixing UI tests, follow these principles:

            **Cause-Effect Over Timeouts**: UI tests should wait for specific conditions rather than arbitrary timeouts.
            - ✅ GOOD: Wait for a button to become visible or enabled before clicking it
            - ✅ GOOD: Wait for specific text to appear in a component
            - ✅ GOOD: Wait for a dialog to be present with a specific accessible name
            - ❌ BAD: Use Thread.sleep() or fixed delays
            - ❌ BAD: Wait for arbitrary timeouts without checking for specific conditions

            Only use timeouts as a maximum wait duration with explicit condition checks (e.g., "wait up to 30 seconds for dialog to appear").
            Tests should be deterministic and based on observable state changes in the UI, not time-based assumptions.

            **Flaky Test = Race Condition**: A test that sometimes passes and sometimes fails has a race condition.
            The fix must ELIMINATE the race, not make it less likely. Increasing timeouts is almost never the correct fix.

            **Wait for UNIQUE State Identifiers**: When waiting for a state transition, find something that:
            - Only exists in the TARGET state (not in previous or intermediate states)
            - Proves the operation COMPLETED (not just started)
            Example: After clicking a button that triggers a notification change, don't wait for an element that exists
            in BOTH the old and new notification. Wait for text/element unique to the NEW state.

            **Understand Framework Built-in Waits**: Before adding explicit waits, check what the framework already does.
            `findText()` already waits up to 5 seconds for elements. Adding `waitFor { hasText(...) }` before `findText()`
            is redundant and indicates misunderstanding of the actual problem.

            **Trace Causality Backwards**: If the failure shows wrong data (e.g., wrong text pasted), trace backwards:
            - Where did the data come from? (e.g., clipboard)
            - When was that data set? (e.g., during a prior click operation)
            - What proves that operation completed? → THIS is your wait condition

            **State Transitions Have Intermediate States**: UI operations often involve: Old State → Transition → New State.
            Elements may briefly exist in both old and new states during transition. Wait for something that proves
            you're in the NEW state, not just that a transition started.

            IMPORTANT: If you have a concrete suggestion for fixing the test, ALWAYS proceed with creating a branch and PR. Never ask for permission - just do it.

            If you have a concrete suggestion for fixing the test:
            1. Create a new branch with a descriptive name (e.g., fix/ui-test-accessible-name)
            2. Apply your suggested fix to the codebase
            3. CRITICAL: When staging changes, NEVER use `git add -A` or `git add .`. Always add modified files explicitly by path (e.g., `git add path/to/file.kt path/to/other.kt`). This prevents accidentally staging unrelated files.
            4. MANDATORY: Verify compilation succeeds with your changes: `gradle compileKotlin compileTestKotlin`
            5. If the fix is for a common issue, ensure it works on both platforms
            6. MANDATORY: Run the specific failing test to verify the fix improves or resolves the issue
               - For IntelliJ IDEA UI tests: `gradle :tests:ui-ij-tests:testUi --tests "YourTestClassName.yourTestMethod"`
               - To run all IntelliJ UI tests: `gradle :tests:ui-ij-tests:testUi`
               - The test MUST either pass completely or show clear improvement (e.g., progressing further before failure)
            7. If the test passes or shows improvement with your fix, create a PR with:
               - Clear title describing the fix
               - Description explaining:
                 * Whether this is a common or platform-specific issue
                 * The root cause and solution
                 * Which platforms were affected
                 * Test results showing the fix works
                 * Reference to the failing CI run
            8. Use the base branch 'master' for the PR
      - name: Upload analysis result
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ai-analysis
          path: analysis-result.txt
        continue-on-error: true