CI/CD Integration Guide for Prompt Engineering¶

# .github/workflows/prompt-regression.yml
name: Prompt Regression Tests

on:
  pull_request:
    paths:
      - 'prompts/**/*.prompt.md'
      - 'prompts/**/tests/**'

jobs:
  prompt-regression:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: pip install openai

      - name: Run prompt regression tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          failed=0
          for suite in prompts/**/tests/*/test_suite.json; do
            echo "Running: $suite"
            python "$(dirname "$suite")/evaluate.py" "$suite" || failed=1
          done
          exit $failed

  schema-validation:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install jsonschema
        run: pip install jsonschema pyyaml

      - name: Validate prompt frontmatter
        run: |
          python scripts/validate-prompt-schema.py prompts/**/*.prompt.md

  prompt-security-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Scan for sensitive patterns
        run: |
          echo "Scanning prompt files for potential secrets..."
          patterns=(
            'sk-[a-zA-Z0-9]{20,}'        # OpenAI API keys
            'ghp_[a-zA-Z0-9]{36}'         # GitHub PATs
            'AKIA[0-9A-Z]{16}'            # AWS access keys
            'password\s*[:=]\s*\S+'       # Hardcoded passwords
          )
          failed=0
          for pattern in "${patterns[@]}"; do
            if grep -rPn "$pattern" prompts/ learn/; then
              echo "FAIL: Found sensitive pattern: $pattern"
              failed=1
            fi
          done
          if [ $failed -eq 0 ]; then
            echo "PASS: No sensitive patterns found."
          fi
          exit $failed

name: Prompt CI

on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'learn/**'
      - 'scripts/**'

  push:
    branches: [main]
    paths:
      - 'prompts/**'
      - 'learn/**'

jobs:
  # --- Lint checks (fast, no API needed) ---
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Lint prompt frontmatter
        run: bash scripts/lint-prompt-frontmatter.sh

      - name: Check markdown links
        uses: gaurav-nelson/github-action-markdown-link-check@v1
        with:
          use-quiet-mode: 'yes'

  # --- Token budget (fast, no API needed) ---
  token-budget:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Estimate token budgets
        run: |
          for f in $(find prompts -name '*.prompt.md'); do
            chars=$(wc -c < "$f")
            tokens=$((chars / 4))
            if [ "$tokens" -gt 4000 ]; then
              echo "WARNING: $f ≈ $tokens tokens (exceeds 4000)"
            else
              echo "OK: $f ≈ $tokens tokens"
            fi
          done

  # --- Security scan (fast, no API needed) ---
  security:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Scan for secrets in prompts
        run: |
          patterns='(sk-[a-zA-Z0-9]{20,}|ghp_[a-zA-Z0-9]{36}|AKIA[0-9A-Z]{16})'
          if grep -rPn "$patterns" prompts/ learn/; then
            echo "::error::Potential secrets found in prompt files"
            exit 1
          fi
          echo "No secrets found."

  # --- Regression tests (requires API key, runs only on prompt changes) ---
  regression:
    if: github.event_name == 'pull_request'
    needs: [lint, security]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - run: pip install openai

      - name: Run prompt regression tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          found=0
          for suite in $(find prompts -name 'test_suite.json' 2>/dev/null); do
            dir=$(dirname "$suite")
            if [ -f "$dir/evaluate.py" ]; then
              echo "Running: $suite"
              python "$dir/evaluate.py" "$suite" || found=1
            fi
          done
          if [ $found -eq 0 ] && [ "$(find prompts -name 'test_suite.json' | wc -l)" -eq 0 ]; then
            echo "No test suites found — skipping regression tests."
          fi
          exit $found