EndpointEvaluator › Quick Start Guide

# .github/workflows/llm-eval.yml
name: LLM Evaluation
on: [pull_request]
jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Generate LLM output
        id: llm
        run: |
          # Your app logic here — produce the text to evaluate
          echo "output=Paris is the capital of France" >> "$GITHUB_OUTPUT"
          echo "reference=The capital of France is Paris" >> "$GITHUB_OUTPUT"

      - name: Evaluate with EndpointEvaluator
        id: eval
        env:
          EE_API_KEY: ${{ secrets.EE_API_KEY }}
        run: |
          RESPONSE=$(curl -sf -X POST https://endpointevaluator.com/api/v1/evaluate \
            -H "X-API-Key: $EE_API_KEY" \
            -H "Content-Type: application/json" \
            -d "$(jq -n \
              --arg out "${{ steps.llm.outputs.output }}" \
              --arg ref "${{ steps.llm.outputs.reference }}" \
              '{"output_text":$out,"reference_text":$ref,"scoring_method":"inferential"}')")

          VERDICT=$(echo "$RESPONSE" | jq -r '.verdict')
          CREDITS=$(echo "$RESPONSE" | jq -r '.credits_consumed')

          echo "verdict=$VERDICT" >> "$GITHUB_OUTPUT"

          # Post results to the GitHub Actions job summary
          echo "### Evaluation Results" >> "$GITHUB_STEP_SUMMARY"
          echo "| Metric | Value |" >> "$GITHUB_STEP_SUMMARY"
          echo "|--------|-------|" >> "$GITHUB_STEP_SUMMARY"
          echo "| Verdict | \`$VERDICT\` |" >> "$GITHUB_STEP_SUMMARY"
          echo "| Credits Used | $CREDITS |" >> "$GITHUB_STEP_SUMMARY"

      - name: Gate — fail on inconsistent
        if: steps.eval.outputs.verdict == 'inconsistent'
        run: |
          echo "::error::LLM output is inconsistent with reference"
          exit 1

# test/fixtures/eval_cases.json — your test fixtures
# [
#   {"name":"capital","output":"Paris is the capital of France","reference":"The capital of France is Paris"},
#   {"name":"date","output":"WWII ended in 1945","reference":"World War II concluded in 1945"}
# ]

# .gitlab-ci.yml
llm-eval:
  stage: test
  image: alpine:latest
  before_script:
    - apk add --no-cache curl jq
  script:
    - |
      CASES=$(cat test/fixtures/eval_cases.json)
      COUNT=$(echo "$CASES" | jq length)
      FAILED=0

      for i in $(seq 0 $(( COUNT - 1 ))); do
        NAME=$(echo "$CASES" | jq -r ".[$i].name")
        OUTPUT=$(echo "$CASES" | jq -r ".[$i].output")
        REF=$(echo "$CASES" | jq -r ".[$i].reference")

        RESPONSE=$(curl -sf -X POST https://endpointevaluator.com/api/v1/evaluate \
          -H "X-API-Key: $EE_API_KEY" \
          -H "Content-Type: application/json" \
          -d "$(jq -n --arg o "$OUTPUT" --arg r "$REF" \
            '{"output_text":$o,"reference_text":$r,"scoring_method":"inferential"}')")

        VERDICT=$(echo "$RESPONSE" | jq -r '.verdict')
        echo "[$NAME] verdict=$VERDICT"

        # Save each result for the report artifact
        echo "$RESPONSE" | jq --arg name "$NAME" '. + {case_name: $name}' >> eval-results.jsonl

        if [ "$VERDICT" = "inconsistent" ]; then
          echo "  FAIL — $NAME is inconsistent"
          FAILED=$(( FAILED + 1 ))
        fi
      done

      echo ""
      echo "Evaluated $COUNT cases: $(( COUNT - FAILED )) passed, $FAILED failed"
      [ "$FAILED" -eq 0 ] || exit 1
  artifacts:
    paths:
      - eval-results.jsonl
    when: always

# .circleci/config.yml
version: 2.1
jobs:
  llm-eval:
    docker:
      - image: cimg/base:current
    environment:
      # Set these in your CircleCI project settings, or pass from a prior step
      LLM_OUTPUT: "Paris is the capital of France"
      REFERENCE_TEXT: "The capital of France is Paris"
    steps:
      - checkout
      - run:
          name: Evaluate LLM output
          command: |
            mkdir -p /tmp/eval-results

            RESPONSE=$(curl -sf -X POST https://endpointevaluator.com/api/v1/evaluate \
              -H "X-API-Key: $EE_API_KEY" \
              -H "Content-Type: application/json" \
              -d "$(jq -n \
                --arg out "$LLM_OUTPUT" \
                --arg ref "$REFERENCE_TEXT" \
                '{"output_text":$out,"reference_text":$ref,"scoring_method":"combined"}')")

            # Save full response as a build artifact
            echo "$RESPONSE" | jq '.' > /tmp/eval-results/evaluation.json

            VERDICT=$(echo "$RESPONSE" | jq -r '.verdict')
            CREDITS=$(echo "$RESPONSE" | jq -r '.credits_consumed')

            # Combined returns per-method verdicts — display them
            echo "Overall verdict: $VERDICT"
            echo "Method verdicts:"
            echo "$RESPONSE" | jq -r '.method_verdicts | to_entries[] | "  \(.key): \(.value)"'
            echo "Credits used: $CREDITS"

            # Fail if the overall verdict is inconsistent
            if [ "$VERDICT" = "inconsistent" ]; then
              echo "FAILED — output is inconsistent with reference"
              exit 1
            fi
            echo "PASSED"
      - store_artifacts:
          path: /tmp/eval-results
          destination: eval-results

workflows:
  evaluate:
    jobs:
      - llm-eval