Skip to content
Prev Previous commit
Next Next commit
Find universally skipped tests across wheel test configurations
  • Loading branch information
mdboom committed Apr 27, 2026
commit 01ebb66c23db2009391f16f87f66ac128e8bc036
111 changes: 1 addition & 110 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -433,116 +433,7 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail

REPO="${GITHUB_REPOSITORY}"
WORKDIR=$(mktemp -d)
trap 'rm -rf "${WORKDIR}"' EXIT

mkdir -p "${WORKDIR}/configs"

# Collect all jobs for this run and group matrix jobs into wheel
# test configurations from ci.yml.
jobs_json=$(gh api --paginate \
"repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \
| jq -s '[.[].jobs[]]')

declare -A CONFIG_PATTERNS=(
[test-linux-64]='^Test linux-64 / '
[test-linux-aarch64]='^Test linux-aarch64 / '
[test-windows]='^Test (win-64|windows) / '
)

configs=(
test-linux-64
test-linux-aarch64
test-windows
)

for cfg in "${configs[@]}"; do
cfg_dir="${WORKDIR}/configs/${cfg}"
mkdir -p "${cfg_dir}/logs"
job_ids=$(echo "${jobs_json}" | jq -r \
--arg pat "${CONFIG_PATTERNS[$cfg]}" \
'.[] | select(.name | test($pat)) | .id')

if [[ -z "${job_ids}" ]]; then
echo "No matrix jobs found for ${cfg}" >&2
: > "${cfg_dir}/skipped.txt"
continue
fi

for job_id in ${job_ids}; do
logfile="${cfg_dir}/logs/${job_id}.log"
# Prefer the job log API; fall back to gh run view if needed.
if ! gh api "repos/${REPO}/actions/jobs/${job_id}/logs" > "${logfile}" 2>/dev/null; then
gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true
fi
done

# Extract pytest node IDs from SKIPPED lines. This tolerates
# timestamped logs, ANSI escapes, and trailing skip reasons.
grep -h 'SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \
| sed -E 's/\x1B\[[0-9;]*[[:alpha:]]//g' \
| sed 's#\\#/#g' \
| sed -nE 's#.*(tests/[[:graph:]]+\.py::[^[:space:]]+).*SKIPPED.*#\1#p' \
| sort -u > "${cfg_dir}/skipped.txt" || true

echo "${cfg}: $(wc -l < "${cfg_dir}/skipped.txt") skipped tests"
done

{
echo "## Universally-skipped tests"
echo ""
} >> "${GITHUB_STEP_SUMMARY}"

available_configs=0
missing_configs=()
for cfg in "${configs[@]}"; do
if [[ -s "${WORKDIR}/configs/${cfg}/skipped.txt" || -n "$(ls -A "${WORKDIR}/configs/${cfg}/logs" 2>/dev/null || true)" ]]; then
available_configs=$((available_configs + 1))
else
missing_configs+=("${cfg}")
fi
done

if [[ ${available_configs} -eq 0 ]]; then
echo "_No test job logs found in this run._" >> "${GITHUB_STEP_SUMMARY}"
exit 0
fi

if [[ ${#missing_configs[@]} -gt 0 ]]; then
{
echo "_Warning: missing logs for configuration(s): ${missing_configs[*]}_"
echo ""
} >> "${GITHUB_STEP_SUMMARY}"
fi

# Start intersection from the first wheel configuration, then narrow.
cp "${WORKDIR}/configs/${configs[0]}/skipped.txt" "${WORKDIR}/intersection.txt"
for cfg in "${configs[@]:1}"; do
comm -12 \
"${WORKDIR}/intersection.txt" \
"${WORKDIR}/configs/${cfg}/skipped.txt" \
> "${WORKDIR}/intersection_new.txt"
mv "${WORKDIR}/intersection_new.txt" "${WORKDIR}/intersection.txt"
done

count=$(wc -l < "${WORKDIR}/intersection.txt")
{
echo "Tests skipped across wheel test configurations (${#configs[@]}):"
echo ""
if [[ ${count} -eq 0 ]]; then
echo "_No tests were skipped in all configurations._"
else
echo "| Test |"
echo "| --- |"
while IFS= read -r test; do
[[ -z "${test}" ]] && continue
echo "| \`${test}\` |"
done < "${WORKDIR}/intersection.txt"
fi
} >> "${GITHUB_STEP_SUMMARY}"
python ci/tools/report_universally_skipped_tests.py
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to check out the repo first


- name: Exit
run: |
Expand Down
Loading
Loading