3b4a225594
Deterministic tiers: core = token overlap with top-20 market hard skills or essential ESCO competences; in-tier ranking by overlap score. Per-source cap 10 (overflow -> audit log, not the package). Source-domain allowlists stop cross-contamination (marketing skill entering an engineering package via a generic 'api' keyword). Flagship selection: 427 -> 73 entries; all 10 real v1 flood negatives excluded, 10 must-have positives included. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
103 lines
3.8 KiB
Python
103 lines
3.8 KiB
Python
"""End-to-end eval for the tiered agent-skill mapping (phase 2c).
|
|
|
|
Runs the REAL p5 selection for the target occupation (catalog scan, rule +
|
|
auto-domain matching, tiering, per-source cap) and checks each labeled case:
|
|
include -> the skill must be in the final selection
|
|
exclude -> the skill must NOT be in the final selection
|
|
Ship criterion: >= 18/20.
|
|
|
|
Usage: python evals/skill-tiering/run_eval.py
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
BASE = os.path.join(os.path.dirname(__file__), "..", "..")
|
|
sys.path.insert(0, os.path.join(BASE, "pipeline"))
|
|
import p5_enrich_ai_skills as p5 # noqa: E402
|
|
|
|
EVAL_SET = os.path.join(os.path.dirname(__file__), "eval-set.json")
|
|
|
|
|
|
def final_selection(slug):
|
|
"""Reproduce the p5 per-occupation selection (matching + tier + cap)."""
|
|
catalog = p5.load_catalog()
|
|
domain_index = {}
|
|
for (src_key, skill_key), entries in catalog.items():
|
|
if src_key not in p5.AUTO_SOURCES:
|
|
continue
|
|
text = (" ".join(e["name"] + " " + e["desc"] for e in entries).lower()
|
|
+ " " + skill_key.lower())
|
|
allowed = p5.SOURCE_DOMAINS.get(src_key)
|
|
for dom in p5.AUTO_DOMAINS:
|
|
if allowed is not None and dom["name"] not in allowed:
|
|
continue
|
|
if any(kw in text for kw in dom["skill_kw"]):
|
|
domain_index.setdefault(dom["name"], []).append((src_key, skill_key))
|
|
|
|
occ = p5.load_occupation(slug)
|
|
matched, seen = {}, set()
|
|
for rule in p5.RULES:
|
|
if not p5.rule_matches(rule, occ):
|
|
continue
|
|
for ref in rule["skills"]:
|
|
for entry in catalog.get(ref, []):
|
|
k = (ref[0], entry["name"])
|
|
if k not in seen:
|
|
seen.add(k)
|
|
matched.setdefault(ref[0], []).append(dict(entry))
|
|
for dom in p5.AUTO_DOMAINS:
|
|
if not p5.rule_matches(dom["occ"], occ):
|
|
continue
|
|
for (src_key, skill_key) in domain_index.get(dom["name"], ()):
|
|
for entry in catalog[(src_key, skill_key)]:
|
|
k = (src_key, entry["name"])
|
|
if k not in seen:
|
|
seen.add(k)
|
|
matched.setdefault(src_key, []).append(dict(entry))
|
|
|
|
core_signals = p5.load_core_signals(slug)
|
|
for src_key in list(matched):
|
|
entries = matched[src_key]
|
|
for e in entries:
|
|
e["tier"], e["score"] = p5.tier_for(e, core_signals)
|
|
entries.sort(key=lambda e: (0 if e["tier"] == "core" else 1,
|
|
-e["score"], e["name"]))
|
|
matched[src_key] = entries[:p5.MAX_PER_SOURCE]
|
|
return matched
|
|
|
|
|
|
def main():
|
|
data = json.load(open(EVAL_SET, encoding="utf-8"))
|
|
matched = final_selection(data["target_slug"])
|
|
included = {(src, e["name"].lower()) for src, es in matched.items() for e in es}
|
|
included_names = {}
|
|
for src, es in matched.items():
|
|
included_names[src] = [e["name"].lower() for e in es]
|
|
|
|
correct = 0
|
|
for case in data["cases"]:
|
|
src = case["source"]
|
|
if "skill" in case:
|
|
hit = (src, case["skill"].lower()) in included
|
|
else:
|
|
frag = case["skill_contains"].lower()
|
|
hit = any(frag in n for n in included_names.get(src, []))
|
|
want_included = case["expected"] == "include"
|
|
ok = (hit == want_included)
|
|
correct += ok
|
|
name = case.get("skill") or case["skill_contains"]
|
|
print(f"{'OK ' if ok else 'MISS'} {case['expected']:7} "
|
|
f"{'in-package' if hit else 'excluded '} {src}/{name}")
|
|
|
|
n = len(data["cases"])
|
|
print(f"\nscore: {correct}/{n} (ship threshold: {data['ship_threshold']})")
|
|
total = sum(len(v) for v in matched.values())
|
|
print(f"final selection size: {total} entries across {len(matched)} sources "
|
|
f"(v1 had 427)")
|
|
sys.exit(0 if correct >= data["ship_threshold"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|