"""Phase 5 — enrich skill packages with the best public AI agent skills.

Maps proven, publicly available agent-skill repositories onto the occupation
catalog and writes, per matched occupation:

  skills/<slug>/references/ai-skills.md   (table per source, full attribution)
  SKILL.md                                (one link line in "How to use this skill")
  manifest.json                           ("enrichment_ai_skills" block)

Nothing is copied from the sources — every entry is name + one-line
description + upstream link, grouped under a source header that names
repository, commit, license and retrieval date. Matching is deterministic
(ISCO prefix + title/competence keywords), so re-runs are reproducible and
idempotent.

Sources (cloned to data/external-skills/ by the operator):
  anthropics/skills   — official Anthropic skills (docs, design, dev)
  obra/superpowers    — software-engineering workflow skills (MIT)
  wshobson/agents     — 80+ domain plugins with skills (MIT)

Run:  python pipeline/p5_enrich_ai_skills.py [--dry-run] [slug]
"""
import json
import os
import re
import sys

BASE = os.path.join(os.path.dirname(__file__), "..")
SKILLS_DIR = os.path.join(BASE, "skills")
EXT_DIR = os.path.join(BASE, "data", "external-skills")

RETRIEVED = "2026-07-07"

# ─── Phase 2c: Tiering + Kappung ─────────────────────────────────────────────
# core     = Skill berührt eine Top-20-Market-Hard-Skill oder eine essentielle
#            ESCO-Kompetenz des Berufs (deterministischer Token-Overlap).
# adjacent = plausibel nützlich (Regel-/Domain-Match), aber nicht core.
# excluded = Kappungs-Überlauf (max. MAX_PER_SOURCE je Quelle) — landet NUR im
#            Audit-Log data/audit/ai-skills-excluded.jsonl, nie im Paket.
# Optionaler LLM-Feinschnitt (--llm-tier, Referenz-Pakete): gemma3 entscheidet
# include/exclude je Kandidat; Eval in evals/skill-tiering/.
MAX_PER_SOURCE = 10
AUDIT_LOG = os.path.join(BASE, "data", "audit", "ai-skills-excluded.jsonl")

# Quell-Domain-Allowlists: eine Skill-Library mappt nur in die Domains ihres
# Zwecks — verhindert Kreuzkontamination (Marketing-Skill via "api"-Keyword
# im dev-general-Fallback bei einem Engineering-Beruf gelandet, v1-Befund).
SOURCE_DOMAINS = {
    "nvidia":      ["ai-ml", "robotics-simulation", "cloud-hpc-devops",
                    "data-analytics", "dev-general"],
    "google":      ["advertising", "marketing", "data-analytics",
                    "frontend-web", "audio-video-media"],
    "pmskills":    ["product-management", "project-management", "data-analytics"],
    "pmskills2":   ["product-management", "project-management"],
    "marketing":   ["marketing", "advertising", "email-communication",
                    "sales-crm", "writing-docs"],
    "advertising": ["advertising", "marketing"],
    "lambdatest":  ["qa-testing"],
    "venice":      ["ai-ml", "dev-general", "audio-video-media",
                    "crypto-web3", "design-ux"],
    "n8n":         ["workflow-automation"],
    "cypress":     ["qa-testing"],
    "angular":     ["frontend-web"],
    "resend":      ["email-communication"],
    "garden":      ["design-ux", "knowledge-research", "frontend-web",
                    "audio-video-media"],
}

_STOP_TOK = {"and", "or", "the", "a", "an", "of", "for", "to", "in", "on",
             "with", "using", "use", "skills", "skill", "data"}


def _tokens(s):
    return {w for w in re.split(r"[^a-z0-9+#]+", (s or "").lower())
            if len(w) > 2 and w not in _STOP_TOK}


def load_core_signals(slug):
    """Top-20 market hard skills (post-gate evidence) + essential ESCO
    competences as token sets — the ground truth for the 'core' tier."""
    signals = []
    # market: top-20 hard skills from the evidence store (if present)
    try:
        from db import connect
        cn = connect()
        cur = cn.cursor()
        cur.execute("""
            SELECT TOP 20 ee.entity FROM evidence_entity ee
            JOIN evidence_job ej ON ej.job_id=ee.job_id
            WHERE ej.occupation_slug=? AND ee.entity_type='hard_skills'
            GROUP BY ee.entity ORDER BY COUNT(DISTINCT ee.job_id) DESC""", slug)
        signals += [r[0] for r in cur.fetchall()]
        cn.close()
    except Exception:
        pass
    # ESCO essential competences (full list from references/skills.md)
    try:
        txt = open(os.path.join(SKILLS_DIR, slug, "references", "skills.md"),
                   encoding="utf-8").read()
        m = re.search(r"(?ms)^## Essential$(.*?)(?=^## |\Z)", txt)
        if m:
            signals += [re.sub(r"\*\*|\(.*?\)", "", l.strip("- ").strip())
                        for l in m.group(1).splitlines() if l.strip().startswith("-")]
    except OSError:
        pass
    return [_tokens(s) for s in signals if s]


def tier_for(entry, core_signals):
    """Returns (tier, score). 'core' if the skill name/desc shares >=2 tokens
    (or one exact short-signal hit) with any core signal; score = summed
    overlap for in-tier ranking (strongest matches survive the cap)."""
    sk_tok = _tokens(entry["name"] + " " + entry["desc"])
    score = 0
    core = False
    for sig in core_signals:
        if not sig:
            continue
        inter = sk_tok & sig
        score += len(inter)
        if len(inter) >= 2 or (len(sig) <= 2 and sig <= sk_tok):
            core = True
    return ("core" if core else "adjacent"), score

SOURCES = {
    "anthropic": {
        "label": "anthropics/skills",
        "repo_url": "https://github.com/anthropics/skills",
        "commit": "9d2f1ae",
        "license": ("Apache-2.0; the document skills (docx/pdf/pptx/xlsx) are "
                    "source-available — see the LICENSE.txt in the upstream skill folder"),
        "skill_root": os.path.join(EXT_DIR, "anthropic-skills", "skills"),
        "link_base": "https://github.com/anthropics/skills/tree/main/skills",
    },
    "superpowers": {
        "label": "obra/superpowers",
        "repo_url": "https://github.com/obra/superpowers",
        "commit": "d884ae0",
        "license": "MIT (c) Jesse Vincent",
        "skill_root": os.path.join(EXT_DIR, "superpowers", "skills"),
        "link_base": "https://github.com/obra/superpowers/tree/main/skills",
    },
    "wshobson": {
        "label": "wshobson/agents",
        "repo_url": "https://github.com/wshobson/agents",
        "commit": "6fd3247",
        "license": "MIT (c) Seth Hobson",
        # plugin level: plugins/<plugin>/skills/<skill>/SKILL.md
        "skill_root": os.path.join(EXT_DIR, "wshobson-agents", "plugins"),
        "link_base": "https://github.com/wshobson/agents/tree/main/plugins",
    },
}

# ─── Erweiterte Quellen (Auto-Domain-Mapping, rekursiver SKILL.md-Scan) ──────
# Links zeigen auf den Commit-Stand (tree/<sha>) = stabile Provenance.
def _auto(label, repo, commit, license_, folder):
    return {"label": label, "repo_url": f"https://github.com/{repo}",
            "commit": commit, "license": license_,
            "skill_root": os.path.join(EXT_DIR, folder),
            "link_base": f"https://github.com/{repo}/tree/{commit}"}

AUTO_SOURCES = {
    "nvidia":      _auto("NVIDIA/skills", "NVIDIA/skills", "153b14b",
                         "CC-BY-4.0 (skills/docs), Apache-2.0 (code)", "nvidia-skills"),
    "google":      _auto("google/skills", "google/skills", "b15f327",
                         "Apache-2.0", "google-skills"),
    "pmskills":    _auto("phuryn/pm-skills", "phuryn/pm-skills", "18468a9",
                         "MIT", "pm-skills"),
    "pmskills2":   _auto("deanpeters/Product-Manager-Skills",
                         "deanpeters/Product-Manager-Skills", "99be43c",
                         "CC BY-NC-SA 4.0 (referenced by link, nothing copied)",
                         "product-manager-skills"),
    "marketing":   _auto("coreyhaines31/marketingskills",
                         "coreyhaines31/marketingskills", "33218ff",
                         "MIT", "marketing-skills"),
    "advertising": _auto("realkimbarrett/advertising-skills",
                         "realkimbarrett/advertising-skills", "45f4a4a",
                         "no explicit license — referenced by link only",
                         "advertising-skills"),
    "lambdatest":  _auto("LambdaTest/agent-skills", "LambdaTest/agent-skills",
                         "54824d6", "MIT", "lambdatest-skills"),
    "venice":      _auto("veniceai/skills", "veniceai/skills", "de089fa",
                         "MIT", "venice-skills"),
    "n8n":         _auto("czlonkowski/n8n-skills", "czlonkowski/n8n-skills",
                         "9ea3aa5", "MIT", "n8n-skills"),
    "cypress":     _auto("cypress-io/ai-toolkit", "cypress-io/ai-toolkit",
                         "9c9038e", "MIT", "cypress-toolkit"),
    "angular":     _auto("angular/skills", "angular/skills", "5dd20da",
                         "no explicit license — referenced by link only",
                         "angular-skills"),
    "resend":      _auto("resend/resend-skills", "resend/resend-skills",
                         "2caefff", "MIT", "resend-skills"),
    "garden":      _auto("ConardLi/garden-skills", "ConardLi/garden-skills",
                         "fbd6453", "MIT", "garden-skills"),
}
ALL_SOURCES = {**SOURCES, **AUTO_SOURCES}

SUPERPOWERS_ALL = [
    "brainstorming", "dispatching-parallel-agents", "executing-plans",
    "finishing-a-development-branch", "receiving-code-review",
    "requesting-code-review", "subagent-driven-development",
    "systematic-debugging", "test-driven-development", "using-git-worktrees",
    "verification-before-completion", "writing-plans", "writing-skills",
]

# ─── Mapping rules ───────────────────────────────────────────────────────────
# An occupation matches a rule if its ISCO group starts with one of `isco`
# OR its title contains one of `title_kw` OR its competences contain one of
# `comp_kw`. Matched occupations receive all skills listed in `skills`.
# ("wshobson", "<plugin>") pulls in every skill of that plugin.

RULES = [
    # ── documents & office (broad knowledge work) ────────────────────────────
    {"name": "office-documents",
     "skills": [("anthropic", "docx"), ("anthropic", "pdf")],
     "isco": ["1", "2", "3", "4"], "title_kw": [], "comp_kw": []},

    {"name": "spreadsheets-analysis",
     "skills": [("anthropic", "xlsx")],
     "isco": ["241", "331", "4311", "4312", "212"],
     "title_kw": ["accountant", "auditor", "financial", "bookkeep", "controller",
                  "payroll", "statistician", "economist", "actuar", "tax advisor",
                  "budget", "analyst", "cost estimator"],
     "comp_kw": []},

    {"name": "presentations",
     "skills": [("anthropic", "pptx")],
     "isco": ["112", "121", "122", "242", "2431"],
     "title_kw": ["consultant", "corporate trainer", "lecturer", "teacher",
                  "marketing", "sales manager"],
     "comp_kw": []},

    {"name": "writing-editing",
     "skills": [("anthropic", "doc-coauthoring")],
     "isco": ["264", "2432"],
     "title_kw": ["writer", "editor", "journalist", "author", "copywriter",
                  "communication officer", "content"],
     "comp_kw": []},

    {"name": "internal-communications",
     "skills": [("anthropic", "internal-comms")],
     "isco": ["2432", "1212"],
     "title_kw": ["public relations", "communications manager", "human resources manager"],
     "comp_kw": []},

    # ── design & creative ────────────────────────────────────────────────────
    {"name": "design-creative",
     "skills": [("anthropic", "canvas-design"), ("anthropic", "theme-factory"),
                ("anthropic", "algorithmic-art"), ("anthropic", "brand-guidelines")],
     "isco": ["2166", "2163", "265"],
     "title_kw": ["designer", "graphic", "illustrator", "artist", "art director",
                  "visual", "animator", "typograph"],
     "comp_kw": []},

    # ── software development core ────────────────────────────────────────────
    {"name": "dev-core",
     "skills": ([("superpowers", s) for s in SUPERPOWERS_ALL] +
                [("anthropic", "mcp-builder"), ("anthropic", "claude-api"),
                 ("anthropic", "webapp-testing"),
                 ("wshobson", "developer-essentials"), ("wshobson", "git-pr-workflows"),
                 ("wshobson", "unit-testing"), ("wshobson", "tdd-workflows"),
                 ("wshobson", "debugging-toolkit"), ("wshobson", "error-debugging"),
                 ("wshobson", "code-refactoring"), ("wshobson", "code-documentation"),
                 ("wshobson", "codebase-cleanup"), ("wshobson", "dependency-management"),
                 ("wshobson", "shell-scripting"), ("wshobson", "python-development"),
                 ("wshobson", "comprehensive-review")]),
     "isco": ["251", "252", "351"], "title_kw": [], "comp_kw": []},

    {"name": "dev-web-mobile",
     "skills": [("anthropic", "frontend-design"), ("anthropic", "web-artifacts-builder"),
                ("wshobson", "frontend-mobile-development"), ("wshobson", "web-scripting"),
                ("wshobson", "javascript-typescript"), ("wshobson", "api-scaffolding"),
                ("wshobson", "backend-development"), ("wshobson", "accessibility-compliance"),
                ("wshobson", "multi-platform-apps"), ("wshobson", "payment-processing"),
                ("wshobson", "ui-design")],
     "isco": ["2513", "2514"],
     "title_kw": ["web develop", "web design", "front-end", "frontend", "full stack",
                  "mobile app", "user interface"],
     "comp_kw": []},

    {"name": "dev-data-ml",
     "skills": [("wshobson", "data-engineering"), ("wshobson", "machine-learning-ops"),
                ("wshobson", "llm-application-dev"), ("wshobson", "data-validation-suite")],
     "isco": ["2511", "2521"],
     "title_kw": ["data scientist", "data engineer", "machine learning",
                  "artificial intelligence", "big data", "business intelligence",
                  "data analyst"],
     "comp_kw": []},

    {"name": "dev-database",
     "skills": [("wshobson", "database-design"), ("wshobson", "database-migrations"),
                ("wshobson", "database-cloud-optimization")],
     "isco": ["2521", "2522"],
     "title_kw": ["database"], "comp_kw": []},

    {"name": "dev-ops-cloud",
     "skills": [("wshobson", "kubernetes-operations"), ("wshobson", "cloud-infrastructure"),
                ("wshobson", "cicd-automation"), ("wshobson", "deployment-strategies"),
                ("wshobson", "deployment-validation"), ("wshobson", "observability-monitoring"),
                ("wshobson", "distributed-debugging"), ("wshobson", "application-performance"),
                ("wshobson", "incident-response")],
     "isco": ["2522", "2523", "3511", "3512", "3513"],
     "title_kw": ["devops", "system administrator", "cloud", "network administrator",
                  "site reliability", "ict operations"],
     "comp_kw": []},

    {"name": "security",
     "skills": [("wshobson", "security-compliance"), ("wshobson", "security-scanning"),
                ("wshobson", "backend-api-security"), ("wshobson", "frontend-mobile-security"),
                ("wshobson", "reverse-engineering"), ("wshobson", "incident-response"),
                ("wshobson", "signed-audit-trails")],
     "isco": ["2529"],
     "title_kw": ["it security", "ict security", "cyber", "information security",
                  "computer security", "penetration", "ethical hacker",
                  "digital forensics", "security analyst", "security engineer"],
     "comp_kw": []},

    {"name": "dev-languages-special",
     "skills": [("wshobson", "julia-development"), ("wshobson", "jvm-languages"),
                ("wshobson", "dotnet-contribution"), ("wshobson", "functional-programming"),
                ("wshobson", "systems-programming")],
     "isco": ["2512", "2514"], "title_kw": [], "comp_kw": []},

    {"name": "dev-embedded",
     "skills": [("wshobson", "arm-cortex-microcontrollers"), ("wshobson", "systems-programming")],
     "isco": [],
     "title_kw": ["embedded", "firmware", "microcontroller", "robotics engineer",
                  "electronics engineer"],
     "comp_kw": []},

    {"name": "testing-qa",
     "skills": [("wshobson", "performance-testing-review"),
                ("wshobson", "api-testing-observability"),
                ("anthropic", "webapp-testing")],
     "isco": ["2519"],
     "title_kw": ["software tester", "quality assurance", "test engineer", "test analyst"],
     "comp_kw": []},

    {"name": "software-architecture",
     "skills": [("wshobson", "c4-architecture"), ("wshobson", "framework-migration"),
                ("wshobson", "full-stack-orchestration")],
     "isco": [],
     "title_kw": ["software architect", "system architect", "systems architect",
                  "solution architect", "enterprise architect", "ict system"],
     "comp_kw": []},

    {"name": "game-development",
     "skills": [("wshobson", "game-development")],
     "isco": [],
     "title_kw": ["game developer", "game designer", "game programmer", "video game",
                  "digital games"],
     "comp_kw": []},

    {"name": "blockchain",
     "skills": [("wshobson", "blockchain-web3")],
     "isco": [],
     "title_kw": ["blockchain", "web3", "crypto"],
     "comp_kw": []},

    # ── business domains ─────────────────────────────────────────────────────
    {"name": "marketing-seo",
     "skills": [("wshobson", "content-marketing"), ("wshobson", "seo-content-creation"),
                ("wshobson", "seo-analysis-monitoring"),
                ("wshobson", "seo-technical-optimization"),
                ("wshobson", "social-publishing"), ("wshobson", "brand-landingpage"),
                ("anthropic", "brand-guidelines")],
     "isco": ["2431", "1221"],
     "title_kw": ["marketing", "seo", "social media", "advertis", "brand",
                  "content manager", "digital media", "e-commerce", "growth hacker"],
     "comp_kw": []},

    {"name": "sales-crm",
     "skills": [("wshobson", "customer-sales-automation")],
     "isco": ["2433", "2434", "3322"],
     "title_kw": ["sales manager", "sales representative", "sales engineer",
                  "sales director", "technical sales", "account manager",
                  "business development", "customer relationship", "key account"],
     "comp_kw": []},

    {"name": "business-analysis",
     "skills": [("wshobson", "business-analytics"), ("wshobson", "startup-business-analyst")],
     "isco": ["2421", "2413"],
     "title_kw": ["business analyst", "management consultant", "strategy",
                  "business intelligence", "market research", "financial analyst"],
     "comp_kw": []},

    {"name": "finance-trading",
     "skills": [("wshobson", "quantitative-trading")],
     "isco": ["3311"],
     "title_kw": ["trader", "trading", "investment", "portfolio", "quantitative",
                  "fund manager", "asset manager", "broker", "hedge fund"],
     "comp_kw": []},

    {"name": "human-resources",
     "skills": [("wshobson", "hr-legal-compliance")],
     "isco": ["2423", "4416", "1212"],
     "title_kw": ["recruit", "human resources", "talent", "personnel officer",
                  "headhunter"],
     "comp_kw": []},

    {"name": "legal",
     "skills": [("wshobson", "hr-legal-compliance")],
     "isco": ["261", "3411"],
     "title_kw": ["lawyer", "legal", "attorney", "solicitor", "barrister",
                  "paralegal", "compliance officer", "notary"],
     "comp_kw": []},

    {"name": "project-team-management",
     "skills": [("wshobson", "team-collaboration")],
     "isco": ["1"],
     "title_kw": ["project manager", "product manager", "scrum", "agile coach",
                  "team leader"],
     "comp_kw": []},

    {"name": "technical-writing",
     "skills": [("wshobson", "documentation-generation"),
                ("wshobson", "documentation-standards")],
     "isco": [],
     "title_kw": ["technical writer", "technical communicator", "documentation"],
     "comp_kw": []},

    {"name": "office-file-handling",
     "skills": [("wshobson", "file-conversion")],
     "isco": ["4"],
     "title_kw": [], "comp_kw": []},
]

# ─── Auto-Domains für die erweiterten Quellen ────────────────────────────────
# Jeder externe Skill wird per Keyword gegen (name + description + pfad)
# klassifiziert; die getroffenen Domains bestimmen via occ-Kriterien (gleiche
# Semantik wie RULES) die Ziel-Berufe. Ein Skill kann mehrere Domains treffen.

AUTO_DOMAINS = [
    {"name": "ai-ml",
     "skill_kw": ["machine learning", "deep learning", "llm", "cuda", "tensorrt",
                  "nemo", "inference", "gpu", "model training", "fine-tun",
                  "embedding", "rag", "vector", "nim ", "triton", "dataset",
                  "prompt", "agentic", "neural"],
     "occ": {"isco": ["2511", "2512", "2521"],
             "title_kw": ["artificial intelligence", "machine learning",
                          "data scientist", "data engineer", "big data"],
             "comp_kw": []}},
    {"name": "robotics-simulation",
     "skill_kw": ["robot", "omniverse", "simulation", "isaac", "physical ai",
                  "simready", "usd ", "digital twin", "cad"],
     "occ": {"isco": [], "title_kw": ["robotics", "automation engineer",
                                      "simulation", "mechatronic"],
             "comp_kw": []}},
    {"name": "cloud-hpc-devops",
     "skill_kw": ["kubernetes", "docker", "cluster", "slurm", "hpc",
                  "infrastructure", "deploy", "networking", "cloud", "devops",
                  "observability", "scaling"],
     "occ": {"isco": ["2522", "2523", "3511", "3512", "3513"],
             "title_kw": ["devops", "system administrator", "cloud",
                          "site reliability", "network"],
             "comp_kw": []}},
    {"name": "dev-general",
     "skill_kw": ["api", "sdk", "code", "debug", "framework", "library",
                  "typescript", "python", "javascript", "git", "refactor",
                  "migration", "webhook", "integration"],
     "occ": {"isco": ["251", "252"], "title_kw": [], "comp_kw": []}},
    {"name": "qa-testing",
     "skill_kw": ["test", "e2e", "selenium", "playwright", "cypress", "appium",
                  "quality assurance", "automation testing", "accessibility"],
     "occ": {"isco": ["2519"],
             "title_kw": ["software tester", "quality assurance",
                          "test engineer", "test analyst"],
             "comp_kw": []}},
    {"name": "frontend-web",
     "skill_kw": ["angular", "react", "frontend", "component", "css",
                  "web app", "responsive", "ssr"],
     "occ": {"isco": ["2513", "2514"],
             "title_kw": ["web develop", "web design", "front-end", "frontend",
                          "user interface"],
             "comp_kw": []}},
    {"name": "marketing",
     "skill_kw": ["marketing", "seo", "campaign", "brand", "growth", "funnel",
                  "churn", "aso", "landing page", "newsletter", "social media",
                  "content strategy", "positioning", "launch"],
     "occ": {"isco": ["2431", "1221"],
             "title_kw": ["marketing", "seo", "social media", "brand",
                          "content manager", "e-commerce", "growth"],
             "comp_kw": []}},
    {"name": "advertising",
     "skill_kw": ["advertis", "ad creative", "ads api", "google ads",
                  "mobile ads", "ppc", "media buying", "adsense", "ad manager"],
     "occ": {"isco": ["2431"],
             "title_kw": ["advertis", "media planner", "media buyer",
                          "marketing"],
             "comp_kw": []}},
    {"name": "product-management",
     "skill_kw": ["product", "prd", "roadmap", "okr", "backlog", "user stor",
                  "prioritization", "discovery", "job stories", "pre-mortem",
                  "a/b test", "ab-test", "user research", "persona"],
     "occ": {"isco": [],
             "title_kw": ["product manager", "product owner",
                          "product developer", "innovation manager"],
             "comp_kw": []}},
    {"name": "project-management",
     "skill_kw": ["project plan", "sprint", "scrum", "stakeholder", "kickoff",
                  "retrospective", "milestone"],
     "occ": {"isco": [],
             "title_kw": ["project manager", "scrum", "agile coach",
                          "programme manager", "project coordinator"],
             "comp_kw": []}},
    {"name": "data-analytics",
     "skill_kw": ["sql", "analytics", "dashboard", "cohort", "metric",
                  "data analysis", "reporting", "kpi", "visualization"],
     "occ": {"isco": ["2413", "2421", "2511", "331"],
             "title_kw": ["analyst", "business intelligence", "controller",
                          "statistician"],
             "comp_kw": []}},
    {"name": "design-ux",
     "skill_kw": ["design", "ui", "ux", "figma", "prototype", "wireframe",
                  "image generation", "visual", "typography", "artwork"],
     "occ": {"isco": ["2166", "2163", "265"],
             "title_kw": ["designer", "graphic", "illustrator", "artist",
                          "art director", "visual", "animator"],
             "comp_kw": []}},
    {"name": "audio-video-media",
     "skill_kw": ["audio", "video", "speech", "transcription", "music",
                  "podcast", "voice"],
     "occ": {"isco": ["2654", "2655", "3521"],
             "title_kw": ["video editor", "sound", "audio", "music producer",
                          "podcast", "broadcast"],
             "comp_kw": []}},
    {"name": "email-communication",
     "skill_kw": ["email", "transactional", "deliverability", "smtp"],
     "occ": {"isco": ["2431", "2432"],
             "title_kw": ["marketing", "communication", "crm"],
             "comp_kw": []}},
    {"name": "sales-crm",
     "skill_kw": ["sales", "crm", "lead gen", "outreach", "cold email",
                  "pipeline review"],
     "occ": {"isco": ["2433", "2434", "3322"],
             "title_kw": ["sales manager", "sales representative",
                          "account manager", "business development",
                          "key account"],
             "comp_kw": []}},
    {"name": "writing-docs",
     "skill_kw": ["writing", "documentation", "article", "blog", "copywriting",
                  "technical writer", "docs"],
     "occ": {"isco": ["264"],
             "title_kw": ["writer", "editor", "journalist", "copywriter",
                          "technical communicator", "documentation"],
             "comp_kw": []}},
    {"name": "crypto-web3",
     "skill_kw": ["crypto", "blockchain", "wallet", "web3", "smart contract"],
     "occ": {"isco": [],
             "title_kw": ["blockchain", "web3", "crypto"],
             "comp_kw": []}},
    {"name": "workflow-automation",
     "skill_kw": ["n8n", "workflow", "automation", "zapier", "no-code",
                  "orchestration"],
     "occ": {"isco": ["251", "252"],
             "title_kw": ["automation", "integration", "process"],
             "comp_kw": []}},
    {"name": "knowledge-research",
     "skill_kw": ["retriev", "knowledge base", "research", "search", "kb-"],
     "occ": {"isco": ["2621", "2622", "243", "2421"],
             "title_kw": ["research", "librarian", "archivist", "information",
                          "knowledge"],
             "comp_kw": []}},
]

# ─── External skill catalog ──────────────────────────────────────────────────

_FM_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n", re.S)


def parse_frontmatter(path):
    """Return (name, description) from a SKILL.md frontmatter. Tolerant YAML-ish."""
    try:
        text = open(path, encoding="utf-8").read()
    except OSError:
        return None, None
    m = _FM_RE.match(text)
    if not m:
        return None, None
    fm = m.group(1)
    name = desc = None
    lines = fm.splitlines()
    for i, line in enumerate(lines):
        if line.startswith("name:") and name is None:
            name = line.split(":", 1)[1].strip().strip("\"'")
        if line.startswith("description:") and desc is None:
            desc = line.split(":", 1)[1].strip()
            if desc in (">", ">-", "|", "|-"):
                desc = ""
            # fold indented continuation lines
            j = i + 1
            while j < len(lines) and (lines[j].startswith(" ") or lines[j] == ""):
                desc += " " + lines[j].strip()
                j += 1
            desc = desc.strip().strip("\"'")
    return name, desc


def shorten(desc, limit=240):
    if not desc:
        return ""
    desc = re.sub(r"\s+", " ", desc).replace("|", "\\|")
    # cut at the first sentence past ~limit? keep simple: hard cut on word
    if len(desc) <= limit:
        return desc
    cut = desc[:limit].rsplit(" ", 1)[0]
    return cut + " …"


def load_catalog():
    """Return {(source_key, skill_key): {name, desc, url}}.
    For wshobson, skill_key is the PLUGIN name and the value holds a LIST of
    its skills; anthropic/superpowers map 1:1."""
    cat = {}
    for key in ("anthropic", "superpowers"):
        src = SOURCES[key]
        root = src["skill_root"]
        if not os.path.isdir(root):
            continue
        for d in sorted(os.listdir(root)):
            p = os.path.join(root, d, "SKILL.md")
            if not os.path.isfile(p):
                continue
            name, desc = parse_frontmatter(p)
            cat[(key, d)] = [{
                "name": name or d,
                "desc": shorten(desc),
                "url": f"{src['link_base']}/{d}",
            }]
    # wshobson: plugin -> [skills]; plugins without skills/ ship agents/*.md
    src = SOURCES["wshobson"]
    root = src["skill_root"]
    if os.path.isdir(root):
        for plugin in sorted(os.listdir(root)):
            entries = []
            sdir = os.path.join(root, plugin, "skills")
            if os.path.isdir(sdir):
                for d in sorted(os.listdir(sdir)):
                    p = os.path.join(sdir, d, "SKILL.md")
                    if not os.path.isfile(p):
                        continue
                    name, desc = parse_frontmatter(p)
                    entries.append({
                        "name": name or d,
                        "desc": shorten(desc),
                        "url": f"{src['link_base']}/{plugin}/skills/{d}",
                    })
            else:
                adir = os.path.join(root, plugin, "agents")
                if os.path.isdir(adir):
                    for f in sorted(os.listdir(adir)):
                        if not f.endswith(".md"):
                            continue
                        name, desc = parse_frontmatter(os.path.join(adir, f))
                        entries.append({
                            "name": (name or f[:-3]) + " (agent)",
                            "desc": shorten(desc),
                            "url": f"{src['link_base']}/{plugin}/agents/{f}",
                        })
            if entries:
                cat[("wshobson", plugin)] = entries

    # Erweiterte Quellen: rekursiver SKILL.md-Scan, Key = relativer Ordnerpfad
    for key, src in AUTO_SOURCES.items():
        root = src["skill_root"]
        if not os.path.isdir(root):
            continue
        for dirpath, dirnames, filenames in os.walk(root):
            dirnames[:] = [d for d in dirnames
                           if d not in (".git", "node_modules", "dist", "website")]
            if "SKILL.md" not in filenames:
                continue
            rel = os.path.relpath(dirpath, root).replace("\\", "/")
            name, desc = parse_frontmatter(os.path.join(dirpath, "SKILL.md"))
            cat[(key, rel)] = [{
                "name": name or os.path.basename(dirpath),
                "desc": shorten(desc),
                "url": f"{src['link_base']}/{rel}" if rel != "." else src["link_base"],
            }]
    return cat


# ─── Occupation loading & matching ───────────────────────────────────────────

def load_occupation(slug):
    base = os.path.join(SKILLS_DIR, slug)
    mpath = os.path.join(base, "manifest.json")
    spath = os.path.join(base, "SKILL.md")
    if not (os.path.isfile(mpath) and os.path.isfile(spath)):
        return None
    m = json.load(open(mpath, encoding="utf-8"))
    text = open(spath, encoding="utf-8").read()
    comp = ""
    cm = re.search(r"(?ms)^## Key competences.*?$(.*?)(?=^## |\Z)", text)
    if cm:
        comp = cm.group(1)
    return {
        "slug": slug,
        "title": (m.get("title") or slug).lower(),
        "isco": str(m.get("ids", {}).get("isco_group") or ""),
        "comp": comp.lower(),
        "manifest": m,
    }


def rule_matches(rule, occ):
    if any(occ["isco"].startswith(p) for p in rule["isco"]):
        return True
    if any(kw in occ["title"] for kw in rule["title_kw"]):
        return True
    if any(kw in occ["comp"] for kw in rule["comp_kw"]):
        return True
    return False


# ─── Writers ─────────────────────────────────────────────────────────────────

SKILL_LINK_LINE = ("- See [references/ai-skills.md](references/ai-skills.md) — "
                   "matched external AI agent skills (per-source attribution).")


def write_ai_skills_md(base, slug, matched):
    """matched: {source_key: [ {name,desc,url}, ... ]} (deduped, sorted)."""
    lines = [
        f"# External AI agent skills — {slug}",
        "",
        "Proven, publicly available AI agent skills mapped to this occupation.",
        "Nothing is copied from the sources: every entry is a name, a one-line",
        "summary and a link to the upstream skill package. Each section names",
        "its source repository, commit, license and retrieval date.",
        "",
        "**Tiers:** `core` = the skill directly exercises a top-20 market hard",
        "skill or an essential ESCO competence of this occupation; `adjacent` =",
        "plausibly useful, secondary. Entries are capped at "
        f"{MAX_PER_SOURCE} per source",
        "(core first); everything beyond the cap is excluded and logged in the",
        "pipeline audit trail, not in this package.",
        "",
        "_Matched deterministically (ISCO group + title/competence keywords,",
        "tiered against market evidence + ESCO essentials) by",
        f"`pipeline/p5_enrich_ai_skills.py` on {RETRIEVED}._",
        "",
    ]
    order = ["anthropic", "superpowers", "wshobson"] + sorted(AUTO_SOURCES)
    for key in order:
        entries = matched.get(key)
        if not entries:
            continue
        src = ALL_SOURCES[key]
        lines += [
            f"## Source: {src['label']}",
            "",
            f"- Repository: [{src['repo_url']}]({src['repo_url']}) "
            f"(commit `{src['commit']}`, retrieved {RETRIEVED})",
            f"- License: {src['license']}",
            "",
            "| Skill | Tier | What it adds | Upstream |",
            "|---|---|---|---|",
        ]
        for e in entries:
            lines.append(f"| `{e['name']}` | {e.get('tier', 'adjacent')} "
                         f"| {e['desc'] or '—'} | [source]({e['url']}) |")
        lines.append("")
    with open(os.path.join(base, "references", "ai-skills.md"), "w",
              encoding="utf-8", newline="\n") as f:
        f.write("\n".join(lines))


def patch_skill_md(base):
    """Insert the ai-skills link line into the 'How to use this skill' list."""
    p = os.path.join(base, "SKILL.md")
    text = open(p, encoding="utf-8").read()
    if "references/ai-skills.md" in text:
        return
    m = re.search(r"(?ms)^## How to use this skill\s*\n(.*?)(?=^## |\Z)", text)
    if not m:
        return
    block = m.group(1)
    # insert after the last list item of the block
    items = list(re.finditer(r"(?m)^- .*$", block))
    if not items:
        return
    insert_at = m.start(1) + items[-1].end()
    text = text[:insert_at] + "\n" + SKILL_LINK_LINE + text[insert_at:]
    open(p, "w", encoding="utf-8", newline="\n").write(text)


def patch_manifest(base, occ, matched):
    m = occ["manifest"]
    m["enrichment_ai_skills"] = {
        "generated": RETRIEVED,
        "method": "deterministic mapping (ISCO prefix + title/competence keywords)",
        "sources": {
            ALL_SOURCES[k]["label"]: {
                "repo": ALL_SOURCES[k]["repo_url"],
                "commit": ALL_SOURCES[k]["commit"],
                "license": ALL_SOURCES[k]["license"],
                "skills": len(v),
            } for k, v in matched.items() if v
        },
        "total_skills": sum(len(v) for v in matched.values()),
        "tiers": {
            "core": sum(1 for v in matched.values() for e in v
                        if e.get("tier") == "core"),
            "adjacent": sum(1 for v in matched.values() for e in v
                            if e.get("tier") != "core"),
        },
    }
    json.dump(m, open(os.path.join(base, "manifest.json"), "w", encoding="utf-8"),
              indent=2)


# ─── Main ────────────────────────────────────────────────────────────────────

def main():
    dry = "--dry-run" in sys.argv
    args = [a for a in sys.argv[1:] if not a.startswith("--")]
    only = args[0] if args else None

    catalog = load_catalog()
    print(f"external catalog: {sum(len(v) for v in catalog.values())} skills "
          f"in {len(catalog)} units from {len(ALL_SOURCES)} sources")

    # Domain-Index: erweiterte Quellen einmalig klassifizieren
    domain_index = {}
    unclassified = 0
    for (src_key, skill_key), entries in catalog.items():
        if src_key not in AUTO_SOURCES:
            continue
        text = " ".join(e["name"] + " " + e["desc"] for e in entries).lower() \
               + " " + skill_key.lower()
        allowed = SOURCE_DOMAINS.get(src_key)
        hit = False
        for dom in AUTO_DOMAINS:
            if allowed is not None and dom["name"] not in allowed:
                continue
            if any(kw in text for kw in dom["skill_kw"]):
                domain_index.setdefault(dom["name"], []).append((src_key, skill_key))
                hit = True
        if not hit:
            unclassified += 1
    print(f"auto-domain index: "
          f"{sum(len(v) for v in domain_index.values())} zuordnungen, "
          f"{unclassified} skills ohne domain (nicht gemappt)")
    missing = set()
    for rule in RULES:
        for ref in rule["skills"]:
            if ref not in catalog:
                missing.add(ref)
    if missing:
        print(f"WARN: {len(missing)} rule references not found in catalog: "
              f"{sorted(missing)[:6]} ...")

    slugs = sorted(d for d in os.listdir(SKILLS_DIR)
                   if os.path.isdir(os.path.join(SKILLS_DIR, d)))
    if only:
        slugs = [s for s in slugs if s == only]

    enriched = 0
    total_links = 0
    for slug in slugs:
        occ = load_occupation(slug)
        if not occ:
            continue
        # collect matches, dedupe by (source, skill name)
        matched = {}
        seen = set()
        for rule in RULES:
            if not rule_matches(rule, occ):
                continue
            for ref in rule["skills"]:
                for entry in catalog.get(ref, []):
                    k = (ref[0], entry["name"])
                    if k in seen:
                        continue
                    seen.add(k)
                    matched.setdefault(ref[0], []).append(entry)
        # Auto-Domains: erweiterte Quellen keyword-basiert zuordnen
        for dom in AUTO_DOMAINS:
            if not rule_matches(dom["occ"], occ):
                continue
            for (src_key, skill_key) in domain_index.get(dom["name"], ()):
                for entry in catalog[(src_key, skill_key)]:
                    k = (src_key, entry["name"])
                    if k in seen:
                        continue
                    seen.add(k)
                    matched.setdefault(src_key, []).append(entry)
        if not matched:
            continue
        # ── Tiering + Kappung (Phase 2c) ────────────────────────────────
        core_signals = load_core_signals(slug)
        audit_rows = []
        for src_key in list(matched):
            entries = matched[src_key]
            for e in entries:
                e["tier"], e["score"] = tier_for(e, core_signals)
            entries.sort(key=lambda e: (0 if e["tier"] == "core" else 1,
                                        -e["score"], e["name"]))
            if len(entries) > MAX_PER_SOURCE:
                for e in entries[MAX_PER_SOURCE:]:
                    audit_rows.append({"slug": slug, "source": src_key,
                                       "skill": e["name"], "tier": e["tier"],
                                       "reason": "per-source cap"})
                matched[src_key] = entries[:MAX_PER_SOURCE]
        if not dry and audit_rows:
            os.makedirs(os.path.dirname(AUDIT_LOG), exist_ok=True)
            with open(AUDIT_LOG, "a", encoding="utf-8") as af:
                for row in audit_rows:
                    af.write(json.dumps(row, ensure_ascii=False) + "\n")
        if not dry:
            base = os.path.join(SKILLS_DIR, slug)
            write_ai_skills_md(base, slug, matched)
            patch_skill_md(base)
            patch_manifest(base, occ, matched)
        enriched += 1
        total_links += sum(len(v) for v in matched.values())

    print(f"enriched {enriched}/{len(slugs)} packages "
          f"with {total_links} skill links total{' (dry run)' if dry else ''}")


if __name__ == "__main__":
    main()