3b4a225594
Deterministic tiers: core = token overlap with top-20 market hard skills or essential ESCO competences; in-tier ranking by overlap score. Per-source cap 10 (overflow -> audit log, not the package). Source-domain allowlists stop cross-contamination (marketing skill entering an engineering package via a generic 'api' keyword). Flagship selection: 427 -> 73 entries; all 10 real v1 flood negatives excluded, 10 must-have positives included. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
925 lines
42 KiB
Python
925 lines
42 KiB
Python
"""Phase 5 — enrich skill packages with the best public AI agent skills.
|
|
|
|
Maps proven, publicly available agent-skill repositories onto the occupation
|
|
catalog and writes, per matched occupation:
|
|
|
|
skills/<slug>/references/ai-skills.md (table per source, full attribution)
|
|
SKILL.md (one link line in "How to use this skill")
|
|
manifest.json ("enrichment_ai_skills" block)
|
|
|
|
Nothing is copied from the sources — every entry is name + one-line
|
|
description + upstream link, grouped under a source header that names
|
|
repository, commit, license and retrieval date. Matching is deterministic
|
|
(ISCO prefix + title/competence keywords), so re-runs are reproducible and
|
|
idempotent.
|
|
|
|
Sources (cloned to data/external-skills/ by the operator):
|
|
anthropics/skills — official Anthropic skills (docs, design, dev)
|
|
obra/superpowers — software-engineering workflow skills (MIT)
|
|
wshobson/agents — 80+ domain plugins with skills (MIT)
|
|
|
|
Run: python pipeline/p5_enrich_ai_skills.py [--dry-run] [slug]
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
BASE = os.path.join(os.path.dirname(__file__), "..")
|
|
SKILLS_DIR = os.path.join(BASE, "skills")
|
|
EXT_DIR = os.path.join(BASE, "data", "external-skills")
|
|
|
|
RETRIEVED = "2026-07-07"
|
|
|
|
# ─── Phase 2c: Tiering + Kappung ─────────────────────────────────────────────
|
|
# core = Skill berührt eine Top-20-Market-Hard-Skill oder eine essentielle
|
|
# ESCO-Kompetenz des Berufs (deterministischer Token-Overlap).
|
|
# adjacent = plausibel nützlich (Regel-/Domain-Match), aber nicht core.
|
|
# excluded = Kappungs-Überlauf (max. MAX_PER_SOURCE je Quelle) — landet NUR im
|
|
# Audit-Log data/audit/ai-skills-excluded.jsonl, nie im Paket.
|
|
# Optionaler LLM-Feinschnitt (--llm-tier, Referenz-Pakete): gemma3 entscheidet
|
|
# include/exclude je Kandidat; Eval in evals/skill-tiering/.
|
|
MAX_PER_SOURCE = 10
|
|
AUDIT_LOG = os.path.join(BASE, "data", "audit", "ai-skills-excluded.jsonl")
|
|
|
|
# Quell-Domain-Allowlists: eine Skill-Library mappt nur in die Domains ihres
|
|
# Zwecks — verhindert Kreuzkontamination (Marketing-Skill via "api"-Keyword
|
|
# im dev-general-Fallback bei einem Engineering-Beruf gelandet, v1-Befund).
|
|
SOURCE_DOMAINS = {
|
|
"nvidia": ["ai-ml", "robotics-simulation", "cloud-hpc-devops",
|
|
"data-analytics", "dev-general"],
|
|
"google": ["advertising", "marketing", "data-analytics",
|
|
"frontend-web", "audio-video-media"],
|
|
"pmskills": ["product-management", "project-management", "data-analytics"],
|
|
"pmskills2": ["product-management", "project-management"],
|
|
"marketing": ["marketing", "advertising", "email-communication",
|
|
"sales-crm", "writing-docs"],
|
|
"advertising": ["advertising", "marketing"],
|
|
"lambdatest": ["qa-testing"],
|
|
"venice": ["ai-ml", "dev-general", "audio-video-media",
|
|
"crypto-web3", "design-ux"],
|
|
"n8n": ["workflow-automation"],
|
|
"cypress": ["qa-testing"],
|
|
"angular": ["frontend-web"],
|
|
"resend": ["email-communication"],
|
|
"garden": ["design-ux", "knowledge-research", "frontend-web",
|
|
"audio-video-media"],
|
|
}
|
|
|
|
_STOP_TOK = {"and", "or", "the", "a", "an", "of", "for", "to", "in", "on",
|
|
"with", "using", "use", "skills", "skill", "data"}
|
|
|
|
|
|
def _tokens(s):
|
|
return {w for w in re.split(r"[^a-z0-9+#]+", (s or "").lower())
|
|
if len(w) > 2 and w not in _STOP_TOK}
|
|
|
|
|
|
def load_core_signals(slug):
|
|
"""Top-20 market hard skills (post-gate evidence) + essential ESCO
|
|
competences as token sets — the ground truth for the 'core' tier."""
|
|
signals = []
|
|
# market: top-20 hard skills from the evidence store (if present)
|
|
try:
|
|
from db import connect
|
|
cn = connect()
|
|
cur = cn.cursor()
|
|
cur.execute("""
|
|
SELECT TOP 20 ee.entity FROM evidence_entity ee
|
|
JOIN evidence_job ej ON ej.job_id=ee.job_id
|
|
WHERE ej.occupation_slug=? AND ee.entity_type='hard_skills'
|
|
GROUP BY ee.entity ORDER BY COUNT(DISTINCT ee.job_id) DESC""", slug)
|
|
signals += [r[0] for r in cur.fetchall()]
|
|
cn.close()
|
|
except Exception:
|
|
pass
|
|
# ESCO essential competences (full list from references/skills.md)
|
|
try:
|
|
txt = open(os.path.join(SKILLS_DIR, slug, "references", "skills.md"),
|
|
encoding="utf-8").read()
|
|
m = re.search(r"(?ms)^## Essential$(.*?)(?=^## |\Z)", txt)
|
|
if m:
|
|
signals += [re.sub(r"\*\*|\(.*?\)", "", l.strip("- ").strip())
|
|
for l in m.group(1).splitlines() if l.strip().startswith("-")]
|
|
except OSError:
|
|
pass
|
|
return [_tokens(s) for s in signals if s]
|
|
|
|
|
|
def tier_for(entry, core_signals):
|
|
"""Returns (tier, score). 'core' if the skill name/desc shares >=2 tokens
|
|
(or one exact short-signal hit) with any core signal; score = summed
|
|
overlap for in-tier ranking (strongest matches survive the cap)."""
|
|
sk_tok = _tokens(entry["name"] + " " + entry["desc"])
|
|
score = 0
|
|
core = False
|
|
for sig in core_signals:
|
|
if not sig:
|
|
continue
|
|
inter = sk_tok & sig
|
|
score += len(inter)
|
|
if len(inter) >= 2 or (len(sig) <= 2 and sig <= sk_tok):
|
|
core = True
|
|
return ("core" if core else "adjacent"), score
|
|
|
|
SOURCES = {
|
|
"anthropic": {
|
|
"label": "anthropics/skills",
|
|
"repo_url": "https://github.com/anthropics/skills",
|
|
"commit": "9d2f1ae",
|
|
"license": ("Apache-2.0; the document skills (docx/pdf/pptx/xlsx) are "
|
|
"source-available — see the LICENSE.txt in the upstream skill folder"),
|
|
"skill_root": os.path.join(EXT_DIR, "anthropic-skills", "skills"),
|
|
"link_base": "https://github.com/anthropics/skills/tree/main/skills",
|
|
},
|
|
"superpowers": {
|
|
"label": "obra/superpowers",
|
|
"repo_url": "https://github.com/obra/superpowers",
|
|
"commit": "d884ae0",
|
|
"license": "MIT (c) Jesse Vincent",
|
|
"skill_root": os.path.join(EXT_DIR, "superpowers", "skills"),
|
|
"link_base": "https://github.com/obra/superpowers/tree/main/skills",
|
|
},
|
|
"wshobson": {
|
|
"label": "wshobson/agents",
|
|
"repo_url": "https://github.com/wshobson/agents",
|
|
"commit": "6fd3247",
|
|
"license": "MIT (c) Seth Hobson",
|
|
# plugin level: plugins/<plugin>/skills/<skill>/SKILL.md
|
|
"skill_root": os.path.join(EXT_DIR, "wshobson-agents", "plugins"),
|
|
"link_base": "https://github.com/wshobson/agents/tree/main/plugins",
|
|
},
|
|
}
|
|
|
|
# ─── Erweiterte Quellen (Auto-Domain-Mapping, rekursiver SKILL.md-Scan) ──────
|
|
# Links zeigen auf den Commit-Stand (tree/<sha>) = stabile Provenance.
|
|
def _auto(label, repo, commit, license_, folder):
|
|
return {"label": label, "repo_url": f"https://github.com/{repo}",
|
|
"commit": commit, "license": license_,
|
|
"skill_root": os.path.join(EXT_DIR, folder),
|
|
"link_base": f"https://github.com/{repo}/tree/{commit}"}
|
|
|
|
AUTO_SOURCES = {
|
|
"nvidia": _auto("NVIDIA/skills", "NVIDIA/skills", "153b14b",
|
|
"CC-BY-4.0 (skills/docs), Apache-2.0 (code)", "nvidia-skills"),
|
|
"google": _auto("google/skills", "google/skills", "b15f327",
|
|
"Apache-2.0", "google-skills"),
|
|
"pmskills": _auto("phuryn/pm-skills", "phuryn/pm-skills", "18468a9",
|
|
"MIT", "pm-skills"),
|
|
"pmskills2": _auto("deanpeters/Product-Manager-Skills",
|
|
"deanpeters/Product-Manager-Skills", "99be43c",
|
|
"CC BY-NC-SA 4.0 (referenced by link, nothing copied)",
|
|
"product-manager-skills"),
|
|
"marketing": _auto("coreyhaines31/marketingskills",
|
|
"coreyhaines31/marketingskills", "33218ff",
|
|
"MIT", "marketing-skills"),
|
|
"advertising": _auto("realkimbarrett/advertising-skills",
|
|
"realkimbarrett/advertising-skills", "45f4a4a",
|
|
"no explicit license — referenced by link only",
|
|
"advertising-skills"),
|
|
"lambdatest": _auto("LambdaTest/agent-skills", "LambdaTest/agent-skills",
|
|
"54824d6", "MIT", "lambdatest-skills"),
|
|
"venice": _auto("veniceai/skills", "veniceai/skills", "de089fa",
|
|
"MIT", "venice-skills"),
|
|
"n8n": _auto("czlonkowski/n8n-skills", "czlonkowski/n8n-skills",
|
|
"9ea3aa5", "MIT", "n8n-skills"),
|
|
"cypress": _auto("cypress-io/ai-toolkit", "cypress-io/ai-toolkit",
|
|
"9c9038e", "MIT", "cypress-toolkit"),
|
|
"angular": _auto("angular/skills", "angular/skills", "5dd20da",
|
|
"no explicit license — referenced by link only",
|
|
"angular-skills"),
|
|
"resend": _auto("resend/resend-skills", "resend/resend-skills",
|
|
"2caefff", "MIT", "resend-skills"),
|
|
"garden": _auto("ConardLi/garden-skills", "ConardLi/garden-skills",
|
|
"fbd6453", "MIT", "garden-skills"),
|
|
}
|
|
ALL_SOURCES = {**SOURCES, **AUTO_SOURCES}
|
|
|
|
SUPERPOWERS_ALL = [
|
|
"brainstorming", "dispatching-parallel-agents", "executing-plans",
|
|
"finishing-a-development-branch", "receiving-code-review",
|
|
"requesting-code-review", "subagent-driven-development",
|
|
"systematic-debugging", "test-driven-development", "using-git-worktrees",
|
|
"verification-before-completion", "writing-plans", "writing-skills",
|
|
]
|
|
|
|
# ─── Mapping rules ───────────────────────────────────────────────────────────
|
|
# An occupation matches a rule if its ISCO group starts with one of `isco`
|
|
# OR its title contains one of `title_kw` OR its competences contain one of
|
|
# `comp_kw`. Matched occupations receive all skills listed in `skills`.
|
|
# ("wshobson", "<plugin>") pulls in every skill of that plugin.
|
|
|
|
RULES = [
|
|
# ── documents & office (broad knowledge work) ────────────────────────────
|
|
{"name": "office-documents",
|
|
"skills": [("anthropic", "docx"), ("anthropic", "pdf")],
|
|
"isco": ["1", "2", "3", "4"], "title_kw": [], "comp_kw": []},
|
|
|
|
{"name": "spreadsheets-analysis",
|
|
"skills": [("anthropic", "xlsx")],
|
|
"isco": ["241", "331", "4311", "4312", "212"],
|
|
"title_kw": ["accountant", "auditor", "financial", "bookkeep", "controller",
|
|
"payroll", "statistician", "economist", "actuar", "tax advisor",
|
|
"budget", "analyst", "cost estimator"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "presentations",
|
|
"skills": [("anthropic", "pptx")],
|
|
"isco": ["112", "121", "122", "242", "2431"],
|
|
"title_kw": ["consultant", "corporate trainer", "lecturer", "teacher",
|
|
"marketing", "sales manager"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "writing-editing",
|
|
"skills": [("anthropic", "doc-coauthoring")],
|
|
"isco": ["264", "2432"],
|
|
"title_kw": ["writer", "editor", "journalist", "author", "copywriter",
|
|
"communication officer", "content"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "internal-communications",
|
|
"skills": [("anthropic", "internal-comms")],
|
|
"isco": ["2432", "1212"],
|
|
"title_kw": ["public relations", "communications manager", "human resources manager"],
|
|
"comp_kw": []},
|
|
|
|
# ── design & creative ────────────────────────────────────────────────────
|
|
{"name": "design-creative",
|
|
"skills": [("anthropic", "canvas-design"), ("anthropic", "theme-factory"),
|
|
("anthropic", "algorithmic-art"), ("anthropic", "brand-guidelines")],
|
|
"isco": ["2166", "2163", "265"],
|
|
"title_kw": ["designer", "graphic", "illustrator", "artist", "art director",
|
|
"visual", "animator", "typograph"],
|
|
"comp_kw": []},
|
|
|
|
# ── software development core ────────────────────────────────────────────
|
|
{"name": "dev-core",
|
|
"skills": ([("superpowers", s) for s in SUPERPOWERS_ALL] +
|
|
[("anthropic", "mcp-builder"), ("anthropic", "claude-api"),
|
|
("anthropic", "webapp-testing"),
|
|
("wshobson", "developer-essentials"), ("wshobson", "git-pr-workflows"),
|
|
("wshobson", "unit-testing"), ("wshobson", "tdd-workflows"),
|
|
("wshobson", "debugging-toolkit"), ("wshobson", "error-debugging"),
|
|
("wshobson", "code-refactoring"), ("wshobson", "code-documentation"),
|
|
("wshobson", "codebase-cleanup"), ("wshobson", "dependency-management"),
|
|
("wshobson", "shell-scripting"), ("wshobson", "python-development"),
|
|
("wshobson", "comprehensive-review")]),
|
|
"isco": ["251", "252", "351"], "title_kw": [], "comp_kw": []},
|
|
|
|
{"name": "dev-web-mobile",
|
|
"skills": [("anthropic", "frontend-design"), ("anthropic", "web-artifacts-builder"),
|
|
("wshobson", "frontend-mobile-development"), ("wshobson", "web-scripting"),
|
|
("wshobson", "javascript-typescript"), ("wshobson", "api-scaffolding"),
|
|
("wshobson", "backend-development"), ("wshobson", "accessibility-compliance"),
|
|
("wshobson", "multi-platform-apps"), ("wshobson", "payment-processing"),
|
|
("wshobson", "ui-design")],
|
|
"isco": ["2513", "2514"],
|
|
"title_kw": ["web develop", "web design", "front-end", "frontend", "full stack",
|
|
"mobile app", "user interface"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "dev-data-ml",
|
|
"skills": [("wshobson", "data-engineering"), ("wshobson", "machine-learning-ops"),
|
|
("wshobson", "llm-application-dev"), ("wshobson", "data-validation-suite")],
|
|
"isco": ["2511", "2521"],
|
|
"title_kw": ["data scientist", "data engineer", "machine learning",
|
|
"artificial intelligence", "big data", "business intelligence",
|
|
"data analyst"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "dev-database",
|
|
"skills": [("wshobson", "database-design"), ("wshobson", "database-migrations"),
|
|
("wshobson", "database-cloud-optimization")],
|
|
"isco": ["2521", "2522"],
|
|
"title_kw": ["database"], "comp_kw": []},
|
|
|
|
{"name": "dev-ops-cloud",
|
|
"skills": [("wshobson", "kubernetes-operations"), ("wshobson", "cloud-infrastructure"),
|
|
("wshobson", "cicd-automation"), ("wshobson", "deployment-strategies"),
|
|
("wshobson", "deployment-validation"), ("wshobson", "observability-monitoring"),
|
|
("wshobson", "distributed-debugging"), ("wshobson", "application-performance"),
|
|
("wshobson", "incident-response")],
|
|
"isco": ["2522", "2523", "3511", "3512", "3513"],
|
|
"title_kw": ["devops", "system administrator", "cloud", "network administrator",
|
|
"site reliability", "ict operations"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "security",
|
|
"skills": [("wshobson", "security-compliance"), ("wshobson", "security-scanning"),
|
|
("wshobson", "backend-api-security"), ("wshobson", "frontend-mobile-security"),
|
|
("wshobson", "reverse-engineering"), ("wshobson", "incident-response"),
|
|
("wshobson", "signed-audit-trails")],
|
|
"isco": ["2529"],
|
|
"title_kw": ["it security", "ict security", "cyber", "information security",
|
|
"computer security", "penetration", "ethical hacker",
|
|
"digital forensics", "security analyst", "security engineer"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "dev-languages-special",
|
|
"skills": [("wshobson", "julia-development"), ("wshobson", "jvm-languages"),
|
|
("wshobson", "dotnet-contribution"), ("wshobson", "functional-programming"),
|
|
("wshobson", "systems-programming")],
|
|
"isco": ["2512", "2514"], "title_kw": [], "comp_kw": []},
|
|
|
|
{"name": "dev-embedded",
|
|
"skills": [("wshobson", "arm-cortex-microcontrollers"), ("wshobson", "systems-programming")],
|
|
"isco": [],
|
|
"title_kw": ["embedded", "firmware", "microcontroller", "robotics engineer",
|
|
"electronics engineer"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "testing-qa",
|
|
"skills": [("wshobson", "performance-testing-review"),
|
|
("wshobson", "api-testing-observability"),
|
|
("anthropic", "webapp-testing")],
|
|
"isco": ["2519"],
|
|
"title_kw": ["software tester", "quality assurance", "test engineer", "test analyst"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "software-architecture",
|
|
"skills": [("wshobson", "c4-architecture"), ("wshobson", "framework-migration"),
|
|
("wshobson", "full-stack-orchestration")],
|
|
"isco": [],
|
|
"title_kw": ["software architect", "system architect", "systems architect",
|
|
"solution architect", "enterprise architect", "ict system"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "game-development",
|
|
"skills": [("wshobson", "game-development")],
|
|
"isco": [],
|
|
"title_kw": ["game developer", "game designer", "game programmer", "video game",
|
|
"digital games"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "blockchain",
|
|
"skills": [("wshobson", "blockchain-web3")],
|
|
"isco": [],
|
|
"title_kw": ["blockchain", "web3", "crypto"],
|
|
"comp_kw": []},
|
|
|
|
# ── business domains ─────────────────────────────────────────────────────
|
|
{"name": "marketing-seo",
|
|
"skills": [("wshobson", "content-marketing"), ("wshobson", "seo-content-creation"),
|
|
("wshobson", "seo-analysis-monitoring"),
|
|
("wshobson", "seo-technical-optimization"),
|
|
("wshobson", "social-publishing"), ("wshobson", "brand-landingpage"),
|
|
("anthropic", "brand-guidelines")],
|
|
"isco": ["2431", "1221"],
|
|
"title_kw": ["marketing", "seo", "social media", "advertis", "brand",
|
|
"content manager", "digital media", "e-commerce", "growth hacker"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "sales-crm",
|
|
"skills": [("wshobson", "customer-sales-automation")],
|
|
"isco": ["2433", "2434", "3322"],
|
|
"title_kw": ["sales manager", "sales representative", "sales engineer",
|
|
"sales director", "technical sales", "account manager",
|
|
"business development", "customer relationship", "key account"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "business-analysis",
|
|
"skills": [("wshobson", "business-analytics"), ("wshobson", "startup-business-analyst")],
|
|
"isco": ["2421", "2413"],
|
|
"title_kw": ["business analyst", "management consultant", "strategy",
|
|
"business intelligence", "market research", "financial analyst"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "finance-trading",
|
|
"skills": [("wshobson", "quantitative-trading")],
|
|
"isco": ["3311"],
|
|
"title_kw": ["trader", "trading", "investment", "portfolio", "quantitative",
|
|
"fund manager", "asset manager", "broker", "hedge fund"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "human-resources",
|
|
"skills": [("wshobson", "hr-legal-compliance")],
|
|
"isco": ["2423", "4416", "1212"],
|
|
"title_kw": ["recruit", "human resources", "talent", "personnel officer",
|
|
"headhunter"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "legal",
|
|
"skills": [("wshobson", "hr-legal-compliance")],
|
|
"isco": ["261", "3411"],
|
|
"title_kw": ["lawyer", "legal", "attorney", "solicitor", "barrister",
|
|
"paralegal", "compliance officer", "notary"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "project-team-management",
|
|
"skills": [("wshobson", "team-collaboration")],
|
|
"isco": ["1"],
|
|
"title_kw": ["project manager", "product manager", "scrum", "agile coach",
|
|
"team leader"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "technical-writing",
|
|
"skills": [("wshobson", "documentation-generation"),
|
|
("wshobson", "documentation-standards")],
|
|
"isco": [],
|
|
"title_kw": ["technical writer", "technical communicator", "documentation"],
|
|
"comp_kw": []},
|
|
|
|
{"name": "office-file-handling",
|
|
"skills": [("wshobson", "file-conversion")],
|
|
"isco": ["4"],
|
|
"title_kw": [], "comp_kw": []},
|
|
]
|
|
|
|
# ─── Auto-Domains für die erweiterten Quellen ────────────────────────────────
|
|
# Jeder externe Skill wird per Keyword gegen (name + description + pfad)
|
|
# klassifiziert; die getroffenen Domains bestimmen via occ-Kriterien (gleiche
|
|
# Semantik wie RULES) die Ziel-Berufe. Ein Skill kann mehrere Domains treffen.
|
|
|
|
AUTO_DOMAINS = [
|
|
{"name": "ai-ml",
|
|
"skill_kw": ["machine learning", "deep learning", "llm", "cuda", "tensorrt",
|
|
"nemo", "inference", "gpu", "model training", "fine-tun",
|
|
"embedding", "rag", "vector", "nim ", "triton", "dataset",
|
|
"prompt", "agentic", "neural"],
|
|
"occ": {"isco": ["2511", "2512", "2521"],
|
|
"title_kw": ["artificial intelligence", "machine learning",
|
|
"data scientist", "data engineer", "big data"],
|
|
"comp_kw": []}},
|
|
{"name": "robotics-simulation",
|
|
"skill_kw": ["robot", "omniverse", "simulation", "isaac", "physical ai",
|
|
"simready", "usd ", "digital twin", "cad"],
|
|
"occ": {"isco": [], "title_kw": ["robotics", "automation engineer",
|
|
"simulation", "mechatronic"],
|
|
"comp_kw": []}},
|
|
{"name": "cloud-hpc-devops",
|
|
"skill_kw": ["kubernetes", "docker", "cluster", "slurm", "hpc",
|
|
"infrastructure", "deploy", "networking", "cloud", "devops",
|
|
"observability", "scaling"],
|
|
"occ": {"isco": ["2522", "2523", "3511", "3512", "3513"],
|
|
"title_kw": ["devops", "system administrator", "cloud",
|
|
"site reliability", "network"],
|
|
"comp_kw": []}},
|
|
{"name": "dev-general",
|
|
"skill_kw": ["api", "sdk", "code", "debug", "framework", "library",
|
|
"typescript", "python", "javascript", "git", "refactor",
|
|
"migration", "webhook", "integration"],
|
|
"occ": {"isco": ["251", "252"], "title_kw": [], "comp_kw": []}},
|
|
{"name": "qa-testing",
|
|
"skill_kw": ["test", "e2e", "selenium", "playwright", "cypress", "appium",
|
|
"quality assurance", "automation testing", "accessibility"],
|
|
"occ": {"isco": ["2519"],
|
|
"title_kw": ["software tester", "quality assurance",
|
|
"test engineer", "test analyst"],
|
|
"comp_kw": []}},
|
|
{"name": "frontend-web",
|
|
"skill_kw": ["angular", "react", "frontend", "component", "css",
|
|
"web app", "responsive", "ssr"],
|
|
"occ": {"isco": ["2513", "2514"],
|
|
"title_kw": ["web develop", "web design", "front-end", "frontend",
|
|
"user interface"],
|
|
"comp_kw": []}},
|
|
{"name": "marketing",
|
|
"skill_kw": ["marketing", "seo", "campaign", "brand", "growth", "funnel",
|
|
"churn", "aso", "landing page", "newsletter", "social media",
|
|
"content strategy", "positioning", "launch"],
|
|
"occ": {"isco": ["2431", "1221"],
|
|
"title_kw": ["marketing", "seo", "social media", "brand",
|
|
"content manager", "e-commerce", "growth"],
|
|
"comp_kw": []}},
|
|
{"name": "advertising",
|
|
"skill_kw": ["advertis", "ad creative", "ads api", "google ads",
|
|
"mobile ads", "ppc", "media buying", "adsense", "ad manager"],
|
|
"occ": {"isco": ["2431"],
|
|
"title_kw": ["advertis", "media planner", "media buyer",
|
|
"marketing"],
|
|
"comp_kw": []}},
|
|
{"name": "product-management",
|
|
"skill_kw": ["product", "prd", "roadmap", "okr", "backlog", "user stor",
|
|
"prioritization", "discovery", "job stories", "pre-mortem",
|
|
"a/b test", "ab-test", "user research", "persona"],
|
|
"occ": {"isco": [],
|
|
"title_kw": ["product manager", "product owner",
|
|
"product developer", "innovation manager"],
|
|
"comp_kw": []}},
|
|
{"name": "project-management",
|
|
"skill_kw": ["project plan", "sprint", "scrum", "stakeholder", "kickoff",
|
|
"retrospective", "milestone"],
|
|
"occ": {"isco": [],
|
|
"title_kw": ["project manager", "scrum", "agile coach",
|
|
"programme manager", "project coordinator"],
|
|
"comp_kw": []}},
|
|
{"name": "data-analytics",
|
|
"skill_kw": ["sql", "analytics", "dashboard", "cohort", "metric",
|
|
"data analysis", "reporting", "kpi", "visualization"],
|
|
"occ": {"isco": ["2413", "2421", "2511", "331"],
|
|
"title_kw": ["analyst", "business intelligence", "controller",
|
|
"statistician"],
|
|
"comp_kw": []}},
|
|
{"name": "design-ux",
|
|
"skill_kw": ["design", "ui", "ux", "figma", "prototype", "wireframe",
|
|
"image generation", "visual", "typography", "artwork"],
|
|
"occ": {"isco": ["2166", "2163", "265"],
|
|
"title_kw": ["designer", "graphic", "illustrator", "artist",
|
|
"art director", "visual", "animator"],
|
|
"comp_kw": []}},
|
|
{"name": "audio-video-media",
|
|
"skill_kw": ["audio", "video", "speech", "transcription", "music",
|
|
"podcast", "voice"],
|
|
"occ": {"isco": ["2654", "2655", "3521"],
|
|
"title_kw": ["video editor", "sound", "audio", "music producer",
|
|
"podcast", "broadcast"],
|
|
"comp_kw": []}},
|
|
{"name": "email-communication",
|
|
"skill_kw": ["email", "transactional", "deliverability", "smtp"],
|
|
"occ": {"isco": ["2431", "2432"],
|
|
"title_kw": ["marketing", "communication", "crm"],
|
|
"comp_kw": []}},
|
|
{"name": "sales-crm",
|
|
"skill_kw": ["sales", "crm", "lead gen", "outreach", "cold email",
|
|
"pipeline review"],
|
|
"occ": {"isco": ["2433", "2434", "3322"],
|
|
"title_kw": ["sales manager", "sales representative",
|
|
"account manager", "business development",
|
|
"key account"],
|
|
"comp_kw": []}},
|
|
{"name": "writing-docs",
|
|
"skill_kw": ["writing", "documentation", "article", "blog", "copywriting",
|
|
"technical writer", "docs"],
|
|
"occ": {"isco": ["264"],
|
|
"title_kw": ["writer", "editor", "journalist", "copywriter",
|
|
"technical communicator", "documentation"],
|
|
"comp_kw": []}},
|
|
{"name": "crypto-web3",
|
|
"skill_kw": ["crypto", "blockchain", "wallet", "web3", "smart contract"],
|
|
"occ": {"isco": [],
|
|
"title_kw": ["blockchain", "web3", "crypto"],
|
|
"comp_kw": []}},
|
|
{"name": "workflow-automation",
|
|
"skill_kw": ["n8n", "workflow", "automation", "zapier", "no-code",
|
|
"orchestration"],
|
|
"occ": {"isco": ["251", "252"],
|
|
"title_kw": ["automation", "integration", "process"],
|
|
"comp_kw": []}},
|
|
{"name": "knowledge-research",
|
|
"skill_kw": ["retriev", "knowledge base", "research", "search", "kb-"],
|
|
"occ": {"isco": ["2621", "2622", "243", "2421"],
|
|
"title_kw": ["research", "librarian", "archivist", "information",
|
|
"knowledge"],
|
|
"comp_kw": []}},
|
|
]
|
|
|
|
# ─── External skill catalog ──────────────────────────────────────────────────
|
|
|
|
_FM_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n", re.S)
|
|
|
|
|
|
def parse_frontmatter(path):
|
|
"""Return (name, description) from a SKILL.md frontmatter. Tolerant YAML-ish."""
|
|
try:
|
|
text = open(path, encoding="utf-8").read()
|
|
except OSError:
|
|
return None, None
|
|
m = _FM_RE.match(text)
|
|
if not m:
|
|
return None, None
|
|
fm = m.group(1)
|
|
name = desc = None
|
|
lines = fm.splitlines()
|
|
for i, line in enumerate(lines):
|
|
if line.startswith("name:") and name is None:
|
|
name = line.split(":", 1)[1].strip().strip("\"'")
|
|
if line.startswith("description:") and desc is None:
|
|
desc = line.split(":", 1)[1].strip()
|
|
if desc in (">", ">-", "|", "|-"):
|
|
desc = ""
|
|
# fold indented continuation lines
|
|
j = i + 1
|
|
while j < len(lines) and (lines[j].startswith(" ") or lines[j] == ""):
|
|
desc += " " + lines[j].strip()
|
|
j += 1
|
|
desc = desc.strip().strip("\"'")
|
|
return name, desc
|
|
|
|
|
|
def shorten(desc, limit=240):
|
|
if not desc:
|
|
return ""
|
|
desc = re.sub(r"\s+", " ", desc).replace("|", "\\|")
|
|
# cut at the first sentence past ~limit? keep simple: hard cut on word
|
|
if len(desc) <= limit:
|
|
return desc
|
|
cut = desc[:limit].rsplit(" ", 1)[0]
|
|
return cut + " …"
|
|
|
|
|
|
def load_catalog():
|
|
"""Return {(source_key, skill_key): {name, desc, url}}.
|
|
For wshobson, skill_key is the PLUGIN name and the value holds a LIST of
|
|
its skills; anthropic/superpowers map 1:1."""
|
|
cat = {}
|
|
for key in ("anthropic", "superpowers"):
|
|
src = SOURCES[key]
|
|
root = src["skill_root"]
|
|
if not os.path.isdir(root):
|
|
continue
|
|
for d in sorted(os.listdir(root)):
|
|
p = os.path.join(root, d, "SKILL.md")
|
|
if not os.path.isfile(p):
|
|
continue
|
|
name, desc = parse_frontmatter(p)
|
|
cat[(key, d)] = [{
|
|
"name": name or d,
|
|
"desc": shorten(desc),
|
|
"url": f"{src['link_base']}/{d}",
|
|
}]
|
|
# wshobson: plugin -> [skills]; plugins without skills/ ship agents/*.md
|
|
src = SOURCES["wshobson"]
|
|
root = src["skill_root"]
|
|
if os.path.isdir(root):
|
|
for plugin in sorted(os.listdir(root)):
|
|
entries = []
|
|
sdir = os.path.join(root, plugin, "skills")
|
|
if os.path.isdir(sdir):
|
|
for d in sorted(os.listdir(sdir)):
|
|
p = os.path.join(sdir, d, "SKILL.md")
|
|
if not os.path.isfile(p):
|
|
continue
|
|
name, desc = parse_frontmatter(p)
|
|
entries.append({
|
|
"name": name or d,
|
|
"desc": shorten(desc),
|
|
"url": f"{src['link_base']}/{plugin}/skills/{d}",
|
|
})
|
|
else:
|
|
adir = os.path.join(root, plugin, "agents")
|
|
if os.path.isdir(adir):
|
|
for f in sorted(os.listdir(adir)):
|
|
if not f.endswith(".md"):
|
|
continue
|
|
name, desc = parse_frontmatter(os.path.join(adir, f))
|
|
entries.append({
|
|
"name": (name or f[:-3]) + " (agent)",
|
|
"desc": shorten(desc),
|
|
"url": f"{src['link_base']}/{plugin}/agents/{f}",
|
|
})
|
|
if entries:
|
|
cat[("wshobson", plugin)] = entries
|
|
|
|
# Erweiterte Quellen: rekursiver SKILL.md-Scan, Key = relativer Ordnerpfad
|
|
for key, src in AUTO_SOURCES.items():
|
|
root = src["skill_root"]
|
|
if not os.path.isdir(root):
|
|
continue
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
dirnames[:] = [d for d in dirnames
|
|
if d not in (".git", "node_modules", "dist", "website")]
|
|
if "SKILL.md" not in filenames:
|
|
continue
|
|
rel = os.path.relpath(dirpath, root).replace("\\", "/")
|
|
name, desc = parse_frontmatter(os.path.join(dirpath, "SKILL.md"))
|
|
cat[(key, rel)] = [{
|
|
"name": name or os.path.basename(dirpath),
|
|
"desc": shorten(desc),
|
|
"url": f"{src['link_base']}/{rel}" if rel != "." else src["link_base"],
|
|
}]
|
|
return cat
|
|
|
|
|
|
# ─── Occupation loading & matching ───────────────────────────────────────────
|
|
|
|
def load_occupation(slug):
|
|
base = os.path.join(SKILLS_DIR, slug)
|
|
mpath = os.path.join(base, "manifest.json")
|
|
spath = os.path.join(base, "SKILL.md")
|
|
if not (os.path.isfile(mpath) and os.path.isfile(spath)):
|
|
return None
|
|
m = json.load(open(mpath, encoding="utf-8"))
|
|
text = open(spath, encoding="utf-8").read()
|
|
comp = ""
|
|
cm = re.search(r"(?ms)^## Key competences.*?$(.*?)(?=^## |\Z)", text)
|
|
if cm:
|
|
comp = cm.group(1)
|
|
return {
|
|
"slug": slug,
|
|
"title": (m.get("title") or slug).lower(),
|
|
"isco": str(m.get("ids", {}).get("isco_group") or ""),
|
|
"comp": comp.lower(),
|
|
"manifest": m,
|
|
}
|
|
|
|
|
|
def rule_matches(rule, occ):
|
|
if any(occ["isco"].startswith(p) for p in rule["isco"]):
|
|
return True
|
|
if any(kw in occ["title"] for kw in rule["title_kw"]):
|
|
return True
|
|
if any(kw in occ["comp"] for kw in rule["comp_kw"]):
|
|
return True
|
|
return False
|
|
|
|
|
|
# ─── Writers ─────────────────────────────────────────────────────────────────
|
|
|
|
SKILL_LINK_LINE = ("- See [references/ai-skills.md](references/ai-skills.md) — "
|
|
"matched external AI agent skills (per-source attribution).")
|
|
|
|
|
|
def write_ai_skills_md(base, slug, matched):
|
|
"""matched: {source_key: [ {name,desc,url}, ... ]} (deduped, sorted)."""
|
|
lines = [
|
|
f"# External AI agent skills — {slug}",
|
|
"",
|
|
"Proven, publicly available AI agent skills mapped to this occupation.",
|
|
"Nothing is copied from the sources: every entry is a name, a one-line",
|
|
"summary and a link to the upstream skill package. Each section names",
|
|
"its source repository, commit, license and retrieval date.",
|
|
"",
|
|
"**Tiers:** `core` = the skill directly exercises a top-20 market hard",
|
|
"skill or an essential ESCO competence of this occupation; `adjacent` =",
|
|
"plausibly useful, secondary. Entries are capped at "
|
|
f"{MAX_PER_SOURCE} per source",
|
|
"(core first); everything beyond the cap is excluded and logged in the",
|
|
"pipeline audit trail, not in this package.",
|
|
"",
|
|
"_Matched deterministically (ISCO group + title/competence keywords,",
|
|
"tiered against market evidence + ESCO essentials) by",
|
|
f"`pipeline/p5_enrich_ai_skills.py` on {RETRIEVED}._",
|
|
"",
|
|
]
|
|
order = ["anthropic", "superpowers", "wshobson"] + sorted(AUTO_SOURCES)
|
|
for key in order:
|
|
entries = matched.get(key)
|
|
if not entries:
|
|
continue
|
|
src = ALL_SOURCES[key]
|
|
lines += [
|
|
f"## Source: {src['label']}",
|
|
"",
|
|
f"- Repository: [{src['repo_url']}]({src['repo_url']}) "
|
|
f"(commit `{src['commit']}`, retrieved {RETRIEVED})",
|
|
f"- License: {src['license']}",
|
|
"",
|
|
"| Skill | Tier | What it adds | Upstream |",
|
|
"|---|---|---|---|",
|
|
]
|
|
for e in entries:
|
|
lines.append(f"| `{e['name']}` | {e.get('tier', 'adjacent')} "
|
|
f"| {e['desc'] or '—'} | [source]({e['url']}) |")
|
|
lines.append("")
|
|
with open(os.path.join(base, "references", "ai-skills.md"), "w",
|
|
encoding="utf-8", newline="\n") as f:
|
|
f.write("\n".join(lines))
|
|
|
|
|
|
def patch_skill_md(base):
|
|
"""Insert the ai-skills link line into the 'How to use this skill' list."""
|
|
p = os.path.join(base, "SKILL.md")
|
|
text = open(p, encoding="utf-8").read()
|
|
if "references/ai-skills.md" in text:
|
|
return
|
|
m = re.search(r"(?ms)^## How to use this skill\s*\n(.*?)(?=^## |\Z)", text)
|
|
if not m:
|
|
return
|
|
block = m.group(1)
|
|
# insert after the last list item of the block
|
|
items = list(re.finditer(r"(?m)^- .*$", block))
|
|
if not items:
|
|
return
|
|
insert_at = m.start(1) + items[-1].end()
|
|
text = text[:insert_at] + "\n" + SKILL_LINK_LINE + text[insert_at:]
|
|
open(p, "w", encoding="utf-8", newline="\n").write(text)
|
|
|
|
|
|
def patch_manifest(base, occ, matched):
|
|
m = occ["manifest"]
|
|
m["enrichment_ai_skills"] = {
|
|
"generated": RETRIEVED,
|
|
"method": "deterministic mapping (ISCO prefix + title/competence keywords)",
|
|
"sources": {
|
|
ALL_SOURCES[k]["label"]: {
|
|
"repo": ALL_SOURCES[k]["repo_url"],
|
|
"commit": ALL_SOURCES[k]["commit"],
|
|
"license": ALL_SOURCES[k]["license"],
|
|
"skills": len(v),
|
|
} for k, v in matched.items() if v
|
|
},
|
|
"total_skills": sum(len(v) for v in matched.values()),
|
|
"tiers": {
|
|
"core": sum(1 for v in matched.values() for e in v
|
|
if e.get("tier") == "core"),
|
|
"adjacent": sum(1 for v in matched.values() for e in v
|
|
if e.get("tier") != "core"),
|
|
},
|
|
}
|
|
json.dump(m, open(os.path.join(base, "manifest.json"), "w", encoding="utf-8"),
|
|
indent=2)
|
|
|
|
|
|
# ─── Main ────────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
dry = "--dry-run" in sys.argv
|
|
args = [a for a in sys.argv[1:] if not a.startswith("--")]
|
|
only = args[0] if args else None
|
|
|
|
catalog = load_catalog()
|
|
print(f"external catalog: {sum(len(v) for v in catalog.values())} skills "
|
|
f"in {len(catalog)} units from {len(ALL_SOURCES)} sources")
|
|
|
|
# Domain-Index: erweiterte Quellen einmalig klassifizieren
|
|
domain_index = {}
|
|
unclassified = 0
|
|
for (src_key, skill_key), entries in catalog.items():
|
|
if src_key not in AUTO_SOURCES:
|
|
continue
|
|
text = " ".join(e["name"] + " " + e["desc"] for e in entries).lower() \
|
|
+ " " + skill_key.lower()
|
|
allowed = SOURCE_DOMAINS.get(src_key)
|
|
hit = False
|
|
for dom in AUTO_DOMAINS:
|
|
if allowed is not None and dom["name"] not in allowed:
|
|
continue
|
|
if any(kw in text for kw in dom["skill_kw"]):
|
|
domain_index.setdefault(dom["name"], []).append((src_key, skill_key))
|
|
hit = True
|
|
if not hit:
|
|
unclassified += 1
|
|
print(f"auto-domain index: "
|
|
f"{sum(len(v) for v in domain_index.values())} zuordnungen, "
|
|
f"{unclassified} skills ohne domain (nicht gemappt)")
|
|
missing = set()
|
|
for rule in RULES:
|
|
for ref in rule["skills"]:
|
|
if ref not in catalog:
|
|
missing.add(ref)
|
|
if missing:
|
|
print(f"WARN: {len(missing)} rule references not found in catalog: "
|
|
f"{sorted(missing)[:6]} ...")
|
|
|
|
slugs = sorted(d for d in os.listdir(SKILLS_DIR)
|
|
if os.path.isdir(os.path.join(SKILLS_DIR, d)))
|
|
if only:
|
|
slugs = [s for s in slugs if s == only]
|
|
|
|
enriched = 0
|
|
total_links = 0
|
|
for slug in slugs:
|
|
occ = load_occupation(slug)
|
|
if not occ:
|
|
continue
|
|
# collect matches, dedupe by (source, skill name)
|
|
matched = {}
|
|
seen = set()
|
|
for rule in RULES:
|
|
if not rule_matches(rule, occ):
|
|
continue
|
|
for ref in rule["skills"]:
|
|
for entry in catalog.get(ref, []):
|
|
k = (ref[0], entry["name"])
|
|
if k in seen:
|
|
continue
|
|
seen.add(k)
|
|
matched.setdefault(ref[0], []).append(entry)
|
|
# Auto-Domains: erweiterte Quellen keyword-basiert zuordnen
|
|
for dom in AUTO_DOMAINS:
|
|
if not rule_matches(dom["occ"], occ):
|
|
continue
|
|
for (src_key, skill_key) in domain_index.get(dom["name"], ()):
|
|
for entry in catalog[(src_key, skill_key)]:
|
|
k = (src_key, entry["name"])
|
|
if k in seen:
|
|
continue
|
|
seen.add(k)
|
|
matched.setdefault(src_key, []).append(entry)
|
|
if not matched:
|
|
continue
|
|
# ── Tiering + Kappung (Phase 2c) ────────────────────────────────
|
|
core_signals = load_core_signals(slug)
|
|
audit_rows = []
|
|
for src_key in list(matched):
|
|
entries = matched[src_key]
|
|
for e in entries:
|
|
e["tier"], e["score"] = tier_for(e, core_signals)
|
|
entries.sort(key=lambda e: (0 if e["tier"] == "core" else 1,
|
|
-e["score"], e["name"]))
|
|
if len(entries) > MAX_PER_SOURCE:
|
|
for e in entries[MAX_PER_SOURCE:]:
|
|
audit_rows.append({"slug": slug, "source": src_key,
|
|
"skill": e["name"], "tier": e["tier"],
|
|
"reason": "per-source cap"})
|
|
matched[src_key] = entries[:MAX_PER_SOURCE]
|
|
if not dry and audit_rows:
|
|
os.makedirs(os.path.dirname(AUDIT_LOG), exist_ok=True)
|
|
with open(AUDIT_LOG, "a", encoding="utf-8") as af:
|
|
for row in audit_rows:
|
|
af.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
if not dry:
|
|
base = os.path.join(SKILLS_DIR, slug)
|
|
write_ai_skills_md(base, slug, matched)
|
|
patch_skill_md(base)
|
|
patch_manifest(base, occ, matched)
|
|
enriched += 1
|
|
total_links += sum(len(v) for v in matched.values())
|
|
|
|
print(f"enriched {enriched}/{len(slugs)} packages "
|
|
f"with {total_links} skill links total{' (dry run)' if dry else ''}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|