Hole Tickets und Kommentare aus Gitea in die km-Ablage
Erster Baustein von spine/core#13: image/km/ingest_gitea.py liest Issues, Pull Requests und Kommentare je Repo oder Organisation und schreibt je Vorgang eine Markdown-Datei mit Frontmatter (Zustand, Labels, Milestone, Beteiligte, Zeitstempel, Verweise auf andere Vorgänge als Kanten) unter <ablage>/<namespace>/vorgaenge/<owner>/<repo>/<nummer>.md. Inkrementell über since je Repo, idempotent, Stand erst nach vollständigem Durchlauf. Zugang aus OpenBao wie beim Gitea-Helfer, nie in der Ausgabe. Schreibt nichts nach Gitea zurück. Gemessen: 1165 Vorgänge in drei Minuten. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
+4
-1
@@ -3,7 +3,10 @@
|
||||
Hier entsteht `git.42i.org/spine/km`: die API, die Ablage, der MCP-Endpunkt und
|
||||
die prüfende Schreibfunktion.
|
||||
|
||||
**Noch leer.** Was hineingehört, steht in [../docs/konzept.md](../docs/konzept.md).
|
||||
**Erster Baustein vorhanden:** `km/ingest_gitea.py` holt Tickets und Kommentare
|
||||
aus Gitea in die Ablage (spine/core#13, Doku in
|
||||
[../docs/ingest-gitea.md](../docs/ingest-gitea.md)). Der Rest steht in
|
||||
[../docs/konzept.md](../docs/konzept.md).
|
||||
Die Reihenfolge, in der es sinnvoll wächst:
|
||||
|
||||
1. `read` und `search` über eine Ablage aus Markdown-Dateien — ohne die beiden
|
||||
|
||||
Binary file not shown.
Executable
+288
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gitea-Ingest fuer km (spine/core#13).
|
||||
|
||||
Liest Issues, Pull Requests und ihre Kommentare aus einer Gitea-Instanz und
|
||||
legt je Vorgang eine Markdown-Datei in der km-Ablage ab:
|
||||
|
||||
<ablage>/<namespace>/vorgaenge/<owner>/<repo>/<number>.md
|
||||
|
||||
Die Datei traegt Frontmatter (Titel, Zustand, Labels, Milestone, Zuweisung,
|
||||
Zeitstempel, URL, erkannte Verweise auf andere Vorgaenge) und darunter den
|
||||
Body und alle Kommentare in Reihenfolge. Sie wird bei jeder Aenderung am
|
||||
Vorgang komplett neu geschrieben -- ein umgeschriebener Body ist eine
|
||||
Aenderung des Stands, kein Anhang.
|
||||
|
||||
Inkrementell ueber `since` je Repo (Gitea filtert nach `updated`); der Stand
|
||||
liegt in einer JSON-Datei ausserhalb der Ablage. Ein Lauf ohne Stand ist die
|
||||
Erstbefuellung. Jeder Lauf ist idempotent: dieselbe Eingabe ergibt dieselbe
|
||||
Datei, also kann man ihn beliebig wiederholen -- das ist der Abgleich aus
|
||||
spine/core#17, nicht nur die Erstbefuellung.
|
||||
|
||||
Zugangsdaten kommen aus OpenBao ueber `secret get <persona> <service>`
|
||||
(HTTP Basic, wie ~/.agents/mcp/gitea/bin/gitea) und werden nie ausgegeben.
|
||||
Der Ingest schreibt nichts nach Gitea zurueck.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
import requests
|
||||
|
||||
# owner/repo#123 -- oder nacktes #123, das dann im eigenen Repo liegt
|
||||
_REF_QUALIFIED = re.compile(r"(?<![\w/])([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)#(\d+)\b")
|
||||
_REF_BARE = re.compile(r"(?<![\w/#&])#(\d+)\b")
|
||||
# "42i/intern#193 / #198": nackte Nummern in einer Aufzaehlung direkt hinter
|
||||
# einem qualifizierten Verweis gehoeren zu dessen Repo, nicht zum eigenen.
|
||||
_REF_CHAIN = re.compile(r"(?<![\w/])([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)#\d+((?:\s*[/,]\s*#\d+)+)")
|
||||
_TLDR = re.compile(r"<!-- tldr:start.*?<!-- tldr:end -->\s*", re.DOTALL)
|
||||
_MENTION_URL = re.compile(r"https?://[^\s)]+/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)/(?:issues|pulls)/(\d+)")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Forge:
|
||||
url: str
|
||||
namespace: str
|
||||
persona: str = "elton"
|
||||
service: str = "git"
|
||||
classification: str = "lan"
|
||||
session: requests.Session = field(default_factory=requests.Session)
|
||||
|
||||
@property
|
||||
def host(self) -> str:
|
||||
return re.sub(r"^https?://", "", self.url).split("/")[0]
|
||||
|
||||
def login(self, secret_bin: str) -> None:
|
||||
def get(fld: str) -> str:
|
||||
r = subprocess.run(
|
||||
[secret_bin, "get", self.persona, self.service, fld],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
user = get("username") or self.persona
|
||||
pw = get("password")
|
||||
if not pw:
|
||||
sys.exit(f"ingest: kein Gitea-Zugang fuer '{self.persona}' in OpenBao "
|
||||
f"(agents/{self.persona}/{self.service})")
|
||||
self.session.auth = (user, pw)
|
||||
self.session.headers["Accept"] = "application/json"
|
||||
|
||||
def get(self, path: str, **params) -> object:
|
||||
for attempt in range(4):
|
||||
r = self.session.get(f"{self.url}/api/v1{path}", params=params, timeout=60)
|
||||
if r.status_code in (429, 502, 503, 504) and attempt < 3:
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
def paged(self, path: str, **params) -> Iterator[dict]:
|
||||
page = 1
|
||||
while True:
|
||||
batch = self.get(path, page=page, limit=50, **params)
|
||||
if not batch:
|
||||
return
|
||||
yield from batch
|
||||
if len(batch) < 50:
|
||||
return
|
||||
page += 1
|
||||
|
||||
def org_repos(self, org: str) -> list[str]:
|
||||
return [f"{org}/{r['name']}" for r in self.paged(f"/orgs/{org}/repos")]
|
||||
|
||||
def issues(self, repo: str, kind: str, since: str | None) -> Iterator[dict]:
|
||||
params = {"state": "all", "type": kind, "sort": "recentupdate"}
|
||||
if since:
|
||||
params["since"] = since
|
||||
try:
|
||||
yield from self.paged(f"/repos/{repo}/issues", **params)
|
||||
except requests.HTTPError as e:
|
||||
# Repo ohne Pull Requests (Einheit abgeschaltet) antwortet mit 404
|
||||
# auf type=pulls -- das ist "keine", kein Fehler.
|
||||
if kind == "pulls" and e.response is not None and e.response.status_code == 404:
|
||||
return
|
||||
raise
|
||||
|
||||
def comments(self, repo: str, number: int) -> list[dict]:
|
||||
return list(self.paged(f"/repos/{repo}/issues/{number}/comments"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Markdown
|
||||
|
||||
def _iso(s: str | None) -> str | None:
|
||||
if not s or s.startswith("0001-"):
|
||||
return None
|
||||
return datetime.fromisoformat(s.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _yaml_str(s: str) -> str:
|
||||
return json.dumps(s, ensure_ascii=False)
|
||||
|
||||
|
||||
def _refs(text: str, own_repo: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for o, r, n in _REF_QUALIFIED.findall(text):
|
||||
out.add(f"{o}/{r}#{n}")
|
||||
chained: set[str] = set()
|
||||
for repo, tail in _REF_CHAIN.findall(text):
|
||||
for n in re.findall(r"#(\d+)", tail):
|
||||
out.add(f"{repo}#{n}")
|
||||
chained.add(n)
|
||||
for n in _REF_BARE.findall(text):
|
||||
if n not in chained:
|
||||
out.add(f"{own_repo}#{n}")
|
||||
for o, r, n in _MENTION_URL.findall(text):
|
||||
out.add(f"{o}/{r}#{n}")
|
||||
return out
|
||||
|
||||
|
||||
def render(forge: Forge, repo: str, issue: dict, comments: list[dict]) -> str:
|
||||
kind = "pull" if issue.get("pull_request") else "issue"
|
||||
number = issue["number"]
|
||||
own = f"{repo}#{number}"
|
||||
body = (issue.get("body") or "").replace("\r\n", "\n").strip()
|
||||
stand = None
|
||||
m = _TLDR.search(body)
|
||||
if m:
|
||||
stand = m.group(0).strip()
|
||||
body = _TLDR.sub("", body).strip()
|
||||
|
||||
refs: set[str] = _refs(body, repo)
|
||||
if stand:
|
||||
refs |= _refs(stand, repo)
|
||||
for c in comments:
|
||||
refs |= _refs(c.get("body") or "", repo)
|
||||
refs.discard(own)
|
||||
|
||||
participants = {issue["user"]["login"]}
|
||||
participants |= {c["user"]["login"] for c in comments if c.get("user")}
|
||||
|
||||
fm = [
|
||||
"---",
|
||||
f"quelle: gitea",
|
||||
f"forge: {forge.host}",
|
||||
f"namespace: {forge.namespace}",
|
||||
f"classification: {forge.classification}",
|
||||
f"repo: {repo}",
|
||||
f"nummer: {number}",
|
||||
f"art: {kind}",
|
||||
f"titel: {_yaml_str(issue['title'])}",
|
||||
f"zustand: {issue['state']}",
|
||||
f"autor: {issue['user']['login']}",
|
||||
f"beteiligte: [{', '.join(sorted(participants))}]",
|
||||
f"labels: [{', '.join(_yaml_str(l['name']) for l in issue.get('labels') or [])}]",
|
||||
f"milestone: {_yaml_str(issue['milestone']['title']) if issue.get('milestone') else 'null'}",
|
||||
f"zugewiesen: [{', '.join(a['login'] for a in issue.get('assignees') or [])}]",
|
||||
f"erstellt: {_iso(issue['created_at'])}",
|
||||
f"aktualisiert: {_iso(issue['updated_at'])}",
|
||||
f"geschlossen: {_iso(issue.get('closed_at')) or 'null'}",
|
||||
f"kommentare: {len(comments)}",
|
||||
f"url: {issue['html_url']}",
|
||||
f"verweise: [{', '.join(_yaml_str(r) for r in sorted(refs))}]",
|
||||
"---",
|
||||
]
|
||||
out = [*fm, "", f"# {own}: {issue['title']}", ""]
|
||||
if stand:
|
||||
out += [stand, ""]
|
||||
out += [body or "_(kein Text)_", ""]
|
||||
if comments:
|
||||
out += ["## Kommentare", ""]
|
||||
for c in comments:
|
||||
when = _iso(c["created_at"])
|
||||
who = c["user"]["login"] if c.get("user") else "?"
|
||||
edited = " (bearbeitet)" if _iso(c.get("updated_at")) != when else ""
|
||||
out += [f"### {who} · {when}{edited}", "",
|
||||
(c.get("body") or "").replace("\r\n", "\n").strip() or "_(leer)_", ""]
|
||||
return "\n".join(out).rstrip() + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Lauf
|
||||
|
||||
def load_state(path: Path) -> dict:
|
||||
if path.exists():
|
||||
return json.loads(path.read_text())
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(path: Path, state: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def ingest_repo(forge: Forge, repo: str, ablage: Path, state: dict, full: bool) -> tuple[int, int]:
|
||||
key = f"{forge.host}/{repo}"
|
||||
since = None if full else state.get(key, {}).get("since")
|
||||
started = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
target = ablage / forge.namespace / "vorgaenge" / repo
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
seen = written = 0
|
||||
for kind in ("issues", "pulls"):
|
||||
for issue in forge.issues(repo, kind, since):
|
||||
seen += 1
|
||||
comments = forge.comments(repo, issue["number"]) if issue.get("comments") else []
|
||||
text = render(forge, repo, issue, comments)
|
||||
f = target / f"{issue['number']}.md"
|
||||
if not f.exists() or f.read_text(encoding="utf-8") != text:
|
||||
f.write_text(text, encoding="utf-8")
|
||||
written += 1
|
||||
# Stand erst nach vollstaendigem Durchlauf setzen -- bricht der Lauf ab,
|
||||
# holt der naechste dieselben Vorgaenge noch einmal (Wiederholen ist billig,
|
||||
# Luecken sind es nicht).
|
||||
state[key] = {"since": started, "letzter_lauf": started, "gesehen": seen, "geschrieben": written}
|
||||
return seen, written
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--url", default=os.environ.get("KM_GITEA_URL", "https://git.42i.org"))
|
||||
ap.add_argument("--namespace", default=os.environ.get("KM_NAMESPACE", "42i"))
|
||||
ap.add_argument("--classification", default=os.environ.get("KM_CLASSIFICATION", "lan"))
|
||||
ap.add_argument("--persona", default=os.environ.get("AGENT_PERSONA", "elton"))
|
||||
ap.add_argument("--secret-bin", default=os.environ.get("SECRET_BIN", str(Path.home() / ".agents/bin/secret")))
|
||||
ap.add_argument("--ablage", required=True, type=Path, help="Wurzel der km-Ablage (Git-Checkout)")
|
||||
ap.add_argument("--state", type=Path, help="Standdatei (Default: <ablage>/../km-ingest-state.json)")
|
||||
ap.add_argument("--repo", action="append", default=[], help="owner/repo, mehrfach")
|
||||
ap.add_argument("--org", action="append", default=[], help="alle Repos einer Organisation")
|
||||
ap.add_argument("--full", action="store_true", help="Stand ignorieren, alles neu lesen")
|
||||
a = ap.parse_args()
|
||||
|
||||
forge = Forge(url=a.url.rstrip("/"), namespace=a.namespace, persona=a.persona,
|
||||
classification=a.classification)
|
||||
forge.login(a.secret_bin)
|
||||
|
||||
repos = list(a.repo)
|
||||
for org in a.org:
|
||||
repos += forge.org_repos(org)
|
||||
if not repos:
|
||||
sys.exit("ingest: --repo oder --org angeben")
|
||||
|
||||
state_path = a.state or (a.ablage.parent / "km-ingest-state.json")
|
||||
state = load_state(state_path)
|
||||
t0 = time.time()
|
||||
total_seen = total_written = 0
|
||||
for repo in dict.fromkeys(repos):
|
||||
seen, written = ingest_repo(forge, repo, a.ablage, state, a.full)
|
||||
total_seen += seen
|
||||
total_written += written
|
||||
print(f"{forge.host}/{repo}: {seen} gesehen, {written} geschrieben", file=sys.stderr)
|
||||
save_state(state_path, state)
|
||||
print(f"fertig: {len(repos)} repos, {total_seen} vorgaenge gesehen, "
|
||||
f"{total_written} dateien geschrieben, {time.time() - t0:.1f}s", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
requests>=2.31
|
||||
Reference in New Issue
Block a user