diff --git a/Dockerfile.pqrst b/Dockerfile.pqrst new file mode 100644 index 0000000..f7cfc3a --- /dev/null +++ b/Dockerfile.pqrst @@ -0,0 +1,5 @@ +# Preserve the grant-aware, frontend-reporting deployed worker byte-for-byte. +FROM forgejo.coulomb.social/coulomb/activity-core@sha256:afd8a7af73132becba432e7b315d70f88d66676020169d5d8811641304136437 +COPY src/activity_core/context_resolvers/pqrst.py /app/src/activity_core/context_resolvers/pqrst.py +RUN printf '\nfrom activity_core.context_resolvers import pqrst # noqa: F401\n' >> /app/src/activity_core/context_resolvers/__init__.py +RUN python -c "import activity_core.context_resolvers; from activity_core.context_resolvers.base import CONTEXT_RESOLVER_REGISTRY as r; from activity_core.repository_grant import RepositoryGrant; assert 'pqrst' in r and 'frontend-patterns' in r; assert RepositoryGrant.model_validate({'version':'1','allowed_paths':['docs/'],'commit_count':{'min':1,'max':1},'publish':False}).payload()['publish'] is False" diff --git a/k8s/gitops/runtime.yaml b/k8s/gitops/runtime.yaml index 9f2c916..f9dad03 100644 --- a/k8s/gitops/runtime.yaml +++ b/k8s/gitops/runtime.yaml @@ -354,6 +354,61 @@ data: priority: medium labels: ["freedom-intelligence", "research-brief", "automated"] ``` + monthly-pqrst-review.md: | + --- + id: monthly-pqrst-review + name: Monthly PQRST Data Review + enabled: false + owner: pqrst-practice + governance: custodian + status: active + trigger: + type: cron + cron_expression: "30 9 1 * *" + timezone: Europe/Berlin + misfire_policy: catchup_latest + catchup_window_seconds: 86400 + context_sources: + - type: pqrst + query: monthly + params: {required: true} + bind_to: context.daily_triage_digest + --- + + # Monthly PQRST Data Review + + At 09:30 Europe/Berlin on the first of each month, review the six completed + calendar months ending with the previous month. Read hall-of-helix's committed + main branch, pinned to one revision per run. Report one datapoint per structurally + valid agent entry, including drafts; select the unique block matching frontmatter + and retain attempt counts. Missing, invalid and ambiguous records do not enter + means. All five dimensions have equal-entry means and month-over-month changes + in percentage points; empty months and comparisons across a gap use null. + + The report preserves source revision, counts, confidence frequencies and data + limitations. No worker/model rankings, target allocations, empirical accuracy or + causal claims. It creates reports, not tasks, and uses no LLM. See + `docs/monthly-pqrst-automation.md` for operation and rollback. + + ```instruction + id: monthly-pqrst-review-report + trusted_fields: [] + model: deterministic + temperature: 0 + max_tokens: 1 + prompt: Deterministic monthly PQRST counts, averages and trends from published hall entries. + output_schema: "" + review_advisory: false + report_sinks: + - type: working-memory + path: /var/custodian/memory/working + filename_template: "pqrst-monthly-{date}-{run_id_short}.md" + timezone: Europe/Berlin + - type: state-hub-progress + event_type: pqrst_monthly_review + author: activity-core + topic_id: c1d199b6-55ee-4db6-b49e-257a9f0f15ac + ``` frontend-patterns-daily.md: | --- id: frontend-patterns-daily @@ -1653,7 +1708,7 @@ spec: name: actcore-runtime-config - secretRef: name: actcore-runtime-secret - image: forgejo.coulomb.social/coulomb/activity-core@sha256:afd8a7af73132becba432e7b315d70f88d66676020169d5d8811641304136437 + image: forgejo.coulomb.social/coulomb/activity-core@sha256:75e240ac23c1c6d7daae14691c2b4137d2d639a9af694aaf0d4c774a7e4d3886 imagePullPolicy: IfNotPresent name: worker ports: diff --git a/k8s/railiance/20-runtime.yaml b/k8s/railiance/20-runtime.yaml index 97d433c..6031068 100644 --- a/k8s/railiance/20-runtime.yaml +++ b/k8s/railiance/20-runtime.yaml @@ -353,6 +353,61 @@ data: priority: medium labels: ["freedom-intelligence", "research-brief", "automated"] ``` + monthly-pqrst-review.md: | + --- + id: monthly-pqrst-review + name: Monthly PQRST Data Review + enabled: false + owner: pqrst-practice + governance: custodian + status: active + trigger: + type: cron + cron_expression: "30 9 1 * *" + timezone: Europe/Berlin + misfire_policy: catchup_latest + catchup_window_seconds: 86400 + context_sources: + - type: pqrst + query: monthly + params: {required: true} + bind_to: context.daily_triage_digest + --- + + # Monthly PQRST Data Review + + At 09:30 Europe/Berlin on the first of each month, review the six completed + calendar months ending with the previous month. Read hall-of-helix's committed + main branch, pinned to one revision per run. Report one datapoint per structurally + valid agent entry, including drafts; select the unique block matching frontmatter + and retain attempt counts. Missing, invalid and ambiguous records do not enter + means. All five dimensions have equal-entry means and month-over-month changes + in percentage points; empty months and comparisons across a gap use null. + + The report preserves source revision, counts, confidence frequencies and data + limitations. No worker/model rankings, target allocations, empirical accuracy or + causal claims. It creates reports, not tasks, and uses no LLM. See + `docs/monthly-pqrst-automation.md` for operation and rollback. + + ```instruction + id: monthly-pqrst-review-report + trusted_fields: [] + model: deterministic + temperature: 0 + max_tokens: 1 + prompt: Deterministic monthly PQRST counts, averages and trends from published hall entries. + output_schema: "" + review_advisory: false + report_sinks: + - type: working-memory + path: /var/custodian/memory/working + filename_template: "pqrst-monthly-{date}-{run_id_short}.md" + timezone: Europe/Berlin + - type: state-hub-progress + event_type: pqrst_monthly_review + author: activity-core + topic_id: c1d199b6-55ee-4db6-b49e-257a9f0f15ac + ``` frontend-patterns-daily.md: | --- id: frontend-patterns-daily @@ -2200,7 +2255,7 @@ spec: name: actcore-runtime-config - secretRef: name: actcore-runtime-secret - image: forgejo.coulomb.social/coulomb/activity-core@sha256:afd8a7af73132becba432e7b315d70f88d66676020169d5d8811641304136437 + image: forgejo.coulomb.social/coulomb/activity-core@sha256:75e240ac23c1c6d7daae14691c2b4137d2d639a9af694aaf0d4c774a7e4d3886 imagePullPolicy: IfNotPresent name: worker ports: diff --git a/src/activity_core/context_resolvers/__init__.py b/src/activity_core/context_resolvers/__init__.py index 35fb7bf..a5b4c01 100644 --- a/src/activity_core/context_resolvers/__init__.py +++ b/src/activity_core/context_resolvers/__init__.py @@ -9,3 +9,5 @@ from activity_core.context_resolvers import ( # noqa: F401 state_hub, reuse_surface, ) + +from activity_core.context_resolvers import pqrst # noqa: F401 diff --git a/src/activity_core/context_resolvers/pqrst.py b/src/activity_core/context_resolvers/pqrst.py new file mode 100644 index 0000000..90a00e6 --- /dev/null +++ b/src/activity_core/context_resolvers/pqrst.py @@ -0,0 +1,231 @@ +"""Monthly, read-only PQRST statistics from a pinned public hall Git tree. + +One selected published block per agent entry; never infer first-attempt success, +weight by time/tokens, execute entry text, or compare workers. Missing observations +are null, not zero. Collection fails closed rather than reporting partial means. +""" +from __future__ import annotations + +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from datetime import date, datetime, timezone +import hashlib +import json +import re +import time +from urllib.parse import quote +from zoneinfo import ZoneInfo + +import httpx +import yaml + +from activity_core.context_resolvers.base import CONTEXT_RESOLVER_REGISTRY, ContextResolver + +HOST = 'https://forgejo.coulomb.social' +REPO = 'coulomb/hall-of-helix' +TZ = ZoneInfo('Europe/Berlin') +LETTERS = 'PQRST' +SHA = re.compile(r'[a-f0-9]{40}') +SIG = re.compile(r'P(\d{1,3}) Q(\d{1,3}) R(\d{1,3}) S(\d{1,3}) T(\d{1,3})') +MAX_ENTRIES = 5000 + + +class StrictLoader(yaml.SafeLoader): + pass + + +def mapping(loader, node, deep=False): + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in result: + raise ValueError('duplicate_metadata') + result[key] = loader.construct_object(value_node, deep=deep) + return result + + +StrictLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping) + + +def signature(value): + match = SIG.fullmatch(value) if isinstance(value, str) else None + return tuple(map(int, match.groups())) if match else None + + +def parse_entry(text): + """Return metadata, date, attempt count and selected structural result.""" + fm = re.match(r'\A---\s*\n(.*?)\n---\s*\n', text, re.S) + if not fm: + raise ValueError('metadata') + meta = yaml.load(fm[1], Loader=StrictLoader) + if not isinstance(meta, dict): + raise ValueError('metadata') + stamp = meta.get('recorded_at') + if isinstance(stamp, datetime): + if stamp.tzinfo is None: + raise ValueError('date') + day = stamp.astimezone(TZ).date() + elif isinstance(stamp, date): + day = stamp + elif isinstance(stamp, str) and re.fullmatch(r'\d{4}-\d{2}-\d{2}', stamp): + day = date.fromisoformat(stamp) + elif isinstance(stamp, str): + parsed = datetime.fromisoformat(stamp.replace('Z', '+00:00')) + if parsed.tzinfo is None: + raise ValueError('date') + day = parsed.astimezone(TZ).date() + else: + raise ValueError('date') + sections = re.findall(r'^## PQRST estimate[^\S\n]*\n(.*?)(?=^## |\Z)', text[fm.end():], re.M | re.S | re.I) + blocks = [m[1] for section in sections for m in re.finditer( + r'^```[^\n]*\n(PQRST-Estimate\n.*?)^```[^\S\n]*$', section, re.M | re.S)] + if not sections and not meta.get('pqrst_estimate'): + return meta, day, len(blocks), None, 'missing' + target = signature(meta.get('pqrst_estimate')) + selected = [] + for block in blocks: + fields = {} + for field in re.finditer(r'^([A-Za-z][A-Za-z ]*):[^\S\n]*(.*?)(?=^[A-Za-z][A-Za-z ]*:|\Z)', block, re.M | re.S): + fields.setdefault(field[1], []).append(field[2].strip()) + sigs = fields.get('Signature', []) + if len(sigs) == 1 and target is not None and signature(sigs[0]) == target: + selected.append(fields) + if len(sections) != 1 or len(selected) != 1: + return meta, day, len(blocks), None, 'invalid' + fields = selected[0] + required = [*LETTERS, 'Sum', 'Confidence', 'Signature', 'Dominant factors'] + if any(len(fields.get(k, [])) != 1 for k in required) or any(k not in required + ['Notes'] for k in fields): + return meta, day, len(blocks), None, 'invalid' + values = [] + for key in LETTERS: + match = re.fullmatch(r'(\d{1,3})%', fields[key][0]) + if not match: + return meta, day, len(blocks), None, 'invalid' + values.append(int(match[1])) + confidence = fields['Confidence'][0] + if (sum(values) != 100 or any(v > 100 for v in values) or tuple(values) != target + or fields['Sum'] != ['100%'] or confidence not in {'low', 'medium', 'high'} + or not fields['Dominant factors'][0]): + return meta, day, len(blocks), None, 'invalid' + return meta, day, len(blocks), (values, confidence), 'included' + + +def month_keys(now): + current = now.astimezone(TZ) + number = current.year * 12 + current.month - 1 + return [f'{(number-i)//12:04d}-{(number-i)%12+1:02d}' for i in range(6, 0, -1)] + + +def summarize(entries, revision, now=None): + now = now or datetime.now(timezone.utc) + if now.tzinfo is None: + raise ValueError('timezone_required') + months = month_keys(now) + counts = Counter(entries=len(entries)) + buckets = {key: {'entries': 0, 'missing': 0, 'invalid': 0, 'attempts': 0, + 'drafts': 0, 'values': [], 'confidence': Counter()} for key in months} + parsed = [] + for path, text in entries: + try: + result = parse_entry(text) + parsed.append(result) + except (ValueError, TypeError, KeyError, OverflowError, yaml.YAMLError): + counts['unclassifiable'] += 1 + identities = Counter(m.get('id') for m, *_ in parsed if isinstance(m.get('id'), str)) + for meta, day, attempts, result, disposition in parsed: + if meta.get('worker_kind') != 'agent-session': + counts['non_agent'] += 1 + continue + key = day.strftime('%Y-%m') + if key not in buckets: + counts['outside_window'] += 1 + continue + bucket = buckets[key] + bucket['entries'] += 1 + bucket['attempts'] += attempts + if not isinstance(meta.get('id'), str) or identities[meta['id']] != 1: + disposition = 'invalid' + if disposition != 'included': + bucket[disposition] += 1 + continue + values, confidence = result + bucket['values'].append(values) + bucket['confidence'][confidence] += 1 + bucket['drafts'] += meta.get('status') == 'draft' + series = [] + previous = None + for month, bucket in buckets.items(): + values = bucket.pop('values') + n = len(values) + means = [sum(v[i] for v in values) / n for i in range(5)] if n else None + delta = [round(a-b, 2) for a, b in zip(means, previous)] if means is not None and previous is not None else None + series.append(dict(month=month, n=n, mean_pp=[round(v, 2) for v in means] if means else None, + change_pp=delta, **bucket)) + previous = means + latest = series[-1] + return {'repo': 'pqrst-practice', 'source': f'{REPO}@{revision}', + 'generated_at': now.isoformat(), 'timezone': 'Europe/Berlin', + 'status': 'needs_review' if counts['unclassifiable'] else 'ok', + 'summary': f"PQRST {latest['month']}: {latest['n']} structurally valid published datapoints; equal-entry monthly means and month-over-month changes in P/Q/R/S/T percentage points.", + 'order': list(LETTERS), 'counts': dict(counts), 'months': series, + 'limits': 'Published agent entries, including drafts; not first attempts or a census. Structural checks cannot certify evidence quality. No worker rankings, causal inference or significance claim; compare sample counts and task mix. Empty means/deltas are null. Rounding may shift totals by 0.01.', + 'model_calls': 0} + + +def collect(now=None): + """Fixed-host, commit-pinned, bounded collection. Never return partial means.""" + deadline = time.monotonic() + 120 + def get(client, url, limit): + if time.monotonic() >= deadline: + raise ValueError('collection_deadline') + with client.stream('GET', url) as response: + response.raise_for_status() + body = bytearray() + for part in response.iter_bytes(): + body.extend(part) + if len(body) > limit or time.monotonic() >= deadline: + raise ValueError('collection_limit') + return bytes(body) + with httpx.Client(timeout=10, follow_redirects=False) as client: + info = json.loads(get(client, f'{HOST}/api/v1/repos/{REPO}/branches/main', 65536)) + sha = info['commit']['id'] + if not isinstance(sha, str) or not SHA.fullmatch(sha): + raise ValueError('source_revision') + tree = json.loads(get(client, f'{HOST}/api/v1/repos/{REPO}/git/trees/{sha}?recursive=true&per_page=10000', 4_000_000)) + if tree.get('truncated') is not False or len(tree['tree']) != tree['total_count']: + raise ValueError('incomplete_tree') + files = [f for f in tree['tree'] if f['path'].startswith('entries/') and f['path'].endswith('.md')] + if len(files) > MAX_ENTRIES or not files: + raise ValueError('entry_count') + def read(f): + path = f['path'] + if (f.get('mode') != '100644' or f.get('type') != 'blob' + or '..' in path.split('/') or not SHA.fullmatch(f.get('sha', ''))): + raise ValueError('entry_path') + data = get(client, f'{HOST}/{REPO}/raw/commit/{sha}/{quote(path, safe="/")}', 131072) + blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest() + if blob != f['sha']: + raise ValueError('entry_hash') + return path, data.decode('utf-8') + with ThreadPoolExecutor(max_workers=8) as pool: + entries = list(pool.map(read, files)) + return summarize(entries, sha, now) + + +class PqrstResolver(ContextResolver): + def resolve(self, query, event, params): + if query != 'monthly': + raise ValueError('unsupported_pqrst_query') + try: + report = collect() + except (ValueError, KeyError, TypeError, UnicodeError, httpx.HTTPError): + report = {'repo': 'pqrst-practice', 'status': 'unavailable', + 'summary': 'PQRST source collection failed; no partial averages published.', + 'months': [], 'model_calls': 0} + encoded = json.dumps(report, sort_keys=True, separators=(',', ':')) + if len(encoded) > 3900: + raise ValueError('pqrst_digest_limit') + return encoded + + +CONTEXT_RESOLVER_REGISTRY['pqrst'] = PqrstResolver diff --git a/tests/test_pqrst_monthly.py b/tests/test_pqrst_monthly.py new file mode 100644 index 0000000..4e24e01 --- /dev/null +++ b/tests/test_pqrst_monthly.py @@ -0,0 +1,79 @@ +from datetime import datetime, timezone +import importlib.util +import json +from pathlib import Path +import pytest + +# Installed tests use the actual registered resolver. +from activity_core.context_resolvers import pqrst as p + +NOW = datetime(2026, 10, 1, 7, 30, tzinfo=timezone.utc) + + +def entry(day='2026-09-15', values=(30, 20, 25, 0, 25), identity='one', extra='', front=None): + sig = ' '.join(k + str(v) for k, v in zip('PQRST', values)) + block = 'PQRST-Estimate\n' + ''.join(f'{k}: {v}%\n' for k, v in zip('PQRST', values)) + block += f'Sum: 100%\nConfidence: medium\nSignature: {sig}\nDominant factors: Changed parser and tested invalid input.\n' + return f'---\nid: {identity}\nworker_kind: agent-session\nstatus: draft\nrecorded_at: "{day}"\npqrst_estimate: "{front or sig}"\n---\n## PQRST estimate\n{extra}```text\n{block}```\n' + + +def test_equal_entry_means_deltas_and_calendar_gaps(): + rows = [('a', entry('2026-08-10', (20, 20, 40, 0, 20), 'a')), + ('b', entry(identity='b')), ('c', entry(values=(40, 10, 30, 10, 10), identity='c')), + ('future', entry('2026-10-01', identity='f'))] + r = p.summarize(rows, 'a'*40, NOW) + assert r['months'][-1]['n'] == 2 + assert r['months'][-1]['mean_pp'] == [35, 15, 27.5, 5, 17.5] + assert r['months'][-1]['change_pp'] == [15, -5, -12.5, 5, -2.5] + assert r['months'][0]['mean_pp'] is None and r['months'][0]['change_pp'] is None + assert r['counts']['outside_window'] == 1 + assert r['months'][-1]['drafts'] == 2 + assert len(json.dumps(r, separators=(',', ':'))) < 3900 + + +def test_failed_attempt_not_counted_or_mixed_with_selected_retry(): + bad = '```text\nPQRST-Estimate\nP: 30%\nQ: 15%\nR: 25%\nS: 0%\nT: 30%\nSum: 100%\nConfidence: medium\nSignature: P30 Q30 R25 S0 T30\n```\n' + r = p.summarize([('a', entry(extra=bad))], 'a'*40, NOW)['months'][-1] + assert r['n'] == 1 and r['attempts'] == 2 and r['mean_pp'] == [30, 20, 25, 0, 25] + + +@pytest.mark.parametrize('change', [ + lambda s:s.replace('Q: 20%', 'Q: 21%'), + lambda s:s.replace('Confidence: medium\n',''), + lambda s:s.replace('Confidence: medium','Confidence: confident'), + lambda s:s.replace('Dominant factors: Changed parser and tested invalid input.','Dominant factors:'), + lambda s:s.replace('P: 30%', 'P: 30%\nP: 30%'), + lambda s:s.replace('Sum: 100%', 'Sum: 100%\nU: 0%'), + lambda s:s.replace('P: 30%', 'P: -30%'), +]) +def test_invalid_excluded_without_zero_imputation(change): + r = p.summarize([('a', change(entry()))], 'a'*40, NOW)['months'][-1] + assert r['invalid'] == 1 and r['n'] == 0 and r['mean_pp'] is None + + +def test_duplicate_ids_and_ambiguous_attempts_are_quarantined(): + r = p.summarize([('a', entry()), ('b', entry())], 'a'*40, NOW)['months'][-1] + assert r['invalid'] == 2 and r['n'] == 0 + block = entry().split('## PQRST estimate\n')[1] + assert p.summarize([('a', entry(extra=block))], 'a'*40, NOW)['months'][-1]['invalid'] == 1 + + +def test_date_timezone_and_year_rollover(): + r = p.summarize([('a', entry('2026-08-31T23:30:00Z'))], 'a'*40, NOW) + assert r['months'][-1]['n'] == 1 + assert p.month_keys(datetime(2027, 1, 1, tzinfo=timezone.utc))[-1] == '2026-12' + assert p.month_keys(datetime(2026, 9, 30, 22, tzinfo=timezone.utc))[-1] == '2026-09' + + +def test_missing_dates_and_missing_records_are_visible(): + a=entry().replace('recorded_at: "2026-09-15"','recorded_at: missing') + b=entry(identity='b').split('## PQRST estimate')[0].replace('pqrst_estimate: "P30 Q20 R25 S0 T25"\n','') + r=p.summarize([('a',a),('b',b)],'a'*40,NOW) + assert r['counts']['unclassifiable']==1 and r['status']=='needs_review' + assert r['months'][-1]['missing']==1 + + +def test_unavailable_does_not_publish_partial_or_zero_means(monkeypatch): + monkeypatch.setattr(p,'collect',lambda: (_ for _ in ()).throw(ValueError('truncated'))) + r=json.loads(p.PqrstResolver().resolve('monthly',None,{})) + assert r['status']=='unavailable' and r['months']==[] and r['model_calls']==0