From 01571ef10484617819e1406b9c41c25b1f0f5188 Mon Sep 17 00:00:00 2001 From: rosspeili Date: Sun, 19 Jul 2026 21:03:29 +0300 Subject: [PATCH] Add card ui_schema validation guard and fix drift (#199) Introduce skillware.core.ui_schema helpers and parametrized CI fixtures for all 13 output-card registry skills. Align mica_module and pii_masker card.json with execute() output, document the workflow in CONTRIBUTING and TESTING, and fix pdf_form_filler maintainer test to use tmp_path on Windows. --- CHANGELOG.md | 13 +++ CONTRIBUTING.md | 3 + docs/TESTING.md | 4 +- docs/contributing/ai_native_workflow.md | 3 +- skills/compliance/mica_module/card.json | 14 ++- skills/compliance/pii_masker/card.json | 14 ++- skillware/core/ui_schema.py | 99 +++++++++++++++++++ templates/python_skill/card.json | 3 +- .../compliance__mica_module.json | 14 +++ .../compliance__pii_masker.json | 13 +++ .../compliance__tos_evaluator.json | 27 +++++ .../card_ui_schema/creative__bg_remover.json | 13 +++ ...data_engineering__synthetic_generator.json | 11 +++ .../card_ui_schema/defi__evm_tx_handler.json | 20 ++++ .../dev_tools__issue_resolver.json | 25 +++++ .../finance__uk_companies_house_handler.json | 14 +++ .../finance__wallet_screening.json | 39 ++++++++ .../monitoring__token_limiter.json | 32 ++++++ .../office__pdf_form_filler.json | 10 ++ .../optimization__prompt_rewriter.json | 10 ++ .../wellness__mental_coach.json | 18 ++++ tests/skills/office/test_pdf_form_filler.py | 39 +++----- tests/test_card_ui_schema.py | 95 ++++++++++++++++++ tests/test_ui_schema.py | 69 +++++++++++++ 24 files changed, 567 insertions(+), 35 deletions(-) create mode 100644 skillware/core/ui_schema.py create mode 100644 tests/fixtures/card_ui_schema/compliance__mica_module.json create mode 100644 tests/fixtures/card_ui_schema/compliance__pii_masker.json create mode 100644 tests/fixtures/card_ui_schema/compliance__tos_evaluator.json create mode 100644 tests/fixtures/card_ui_schema/creative__bg_remover.json create mode 100644 tests/fixtures/card_ui_schema/data_engineering__synthetic_generator.json create mode 100644 tests/fixtures/card_ui_schema/defi__evm_tx_handler.json create mode 100644 tests/fixtures/card_ui_schema/dev_tools__issue_resolver.json create mode 100644 tests/fixtures/card_ui_schema/finance__uk_companies_house_handler.json create mode 100644 tests/fixtures/card_ui_schema/finance__wallet_screening.json create mode 100644 tests/fixtures/card_ui_schema/monitoring__token_limiter.json create mode 100644 tests/fixtures/card_ui_schema/office__pdf_form_filler.json create mode 100644 tests/fixtures/card_ui_schema/optimization__prompt_rewriter.json create mode 100644 tests/fixtures/card_ui_schema/wellness__mental_coach.json create mode 100644 tests/test_card_ui_schema.py create mode 100644 tests/test_ui_schema.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 63c39c9..6c80420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ Contributors add user-facing entries under `[Unreleased]` in the same PR. Mainta ## [Unreleased] +### Added + +- **Framework:** `skillware/core/ui_schema.py` helpers to resolve dot paths and validate output-card field keys (#199). +- **CI:** Parametrized guard `tests/test_card_ui_schema.py` — every registry skill with output-card `ui_schema` must ship a fixture under `tests/fixtures/card_ui_schema/` whose samples cover all field keys (#199). + +### Fixed + +- **`compliance/mica_module`**, **`compliance/pii_masker`**: Align `card.json` output fields with actual `execute()` return shape (#199). + +### Changed + +- **Documentation:** CONTRIBUTING and contributor workflow checklists — update `card.json` and output fixtures together when changing `execute()` output (#199). + ## [0.4.5] - 2026-07-16 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fca1eeb..4b71d6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -269,6 +269,7 @@ requirements: - Catch internal errors and return a structured error report; do not crash the host agent. - Do **not** print to stdout or stderr for normal operation. - Do **not** embed open-ended LLM code generation as the skill implementation. +- When you change the JSON shape returned by `execute()`, update `card.json` output fields (if present) and the matching fixture under `tests/fixtures/card_ui_schema/` in the same PR. ### 3. `instructions.md` (cognition) @@ -284,6 +285,8 @@ The primary guide for the host LLM. - Optional but recommended for user-facing agents and catalog UIs. - Describes UI presentation (`name`, `description`, `icon`, `ui_schema`, and similar). - When present, include an `issuer` object that matches `manifest.yaml` (`name` and `email` at minimum; copy `github` and `org` when used). +- For output cards (`ui_schema.type` = `card`), each `ui_schema.fields[].key` must be a dot path into the JSON returned by `execute()` (for example `metadata.wallet_address`, `preview.you_pay`). Update `card.json` in the same PR when you change the output shape. +- Add or refresh a representative output fixture at `tests/fixtures/card_ui_schema/__.json` (one object or a `{"samples": [...]}` list when multiple execute paths surface different fields). CI validates keys via `tests/test_card_ui_schema.py` (#199). ### 5. `test_skill.py` (bundle test) diff --git a/docs/TESTING.md b/docs/TESTING.md index 96e3691..db1b9d5 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -19,6 +19,7 @@ Tests fall into four layers: **bundle**, **framework**, **maintainer**, and **ex | GitHub label policy test (`test_github_labels.py`) | Done | | PyPI wheel packaging smoke test (`scripts/wheel_smoke_test.py`) | Done | | Optional extras sync (`scripts/sync_extras.py`, `tests/test_extras_sync.py`) | Done | +| Card UI schema vs execute output (`tests/test_card_ui_schema.py`) | Done | Every pull request runs `black --check`, `flake8`, `pytest skills/`, `pytest tests/`, and a **wheel-smoke** job that builds a wheel, installs it in a fresh venv (base install only — no `[all]` or per-skill extras), and verifies every bundled registry skill is present and loadable. Bundle tests gate merge the same as framework and maintainer tests. @@ -62,6 +63,7 @@ pip install -r requirements.txt - Core engine health: loader, CLI, issuer rules, version policy, parameter schema validation (`tests/test_validate_params.py`). - `tests/test_skill_issuer.py` also enforces registry packaging (`__init__.py`), issuer metadata, presence of `test_skill.py` in every skill bundle, and rejects legacy manifest `output:` keys. +- `tests/test_card_ui_schema.py` validates output-card `ui_schema.fields[].key` dot paths against fixtures in `tests/fixtures/card_ui_schema/` (#199). - `tests/test_registry_docs.py` enforces doc-drift parity: skill catalog index matches manifests, examples README matches scripts on disk, and agent-loops.md references every registered skill. - Lives at the **root of `tests/`** only (`tests/test_loader.py`, `tests/test_cli.py`, …). - Clone-repo only; runs in CI via `pytest tests/` together with maintainer tests below. @@ -84,7 +86,7 @@ pip install -r requirements.txt | :--- | :--- | :--- | | Manifest + execute contract for one skill | Bundle test | `skills/compliance/tos_evaluator/test_skill.py` | | Loader path + mocked externals (optional depth) | Maintainer test | `tests/skills/compliance/test_tos_evaluator.py` | -| Loader, CLI, registry issuer rules, param validation | Framework test | `tests/test_loader.py`, `tests/test_skill_issuer.py`, `tests/test_validate_params.py`, `tests/test_registry_docs.py` | +| Loader, CLI, registry issuer rules, param validation | Framework test | `tests/test_loader.py`, `tests/test_skill_issuer.py`, `tests/test_validate_params.py`, `tests/test_registry_docs.py`, `tests/test_card_ui_schema.py` | | End-to-end provider demo script | Usage example | `examples/gemini_tos_evaluator.py` | **Rule of thumb:** if it ships with the skill and must pass before merge → **bundle test** (CI + local). If it is extra regression depth for clone-repo work → **maintainer test** (optional). If it proves provider integration → **example**, not pytest. diff --git a/docs/contributing/ai_native_workflow.md b/docs/contributing/ai_native_workflow.md index 67c4af6..026b3b1 100644 --- a/docs/contributing/ai_native_workflow.md +++ b/docs/contributing/ai_native_workflow.md @@ -239,6 +239,7 @@ These align with [CONTRIBUTING.md](../../CONTRIBUTING.md). Violations block merg - `manifest.yaml` `name` must equal `category/skill_name` (matches folder path); loader warns on mismatch for registry layout - `issuer.name` and `issuer.email` required; `github` and `org` optional; no template placeholders in registry paths - `card.json` issuer must match manifest `name` and `email` when present +- Output-card `ui_schema.fields[].key` values must resolve in `execute()` JSON; keep `tests/fixtures/card_ui_schema/__.json` in sync (#199) - Update `docs/skills/.md` and `docs/skills/README.md` - On each catalog page, add a **Usage Examples** section (Gemini, Claude, OpenAI, DeepSeek, Ollama prompt mode) per [skill usage template](../usage/skill_usage_template.md). Keep provider mechanics in `docs/usage/`; put skill-specific paths, sample user messages, and `execute` payloads on the skill page. - Categories: `compliance`, `creative`, `data_engineering`, `defi`, `dev_tools`, `finance`, `monitoring`, `office`, `optimization`, `wellness` — see [Skill library](../skills/README.md) for the live registry; [Choosing a category](../../CONTRIBUTING.md#choosing-a-category) in CONTRIBUTING.md (issue first for new top-level folders) @@ -276,7 +277,7 @@ Complete the checklist that matches your issue during Stage 5. - [ ] Optional: `short_description` field (~80 chars) for a concise one-line summary in `skillware list` - [ ] `skill.py`: exactly one `BaseSkill` subclass (auto-discovered as `bundle["class"]`); deterministic, JSON-serializable returns, safe error handling - [ ] `instructions.md`: when to use, how to interpret output, limitations -- [ ] `card.json`: `issuer` matches manifest +- [ ] `card.json`: `issuer` matches manifest; output-card `ui_schema.fields[].key` paths resolve in `tests/fixtures/card_ui_schema/__.json` (update fixture when `execute()` output changes) - [ ] `test_skill.py` (bundle test) passes — `pytest skills///test_skill.py` or `skillware test /` - [ ] Bundle tests mock all network calls and model downloads; CI does not download models. - [ ] `docs/skills/.md` and catalog row in `docs/skills/README.md` (include **Recommended install:** `pip install "skillware[_]"` per [install_extras.md](../usage/install_extras.md)) diff --git a/skills/compliance/mica_module/card.json b/skills/compliance/mica_module/card.json index f071766..eb42677 100644 --- a/skills/compliance/mica_module/card.json +++ b/skills/compliance/mica_module/card.json @@ -13,12 +13,18 @@ "type": "card", "fields": [ { - "key": "run_evaluator", - "label": "Enable Evaluator" + "key": "policy_status", + "label": "Policy Status", + "type": "status_badge" }, { - "key": "evaluator_model", - "label": "Evaluator Model" + "key": "gemini_evaluator_feedback.grade", + "label": "Evaluator Grade" + }, + { + "key": "retrieved_sections", + "label": "Sections", + "type": "tags" } ] } diff --git a/skills/compliance/pii_masker/card.json b/skills/compliance/pii_masker/card.json index 1a5317b..762c4ac 100644 --- a/skills/compliance/pii_masker/card.json +++ b/skills/compliance/pii_masker/card.json @@ -13,12 +13,18 @@ "type": "card", "fields": [ { - "key": "mode", - "label": "Redaction Mode" + "key": "metadata.entity_count", + "label": "Entities Found", + "type": "count" }, { - "key": "ollama_url", - "label": "Ollama Endpoint" + "key": "metadata.detected_entities", + "label": "Entity Types", + "type": "tags" + }, + { + "key": "metadata.security_level", + "label": "Security Level" } ] } diff --git a/skillware/core/ui_schema.py b/skillware/core/ui_schema.py new file mode 100644 index 0000000..3042410 --- /dev/null +++ b/skillware/core/ui_schema.py @@ -0,0 +1,99 @@ +"""Helpers for validating card.json ui_schema field keys against skill output.""" + +from __future__ import annotations + +from typing import Any, Dict, Iterable, List, Mapping, Sequence, Union + +JsonObject = Dict[str, Any] + + +def resolve_dot_path(data: Mapping[str, Any], path: str) -> Any: + """Resolve a dot-separated path against a nested mapping. + + Raises: + KeyError: if any segment is missing. + TypeError: if a segment is not a mapping when further segments remain. + """ + if not path or not path.strip(): + raise KeyError("empty path") + + current: Any = data + for segment in path.split("."): + if not isinstance(current, Mapping): + raise KeyError(path) + if segment not in current: + raise KeyError(path) + current = current[segment] + return current + + +def path_exists(data: Mapping[str, Any], path: str) -> bool: + """Return True when ``path`` resolves in ``data``.""" + try: + resolve_dot_path(data, path) + except (KeyError, TypeError): + return False + return True + + +def is_output_card_ui_schema(ui_schema: Any) -> bool: + """True when ui_schema declares output card fields (type=card + fields list).""" + if not isinstance(ui_schema, Mapping): + return False + if ui_schema.get("type") != "card": + return False + fields = ui_schema.get("fields") + return isinstance(fields, list) and bool(fields) + + +def extract_card_field_keys(ui_schema: Mapping[str, Any]) -> List[str]: + """Return ui_schema.fields[].key values for output card schemas.""" + if not is_output_card_ui_schema(ui_schema): + return [] + + keys: List[str] = [] + for field in ui_schema.get("fields", []): + if not isinstance(field, Mapping): + continue + key = field.get("key") + if isinstance(key, str) and key.strip(): + keys.append(key.strip()) + return keys + + +def missing_keys_for_samples( + samples: Sequence[Mapping[str, Any]], keys: Iterable[str] +) -> List[str]: + """Return field keys that do not resolve in any sample output.""" + key_list = list(keys) + missing: List[str] = [] + for key in key_list: + if not any(path_exists(sample, key) for sample in samples): + missing.append(key) + return missing + + +def normalize_fixture_samples( + payload: Union[JsonObject, Sequence[JsonObject]], +) -> List[JsonObject]: + """Accept a single output object or {"samples": [...]} fixture payload.""" + if isinstance(payload, Mapping) and "samples" in payload: + raw_samples = payload.get("samples") + if not isinstance(raw_samples, list) or not raw_samples: + raise ValueError("fixture samples must be a non-empty list") + return [sample for sample in raw_samples if isinstance(sample, Mapping)] + + if isinstance(payload, Mapping): + return [payload] + + raise ValueError("fixture must be an object or a samples list wrapper") + + +def validate_card_ui_schema( + ui_schema: Mapping[str, Any], samples: Sequence[Mapping[str, Any]] +) -> List[str]: + """Validate output card field keys against one or more execute() samples.""" + keys = extract_card_field_keys(ui_schema) + if not keys: + return [] + return missing_keys_for_samples(samples, keys) diff --git a/templates/python_skill/card.json b/templates/python_skill/card.json index 52b5462..854fcc6 100644 --- a/templates/python_skill/card.json +++ b/templates/python_skill/card.json @@ -14,7 +14,8 @@ "fields": [ { "key": "status", - "label": "Status" + "label": "Status", + "type": "status_badge" } ] } diff --git a/tests/fixtures/card_ui_schema/compliance__mica_module.json b/tests/fixtures/card_ui_schema/compliance__mica_module.json new file mode 100644 index 0000000..5709598 --- /dev/null +++ b/tests/fixtures/card_ui_schema/compliance__mica_module.json @@ -0,0 +1,14 @@ +{ + "samples": [ + { + "retrieved_sections": ["Title V / Article 59"], + "policy_status": "CAUTION", + "gemini_evaluator_feedback": { + "grade": "N/A", + "holes_found": "Evaluator disabled by parameter.", + "suggestion": "Proceed manually integrating the extracted logic." + }, + "final_context_for_agent": "Output your final answer seamlessly integrating MiCA rules." + } + ] +} diff --git a/tests/fixtures/card_ui_schema/compliance__pii_masker.json b/tests/fixtures/card_ui_schema/compliance__pii_masker.json new file mode 100644 index 0000000..6ad8ba7 --- /dev/null +++ b/tests/fixtures/card_ui_schema/compliance__pii_masker.json @@ -0,0 +1,13 @@ +{ + "samples": [ + { + "sanitized_text": "Contact [EMAIL_1] for support.", + "metadata": { + "detected_entities": ["EMAIL"], + "entity_count": 1, + "security_level": "local-only", + "model": "arpacorp/micro-f1-mask" + } + } + ] +} diff --git a/tests/fixtures/card_ui_schema/compliance__tos_evaluator.json b/tests/fixtures/card_ui_schema/compliance__tos_evaluator.json new file mode 100644 index 0000000..e4e7d55 --- /dev/null +++ b/tests/fixtures/card_ui_schema/compliance__tos_evaluator.json @@ -0,0 +1,27 @@ +{ + "samples": [ + { + "is_safe_to_proceed": true, + "confidence_score": 0.72, + "verdict": "SAFE", + "reason": "Discovered policy text includes language that appears permissive for the requested behavior.", + "recommended_next_step": "Proceed conservatively and continue honoring crawl delays and rate limits.", + "action_type": "read", + "robots_assessment": { + "can_fetch": true, + "reason": "Allowed by robots.txt" + }, + "tos_assessment": { + "status": "allowed", + "summary": "Permissive language found." + }, + "llm_assessment": { + "status": "not_used" + }, + "discovered_policy_urls": { + "candidates": ["https://example.com/terms"] + }, + "evidence": [] + } + ] +} diff --git a/tests/fixtures/card_ui_schema/creative__bg_remover.json b/tests/fixtures/card_ui_schema/creative__bg_remover.json new file mode 100644 index 0000000..40c0423 --- /dev/null +++ b/tests/fixtures/card_ui_schema/creative__bg_remover.json @@ -0,0 +1,13 @@ +{ + "samples": [ + { + "success": true, + "image_base64": "iVBORw0KGgo=", + "mime_type": "image/png", + "output_path": "/tmp/bg_removed.png", + "width": 640, + "height": 480, + "model_used": "u2net" + } + ] +} diff --git a/tests/fixtures/card_ui_schema/data_engineering__synthetic_generator.json b/tests/fixtures/card_ui_schema/data_engineering__synthetic_generator.json new file mode 100644 index 0000000..90e48ff --- /dev/null +++ b/tests/fixtures/card_ui_schema/data_engineering__synthetic_generator.json @@ -0,0 +1,11 @@ +{ + "samples": [ + { + "samples": [{"id": 1, "value": "sample-a"}], + "entropy_score": 4.2, + "status": "success", + "provider_used": "gemini", + "samples_generated": 1 + } + ] +} diff --git a/tests/fixtures/card_ui_schema/defi__evm_tx_handler.json b/tests/fixtures/card_ui_schema/defi__evm_tx_handler.json new file mode 100644 index 0000000..74a40ca --- /dev/null +++ b/tests/fixtures/card_ui_schema/defi__evm_tx_handler.json @@ -0,0 +1,20 @@ +{ + "samples": [ + { + "status": "ready", + "preview": { + "you_pay": {"asset": "USDC", "amount": "100.0"}, + "you_receive": {"asset": "WETH", "amount": "0.04"} + }, + "quote": {}, + "requires_confirmation": true + }, + { + "status": "confirmed", + "tx_hash": "0xabc123def456", + "explorer_url": "https://etherscan.io/tx/0xabc123def456", + "receipt": {"success": true}, + "quote": {} + } + ] +} diff --git a/tests/fixtures/card_ui_schema/dev_tools__issue_resolver.json b/tests/fixtures/card_ui_schema/dev_tools__issue_resolver.json new file mode 100644 index 0000000..1e2ba96 --- /dev/null +++ b/tests/fixtures/card_ui_schema/dev_tools__issue_resolver.json @@ -0,0 +1,25 @@ +{ + "samples": [ + { + "status": "ready", + "action": "prepare", + "workflow_version": "1.0", + "issue": { + "url": "https://github.com/ARPAHLS/skillware/issues/199", + "api_url": "https://api.github.com/repos/ARPAHLS/skillware/issues/199", + "owner": "ARPAHLS", + "repo": "skillware", + "number": 199 + }, + "repository": { + "html_url": "https://github.com/ARPAHLS/skillware", + "api_url": "https://api.github.com/repos/ARPAHLS/skillware" + }, + "auth": { + "token_provided": true, + "note": "Include the Authorization header." + }, + "next_step": "Call action workflow_overview." + } + ] +} diff --git a/tests/fixtures/card_ui_schema/finance__uk_companies_house_handler.json b/tests/fixtures/card_ui_schema/finance__uk_companies_house_handler.json new file mode 100644 index 0000000..a04f44c --- /dev/null +++ b/tests/fixtures/card_ui_schema/finance__uk_companies_house_handler.json @@ -0,0 +1,14 @@ +{ + "samples": [ + { + "status": "ready", + "source": "companies_house_api", + "fetched_at": "2026-07-19T12:00:00Z", + "company_number": "01234567", + "company_name": "Example Ltd", + "company_status": "active", + "company_type": "ltd", + "next_actions": ["get_officers"] + } + ] +} diff --git a/tests/fixtures/card_ui_schema/finance__wallet_screening.json b/tests/fixtures/card_ui_schema/finance__wallet_screening.json new file mode 100644 index 0000000..097ae1a --- /dev/null +++ b/tests/fixtures/card_ui_schema/finance__wallet_screening.json @@ -0,0 +1,39 @@ +{ + "samples": [ + { + "metadata": { + "screening_time": "2026-07-19T12:00:00", + "wallet_address": "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045", + "data_sources_count": 2 + }, + "summary": { + "risk_flag": false, + "sanctioned_entity_match": false, + "malicious_interaction_count": 0, + "balance_eth": 1.5, + "balance_usd": 4500.0, + "total_transactions": 42 + }, + "financial_analysis": { + "value_in_eth": 10.0, + "value_in_usd": 30000.0, + "value_out_eth": 8.0, + "value_out_usd": 24000.0, + "gas_paid_eth": 0.05, + "pnl_eth": 1.95, + "pnl_usd": 5850.0, + "pnl_percent": 19.5, + "value_in_eur": 28000.0, + "value_out_eur": 22400.0 + }, + "risk_details": { + "sanctions_hits": [], + "malicious_interactions": [] + }, + "network_analysis": { + "most_interacted_wallet": "0xabc", + "top_10_counterparties": [] + } + } + ] +} diff --git a/tests/fixtures/card_ui_schema/monitoring__token_limiter.json b/tests/fixtures/card_ui_schema/monitoring__token_limiter.json new file mode 100644 index 0000000..232473d --- /dev/null +++ b/tests/fixtures/card_ui_schema/monitoring__token_limiter.json @@ -0,0 +1,32 @@ +{ + "samples": [ + { + "status": "ready", + "schema_version": "1.0", + "action": "CONTINUE", + "task_id": "demo-task", + "reason": "Token utilization at 10.0% (10/100).", + "budget": { + "current_token_count": 10, + "max_allowed_tokens": 100, + "utilization_pct": 10.0, + "tokens_over_budget": 0, + "soft_threshold_pct": 80.0, + "soft_limit_tokens": 80 + }, + "cost": { + "incurred_usd": 0.001, + "model_id": "gemini-2.5-flash-lite", + "pricing_source": "data/model_pricing.json", + "pricing_last_updated": "2026-01-01" + }, + "roi": {}, + "metadata": { + "warnings": [], + "turn_id": null, + "cache_hit": false + }, + "timestamp": "2026-07-19T12:00:00Z" + } + ] +} diff --git a/tests/fixtures/card_ui_schema/office__pdf_form_filler.json b/tests/fixtures/card_ui_schema/office__pdf_form_filler.json new file mode 100644 index 0000000..6d40d2e --- /dev/null +++ b/tests/fixtures/card_ui_schema/office__pdf_form_filler.json @@ -0,0 +1,10 @@ +{ + "samples": [ + { + "status": "success", + "output_path": "/tmp/filled.pdf", + "filled_fields": ["page0_name", "page0_email"], + "message": "Successfully filled 2 fields." + } + ] +} diff --git a/tests/fixtures/card_ui_schema/optimization__prompt_rewriter.json b/tests/fixtures/card_ui_schema/optimization__prompt_rewriter.json new file mode 100644 index 0000000..88be2bc --- /dev/null +++ b/tests/fixtures/card_ui_schema/optimization__prompt_rewriter.json @@ -0,0 +1,10 @@ +{ + "samples": [ + { + "compressed_text": "Analyze quarterly revenue trends.", + "original_tokens": 12, + "new_tokens": 6, + "tokens_saved": 6 + } + ] +} diff --git a/tests/fixtures/card_ui_schema/wellness__mental_coach.json b/tests/fixtures/card_ui_schema/wellness__mental_coach.json new file mode 100644 index 0000000..87cf801 --- /dev/null +++ b/tests/fixtures/card_ui_schema/wellness__mental_coach.json @@ -0,0 +1,18 @@ +{ + "samples": [ + { + "policy_status": "CAUTION", + "scope": "coaching", + "retrieved_sections": ["grounding_chunk_1"], + "citations": [], + "hard_constraints_applied": [], + "disclaimers_required": ["not_medical_advice"], + "evaluator_feedback": { + "grade": "B", + "holes_found": "None", + "suggestion": "Proceed with empathy." + }, + "final_context_for_agent": "Coach the user using retrieved grounding." + } + ] +} diff --git a/tests/skills/office/test_pdf_form_filler.py b/tests/skills/office/test_pdf_form_filler.py index fde82b8..7b1d34e 100644 --- a/tests/skills/office/test_pdf_form_filler.py +++ b/tests/skills/office/test_pdf_form_filler.py @@ -77,7 +77,7 @@ def test_apply_edits_checkbox(): @patch("skills.office.pdf_form_filler.skill.anthropic.Anthropic") -def test_skill_execute_mocked(mock_anthropic_cls): +def test_skill_execute_mocked(mock_anthropic_cls, tmp_path): """Test the full execution flow with mocked LLM.""" # Setup Mock mock_client = mock_anthropic_cls.return_value @@ -98,29 +98,20 @@ def test_skill_execute_mocked(mock_anthropic_cls): # 7 is TEXT widget.field_type = 7 page.add_widget(widget) - temp_pdf_path = "temp_test_skill.pdf" + temp_pdf_path = tmp_path / "temp_test_skill.pdf" doc.save(temp_pdf_path) doc.close() - try: - # Execute - result = skill.execute( - { - "pdf_path": os.path.abspath(temp_pdf_path), - "instructions": "Fill test field with Hello World", - } - ) - - if "error" in result: - pytest.fail(f"Skill execution returned error: {result['error']}") - - assert result["status"] == "success" - assert "page0_test_field" in result["filled_fields"] - assert os.path.exists(result["output_path"]) - - finally: - # Cleanup - if os.path.exists(temp_pdf_path): - os.remove(temp_pdf_path) - if "result" in locals() and os.path.exists(result.get("output_path", "")): - os.remove(result["output_path"]) + result = skill.execute( + { + "pdf_path": str(temp_pdf_path), + "instructions": "Fill test field with Hello World", + } + ) + + if "error" in result: + pytest.fail(f"Skill execution returned error: {result['error']}") + + assert result["status"] == "success" + assert "page0_test_field" in result["filled_fields"] + assert os.path.exists(result["output_path"]) diff --git a/tests/test_card_ui_schema.py b/tests/test_card_ui_schema.py new file mode 100644 index 0000000..f292f5e --- /dev/null +++ b/tests/test_card_ui_schema.py @@ -0,0 +1,95 @@ +"""Registry card.json ui_schema field keys must resolve in execute() output samples.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from skillware.core.ui_schema import ( + is_output_card_ui_schema, + normalize_fixture_samples, + validate_card_ui_schema, +) + +REPO_ROOT = Path(__file__).resolve().parent.parent +SKILLS_ROOT = REPO_ROOT / "skills" +FIXTURES_ROOT = REPO_ROOT / "tests" / "fixtures" / "card_ui_schema" + + +def _discover_skill_dirs(): + if not SKILLS_ROOT.is_dir(): + return [] + return sorted(p.parent for p in SKILLS_ROOT.rglob("manifest.yaml")) + + +def _skill_id(skill_dir: Path) -> str: + return skill_dir.relative_to(SKILLS_ROOT).as_posix() + + +def _fixture_path(skill_dir: Path) -> Path: + return FIXTURES_ROOT / f"{_skill_id(skill_dir).replace('/', '__')}.json" + + +def _load_card(skill_dir: Path) -> dict | None: + card_path = skill_dir / "card.json" + if not card_path.is_file(): + return None + with open(card_path, encoding="utf-8") as handle: + return json.load(handle) + + +def _skills_with_output_card_ui_schema(): + cases = [] + for skill_dir in _discover_skill_dirs(): + card = _load_card(skill_dir) + if not card: + continue + ui_schema = card.get("ui_schema") + if is_output_card_ui_schema(ui_schema): + cases.append((_skill_id(skill_dir), skill_dir)) + assert cases, "expected at least one registry skill with output card ui_schema" + return cases + + +OUTPUT_CARD_SKILLS = _skills_with_output_card_ui_schema() + + +@pytest.mark.parametrize( + "skill_id,skill_dir", + OUTPUT_CARD_SKILLS, + ids=[skill_id for skill_id, _ in OUTPUT_CARD_SKILLS], +) +def test_card_ui_schema_keys_match_execute_output_fixture(skill_id, skill_dir): + rel = skill_dir.relative_to(REPO_ROOT).as_posix() + card = _load_card(skill_dir) + ui_schema = card["ui_schema"] + + fixture_path = _fixture_path(skill_dir) + assert fixture_path.is_file(), ( + f"{rel}: add execute() output fixture at " + f"{fixture_path.relative_to(REPO_ROOT).as_posix()} " + f"for card.json ui_schema validation (#199)" + ) + + with open(fixture_path, encoding="utf-8") as handle: + payload = json.load(handle) + + samples = normalize_fixture_samples(payload) + missing = validate_card_ui_schema(ui_schema, samples) + assert not missing, ( + f"{rel} card.json ui_schema keys missing from fixture samples: " + f"{', '.join(missing)}" + ) + + +def test_output_card_skills_have_fixture_files(): + """Every output-card skill must ship a co-located CI fixture under tests/fixtures/.""" + missing_fixtures = [] + for skill_id, skill_dir in _skills_with_output_card_ui_schema(): + if not _fixture_path(skill_dir).is_file(): + missing_fixtures.append(skill_id) + assert not missing_fixtures, "missing card ui_schema fixtures for: " + ", ".join( + missing_fixtures + ) diff --git a/tests/test_ui_schema.py b/tests/test_ui_schema.py new file mode 100644 index 0000000..d1d3a19 --- /dev/null +++ b/tests/test_ui_schema.py @@ -0,0 +1,69 @@ +"""Unit tests for skillware.core.ui_schema helpers.""" + +from skillware.core.ui_schema import ( + extract_card_field_keys, + is_output_card_ui_schema, + missing_keys_for_samples, + normalize_fixture_samples, + path_exists, + resolve_dot_path, + validate_card_ui_schema, +) + + +def test_resolve_dot_path_nested(): + data = {"metadata": {"wallet_address": "0xabc"}, "summary": {"risk_flag": True}} + assert resolve_dot_path(data, "metadata.wallet_address") == "0xabc" + assert resolve_dot_path(data, "summary.risk_flag") is True + + +def test_path_exists_false_on_missing_segment(): + data = {"status": "ready"} + assert path_exists(data, "preview.you_pay") is False + + +def test_is_output_card_ui_schema(): + assert is_output_card_ui_schema({"type": "card", "fields": [{"key": "status"}]}) + assert not is_output_card_ui_schema({"dataset_chunk": {"ui:widget": "textarea"}}) + assert not is_output_card_ui_schema({"type": "card", "fields": []}) + + +def test_validate_card_ui_schema_across_samples(): + ui_schema = { + "type": "card", + "fields": [ + {"key": "preview.you_pay"}, + {"key": "tx_hash"}, + ], + } + samples = [ + {"status": "ready", "preview": {"you_pay": {"asset": "ETH"}}}, + {"status": "confirmed", "tx_hash": "0x1"}, + ] + assert validate_card_ui_schema(ui_schema, samples) == [] + + +def test_validate_card_ui_schema_reports_missing_keys(): + ui_schema = {"type": "card", "fields": [{"key": "missing.path"}]} + samples = [{"status": "ready"}] + assert validate_card_ui_schema(ui_schema, samples) == ["missing.path"] + + +def test_normalize_fixture_samples_accepts_wrapper_or_object(): + single = {"status": "success"} + assert normalize_fixture_samples(single) == [single] + wrapped = normalize_fixture_samples({"samples": [single, {"status": "error"}]}) + assert len(wrapped) == 2 + + +def test_extract_card_field_keys_skips_blank_entries(): + ui_schema = { + "type": "card", + "fields": [{"key": " status "}, {"label": "No key"}, {"key": ""}], + } + assert extract_card_field_keys(ui_schema) == ["status"] + + +def test_missing_keys_for_samples(): + samples = [{"a": {"b": 1}}, {"c": 2}] + assert missing_keys_for_samples(samples, ["a.b", "c", "missing"]) == ["missing"]