{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "ai_document.schema.json",
  "title": "AI document",
  "description": "One cleaned web page, ready for search, RAG chunking, analytics or training. One JSON object per line in the output JSONL.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "id", "url", "canonical_url", "domain", "title", "description", "content_type", "language",
    "tags", "published_at", "modified_at", "fetched_at", "first_seen_at", "last_changed_at",
    "word_count", "char_count", "reading_time_minutes", "link_density", "quality_score",
    "content_hash", "extraction_method", "crawl_depth", "license", "body_text", "sections"
  ],
  "$defs": {
    "date": {"type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$"},
    "timestamp": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"}
  },
  "properties": {
    "id": {"type": "string", "pattern": "^[0-9a-f]{16}$", "description": "First 16 hex chars of sha256(canonical URL without scheme or trailing slash). Stable across runs."},
    "url": {"type": "string", "format": "uri", "description": "URL the page was served from, after redirects."},
    "canonical_url": {"type": "string", "format": "uri", "description": "The page's rel=canonical URL when it points to the same site, else the fetched URL."},
    "domain": {"type": "string"},
    "title": {"type": "string", "description": "The page's <h1>, falling back to og:title or <title> without the site name."},
    "description": {"type": ["string", "null"], "description": "Meta description, if any."},
    "content_type": {"type": "string", "description": "From site rules (e.g. regulation_section, official_interpretation, qa_answer, guide) or page signals (article, doc_page, product_page, other)."},
    "language": {"type": "string", "description": "ISO 639-1 code from stopword frequencies, falling back to <html lang>; 'und' if unknown."},
    "tags": {"type": "array", "items": {"type": "string"}, "description": "Lowercased site-rule tags followed by the page's breadcrumb trail."},
    "published_at": {"$ref": "#/$defs/date", "description": "Publication date from meta tags, JSON-LD or a visible 'published' stamp."},
    "modified_at": {"$ref": "#/$defs/date", "description": "Last change or editorial review date from meta tags, JSON-LD or a visible 'last reviewed/updated' stamp."},
    "fetched_at": {"$ref": "#/$defs/timestamp", "description": "When this version was fetched (UTC)."},
    "first_seen_at": {"$ref": "#/$defs/timestamp", "description": "When the crawler first collected this document; preserved across runs."},
    "last_changed_at": {"$ref": "#/$defs/timestamp", "description": "Last run in which content_hash changed."},
    "word_count": {"type": "integer", "minimum": 0},
    "char_count": {"type": "integer", "minimum": 0},
    "reading_time_minutes": {"type": "integer", "minimum": 1, "description": "word_count / 230, rounded, at least 1."},
    "link_density": {"type": "number", "minimum": 0, "maximum": 1, "description": "Share of the main content's text that is link text."},
    "quality_score": {"type": "number", "minimum": 0, "maximum": 1, "description": "Heuristic 0-1 score from length, link density and structure; for ranking and filtering."},
    "content_hash": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "sha256 of body_text; detects changed and duplicate content."},
    "extraction_method": {"type": "string", "enum": ["selector", "semantic", "density", "body"], "description": "Which main-content strategy produced body_text."},
    "crawl_depth": {"type": "integer", "minimum": 0, "description": "Link hops from the nearest start URL."},
    "license": {"type": ["string", "null"], "description": "Usage terms of the source, from the site profile."},
    "body_text": {"type": "string", "description": "Clean main-content text; paragraphs separated by a blank line, list items prefixed with '- '."},
    "sections": {
      "type": "array",
      "description": "body_text split at headings (and at site-specific section starts), for chunking.",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": ["heading_path", "anchor", "text"],
        "properties": {
          "heading_path": {"type": "array", "items": {"type": "string"}, "description": "Headings this text sits under, outermost first; empty for text before the first heading."},
          "anchor": {"type": ["string", "null"], "description": "Element id of the nearest heading, for deep links (url#anchor)."},
          "text": {"type": "string"}
        }
      }
    }
  }
}