# Debt-collection corpus from consumerfinance.gov (U.S. Consumer Financial Protection Bureau):
# the federal rule for debt collectors (Regulation F, 12 CFR Part 1006) with its official
# interpretations, and the CFPB's plain-language answers about debt collection.
#
#   scrape_site --config configs/cfpb_debt_collection.toml --output data/cfpb_debt_collection.jsonl

start_urls = [
  "https://www.consumerfinance.gov/consumer-tools/debt-collection/",
  "https://www.consumerfinance.gov/rules-policy/regulations/1006/",
]
max_pages = 150
# Depth 3 mostly added off-topic answers (credit reports, credit cards, auto loans)
# reached through "related questions" links: 35 of 36 new answers in a test run.
max_depth = 2
delay_seconds = 1.0

# Header and footer menus link to the whole agency site; links inside <main> stay on topic
# (the regulation's table of contents and the answers' related questions live there).
link_scope = "main"

include = [
  "^/consumer-tools/debt-collection/",
  "^/ask-cfpb/[^/]+-en-\\d+/$",
  "^/rules-policy/regulations/1006/",
]
exclude = [
  "^/rules-policy/regulations/1006/\\d{4}-\\d{2}-\\d{2}/",  # superseded versions of the rule
  "^/rules-policy/regulations/1006/versions/",
]

# The center column of CFPB's page layout (no sidebar, no hero banner).
content_selectors = [".u-layout-grid__main"]
# Collapsed copies of the official interpretations (each has its own page, so keeping both
# would duplicate text) and the "current version / search this regulation" bar.
drop_selectors = [".inline-interpretation", ".regulation-meta"]
# Lettered paragraphs of the rule, e.g. "(b) Repeated or continuous telephone calls ...",
# so each section can be cited and deep-linked (url#b).
section_selectors = ["p.regdown-block.level-0"]

license = "U.S. Government work, public domain in the U.S. (17 U.S.C. 105)"

[[rules]]
match = "^/rules-policy/regulations/1006/$"
content_type = "regulation_overview"
tags = ["regulation f", "12 cfr 1006", "legal text"]

[[rules]]
match = "^/rules-policy/regulations/1006/interp-"
content_type = "official_interpretation"
tags = ["regulation f", "12 cfr 1006", "legal text"]

[[rules]]
match = "^/rules-policy/regulations/1006/[a-z]/$"
content_type = "regulation_appendix"
tags = ["regulation f", "12 cfr 1006", "legal text"]

[[rules]]
match = "^/rules-policy/regulations/1006/\\d+/$"
content_type = "regulation_section"
tags = ["regulation f", "12 cfr 1006", "legal text"]

[[rules]]
match = "^/ask-cfpb/"
content_type = "qa_answer"
tags = ["consumer guidance"]

[[rules]]
match = "^/consumer-tools/debt-collection/"
content_type = "guide"
tags = ["consumer guidance", "debt collection"]