# Product Registry — LLMs & Crawler Guidance
# Location: /llms.txt
# Version: 1.0
# Last-Updated: 2025-09-16
# Base-URL: https://productregistry.org/

# --- Identification & Contact ---
Agent-Policy: welcome
Contact: support@productregistry.org
Documentation: https://productregistry.org/         # human overview
Robots-Txt: https://productregistry.org/robots.txt  # must also be respected

# --- Discovery ---
Sitemap-Index: https://productregistry.org/sitemap.xml
Sitemap: https://productregistry.org/sitemaps/products.xml
Sitemap: https://productregistry.org/sitemaps/merchants.xml
WellKnown: https://productregistry.org/.well-known/

# --- Canonical URL Patterns ---
# Public, long-lived pages with stable links:
Pattern: /product/{product_id}         ; Type=ProductPage ; Canonical=true
Pattern: /merchant/{merchant_domain}   ; Type=MerchantListing ; Canonical=true
Pattern: /                             ; Type=Index

# --- Access Rules ---
Allow: /$
Allow: /product/
Allow: /merchant/
Allow: /sitemap.xml
Allow: /sitemaps/
Allow: /.well-known/
Disallow: /api/

# --- Rate Limits & Concurrency ---
# Follow stricter of robots.txt and these hints:
Max-Requests-Per-Minute: 60
Max-Parallel-Requests: 4
Retry-After-429: 120 # seconds; exponential backoff advised
Prefetch: disallowed # do not prefetch linked pages aggressively

# --- Freshness & Caching ---
Honor-Headers: ETag, Last-Modified, Cache-Control
Default-Cache-TTL: 86400 # seconds (24h) if headers missing
Change-Frequency:
  /product/*: daily
  /merchant/*: weekly
  /: monthly

# --- Content Semantics ---
Identifiers:
  GTIN: supported
  UPC: supported
  EAN: supported
  SKU: supported
  ModelNumber: supported
Product-Identity:
  Key: product_id                 # stable, opaque registry ID
  Canonical-Param: product_id
Merchant-Identity:
  Key: merchant_domain            # FQDN, case-insensitive

# --- Structured Data ---
# Pages embed JSON-LD via <script type="application/ld+json">.
JSON-LD:
  /product/*:
    Types: ["Product"]
    Includes: ["name","description","brand","sku","gtin","offers","image","category","url"]
  /merchant/*:
    Types: ["ItemList","BreadcrumbList"]
    Items-Link: registry_product_url

# --- Pagination Rules ---
Pagination:
  MerchantList:
    Method: keyset
    Param: after                  # exclusive cursor for merchant_domain
    Page-Size: 50                 # typical; may vary via ?per_page=
    Next-Rel: link[rel=next] or JSON-LD ItemList
  MerchantProducts:
    Method: page/per_page
    Params: page, per_page, sort  # sort in {title,price,updated_at,-updated_at}

# --- Canonicality & Duplicates ---
Canonical-Header: Link <https://productregistry.org...>; rel="canonical"
Duplicate-Detection:
  Prefer: canonical URL over UTM variants
  Ignore-Params: ["utm_source","utm_medium","utm_campaign","utm_term","utm_content","ref","src"]

# --- Request Etiquette ---
Accept-Languages: ["en"]          # more may be added in future
Accept-Encoding: gzip, br
User-Agent-Identification: required   # set to org+crawler name and contact URL
Respect-Delays: true

# --- Usage & Attribution ---
Attribution: optional
Cache-And-Reuse: permitted         # you may cache and reuse extracted facts
Prohibited: model-training on PII, account enumeration, rate abuse

# --- Error Handling ---
HTTP-429: backoff-exponential
HTTP-5xx: retry with jitter
Timeout-Sec: 20
Max-Retry: 3

# --- Quick Links (machine friendly) ---
Index: https://productregistry.org/
Sitemaps:
  - https://productregistry.org/sitemap.xml
  - https://productregistry.org/sitemaps/products.xml
Robots: https://productregistry.org/robots.txt

# --- Examples ---
# 1) Discover all products:
#    GET https://productregistry.org/sitemaps/products.xml
# 2) Fetch a product page + JSON-LD:
#    GET https://productregistry.org/product/{product_id}
#    Extract <script type="application/ld+json"> … </script>
# 3) Iterate merchants with keyset pagination:
#    GET https://productregistry.org/merchant
#    If  "Link: <.../merchant?after={cursor}>; rel=next" present, continue.
# 4) De-duplicate:
#    Normalize query by removing UTM parameters listed above.

# --- Future Expansion (reserved keys) ---
Reserved:
  Feed-JSON: https://productregistry.org/feeds/products.json
  GraphQL: https://productregistry.org/api/graphql
  Bulk-Export: https://productregistry.org/exports/products.ndjson