#!/usr/bin/env python3
"""
ai_audit.py - free "Can ChatGPT, Claude and Perplexity read your site?" checker.

Standard library only (Python 3.8+). No API keys, no installs.
Source and how-to: https://astorventuresllc.com/ai-visibility-audit
Free to use, copy and modify. Provided as is, without warranty.

Usage:
    python3 ai_audit.py https://example.com
    python3 ai_audit.py https://example.com --out ./reports --timeout 20

What it checks for ONE page (the URL you pass) and its domain:
  1. robots.txt rules for each AI / search crawler (GPTBot, OAI-SearchBot,
     ChatGPT-User, ClaudeBot, Claude-User, Claude-SearchBot, PerplexityBot,
     Perplexity-User, Google-Extended, Googlebot, Bingbot, CCBot, ...)
  2. A live fetch of the page with each crawler's user-agent string
     (status code, bytes, blocked or not)
  3. How much text is in the raw HTML (what a crawler that does not run
     JavaScript sees) and whether the main content is there
  4. JSON-LD structured data types, title, meta description, canonical,
     headings, lang, meta robots / X-Robots-Tag
  5. sitemap.xml and llms.txt (llms.txt is informational only: there is
     no public evidence that the major AI crawlers use it)
  6. A 0-100 score with plain-English findings

Outputs <host>.json and <host>.md in the output folder.

Limits (read these before trusting the score):
  * The fetch comes from YOUR IP with a copied user-agent string. Real AI
    crawlers come from their own published IP ranges. A firewall or CDN
    (e.g. Cloudflare bot rules) can treat the real bot differently from
    this test, in either direction. A 200 here is a good sign, not proof.
  * Google-Extended and Applebot-Extended are robots.txt tokens only; they
    never fetch pages, so only their robots.txt rule is checked.
  * Only the URL you pass is analysed for content, not the whole site.
  * The score measures technical readability. It does not and cannot
    guarantee that any AI assistant will cite or recommend the site.
"""
import argparse
import datetime as _dt
import gzip
import json
import os
import re
import ssl
import sys
import urllib.error
import urllib.parse
import urllib.request
import zlib
from html.parser import HTMLParser

VERSION = "1.0.0"
MAX_BYTES = 5_000_000

# User-agent strings follow each vendor's published examples where one exists
# (OpenAI, Perplexity, Google, Bing docs, checked 2026-10-24). Versions change over time.
# name, robots.txt token, user-agent string (None = robots token only), role, weight
AGENTS = [
    ("GPTBot", "GPTBot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
     "OpenAI training crawler", 1.0),
    ("OAI-SearchBot", "OAI-SearchBot",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
     "ChatGPT search index", 2.0),
    ("ChatGPT-User", "ChatGPT-User",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
     "ChatGPT fetching a page a user asked about", 2.0),
    ("ClaudeBot", "ClaudeBot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
     "Anthropic training crawler", 1.0),
    ("Claude-SearchBot", "Claude-SearchBot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com)",
     "Claude search index", 1.5),
    ("Claude-User", "Claude-User",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)",
     "Claude fetching a page a user asked about", 1.5),
    ("PerplexityBot", "PerplexityBot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
     "Perplexity search index", 2.0),
    ("Perplexity-User", "Perplexity-User",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
     "Perplexity fetching a page a user asked about", 1.5),
    ("Google-Extended", "Google-Extended", None,
     "robots.txt token: Gemini training/grounding opt-out (never fetches)", 0.5),
    ("Googlebot", "Googlebot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/120.0.0.0 Safari/537.36",
     "Google Search (also feeds AI Overviews)", 2.0),
    ("Bingbot", "bingbot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/120.0.0.0 Safari/537.36",
     "Bing index (also used by Copilot and others)", 2.0),
    ("Applebot-Extended", "Applebot-Extended", None,
     "robots.txt token: Apple AI training opt-out (never fetches)", 0.25),
    ("CCBot", "CCBot", "CCBot/2.0 (https://commoncrawl.org/faq/)",
     "Common Crawl (open dataset used by many AI models)", 0.5),
    ("Meta-ExternalAgent", "meta-externalagent",
     "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
     "Meta AI crawler", 0.25),
    ("Amazonbot", "Amazonbot",
     "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36",
     "Amazon (Alexa / Rufus)", 0.25),
    ("DuckAssistBot", "DuckAssistBot",
     "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
     "DuckDuckGo AI answers", 0.25),
]
BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
              "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36")
JS_SHELL_IDS = ("root", "app", "__next", "__nuxt", "svelte", "q-app", "___gatsby")
QUESTION_WORDS = ("how", "what", "why", "when", "where", "which", "who", "can", "does",
                  "do", "is", "are", "should", "will")


# --------------------------------------------------------------------------- HTTP
def fetch(url, ua, timeout=20, method="GET", max_bytes=MAX_BYTES):
    """Return dict(status, final_url, headers, body(bytes), error, redirects)."""
    req = urllib.request.Request(url, method=method, headers={
        "User-Agent": ua,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.7",
        "Accept-Language": "en-US,en;q=0.8",
        "Accept-Encoding": "gzip, deflate",
    })
    ctx = ssl.create_default_context()
    out = {"status": None, "final_url": url, "headers": {}, "body": b"", "error": None}
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
            out["status"] = r.status
            out["final_url"] = r.geturl()
            out["headers"] = {k.lower(): v for k, v in r.headers.items()}
            out["body"] = r.read(max_bytes) if method == "GET" else b""
    except urllib.error.HTTPError as e:
        out["status"] = e.code
        out["final_url"] = e.geturl() or url
        out["headers"] = {k.lower(): v for k, v in (e.headers or {}).items()}
        try:
            out["body"] = e.read(max_bytes)
        except Exception:
            pass
    except Exception as e:  # DNS, TLS, timeout, connection reset ...
        out["error"] = f"{type(e).__name__}: {e}"
        return out
    enc = out["headers"].get("content-encoding", "").lower()
    try:
        if "gzip" in enc:
            out["body"] = gzip.decompress(out["body"])
        elif "deflate" in enc:
            out["body"] = zlib.decompress(out["body"])
    except Exception:
        pass
    return out


def decode(body, headers):
    m = re.search(r"charset=([\w-]+)", headers.get("content-type", ""), re.I)
    for cs in ([m.group(1)] if m else []) + ["utf-8", "latin-1"]:
        try:
            return body.decode(cs)
        except Exception:
            continue
    return body.decode("utf-8", "replace")


# --------------------------------------------------------------------- robots.txt
def parse_robots(text):
    """Groups as list of (agents[list], rules[list of (allow:bool, path)]); sitemaps list."""
    groups, sitemaps = [], []
    cur_agents, cur_rules, last_was_agent = [], [], False
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        key, val = line.split(":", 1)
        key, val = key.strip().lower(), val.strip()
        if key == "user-agent":
            if not last_was_agent and cur_agents:
                groups.append((cur_agents, cur_rules))
                cur_agents, cur_rules = [], []
            cur_agents.append(val.lower())
            last_was_agent = True
        elif key in ("allow", "disallow"):
            last_was_agent = False
            if cur_agents:
                cur_rules.append((key == "allow", val))
        elif key == "sitemap":
            sitemaps.append(val)
        else:
            last_was_agent = False
    if cur_agents:
        groups.append((cur_agents, cur_rules))
    return groups, sitemaps


def _rule_regex(path):
    anchored = path.endswith("$")
    if anchored:
        path = path[:-1]
    rx = "".join(".*" if c == "*" else re.escape(c) for c in path)
    return re.compile("^" + rx + ("$" if anchored else ""))


def robots_verdict(groups, token, path):
    """Google-style: pick the group(s) whose user-agent best matches the token
    (exact product token, case-insensitive), else '*'. Longest matching rule
    wins; on a tie Allow wins. Returns (allowed, matched_group, matched_rule)."""
    token = token.lower()
    rules, matched = [], None
    for agents, grules in groups:
        if token in agents:
            rules.extend(grules)
            matched = token
    if matched is None:
        for agents, grules in groups:
            if "*" in agents:
                rules.extend(grules)
                matched = "*"
    if matched is None:
        return True, None, None
    best = None  # (length, allow, rule)
    for allow, rpath in rules:
        if rpath == "":
            continue  # "Disallow:" (empty) = allow everything
        if _rule_regex(rpath).match(path):
            ln = len(rpath)
            if best is None or ln > best[0] or (ln == best[0] and allow and not best[1]):
                best = (ln, allow, ("Allow: " if allow else "Disallow: ") + rpath)
    if best is None:
        return True, matched, None
    return best[1], matched, best[2]


# ------------------------------------------------------------------- HTML parsing
class PageParser(HTMLParser):
    SKIP = {"script", "style", "noscript", "template", "svg", "head"}

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.skip_depth = 0
        self.stack = []
        self.text = []
        self.main_text = []
        self.in_main = 0
        self.title = None
        self._in_title = False
        self.metas = []
        self.links = []
        self.headings = []
        self._h = None
        self.jsonld = []
        self._in_jsonld = False
        self._buf = []
        self.html_lang = None
        self.itemtypes = []
        self.shell_ids = []
        self.script_count = 0
        self.a_count = 0
        self.noscript_msgs = []
        self._in_noscript = False

    def handle_starttag(self, tag, attrs):
        a = {k.lower(): (v or "") for k, v in attrs}
        if tag == "html":
            self.html_lang = a.get("lang")
        if tag == "title" and self.title is None:
            self._in_title = True
        if tag == "meta":
            self.metas.append(a)
        if tag == "link":
            self.links.append(a)
        if tag == "a":
            self.a_count += 1
        if "itemtype" in a:
            self.itemtypes.append(a["itemtype"])
        if a.get("id", "").lower() in JS_SHELL_IDS:
            self.shell_ids.append(a["id"])
        if tag == "script":
            self.script_count += 1
            if "ld+json" in a.get("type", "").lower():
                self._in_jsonld, self._buf = True, []
        if tag == "noscript":
            self._in_noscript = True
        if tag in self.SKIP:
            self.skip_depth += 1
        if tag in ("main", "article") or a.get("role") == "main":
            self.in_main += 1
            self.stack.append(("main", tag))
        if re.fullmatch(r"h[1-6]", tag):
            self._h = [tag, []]

    def handle_endtag(self, tag):
        if tag == "title":
            self._in_title = False
        if tag == "script" and self._in_jsonld:
            self.jsonld.append("".join(self._buf))
            self._in_jsonld = False
        if tag == "noscript":
            self._in_noscript = False
        if tag in self.SKIP and self.skip_depth:
            self.skip_depth -= 1
        if tag in ("main", "article") and self.stack and self.stack[-1][1] == tag:
            self.stack.pop()
            self.in_main = max(0, self.in_main - 1)
        if self._h and tag == self._h[0]:
            self.headings.append((self._h[0], " ".join("".join(self._h[1]).split())))
            self._h = None

    def handle_data(self, data):
        if self._in_jsonld:
            self._buf.append(data)
            return
        if self._in_title:
            self.title = (self.title or "") + data
        if self._in_noscript and data.strip():
            self.noscript_msgs.append(data.strip())
        if self.skip_depth:
            return
        self.text.append(data)
        if self.in_main:
            self.main_text.append(data)
        if self._h:
            self._h[1].append(data)


def words(s):
    return re.findall(r"[A-Za-z\u00C0-\u024F0-9][\w'\u00C0-\u024F-]*", s)


def jsonld_types(blobs):
    types, errors, has = [], 0, {}
    def walk(o):
        if isinstance(o, dict):
            t = o.get("@type")
            if isinstance(t, str):
                types.append(t)
            elif isinstance(t, list):
                types.extend(x for x in t if isinstance(x, str))
            for k in ("datePublished", "dateModified", "author", "publisher", "sameAs"):
                if k in o:
                    has[k] = True
            for v in o.values():
                walk(v)
        elif isinstance(o, list):
            for v in o:
                walk(v)
    for b in blobs:
        try:
            walk(json.loads(b.strip()))
        except Exception:
            errors += 1
    return types, errors, has


# --------------------------------------------------------------------- the audit
def audit(url, timeout=20):
    if not re.match(r"^https?://", url):
        url = "https://" + url
    p = urllib.parse.urlsplit(url)
    origin = f"{p.scheme}://{p.netloc}"
    path = (p.path or "/") + (("?" + p.query) if p.query else "")
    now = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0)
    R = {"tool": f"ai_audit.py {VERSION}", "url": url, "origin": origin,
         "checked_at_utc": now.isoformat().replace("+00:00", "Z")}

    # robots.txt
    rb = fetch(origin + "/robots.txt", BROWSER_UA, timeout)
    robots_text = decode(rb["body"], rb["headers"]) if rb["status"] == 200 else ""
    if robots_text.lstrip().lower().startswith(("<!doctype", "<html")):
        robots_text = ""  # an HTML page served at /robots.txt is not a robots file
    groups, sitemaps = parse_robots(robots_text)
    R["robots"] = {"status": rb["status"], "error": rb["error"], "bytes": len(rb["body"]),
                   "groups": len(groups), "sitemaps": sitemaps,
                   "named_agents": sorted({a for ag, _ in groups for a in ag if a != "*"})}

    # browser baseline
    base = fetch(url, BROWSER_UA, timeout)
    base_len = len(base["body"])
    R["browser"] = {"status": base["status"], "bytes": base_len, "final_url": base["final_url"],
                    "error": base["error"]}

    # per-agent
    agents = []
    for name, token, ua, role, w in AGENTS:
        allowed, grp, rule = robots_verdict(groups, token, path)
        row = {"agent": name, "role": role, "weight": w, "robots_token": token,
               "robots_allowed": allowed, "robots_group": grp, "robots_rule": rule}
        if ua:
            f = fetch(url, ua, timeout)
            b = f["body"]
            txt = decode(b, f["headers"]).lower() if b else ""
            challenge = any(s in txt for s in ("cf-chl", "challenge-platform", "just a moment...",
                                               "attention required", "captcha", "access denied"))
            ok = f["status"] == 200 and not challenge and (base_len == 0 or len(b) >= 0.3 * base_len)
            row.update({"http_status": f["status"], "bytes": len(b), "error": f["error"],
                        "challenge_page": challenge, "fetch_ok": ok,
                        "x_robots_tag": f["headers"].get("x-robots-tag")})
        else:
            row.update({"http_status": None, "bytes": None, "fetch_ok": None,
                        "note": "robots.txt token only; never fetches pages"})
        agents.append(row)
    R["agents"] = agents

    # page content (from the browser-UA fetch; raw HTML, no JavaScript run)
    html_text = decode(base["body"], base["headers"]) if base["body"] else ""
    pp = PageParser()
    try:
        pp.feed(html_text)
    except Exception:
        pass
    all_words = words(" ".join(pp.text))
    main_words = words(" ".join(pp.main_text))
    meta = {}
    for m in pp.metas:
        k = (m.get("name") or m.get("property") or "").lower()
        if k and "content" in m:
            meta.setdefault(k, m["content"])
    canon = next((l.get("href") for l in pp.links if "canonical" in l.get("rel", "").lower()), None)
    types, ld_err, ld_has = jsonld_types(pp.jsonld)
    h1 = [t for h, t in pp.headings if h == "h1"]
    qheads = [t for h, t in pp.headings if h in ("h2", "h3") and
              (t.strip().endswith("?") or t.strip().lower().split(" ")[0] in QUESTION_WORDS)]
    js_msg = any(re.search(r"enable javascript|requires javascript|need to enable", s, re.I)
                 for s in pp.noscript_msgs) or bool(re.search(
                 r"you need to enable javascript to run this app", html_text, re.I))
    content_type = base["headers"].get("content-type", "")
    R["content"] = {
        "content_type": content_type,
        "html_bytes": base_len,
        "raw_html_words": len(all_words),
        "main_or_article_present": bool(pp.main_text) or "<main" in html_text.lower(),
        "main_words": len(main_words),
        "js_app_shell_ids": pp.shell_ids,
        "noscript_enable_js_message": js_msg,
        "script_tags": pp.script_count,
        "links": pp.a_count,
        "sample_text": " ".join(all_words[:60]),
    }
    R["meta"] = {
        "title": " ".join((pp.title or "").split()) or None,
        "title_len": len(" ".join((pp.title or "").split())),
        "description": meta.get("description"),
        "description_len": len(meta.get("description") or ""),
        "canonical": canon,
        "meta_robots": meta.get("robots"),
        "x_robots_tag": base["headers"].get("x-robots-tag"),
        "html_lang": pp.html_lang,
        "og_title": meta.get("og:title"), "og_image": meta.get("og:image"),
        "h1": h1, "h2_count": sum(1 for h, _ in pp.headings if h == "h2"),
        "question_headings": qheads[:15], "question_heading_count": len(qheads),
    }
    R["schema"] = {"jsonld_blocks": len(pp.jsonld), "jsonld_parse_errors": ld_err,
                   "types": sorted(set(types)), "microdata_itemtypes": sorted(set(pp.itemtypes)),
                   "has_dates": bool(ld_has.get("datePublished") or ld_has.get("dateModified")),
                   "has_author_or_publisher": bool(ld_has.get("author") or ld_has.get("publisher")),
                   "has_sameAs": bool(ld_has.get("sameAs"))}

    # sitemap
    sm_urls = sitemaps or [origin + "/sitemap.xml"]
    sm = fetch(sm_urls[0], BROWSER_UA, timeout)
    sm_txt = decode(sm["body"], sm["headers"]) if sm["body"] else ""
    R["sitemap"] = {"url": sm_urls[0], "status": sm["status"], "declared_in_robots": bool(sitemaps),
                    "locs": len(re.findall(r"<loc>", sm_txt, re.I)),
                    "is_index": "<sitemapindex" in sm_txt.lower()}

    # llms.txt (informational)
    lt = fetch(origin + "/llms.txt", BROWSER_UA, timeout)
    lt_txt = decode(lt["body"], lt["headers"]) if lt["body"] else ""
    is_txt = lt["status"] == 200 and not lt_txt.lstrip().lower().startswith(("<!doctype", "<html"))
    R["llms_txt"] = {"status": lt["status"], "present": is_txt, "bytes": len(lt["body"]) if is_txt else 0,
                     "note": "Informational only. No major AI crawler has documented that it reads llms.txt; it does not affect the score."}

    score_and_findings(R)
    return R


# ------------------------------------------------------------------------ scoring
def score_and_findings(R):
    F = []  # (severity, area, text)
    S = {}
    ag = R["agents"]
    br = R["browser"]

    # 1. robots.txt access (25)
    tw = sum(a["weight"] for a in ag)
    allowed_w = sum(a["weight"] for a in ag if a["robots_allowed"])
    S["robots_access"] = round(25 * allowed_w / tw, 1)
    blocked = [a["agent"] for a in ag if not a["robots_allowed"]]
    if R["robots"]["status"] != 200:
        F.append(("info", "robots.txt", f"No robots.txt found (HTTP {R['robots']['status']}). That means every crawler is allowed, which is fine, but you have nowhere to list your sitemap."))
    if blocked:
        key = [b for b in blocked if b in ("OAI-SearchBot", "ChatGPT-User", "PerplexityBot", "Perplexity-User", "Claude-SearchBot", "Claude-User", "Googlebot", "Bingbot")]
        sev = "high" if key else "medium"
        F.append((sev, "robots.txt", f"robots.txt blocks this page for: {', '.join(blocked)}." + (
            " These include bots that decide whether you can appear in AI search answers." if key else
            " These are mostly training crawlers; blocking them is a business choice, not an error.")))
    else:
        F.append(("good", "robots.txt", f"robots.txt allows all {len(ag)} AI and search crawlers checked to read this page."))

    # 2. live fetch (15)
    fetchers = [a for a in ag if a["fetch_ok"] is not None]
    fw = sum(a["weight"] for a in fetchers)
    ok_w = sum(a["weight"] for a in fetchers if a["fetch_ok"])
    S["live_fetch"] = round(15 * ok_w / fw, 1) if fw else 0
    bad = [f"{a['agent']} (HTTP {a['http_status'] or a.get('error')}{', challenge page' if a.get('challenge_page') else ''})"
           for a in fetchers if not a["fetch_ok"]]
    if br["status"] != 200:
        F.append(("high", "fetch", f"The page itself returned HTTP {br['status'] or br['error']} to a normal browser user-agent."))
    if bad:
        F.append(("high", "fetch", "The server or firewall answered differently when the request looked like an AI bot: " + "; ".join(bad) + ". Check your CDN/WAF bot settings (for example Cloudflare's 'Block AI bots' or 'AI Labyrinth')."))
    elif fetchers:
        F.append(("good", "fetch", f"All {len(fetchers)} crawler user-agents got HTTP 200 and the full page (tested from our IP; real bots use their own IPs)."))

    # 3. no-JS content (20)
    c = R["content"]
    w = c["raw_html_words"]
    pts = 0
    pts += 10 if w >= 300 else 7 if w >= 150 else 3 if w >= 50 else 0
    pts += 5 if c["main_or_article_present"] and c["main_words"] >= 100 else 2 if c["main_or_article_present"] else 0
    shell = (w < 80 and (c["js_app_shell_ids"] or c["noscript_enable_js_message"]))
    pts += 0 if shell else 5
    S["no_js_content"] = pts
    if shell:
        F.append(("high", "content", f"The raw HTML has only {w} words and looks like a JavaScript app shell ({', '.join(c['js_app_shell_ids']) or 'enable-JavaScript message'}). Most AI crawlers do not run JavaScript, so they see an almost empty page. Server-side rendering or pre-rendering fixes this."))
    elif w < 150:
        F.append(("medium", "content", f"Only {w} words of text are in the raw HTML. AI crawlers that don't run JavaScript have little to quote. Put the key answer, prices and facts in the HTML itself."))
    else:
        F.append(("good", "content", f"{w:,} words of readable text are in the raw HTML, so crawlers that don't run JavaScript can still read the page."))
    if not c["main_or_article_present"]:
        F.append(("low", "content", "No <main> or <article> element. Adding one helps parsers find the main content and skip menus and footers."))

    # 4. structured data (15)
    sc = R["schema"]
    t = set(sc["types"])
    pts = 0
    if sc["jsonld_blocks"]:
        pts += 5
    if t & {"Organization", "LocalBusiness", "WebSite", "Person", "Store", "OnlineStore"} or any(x.endswith("Business") for x in t):
        pts += 3
    if t & {"Product", "Offer", "Service", "Article", "BlogPosting", "NewsArticle", "FAQPage", "HowTo", "QAPage", "Event", "Recipe", "SoftwareApplication", "VideoObject", "Course", "ItemList"}:
        pts += 4
    if sc["has_dates"] or sc["has_author_or_publisher"]:
        pts += 2
    if sc["jsonld_parse_errors"]:
        pts = max(0, pts - 3)
        F.append(("medium", "schema", f"{sc['jsonld_parse_errors']} JSON-LD block(s) are not valid JSON, so search engines ignore them."))
    if not sc["jsonld_blocks"] and sc["microdata_itemtypes"]:
        pts += 3
    if sc["has_sameAs"]:
        pts += 1
    S["structured_data"] = min(15, pts)
    if not sc["jsonld_blocks"] and not sc["microdata_itemtypes"]:
        F.append(("medium", "schema", "No structured data (JSON-LD) on this page. Add at least Organization plus the type that matches the page (Product, Article, FAQPage, LocalBusiness...)."))
    else:
        F.append(("good" if S["structured_data"] >= 10 else "low", "schema", "Structured data found: " + (", ".join(sorted(t)) or ", ".join(sc["microdata_itemtypes"])) + "."))
    if sc["jsonld_blocks"] and not sc["has_sameAs"]:
        F.append(("low", "schema", "No sameAs links in the schema. Linking your official social or directory profiles helps AI systems confirm who you are."))

    # 5. meta basics (15)
    m = R["meta"]
    pts = 0
    if m["title"] and 10 <= m["title_len"] <= 70:
        pts += 3
    elif m["title"]:
        pts += 2
        F.append(("low", "meta", f"Title is {m['title_len']} characters; 30-65 is the usual sweet spot so it isn't cut off."))
    else:
        F.append(("high", "meta", "No <title>."))
    if m["description"] and 50 <= m["description_len"] <= 170:
        pts += 3
    elif m["description"]:
        pts += 2
        F.append(("low", "meta", f"Meta description is {m['description_len']} characters; aim for about 120-160."))
    else:
        F.append(("medium", "meta", "No meta description. Search and AI snippets then guess one from the page."))
    if m["canonical"]:
        pts += 2
    else:
        F.append(("low", "meta", "No canonical link. Add <link rel=\"canonical\"> so duplicate URLs consolidate to one."))
    if len(m["h1"]) == 1:
        pts += 2
    elif not m["h1"]:
        F.append(("medium", "meta", "No H1 heading in the raw HTML."))
    else:
        pts += 1
        F.append(("low", "meta", f"{len(m['h1'])} H1 headings; one clear H1 is easier to parse."))
    if m["html_lang"]:
        pts += 1
    noindex = any("noindex" in (x or "").lower() for x in (m["meta_robots"], m["x_robots_tag"]))
    if noindex:
        F.append(("high", "meta", "The page has noindex (meta robots or X-Robots-Tag). Search engines and AI search indexes are told to drop it."))
    else:
        pts += 4
    S["meta_basics"] = pts

    # 6. sitemap (5)
    smp = R["sitemap"]
    pts = 0
    if smp["status"] == 200 and (smp["locs"] or smp["is_index"]):
        pts += 3
        if smp["declared_in_robots"]:
            pts += 2
        else:
            F.append(("low", "sitemap", "sitemap.xml exists but is not listed in robots.txt (add a 'Sitemap:' line)."))
    else:
        F.append(("medium", "sitemap", f"No usable sitemap at {smp['url']} (HTTP {smp['status']})."))
    S["sitemap"] = pts

    # 7. citation readiness (5)
    pts = 0
    if m["question_heading_count"] >= 2:
        pts += 2
    elif m["question_heading_count"] == 1:
        pts += 1
    if "FAQPage" in t or "QAPage" in t:
        pts += 1
    if sc["has_dates"]:
        pts += 1
    if w >= 300:
        pts += 1
    S["citation_readiness"] = pts
    if m["question_heading_count"] < 2:
        F.append(("low", "citations", "Few or no headings phrased as the questions buyers ask. AI answers often lift a short answer that sits right under a question heading."))
    if not sc["has_dates"]:
        F.append(("low", "citations", "No published/updated date in the schema. Freshness signals help assistants trust and cite a page."))

    # llms.txt (informational)
    if R["llms_txt"]["present"]:
        F.append(("info", "llms.txt", f"llms.txt is present ({R['llms_txt']['bytes']:,} bytes). Harmless, but there is no public evidence the major AI crawlers read it, so it is not scored."))
    else:
        F.append(("info", "llms.txt", "No llms.txt. Optional: there is no public evidence the major AI crawlers read it, so it is not scored."))

    total = round(sum(S.values()))
    R["score"] = {"total": total, "max": 100, "parts": S,
                  "part_max": {"robots_access": 25, "live_fetch": 15, "no_js_content": 20,
                               "structured_data": 15, "meta_basics": 15, "sitemap": 5,
                               "citation_readiness": 5},
                  "grade": "A" if total >= 90 else "B" if total >= 75 else "C" if total >= 60 else "D" if total >= 40 else "F"}
    order = {"high": 0, "medium": 1, "low": 2, "info": 3, "good": 4}
    R["findings"] = [{"severity": s, "area": a, "text": x} for s, a, x in sorted(F, key=lambda f: order[f[0]])]


# ----------------------------------------------------------------------- markdown
def to_markdown(R):
    s = R["score"]
    L = [f"# AI visibility audit: {R['url']}", "",
         f"Checked {R['checked_at_utc']} with `{R['tool']}` (free, standard-library Python).", "",
         f"## Score: {s['total']}/100 (grade {s['grade']})", "",
         "| Area | Points |", "|---|---|"]
    names = {"robots_access": "AI crawler access in robots.txt", "live_fetch": "Live fetch as each bot",
             "no_js_content": "Content visible without JavaScript", "structured_data": "Structured data (schema)",
             "meta_basics": "Title, description, canonical, H1, indexable", "sitemap": "Sitemap",
             "citation_readiness": "Citation readiness"}
    for k, v in s["parts"].items():
        L.append(f"| {names[k]} | {v} / {s['part_max'][k]} |")
    L += ["", "The score measures whether AI crawlers can technically reach and read the page. It does not guarantee that any assistant will cite or recommend the site.", "",
          "## Findings", ""]
    icon = {"high": "HIGH", "medium": "MEDIUM", "low": "LOW", "info": "INFO", "good": "OK"}
    for f in R["findings"]:
        L.append(f"- **{icon[f['severity']]}** ({f['area']}): {f['text']}")
    L += ["", "## Crawler by crawler", "",
          "| Crawler | What it does | robots.txt | Live fetch | Bytes |", "|---|---|---|---|---|"]
    for a in R["agents"]:
        rob = "allowed" if a["robots_allowed"] else f"BLOCKED ({a['robots_rule']})"
        if a["fetch_ok"] is None:
            live = "n/a (token only)"
        else:
            live = f"{a['http_status'] or a.get('error')}" + (" OK" if a["fetch_ok"] else " PROBLEM")
        L.append(f"| {a['agent']} | {a['role']} | {rob} | {live} | {a['bytes'] if a['bytes'] is not None else ''} |")
    c, m, sc = R["content"], R["meta"], R["schema"]
    L += ["", "## Page details", "",
          f"- Browser fetch: HTTP {R['browser']['status']}, {R['browser']['bytes']:,} bytes, final URL {R['browser']['final_url']}",
          f"- Words in raw HTML (no JavaScript): {c['raw_html_words']:,}; inside <main>/<article>: {c['main_words']:,}",
          f"- JS app shell markers: {', '.join(c['js_app_shell_ids']) or 'none'}; 'enable JavaScript' message: {'yes' if c['noscript_enable_js_message'] else 'no'}",
          f"- Title ({m['title_len']} chars): {m['title'] or '(none)'}",
          f"- Meta description ({m['description_len']} chars): {m['description'] or '(none)'}",
          f"- Canonical: {m['canonical'] or '(none)'}",
          f"- H1: {' | '.join(m['h1']) or '(none)'}; H2 count: {m['h2_count']}; question-style headings: {m['question_heading_count']}",
          f"- lang: {m['html_lang'] or '(none)'}; meta robots: {m['meta_robots'] or '(none)'}; X-Robots-Tag: {m['x_robots_tag'] or '(none)'}",
          f"- JSON-LD blocks: {sc['jsonld_blocks']} ({sc['jsonld_parse_errors']} invalid); types: {', '.join(sc['types']) or '(none)'}",
          f"- robots.txt: HTTP {R['robots']['status']}, {R['robots']['groups']} group(s), named agents: {', '.join(R['robots']['named_agents']) or 'none'}",
          f"- Sitemap: {R['sitemap']['url']} HTTP {R['sitemap']['status']}, {R['sitemap']['locs']} URLs, listed in robots.txt: {'yes' if R['sitemap']['declared_in_robots'] else 'no'}",
          f"- llms.txt: {'present, ' + format(R['llms_txt']['bytes'], ',') + ' bytes' if R['llms_txt']['present'] else 'not present'} (informational only, not scored)",
          "", "## Limits", "",
          "- Requests came from one IP with copied user-agent strings. Real crawlers use their own IP ranges, and CDNs or firewalls may treat them differently.",
          "- Only this one URL was analysed for content. Other pages can differ.",
          "- Google-Extended and Applebot-Extended are robots.txt tokens only; they never fetch pages.",
          "- llms.txt is reported but not scored: no major AI crawler has documented that it uses it.", ""]
    return "\n".join(L)


def main():
    ap = argparse.ArgumentParser(description="Check whether AI crawlers can reach and read a web page.")
    ap.add_argument("url")
    ap.add_argument("--out", default=".", help="output folder (default: current folder)")
    ap.add_argument("--timeout", type=int, default=20)
    ap.add_argument("--quiet", action="store_true")
    a = ap.parse_args()
    R = audit(a.url, a.timeout)
    os.makedirs(a.out, exist_ok=True)
    host = urllib.parse.urlsplit(R["url"]).netloc.replace(":", "_")
    jp, mp = os.path.join(a.out, host + ".json"), os.path.join(a.out, host + ".md")
    with open(jp, "w", encoding="utf-8") as f:
        json.dump(R, f, indent=2, ensure_ascii=False)
    with open(mp, "w", encoding="utf-8") as f:
        f.write(to_markdown(R))
    if not a.quiet:
        print(f"{R['url']}: {R['score']['total']}/100 (grade {R['score']['grade']})")
        for x in R["findings"]:
            if x["severity"] in ("high", "medium"):
                print(f"  [{x['severity']}] {x['text']}")
        print(f"Saved {jp} and {mp}")


if __name__ == "__main__":
    main()