audit: BreadcrumbList, robots policy, og:title, sitemap priorities

src/theme/DocBreadcrumbs/index.js

@@ -51,9 +51,14 @@ export default function DocBreadcrumbs() {
   const {siteConfig} = useDocusaurusContext();
   const {pathname} = useLocation();
 
-  if (!breadcrumbs) {
-    return null;
-  }
+  // LIVE-20 fix. Previously this component early-returned when
+  // useSidebarBreadcrumbs() returned null/undefined, which caused
+  // glossary and reference pages not in the sidebar config to ship
+  // with zero BreadcrumbList schema (audited 2026-04-14 on
+  // /docs/concepts/reference/glossary/idempotency/).
+  // Now we treat null/undefined as "no sidebar trail, emit Home + Docs
+  // schema anyway" so AI crawlers always get a hierarchy signal.
+  const sidebarTrail = Array.isArray(breadcrumbs) ? breadcrumbs : [];
 
   const toAbsoluteUrl = (baseUrl, url) => {
     if (!url) {
@@ -89,9 +94,9 @@ export default function DocBreadcrumbs() {
     }
   }
 
-  if (breadcrumbs.length > 0) {
-    breadcrumbs.forEach((crumb, index) => {
-      const isLast = index === breadcrumbs.length - 1;
+  if (sidebarTrail.length > 0) {
+    sidebarTrail.forEach((crumb, index) => {
+      const isLast = index === sidebarTrail.length - 1;
       const href =
         crumb.type === "category" && crumb.linkUnlisted
           ? undefined
@@ -130,35 +135,37 @@ export default function DocBreadcrumbs() {
           </script>
         </Head>
       )}
-      <nav
-        className={clsx(
-          ThemeClassNames.docs.docBreadcrumbs,
-          styles.breadcrumbsContainer
-        )}
-        aria-label={translate({
-          id: "theme.docs.breadcrumbs.navAriaLabel",
-          message: "Breadcrumbs",
-          description: "The ARIA label for the breadcrumbs",
-        })}
-      >
-        <ul className="breadcrumbs">
-          {homePageRoute && <HomeBreadcrumbItem />}
-          {breadcrumbs.map((item, idx) => {
-            const isLast = idx === breadcrumbs.length - 1;
-            const href =
-              item.type === "category" && item.linkUnlisted
-                ? undefined
-                : item.href;
-            return (
-              <BreadcrumbsItem key={idx} active={isLast}>
-                <BreadcrumbsItemLink href={href} isLast={isLast}>
-                  {item.label}
-                </BreadcrumbsItemLink>
-              </BreadcrumbsItem>
-            );
+      {sidebarTrail.length > 0 && (
+        <nav
+          className={clsx(
+            ThemeClassNames.docs.docBreadcrumbs,
+            styles.breadcrumbsContainer
+          )}
+          aria-label={translate({
+            id: "theme.docs.breadcrumbs.navAriaLabel",
+            message: "Breadcrumbs",
+            description: "The ARIA label for the breadcrumbs",
           })}
-        </ul>
-      </nav>
+        >
+          <ul className="breadcrumbs">
+            {homePageRoute && <HomeBreadcrumbItem />}
+            {sidebarTrail.map((item, idx) => {
+              const isLast = idx === sidebarTrail.length - 1;
+              const href =
+                item.type === "category" && item.linkUnlisted
+                  ? undefined
+                  : item.href;
+              return (
+                <BreadcrumbsItem key={idx} active={isLast}>
+                  <BreadcrumbsItemLink href={href} isLast={isLast}>
+                    {item.label}
+                  </BreadcrumbsItemLink>
+                </BreadcrumbsItem>
+              );
+            })}
+          </ul>
+        </nav>
+      )}
     </>
   );
 }

static/robots.txt

@@ -1,12 +1,108 @@
-# Block specific bot
+# Keploy docs robots.txt
+# Policy: allow AI search/answer engines, block training-only crawlers,
+# block Bytespider. Search bots drive visibility in ChatGPT, Claude,
+# Perplexity, Copilot, Gemini answers. Training bots feed future model
+# weights and provide nothing back.
+# Reference: Speedscale / Katalon / Testsigma split policy (2026 competitor audit)
+
+# =============================================================================
+# ALLOW — AI search / answer engines
+# =============================================================================
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: Claude-SearchBot
+Allow: /
+
+User-agent: Claude-User
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Perplexity-User
+Allow: /
+
+User-agent: Gemini-Deep-Research
+Allow: /
+
+User-agent: GoogleOther
+Allow: /
+
+User-agent: Applebot
+Allow: /
+
+User-agent: DuckAssistBot
+Allow: /
+
+User-agent: Amazonbot
+Allow: /
+
+# =============================================================================
+# DISALLOW — Training-only crawlers
+# =============================================================================
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: cohere-ai
+Disallow: /
+
+User-agent: Diffbot
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: ImagesiftBot
+Disallow: /
+
+# Always-block scraper
 User-agent: Bytespider
 Disallow: /
 
-# Default rules — apply to all crawlers including AI bots
+# =============================================================================
+# DEFAULT — Googlebot, Bingbot, and all other crawlers
+# =============================================================================
+
 User-agent: *
 Allow: /
 Crawl-delay: 5
 Disallow: /cgi-bin/
 
+# Block unmaintained legacy doc versions (already set via noindex + canonical,
+# belt-and-braces for crawlers that ignore those signals).
+Disallow: /docs/1.0.0/
+Disallow: /docs/2.0.0/
+Disallow: /docs/3.0.0/
+
+# =============================================================================
 # Sitemap
+# =============================================================================
+
 Sitemap: https://keploy.io/docs/sitemap.xml