From 0ea321a009143d5985d0d1cbb5f89b696d5f2457 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Mon, 22 Dec 2025 15:54:13 +1100 Subject: [PATCH 1/2] Add sitemap to robots.txt --- apps/page/pages/api/robots.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/page/pages/api/robots.ts b/apps/page/pages/api/robots.ts index 1c45980..9f6dc98 100644 --- a/apps/page/pages/api/robots.ts +++ b/apps/page/pages/api/robots.ts @@ -1,10 +1,12 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { fetchRenderData, translateHostToPageIdentifier } from "../../lib/data"; -const ALLOW = `User-agent: * +const getAllowRobots = (hostname: string) => `User-agent: * Allow: / Disallow: /api/ -Disallow: /_next/`; +Disallow: /_next/ + +Sitemap: https://${hostname}/sitemap.xml`; const DISALLOW = `User-agent: * Disallow: /`; @@ -27,7 +29,7 @@ async function handler( if (!page) throw new Error("Page not found"); if (!settings) throw new Error("Settings not found"); - res.status(200).send(settings?.hide_search_engine ? DISALLOW : ALLOW); + res.status(200).send(settings?.hide_search_engine ? DISALLOW : getAllowRobots(hostname)); } catch (e: unknown) { console.log("robots.txt [Error]", e); res.status(200).send(DISALLOW); From f0f6e070491bc497aff9b71d804df2a85df6d380 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Mon, 22 Dec 2025 16:00:47 +1100 Subject: [PATCH 2/2] Build page url --- apps/page/pages/api/robots.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/page/pages/api/robots.ts b/apps/page/pages/api/robots.ts index 9f6dc98..a2c4668 100644 --- a/apps/page/pages/api/robots.ts +++ b/apps/page/pages/api/robots.ts @@ -18,6 +18,7 @@ async function handler( const hostname = String(req?.headers?.host); const { domain, page: url_slug } = translateHostToPageIdentifier(hostname); + const pageUrl = domain ?? `${url_slug}.changes.page`; res.setHeader("Content-Type", "text/plain; charset=UTF-8"); @@ -29,7 +30,9 @@ async function handler( if (!page) throw new Error("Page not found"); if (!settings) throw new Error("Settings not found"); - res.status(200).send(settings?.hide_search_engine ? DISALLOW : getAllowRobots(hostname)); + res + .status(200) + .send(settings?.hide_search_engine ? DISALLOW : getAllowRobots(pageUrl)); } catch (e: unknown) { console.log("robots.txt [Error]", e); res.status(200).send(DISALLOW);