diff --git a/apps/page/pages/api/robots.ts b/apps/page/pages/api/robots.ts index dc66844..1c45980 100644 --- a/apps/page/pages/api/robots.ts +++ b/apps/page/pages/api/robots.ts @@ -2,14 +2,16 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { fetchRenderData, translateHostToPageIdentifier } from "../../lib/data"; const ALLOW = `User-agent: * -Allow: /`; +Allow: / +Disallow: /api/ +Disallow: /_next/`; const DISALLOW = `User-agent: * Disallow: /`; async function handler( req: NextApiRequest, - res: NextApiResponse + res: NextApiResponse, ) { const hostname = String(req?.headers?.host); @@ -19,7 +21,7 @@ async function handler( try { const { page, settings } = await fetchRenderData( - String(domain || url_slug) + String(domain || url_slug), ); if (!page) throw new Error("Page not found"); diff --git a/apps/web/next-sitemap.config.js b/apps/web/next-sitemap.config.js index d6b4f91..8c8930e 100644 --- a/apps/web/next-sitemap.config.js +++ b/apps/web/next-sitemap.config.js @@ -4,4 +4,21 @@ module.exports = { generateRobotsTxt: true, // (optional) // ...other options exclude: ["/pages", "/pages/*", "/account/*"], + robotsTxtOptions: { + policies: [ + { + userAgent: "*", + allow: "/", + disallow: [ + "/pages", + "/pages/*", + "/account", + "/account/*", + "/api/*", + "/login", + "/_next/*", + ], + }, + ], + }, };