From b288bca6b84e9081db5eebddb876f3d079b7fb47 Mon Sep 17 00:00:00 2001
From: gbuomprisco <giancarlopsk@gmail.com>
Date: Thu, 20 Jun 2024 19:28:47 +0800
Subject: [PATCH] Update documentation loader limit and optimize sitemap
 generation

The documentation loader has been updated to limit the number of items to 500 from the previous limit of 'Number.MAX_SAFE_INTEGER'. Additionally, the handling of URL generation and item retrieval in 'server-sitemap.xml/route.ts' has been refactored for more effective sitemap generation. A new 'robots.ts' file was created to set up default rules for web robots accessing the site.
---
 .../docs/_components/docs-navigation.tsx      |  2 +-
 .../docs/_lib/server/docs.loader.ts           |  2 +-
 apps/web/app/robots.ts                        | 13 ++++++++++++
 apps/web/app/server-sitemap.xml/route.ts      | 20 +++++++++----------
 4 files changed, 25 insertions(+), 12 deletions(-)
 create mode 100644 apps/web/app/robots.ts

diff --git a/apps/web/app/(marketing)/docs/_components/docs-navigation.tsx b/apps/web/app/(marketing)/docs/_components/docs-navigation.tsx
index 4e572e4b9..99385ac0b 100644
--- a/apps/web/app/(marketing)/docs/_components/docs-navigation.tsx
+++ b/apps/web/app/(marketing)/docs/_components/docs-navigation.tsx
@@ -54,7 +54,7 @@ function Node({
   activePath: string;
 }) {
   const pathPrefix = `/docs`;
-  const url = `${pathPrefix}/${node.url}`;
+  const url = `${pathPrefix}/${node.slug}`;
 
   return (
     <>
diff --git a/apps/web/app/(marketing)/docs/_lib/server/docs.loader.ts b/apps/web/app/(marketing)/docs/_lib/server/docs.loader.ts
index cc59c4c3c..88cec4ced 100644
--- a/apps/web/app/(marketing)/docs/_lib/server/docs.loader.ts
+++ b/apps/web/app/(marketing)/docs/_lib/server/docs.loader.ts
@@ -15,7 +15,7 @@ async function docsLoader(language: string | undefined) {
   const { items: pages } = await cms.getContentItems({
     collection: 'documentation',
     language,
-    limit: Number.MAX_SAFE_INTEGER,
+    limit: 500
   });
 
   return pages;
diff --git a/apps/web/app/robots.ts b/apps/web/app/robots.ts
new file mode 100644
index 000000000..495a8199d
--- /dev/null
+++ b/apps/web/app/robots.ts
@@ -0,0 +1,13 @@
+import { MetadataRoute } from 'next';
+
+import appConfig from '~/config/app.config';
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: '*',
+      allow: '/',
+    },
+    sitemap: `${appConfig.url}/server-sitemap.xml`,
+  };
+}
diff --git a/apps/web/app/server-sitemap.xml/route.ts b/apps/web/app/server-sitemap.xml/route.ts
index 08ea71960..2c896d912 100644
--- a/apps/web/app/server-sitemap.xml/route.ts
+++ b/apps/web/app/server-sitemap.xml/route.ts
@@ -5,13 +5,12 @@ import { createCmsClient } from '@kit/cms';
 import appConfig from '~/config/app.config';
 
 export async function GET() {
-  const urls = getSiteUrls();
-
-  const items = await getAllItems();
+  const paths = getPaths();
+  const contentItems = await getContentItems();
 
   return getServerSideSitemap([
-    ...urls,
-    ...items.map((path) => {
+    ...paths,
+    ...contentItems.map((path) => {
       return {
         loc: new URL(path, appConfig.url).href,
         lastmod: new Date().toISOString(),
@@ -20,8 +19,8 @@ export async function GET() {
   ]);
 }
 
-function getSiteUrls() {
-  const urls = [
+function getPaths() {
+  const paths = [
     '/',
     '/faq',
     '/blog',
@@ -31,17 +30,18 @@ function getSiteUrls() {
     '/cookie-policy',
     '/terms-of-service',
     '/privacy-policy',
+    // add more paths here
   ];
 
-  return urls.map((url) => {
+  return paths.map((path) => {
     return {
-      loc: new URL(url, appConfig.url).href,
+      loc: new URL(path, appConfig.url).href,
       lastmod: new Date().toISOString(),
     };
   });
 }
 
-async function getAllItems() {
+async function getContentItems() {
   const client = await createCmsClient();
 
   const posts = client