Update documentation loader limit and optimize sitemap generation

The documentation loader has been updated to limit the number of items to 500 from the previous limit of 'Number.MAX_SAFE_INTEGER'. Additionally, the handling of URL generation and item retrieval in 'server-sitemap.xml/route.ts' has been refactored for more effective sitemap generation. A new 'robots.ts' file was created to set up default rules for web robots accessing the site.

apps/web/app/(marketing)/docs/_components/docs-navigation.tsx

@@ -54,7 +54,7 @@ function Node({
   activePath: string;
 }) {
   const pathPrefix = `/docs`;
-  const url = `${pathPrefix}/${node.url}`;
+  const url = `${pathPrefix}/${node.slug}`;
 
   return (
     <>

apps/web/app/(marketing)/docs/_lib/server/docs.loader.ts

@@ -15,7 +15,7 @@ async function docsLoader(language: string | undefined) {
   const { items: pages } = await cms.getContentItems({
     collection: 'documentation',
     language,
-    limit: Number.MAX_SAFE_INTEGER,
+    limit: 500
   });
 
   return pages;

apps/web/app/robots.ts

@@ -0,0 +1,13 @@
+import { MetadataRoute } from 'next';
+
+import appConfig from '~/config/app.config';
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: '*',
+      allow: '/',
+    },
+    sitemap: `${appConfig.url}/server-sitemap.xml`,
+  };
+}

apps/web/app/server-sitemap.xml/route.ts

@@ -5,13 +5,12 @@ import { createCmsClient } from '@kit/cms';
 import appConfig from '~/config/app.config';
 
 export async function GET() {
-  const urls = getSiteUrls();
-
-  const items = await getAllItems();
+  const paths = getPaths();
+  const contentItems = await getContentItems();
 
   return getServerSideSitemap([
-    ...urls,
-    ...items.map((path) => {
+    ...paths,
+    ...contentItems.map((path) => {
       return {
         loc: new URL(path, appConfig.url).href,
         lastmod: new Date().toISOString(),
@@ -20,8 +19,8 @@ export async function GET() {
   ]);
 }
 
-function getSiteUrls() {
-  const urls = [
+function getPaths() {
+  const paths = [
     '/',
     '/faq',
     '/blog',
@@ -31,17 +30,18 @@ function getSiteUrls() {
     '/cookie-policy',
     '/terms-of-service',
     '/privacy-policy',
+    // add more paths here
   ];
 
-  return urls.map((url) => {
+  return paths.map((path) => {
     return {
-      loc: new URL(url, appConfig.url).href,
+      loc: new URL(path, appConfig.url).href,
       lastmod: new Date().toISOString(),
     };
   });
 }
 
-async function getAllItems() {
+async function getContentItems() {
   const client = await createCmsClient();
 
   const posts = client