Implementing llms.txt in Next.js 15 with Sanity CMS

Last month I needed to ship llms.txt support for a Next.js 15 blog backed by Sanity. The goal was simple: every post should have a Markdown twin, and llms.txt should list those twins so LLM crawlers could digest them without scraping HTML. This guide walks through the exact implementation I put into production.

I’ll assume you already have a Next.js App Router project connected to Sanity, and that blog posts store Markdown in markdownContent. We’re going to:

1. Expose Markdown-ready posts from Sanity via the Next cache layer.
2. Serve /blog/md/{slug} as static Markdown.
3. Link the Markdown twin from the HTML article.
4. Generate /llms.txt.
5. Expand the sitemap and robots directives.

1. Cache Markdown-capable posts

Next.js ships with unstable_cache for memoising expensive fetches. Start by creating a dedicated Sanity query and cache helper that flags posts with populated Markdown.

// File: src/lib/sanity/queries/queries.ts
export const MARKDOWN_POSTS_QUERY = defineQuery(`*[_type == "post" && defined(slug.current)] | order(publishedAt desc) {
  _id,
  title,
  subtitle,
  metaDescription,
  slug,
  publishedAt,
  dateModified,
  _updatedAt,
  excerpt,
  "hasMarkdown": defined(markdownContent) && markdownContent != "",
  "categories": categories[]->{
    title,
    slug
  },
  "primaryCategory": categories[0]->{
    title,
    slug
  }
}`)

Pulling only the fields llms.txt needs keeps the resulting payload under Next.js’ 2 MB cache limit.

// File: src/lib/sanity/post-cache.ts
export interface MarkdownPostSummary {
  _id: string
  title?: string | null
  subtitle?: string | null
  metaDescription?: string | null
  excerpt?: string | null
  slug?: { current?: string | null } | string | null
  publishedAt?: string | null
  dateModified?: string | null
  _updatedAt?: string | null
  hasMarkdown?: boolean | null
  categories?: Array<{
    title?: string | null
    slug?: {
      current?: string | null
    } | string | null
  }> | null
  primaryCategory?: {
    title?: string | null
    slug?: {
      current?: string | null
    } | string | null
  } | null
}

function resolveSlugField(
  slug: string | { current?: string | null } | null | undefined
): string | null {
  if (!slug) {
    return null
  }

  if (typeof slug === 'string') {
    const trimmed = slug.trim()
    return trimmed.length > 0 ? trimmed : null
  }

  if (typeof slug === 'object' && typeof slug.current === 'string') {
    const trimmed = slug.current.trim()
    return trimmed.length > 0 ? trimmed : null
  }

  return null
}

const fetchMarkdownPosts = unstable_cache(
  async (): Promise<MarkdownPostSummary[] | null> => {
    const posts = await client.fetch<MarkdownPostSummary[] | null>(MARKDOWN_POSTS_QUERY)

    if (!posts) {
      return null
    }

    const published = filterPublishedPosts(posts)

    return published.filter((post) => Boolean(resolveSlugField(post.slug) && post.hasMarkdown))
  },
  ['sanity-posts-markdown-v3'],
  {
    revalidate: REVALIDATE_SECONDS,
    tags: ['post'],
  }
)

export async function getMarkdownPosts(): Promise<MarkdownPostSummary[]> {
  const posts = await fetchMarkdownPosts()

  if (!posts) {
    return []
  }

  return posts.filter((post) => Boolean(post.hasMarkdown && resolveSlugField(post.slug)))
}

export async function getMarkdownPostSlugs(): Promise<string[]> {
  const posts = await getMarkdownPosts()

  return posts
    .map((post) => resolveSlugField(post.slug))
    .filter((slug): slug is string => typeof slug === 'string' && slug.length > 0)
}

This gives the rest of the app a cheap way to list Markdown-ready articles without hammering Sanity.

2. Serve Markdown at /blog/md/[slug]

With the helpers in place, create a static route handler that emits Markdown, complete with metadata and a footer for categories and keywords.

// File: src/app/(non-intl)/blog/md/[slug]/route.ts
import type { POST_QUERYResult } from '@/../sanity.types'
import { getMarkdownPostSlugs, getPostBySlug } from '@/lib/sanity/post-cache'

export const dynamic = 'force-static'
export const revalidate = 3600

type RouteParams = { slug: string }

export async function generateStaticParams() {
  const slugs = await getMarkdownPostSlugs()
  return slugs.map((slug) => ({ slug }))
}

type MarkdownReadyPost = NonNullable<POST_QUERYResult> & { markdownContent: string }

function formatDate(input?: string | null) {
  if (!input) {
    return null
  }
  const parsed = new Date(input)
  if (Number.isNaN(parsed.getTime())) {
    return null
  }
  return parsed.toISOString().split('T')[0]
}

function buildMarkdownDocument(post: MarkdownReadyPost): string {
  const lines: string[] = []
  const title = post.title?.trim() || 'Untitled Post'
  lines.push(`# ${title}`)

  const description = post.metaDescription?.trim() || post.subtitle?.trim()
  if (description) {
    lines.push(`> ${description.replace(/\s+/g, ' ').trim()}`)
  }

  const metaParts: string[] = []
  const author = post.author?.name?.trim()
  const published = formatDate(post.publishedAt)
  if (author) metaParts.push(author)
  if (published) metaParts.push(published)
  if (metaParts.length) lines.push(metaParts.join(' · '))

  lines.push('')
  lines.push(post.markdownContent.trim())

  const categories = Array.isArray(post.categories)
    ? post.categories
        .map((category) => {
          if (!category) return null
          if (typeof category.title === 'string' && category.title.trim()) {
            return category.title.trim()
          }
          if (typeof category.slug === 'object' && category.slug?.current) {
            return category.slug.current
          }
          if (typeof category.slug === 'string' && category.slug.trim()) {
            return category.slug.trim()
          }
          return null
        })
        .filter(Boolean)
    : []

  const keywords = Array.isArray(post.keywords)
    ? post.keywords.filter((keyword): keyword is string => typeof keyword === 'string' && keyword.trim().length > 0)
    : []

  const footerSegments: string[] = []
  if (categories.length) footerSegments.push(`**Categories:** ${categories.join(', ')}`)
  if (keywords.length) footerSegments.push(`**Keywords:** ${keywords.join(', ')}`)

  const updated = formatDate(post.dateModified || post.publishedAt)
  if (updated) footerSegments.push(`**Last Updated:** ${updated}`)

  if (footerSegments.length) {
    lines.push('')
    lines.push('---')
    lines.push(footerSegments.join('  \n'))
  }

  return lines.join('\n')
}

export async function GET(
  _request: Request,
  { params }: { params: Promise<RouteParams> }
) {
  const { slug } = await params
  if (!slug) {
    return new Response('Not Found', { status: 404 })
  }

  const post = await getPostBySlug(slug)
  if (!post || typeof post.markdownContent !== 'string' || post.markdownContent.trim().length === 0) {
    return new Response('Not Found', { status: 404 })
  }

  const markdown = buildMarkdownDocument(post as MarkdownReadyPost)

  return new Response(markdown, {
    status: 200,
    headers: {
      'Content-Type': 'text/markdown; charset=utf-8',
      'Cache-Control': `public, max-age=0, s-maxage=${revalidate}`,
    },
  })
}

The handler stays static, capped by a one-hour revalidation window, and returns clean Markdown ready for LLM ingestion.

3. Link to the Markdown twin from the HTML page

Expose the alternate format to both browsers and crawlers. First, add a text/markdown alternate in generateMetadata.

// File: src/app/(non-intl)/blog/[slug]/page.tsx
alternates: {
  canonical: `https://buildwithmatija.com/blog/${post.slug?.current || ''}`,
  ...(post.markdownContent && post.slug?.current
    ? {
        types: {
          'text/markdown': `https://buildwithmatija.com/blog/md/${post.slug.current}`,
        },
      }
    : {}),
},

Then drop a footer link at the bottom of the article so readers can grab the Markdown version.

// File: src/app/(non-intl)/blog/[slug]/page.tsx
{post.markdownContent && post.slug?.current ? (
  <div className="mt-12 border-t pt-6 text-sm">
    <a
      href={`/blog/md/${post.slug.current}`}
      className="inline-flex items-center gap-2 font-medium text-primary-600 hover:text-primary-500 dark:text-primary-400"
    >
      <span aria-hidden>📄</span>
      <span>View markdown version</span>
    </a>
  </div>
) : null}

4. Generate /llms.txt

With Markdown posts available, build the llms.txt route by grouping posts into categories and listing the top 30 entries. Each item links directly to the Markdown twin.

// File: src/app/llms.txt/route.ts
const getRecentMarkdownPosts = cache(async () => {
  const posts = await getMarkdownPosts()
  return posts
    .filter((post) => post.hasMarkdown)
    .slice(0, 30)
    .map((post) => post as MarkdownPostEntry)
})

// …helpers trimmed for brevity…

export async function GET() {
  const posts = await getRecentMarkdownPosts()

  const grouped = new Map<string, MarkdownPostEntry[]>()
  posts.forEach((post) => {
    const categoryTitle = normalizeCategoryTitle(post)
    const list = grouped.get(categoryTitle) ?? []
    list.push(post)
    grouped.set(categoryTitle, list)
  })

  const categorySections = Array.from(grouped.entries())
    .sort(([a], [b]) => a.localeCompare(b, undefined, { sensitivity: 'base' }))
    .map(([category, items]) => {
      const lines: string[] = []
      lines.push(`### ${category}`)

      items
        .sort((a, b) => {
          const aDate = a.publishedAt ? new Date(a.publishedAt).getTime() : 0
          const bDate = b.publishedAt ? new Date(b.publishedAt).getTime() : 0
          return bDate - aDate
        })
        .forEach((post) => {
          const slug = resolveSlugValue(post.slug)
          if (!slug) {
            return
          }

          const url = `${siteOrigin}/blog/md/${slug}`
          const title = post.title?.trim() || 'Untitled Post'
          const date = formatDate(post.publishedAt)
          const summary = summarizePost(post)
          const dateSuffix = date ? ` (${date})` : ''
          lines.push(`- [${title}](${url})${dateSuffix}: ${summary}`)
        })

      lines.push('')
      return lines.join('\n')
    })
    .filter(Boolean)

  const documentLines: string[] = []
  documentLines.push(`# ${siteTitle}`)
  documentLines.push(`> ${siteDescription}`)
  documentLines.push('')

  documentLines.push('## About Matija Ziberna')
  documentLines.push('Matija Ziberna is a full-stack developer and technical founder partnering with teams to ship production-ready products, automation, and AI-enabled workflows.')
  documentLines.push('')

  documentLines.push('## Services')
  services.forEach((service) => {
    documentLines.push(`- [${service.title}](${service.url}): ${service.blurb}`)
  })
  documentLines.push('')

  documentLines.push('## Markdown Content Index')
  documentLines.push('All blog posts listed below have dedicated Markdown versions designed for LLM consumption. Use the category sections to find content relevant to your task.')
  documentLines.push('')

  documentLines.push('## Recent Posts by Category')
  if (categorySections.length === 0) {
    documentLines.push('- No markdown posts found. Add markdownContent in Sanity to populate this section.')
  } else {
    documentLines.push(...categorySections)
  }

  documentLines.push('')
  documentLines.push('## Discovery Notes')
  documentLines.push(`- Sitemap: ${siteOrigin}/sitemap.xml`)
  documentLines.push(`- Markdown endpoints follow the pattern ${siteOrigin}/blog/md/{slug}`)

  const body = documentLines.join('\n')

  return new Response(body, {
    headers: {
      'Content-Type': 'text/plain; charset=utf-8',
      'Cache-Control': `public, max-age=0, s-maxage=${revalidate}`,
    },
  })
}

5. Expand sitemap and robots directives

Surface the Markdown endpoints in the sitemap alongside the canonical HTML pages and the llms.txt manifest.

// File: src/app/sitemap.ts
const markdownPosts = await getMarkdownPosts()

const markdownPostUrls = markdownPosts
  .map((post) => {
    const slug = resolveSlug(post.slug as { current?: string | null } | string | null)
    if (!slug) {
      return null
    }

    const lastModifiedSource = post.dateModified || post._updatedAt || post.publishedAt
    const lastModified = lastModifiedSource ? new Date(lastModifiedSource) : new Date()

    return {
      url: `${baseUrl}/blog/md/${slug}`,
      lastModified,
      changeFrequency: 'monthly' as const,
      priority: 0.65 as const,
    }
  })
  .filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))

return [
  {
    url: baseUrl,
    lastModified: new Date(),
    changeFrequency: 'daily',
    priority: 1,
  },
  // …existing entries…
  ...postUrls,
  ...markdownPostUrls,
  ...categoryUrls,
  ...commandUrls,
  {
    url: `${baseUrl}/llms.txt`,
    lastModified: new Date(),
    changeFrequency: 'weekly',
    priority: 0.9,
  },
]

Finally, allow the major LLM bots and reference llms.txt directly in robots.ts.

// File: src/app/robots.ts
const llmAgents = ['GPTBot', 'ClaudeBot', 'anthropic-ai', 'PerplexityBot', 'Googlebot']

export default function robots(): MetadataRoute.Robots {
  return {
    sitemap: `${baseUrl}/sitemap.xml`,
    rules: [
      {
        userAgent: '*',
        allow: ['/', '/llms.txt'],
        disallow: ['/studio', '/api/', '/wp-admin'],
      },
      ...llmAgents.map((userAgent) => ({
        userAgent,
        allow: ['/', '/llms.txt'],
      })),
    ],
  }
}

6. Verify the build

Finish by type-checking and manually curling your new routes.

npx tsc --noEmit

If /blog/md/{slug} and /llms.txt both respond with static content, you’re done. You now have Markdown mirrors of every post, llms.txt points to them, and crawlers can discover the alternate format through the sitemap and robots directives.

Let me know in the comments if you have questions, and subscribe for more practical development guides.

Thanks, Matija