Using Database as Remote file Source

[sagarchoudhary96]

The Documentation mentions we can use remote Sources such as database, API etc for the files. i have a use case where i want to use supabase db to manage these files data but unable to find any example/implementation or reference on how we can use database as remote File Source.

if anyone can help/guide with it, it will be great :)

[sagarchoudhary96]

Can be done as below, the example below is with using supabase as db and reading from a posts table:

import { createClient } from '@supabase/supabase-js';
import { writeFile } from 'fs/promises';
import { join } from 'path';
import { defineDocumentType } from '@contentlayer/source-files';
import { makeSource } from '@contentlayer/source-remote-files';

// Initialize Supabase client
const supabase = createClient('YOUR_SUPABASE_URL', 'YOUR_ANON_KEY');

const Post = defineDocumentType(() => ({
  name: 'Post',
  filePathPattern: `posts/**/*.mdx`,
  fields: {
    title: { type: 'string', required: false },
    content: { type: 'string', required: false },
  },
}));

const syncContentFromSupabase = async (contentDir: string) => {
  let wasCancelled = false;
  let syncInterval;

  const fetchAndWriteData = async () => {
    // Logic to read the  data from the source.
    const { data, error } = await supabase.from('posts').select('*');

    if (error) {
      console.error('Error fetching data: ', error);
      return;
    }

    if (data) {
      // create files for each of the post in the contentDir
      for (const post of data) {
        const filePath = join(contentDir, `${post.id}.mdx`);
        const fileContent = `---
title: "${post.title}"
---
${post.content}
`;
        await writeFile(filePath, fileContent);
      }
    }
  };

  const syncLoop = async () => {
    await fetchAndWriteData();

    if (wasCancelled) return;

    syncInterval = setTimeout(syncLoop, 1000 * 60);
  };

  // Block until the first sync is done
  await syncLoop();

  return () => {
    wasCancelled = true;
    clearTimeout(syncInterval);
  };
};

export default makeSource({
  syncFiles: syncContentFromSupabase,
  contentDirPath: 'content', // The directory where MDX files will be stored
  documentTypes: [Post],
  disableImportAliasWarning: true,
});