[Feature] Generate Python API docs from module discovery

.github/workflows/website-build.yml

@@ -38,6 +38,9 @@ jobs:
           cache: 'npm'
           cache-dependency-path: website/package-lock.json
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+
       - name: Install dependencies
         working-directory: website
         run: npm ci

.github/workflows/website.yml

@@ -42,6 +42,9 @@ jobs:
           cache: 'npm'
           cache-dependency-path: website/package-lock.json
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+
       - name: Install dependencies
         working-directory: website
         run: npm ci

CONTRIBUTING.md

@@ -26,6 +26,7 @@ This document describes **repository-wide** setup and workflow. For **subproject
 - [Quick Start](#quick-start)
 - [Development Workflow](#development-workflow)
 - [Testing](#testing)
+- [API Documentation](#api-documentation)
 - [Project-Specific Guides](#project-specific-guides)
 - [Troubleshooting](#troubleshooting)
 - [References](#references)
@@ -133,6 +134,19 @@ See [testing/README.md](testing/README.md) for more options and details.
 
 ---
 
+## API Documentation
+
+API reference documentation is generated from source code. To update the Python API reference content, edit the relevant Python docstrings. Use `docs/api/` only for generation settings, page metadata, or other API documentation structure changes. The generated pages are written under `website/docs/api/` for the Docusaurus site.
+
+**Preview documentation locally:**
+
+```bash
+cd website
+npm run start
+```
+
+---
+
 ## Project-Specific Guides
 
 Apache Mahout includes several subprojects. Use the root workflow above for issues, branches, and pull requests; use the guides below for **build, run, and test** in each area.

docs/api/qumat_qdp.yml

@@ -25,22 +25,14 @@ loaders:
       - qumat_qdp.loader
       - qumat_qdp.triton_amd
 
-# TODO: Trim this filter once the Python package exposes a cleaner public
-# surface for documentation generation.
+# Generated QDP docs intentionally load implementation modules so pydoc-markdown
+# can render full class and method details. Source docstrings define the public
+# reference surface: documented public objects are rendered, undocumented fields
+# and internal helpers are omitted.
 processors:
   - type: filter
-    documented_only: false
+    documented_only: true
     skip_empty_modules: true
-    expression: >-
-      (not name.startswith('_') or name == '__init__')
-      and name not in (
-        'annotations', 'Mapping', 'import_module', 'ModuleType', 'Any',
-        'dataclass', 'time', 'math', 'os', 'sys', 'warnings', 'Iterator',
-        'TYPE_CHECKING', 'CudaBackendEngine', 'AmdBackendEngine', 'lru_cache',
-        'enum', 'field', 'torch_mod', 'triton_mod', 'triton_lang', 'get_qdp',
-        'get_torch', 'get_backend', 'require_backend'
-      )
-      and default()
   - type: smart
 
 renderer:
@@ -50,11 +42,9 @@ renderer:
   add_module_prefix: false
   add_method_class_prefix: false
   render_toc: false
+  render_module_header: false
   header_level_by_type:
-    Module: 2
     Class: 3
     Function: 3
     Variable: 3
     Method: 4
-  render_module_header_template: |
-    ## {module_name}

website/docusaurus.config.ts

@@ -41,7 +41,7 @@ const config: Config = {
       'classic',
       {
         docs: {
-          path: '../docs/',
+          path: './docs/',
           sidebarPath: './sidebars.ts',
           editUrl: 'https://github.com/apache/mahout/tree/main/docs/',
           remarkPlugins: [remarkMath],

website/scripts/sync-docs.js

@@ -16,12 +16,16 @@
 
 const fs = require('fs');
 const path = require('path');
+const { execFileSync } = require('child_process');
 
 // Configuration
+const REPO_ROOT = path.resolve(__dirname, '../..');
 const SOURCE_DIR = path.resolve(__dirname, '../../docs');
 const DEST_DIR = path.resolve(__dirname, '../docs');
 const BLOG_SOURCE_DIR = path.resolve(__dirname, '../../docs/blog');
 const BLOG_DEST_DIR = path.resolve(__dirname, '../blog');
+const API_SOURCE_DIR = path.resolve(__dirname, '../../docs/api');
+const API_DEST_DIR = path.resolve(__dirname, '../docs/api');
 
 // Files that should be preserved during sync (not deleted)
 const PRESERVE_FILES = ['.gitignore'];
@@ -33,6 +37,7 @@ const EXCLUDE_PATTERNS = [
   /\.pyc$/,
   /^__pycache__$/,
   /^blog$/, // Blog is synced separately to website/blog
+  /^api$/, // API generator config is rendered separately into website/docs/api
 ];
 
 /**
@@ -197,6 +202,54 @@ function copyFile(srcPath, destPath) {
   fs.copyFileSync(srcPath, destPath);
 }
 
+/**
+ * Run a repository-level command and inherit stdio by default.
+ */
+function runCommand(command, args, options = {}) {
+  return execFileSync(command, args, {
+    cwd: REPO_ROOT,
+    stdio: options.capture ? 'pipe' : 'inherit',
+    encoding: options.capture ? 'utf-8' : undefined,
+  });
+}
+
+/**
+ * Generate Python API reference docs with pydoc-markdown.
+ */
+function generateApiDocs() {
+  fs.rmSync(API_DEST_DIR, { recursive: true, force: true });
+  ensureDir(API_DEST_DIR);
+
+  copyFile(path.join(API_SOURCE_DIR, 'index.md'), path.join(API_DEST_DIR, 'index.md'));
+
+  const pages = [
+    {
+      header: 'qumat_header.md',
+      config: 'qumat.yml',
+      output: 'qumat.md',
+    },
+    {
+      header: 'qumat_qdp_header.md',
+      config: 'qumat_qdp.yml',
+      output: 'qumat_qdp.md',
+    },
+  ];
+
+  runCommand('uv', ['sync', '--frozen', '--group', 'dev']);
+
+  for (const page of pages) {
+    const outputPath = path.join(API_DEST_DIR, page.output);
+    copyFile(path.join(API_SOURCE_DIR, page.header), outputPath);
+
+    const generated = runCommand(
+      'uv',
+      ['run', '--frozen', '--group', 'dev', 'pydoc-markdown', path.join('docs/api', page.config)],
+      { capture: true },
+    );
+    fs.appendFileSync(outputPath, generated);
+  }
+}
+
 /**
  * Recursively sync a directory
  */
@@ -260,6 +313,9 @@ function main() {
   console.log('\nSyncing blog posts from /docs/blog...');
   const blogStats = syncDirectory(BLOG_SOURCE_DIR, BLOG_DEST_DIR);
 
+  console.log('\nGenerating Python API reference...');
+  generateApiDocs();
+
   console.log(`\nSync complete!`);
   console.log(`  Docs: ${docsStats.files} files, ${docsStats.dirs} directories`);
   console.log(`  Blog: ${blogStats.files} files, ${blogStats.dirs} directories`);

website/sidebars.ts

@@ -56,6 +56,16 @@ const sidebars: SidebarsConfig = {
         'advanced/gap-analysis',
       ],
     },
+    {
+      type: 'category',
+      label: 'API Reference (experiments)',
+      collapsed: true,
+      link: {type: 'doc', id: 'api/index'},
+      items: [
+        'api/qumat',
+        'api/qumat_qdp',
+      ],
+    },
     {
       type: 'category',
       label: 'Quantum Computing Primer',