From 8345abd44b7dace7b07b170bd61efc608f502644 Mon Sep 17 00:00:00 2001 From: sethwebster Date: Tue, 13 Oct 2026 04:20:11 -0400 Subject: [PATCH 1/2] DX gate audits: API docs coverage, CLI, templates, reproducible builds (#171) - api-docs audit and tests/api-docs-coverage.test.ts: every public export has TSDoc and no any type, every subpath a docs page; docs/api-undocumented.json lists the known gaps and only shrinks - cli audit: help for every command, --version, one-line errors; spark sdk/add/updates without a subcommand name the subcommands; spark dev --help works outside an app - templates audit: spark create from the packed SDK (macOS, Windows, universal), headless, type-checked; two clean release builds of the macOS template compared file by file - gate audit steps run one at a time; e2e:audit --audit --- docs/api-undocumented.json | 925 +++++++++++++++++++++++++++ docs/e2e-flows.md | 62 +- e2e/checks/developer-experience.yaml | 53 ++ e2e/schema/checks.schema.json | 5 +- packages/cli/src/cli-arguments.ts | 7 +- packages/cli/src/dev-command.ts | 5 +- packages/cli/src/index.ts | 4 +- packages/cli/src/local.ts | 16 +- scripts/api-docs.ts | 43 ++ scripts/api-surface.ts | 28 +- scripts/e2e/audit/api-docs.ts | 18 + scripts/e2e/audit/cli.ts | 62 ++ scripts/e2e/audit/steps.ts | 7 +- scripts/e2e/audit/templates.ts | 169 +++++ scripts/e2e/cli.ts | 2 +- scripts/e2e/format/primitives.ts | 2 +- scripts/e2e/gate/gate.ts | 19 +- tests/api-docs-coverage.test.ts | 26 + tests/cli-arguments.test.ts | 6 + tests/e2e-gate.test.ts | 22 +- 20 files changed, 1449 insertions(+), 32 deletions(-) create mode 100644 docs/api-undocumented.json create mode 100644 e2e/checks/developer-experience.yaml create mode 100644 scripts/api-docs.ts create mode 100644 scripts/e2e/audit/api-docs.ts create mode 100644 scripts/e2e/audit/cli.ts create mode 100644 scripts/e2e/audit/templates.ts create mode 100644 tests/api-docs-coverage.test.ts diff --git a/docs/api-undocumented.json b/docs/api-undocumented.json new file mode 100644 index 00000000..95770b68 --- /dev/null +++ b/docs/api-undocumented.json @@ -0,0 +1,925 @@ +{ + "pageless": [ + "./contracts", + "./ui/classnames", + "./ui/popover", + "./ui/search", + "./ui/sidebar", + "./ui/swipe", + "./ui/symbol", + "./ui/tab-view", + "./webview" + ], + "undocumented": { + "./ai": [ + "AIAvailabilityOptions", + "AICodexInvocationOptions", + "AICommandAvailability", + "AIErrorOutputOptions", + "AIInvocation", + "AIInvocationOptions", + "AIToolId", + "AIToolRunResult", + "buildAIInvocation", + "extractJsonCandidate", + "formatAIErrorOutput", + "getAICommandAvailability", + "parseAIJson", + "runAITool", + "RunAIToolOptions", + "selectPreferredAITool" + ], + "./ai/codex": [ + "CodexAvailability", + "CodexRunOptions", + "CodexRunResult", + "runCodexPrompt" + ], + "./app": [ + "activate", + "AppContext", + "AppEvent", + "AppEventMap", + "AppRuntime", + "AsyncRegistration", + "BeforeQuitOptions", + "getAppContext", + "getAppLanguages", + "hide", + "QuitResult", + "RelaunchOptions", + "Subscription" + ], + "./app-intents": [ + "AppIntentHandlerOptions", + "AppIntentInvocation", + "AppIntentParameter", + "AppIntentParameterType", + "AppIntentValue", + "AsyncRegistration", + "Availability" + ], + "./app/documents": [ + "AsyncRegistration", + "clearRecentDocuments", + "createDocumentSession", + "DocumentAppController", + "DocumentAppControllerOptions", + "DocumentAppControllerState", + "DocumentReloadOptions", + "DocumentSaveOptions", + "DocumentSaveResult", + "DocumentSessionOptions", + "DocumentState", + "getLaunchDocumentPath", + "GetLaunchDocumentPathOptions", + "getRecentDocuments", + "noteRecentDocument", + "OpenRequest", + "openSelectedDocumentPath", + "OpenSelectedDocumentPathOptions", + "RecentDocument", + "Subscription", + "UseDocumentAppControllerOptions", + "useWatchedDocumentReload", + "UseWatchedDocumentReloadOptions" + ], + "./apple-authentication": [ + "AppleAuthenticationScope", + "AppleIDCredential", + "AppleIDCredentialState", + "AppleIDFullName", + "AppleSignInOptions", + "AppleSignInResult", + "Availability" + ], + "./audio": [ + "AudioMetadata", + "AudioPlayer", + "AudioPlayerHookOptions", + "AudioPlayerOptions", + "AudioPlayerState", + "AudioSource", + "createAudioPlayer", + "MediaCommand", + "MediaCommandName", + "MediaSession", + "MediaSessionOptions", + "MediaSessionUpdate", + "sessionOptions", + "sessionSnapshot", + "Subscription", + "validateMetadata", + "validateSession", + "validateVolume", + "validMediaCommand" + ], + "./auth-session": [ + "authorize", + "AuthSession", + "AuthSessionOptions", + "AuthSessionResult", + "callbackMatches", + "CompleteAuthSessionResult", + "createAuthSession", + "digestStringAsync", + "getRandomBytesAsync", + "timeout", + "validateCount", + "validateDigest", + "validateRedirect" + ], + "./biometrics": [ + "Availability", + "BiometricAuthOptions", + "BiometricAuthResult", + "BiometricErrorReason", + "BiometricPromptOptions", + "BiometricSecretOptions", + "BiometricStatus" + ], + "./capture": [ + "AudioRecordingOptions", + "Availability", + "CameraContentFit", + "CameraOptions", + "CameraSession", + "CameraStatus", + "CameraViewProps", + "CaptureDevice", + "CaptureDeviceKind", + "CaptureRecordingOutcome", + "CaptureRecordingSession", + "ScreenCaptureOptions", + "ScreenDisplaySource", + "ScreenRecordingOptions", + "ScreenshotOptions", + "ScreenWindowSource" + ], + "./clipboard": [ + "clearClipboard", + "ClipboardChangeEvent", + "ClipboardChangeOptions", + "ClipboardData", + "ClipboardImage", + "getClipboardFormats", + "GetStringOptions", + "hasStringAsync", + "PromisedClipboardContent", + "PromisedClipboardOptions", + "readClipboard", + "setStringAsync", + "SetStringOptions", + "StringFormat", + "writeClipboard" + ], + "./context-menu": [ + "ContextMenuOptions", + "ContextMenuResult", + "getContextMenuAvailability", + "MenuAction", + "MenuIcon", + "MenuItem" + ], + "./contracts": [ + "AsyncRegistration", + "Availability", + "SparkError", + "Subscription" + ], + "./devtools": [ + "NetworkFrame", + "NetworkInspectorOptions", + "OpenDevToolsOptions", + "ProfilingResult", + "StartProfilingOptions", + "StopProfilingOptions" + ], + "./diagnostics": [ + "CrashReport", + "HangReport", + "HangStackFrame", + "HangWatchdogOptions", + "StartupTiming" + ], + "./dialogs": [ + "confirm", + "ConfirmOptions", + "DialogAccessoryOption", + "FileDialogOptions", + "MessageDialogButton", + "MessageDialogOptions", + "MessageDialogResult", + "openFileDialog", + "OpenFileDialogOptions", + "PromptOptions", + "saveFileDialog", + "SaveFileDialogOptions", + "SaveFileDialogResult", + "showMessage" + ], + "./displays": [ + "AsyncRegistration", + "DisplayColorSpace", + "DisplayDetails", + "DisplayField", + "DisplayId", + "DisplayInfo", + "DisplayInsets", + "DisplayListenerOptions" + ], + "./drag-drop": [ + "DragDropView", + "DragDropViewProps", + "DragEndEvent", + "DragOperation", + "DragOptions", + "DropEvent", + "SpringLoadHighlight", + "SpringLoadHighlightEvent" + ], + "./files": [ + "AsyncRegistration", + "CopyMoveOptions", + "CreateBookmarkOptions", + "CreateDirectoryOptions", + "DirectoryEntry", + "exists", + "FileApplication", + "FileCleanupError", + "FileHandle", + "FileInfo", + "FileMode", + "FileScanBatch", + "FileScanOptions", + "FileScanProgress", + "FileScanResult", + "FileScanSkipEntry", + "getAppIconAvailability", + "getDirectory", + "getDiskSpaceAvailability", + "getFileIconAvailability", + "getFileScanAvailability", + "getFileSystemAvailability", + "getOpenWithAvailability", + "getQuarantineAvailability", + "getQuickLookAvailability", + "getThumbnailAvailability", + "mkdir", + "OpenFileOptions", + "readBytes", + "ReadChunksOptions", + "RemoveOptions", + "scanFiles", + "ScannedFile", + "setExtendedAttribute", + "WatchOptions", + "writeBytes", + "WriteChunksOptions" + ], + "./global-shortcuts": [ + "AsyncRegistration", + "getGlobalShortcutAvailability", + "GlobalShortcutOptions" + ], + "./links": [ + "Availability", + "canOpenURL", + "Subscription", + "URLListener" + ], + "./logging": [ + "DiagnosticsExport", + "ExportDiagnosticsOptions", + "LogFile", + "LogFileOptions", + "LoggerOptions", + "LoggingOptions" + ], + "./menus": [ + "addDeveloperMenuListener", + "AsyncRegistration", + "createMenu", + "DeveloperMenuState", + "getMenuAvailability", + "Menu", + "MenuAction", + "MenuFeature", + "MenuIcon", + "MenuItem", + "MenuLifecycleEvent", + "MenuLocation", + "MenuOptions", + "MenuPlacement", + "MenuRole", + "MenuRootItem", + "MenuState", + "MenuTarget", + "MenuUpdate", + "UseMenuOptions" + ], + "./motion": [ + "AnimateOptions", + "AnimationOptions", + "CubicBezier", + "decay", + "DecayOptions", + "DecaySpec", + "Easing", + "getMotionAvailability", + "KeyframesOptions", + "KeyframesSpec", + "MotionAnimation", + "MotionColorProperty", + "MotionEmitterProps", + "MotionFrameReport", + "MotionMaterialProps", + "MotionNumberProps", + "MotionPathOptions", + "MotionPathValues", + "MotionPresenceProps", + "MotionProperty", + "MotionShaderEffect", + "MotionShaderProps", + "MotionShapeFillRule", + "MotionShapeLineCap", + "MotionShapeLineJoin", + "MotionShapeProps", + "MotionState", + "MotionSymbolProps", + "MotionTextProps", + "MotionValues", + "MotionViewProps", + "ParticleEmitter", + "ParticleOptions", + "ParticlePlayback", + "ScrollLinkOptions", + "SequenceOptions", + "SequenceSpec", + "ShaderEffect", + "SpringPreset", + "SpringSpec", + "StaggerOptions", + "SymbolEffect", + "SymbolEffectDirection", + "SymbolEffectOptions", + "SymbolEffectPlayback", + "TimelineStep", + "TimingOptions", + "TimingSpec" + ], + "./notifications": [ + "cancelAllNotifications", + "dismissAllNotifications", + "getDeliveredNotifications", + "getNotificationAvailability", + "getNotificationPermission", + "getPendingNotifications", + "NotificationAction", + "NotificationContent", + "NotificationPermission", + "NotificationTrigger", + "scheduleNotification", + "ScheduleNotificationOptions", + "ShowNotificationOptions", + "Subscription" + ], + "./passkeys": [ + "Availability", + "PasskeyAssertion", + "PasskeyAssertionOptions", + "PasskeyAssertionResult", + "PasskeyRegistration", + "PasskeyRegistrationOptions", + "PasskeyRegistrationResult", + "PasskeyUserVerification" + ], + "./permissions": [ + "AsyncRegistration", + "Availability", + "PermissionChangeEvent", + "PermissionListenerOptions", + "PermissionOptions", + "PermissionResponse" + ], + "./plugins": [ + "Availability", + "PLUGIN_CAPABILITIES", + "PluginCommand", + "PluginFileHandler", + "PluginHost", + "PluginHostOptions", + "PluginInfo", + "PluginManifest", + "PluginMenuItem", + "PluginState" + ], + "./pointer": [ + "CustomCursor", + "HapticFeedbackOptions", + "PointerViewProps" + ], + "./printing": [ + "ExportPDFOptions", + "ExportPDFResult", + "PageOrientation", + "PageSetupOptions", + "PageSetupResult", + "PrintOptions" + ], + "./privileged-helper": [ + "AdminAuthorizationOptions", + "AdminAuthorizationResult", + "Availability", + "PrivilegedHelperMessageOptions" + ], + "./processes": [ + "getProcessAvailability", + "ProcessExit", + "ProcessHandle", + "ProcessInput", + "ProcessOptions", + "ProcessOutput", + "ProcessResult", + "ProcessTarget", + "resolveCommand", + "runCommand", + "RunCommandOptions", + "spawn" + ], + "./secure-storage": [ + "deleteItemAsync", + "isAvailableAsync", + "setItemAsync" + ], + "./settings": [ + "Json", + "settings", + "SettingsReadOptions", + "SettingsStorage", + "SettingsStore", + "SettingsStoreOptions" + ], + "./settings/observable": [ + "ObservableFile", + "ObservableFileOptions", + "ObservableSettingsField", + "ObservableSettingsFields", + "ObservableSettingsOptions", + "ObservableSettingsValues" + ], + "./settings/window": [ + "CreateSettingsWindowOptionsInput", + "SettingsCard", + "SettingsCardProps", + "SettingsPage", + "SettingsPageProps", + "SettingsRow", + "SettingsRowGroup", + "SettingsRowGroupProps", + "SettingsRowProps", + "SettingsSection", + "SettingsSectionProps", + "SettingsSidebar", + "SettingsSidebarProps", + "SettingsWindow", + "SettingsWindowBaseProps", + "SettingsWindowPage", + "SettingsWindowProps", + "SettingsWindowSelectionProps", + "VirtualizedSettingsWindow", + "VirtualizedSettingsWindowProps" + ], + "./sharing": [ + "AsyncRegistration", + "Availability", + "ListenerOptions", + "ShareOptions", + "SharingFeature", + "SpotlightDeletion", + "SpotlightItem", + "SpotlightOpenEvent", + "Subscription" + ], + "./shortcuts": [ + "Accelerator", + "AsyncRegistration", + "getShortcutAvailability", + "parseAccelerator", + "ShortcutOptions" + ], + "./shortcuts/commands": [ + "createHotkeyRouter", + "createHotkeyStore", + "createShortcutMap", + "formatHotkey", + "getDefaultHotkeyBindings", + "getHotkeyBindingConflicts", + "HotkeyBindingLimitOptions", + "hotkeyBindingListsEqual", + "HotkeyBindingsSettingsContent", + "HotkeyBindingsSettingsPage", + "HotkeyBindingsSettingsPageProps", + "HotkeyBindingState", + "HotkeyCapture", + "HotkeyCaptureProps", + "HotkeyDefinition", + "HotkeyFile", + "hotkeyFileVersion", + "HotkeyHandlerContext", + "HotkeyMatchOptions", + "HotkeyRegistration", + "HotkeyRegistrationOptions", + "HotkeyRegistrationState", + "HotkeyRouter", + "HotkeyScope", + "HotkeyStoreOptions", + "HotkeySuspensionOptions", + "HotkeyValue", + "matchesHotkey", + "normalizeHotkeyBindings", + "normalizeHotkeyFile", + "RoutedHotkeyHandlers", + "serializeHotkeyFile", + "ShortcutCommand", + "ShortcutCommandEvent", + "ShortcutConflict", + "ShortcutConflictParty", + "ShortcutMap", + "ShortcutMapChange", + "ShortcutMapCommandState", + "ShortcutMapHandlers", + "ShortcutMapOptions", + "ShortcutMapRegisterOptions", + "ShortcutMapSnapshot", + "shortcutMapVersion", + "ShortcutPlatform", + "useHotkeySuspension", + "useRoutedHotkeys", + "UseRoutedHotkeysOptions" + ], + "./shortcuts/keyboard": [ + "AsyncRegistration", + "createModifierMask", + "getKeyboardAvailability", + "hasModifier", + "KeyboardEvent", + "KeyboardEventListener", + "KeyboardListenerOptions", + "KeyCodes", + "KeyText" + ], + "./speech": [ + "Availability", + "getSpeakingStatus", + "pauseSpeaking", + "PauseSpeakingOptions", + "resumeSpeaking", + "SpeakingStatus", + "SpeakOptions", + "SpeechRecognitionOptions", + "SpeechRecognitionOutcome", + "SpeechRecognitionSegment", + "SpeechRecognitionSession", + "SpeechRecognizerStatus", + "SpeechTranscript", + "SpeechVoice" + ], + "./sqlite": [ + "Database", + "DatabaseOptions", + "IntegerMode", + "RunResult", + "SqlExecutor", + "SqlRow", + "SqlValue" + ], + "./system": [ + "AsyncRegistration", + "AttentionOptions", + "createDockMenu", + "createTaskbarMenu", + "DockMenuItem", + "DockMenuOptions", + "getLoginItemStatus", + "getSystemInfo", + "LoginItemStatus", + "MemoryPressureLevel", + "NetworkInterfaceType", + "NetworkStatus", + "onSystemEvent", + "preventSleep", + "PreventSleepOptions", + "requestAttention", + "setLaunchAtLogin", + "SystemEventOptions", + "SystemInfo", + "TaskbarMenuItem", + "TaskbarMenuOptions", + "ThermalState" + ], + "./tray": [ + "AsyncRegistration", + "createTray", + "getTrayAvailability", + "MenuAction", + "MenuIcon", + "MenuItem", + "Tray", + "TrayAction", + "TrayOptions", + "TrayUpdate" + ], + "./ui": [ + "Button", + "ButtonAvailabilityOptions", + "ButtonVariant", + "Checkbox", + "CheckboxValue", + "ColorPanelChangeEvent", + "ColorPanelOptions", + "ColorWellProps", + "ColorWellVariant", + "ComboBox", + "ComboBoxProps", + "ControlKind", + "ControlledTextInputProps", + "ControlProps", + "ControlSize", + "DatePickerMode", + "DatePickerProps", + "DatePickerVariant", + "DisclosureTriangle", + "DisclosureTriangleProps", + "FontDescriptor", + "FontPanelChangeEvent", + "FontPanelOptions", + "FontPanelSession", + "FontWeight", + "getControlAvailability", + "getFontPanelAvailability", + "LevelIndicator", + "LevelIndicatorProps", + "NumericControlProps", + "PanelCloseEvent", + "PathControl", + "PrimitiveControlProps", + "PrimitiveKind", + "Progress", + "ProgressProps", + "RadioGroup", + "RadioGroupProps", + "ScreenColorResult", + "SegmentedControl", + "SegmentedControlProps", + "Select", + "SelectOption", + "SelectProps", + "Slider", + "SliderProps", + "Stepper", + "StepperProps", + "Switch", + "SwitchProps", + "TextInput", + "TextInputProps", + "TokenField", + "TokenFieldProps", + "UncontrolledTextInputProps" + ], + "./ui/classnames": [ + "ClassValue", + "cn" + ], + "./ui/glass": [ + "getGlassAvailability", + "GlassStyle", + "GlassViewProps" + ], + "./ui/popover": [ + "getPopoverAvailability", + "PopoverAnchor", + "PopoverProps" + ], + "./ui/search": [ + "getSearchAvailability", + "TextInputSearch", + "TextInputSearchProps", + "TextInputSearchRef" + ], + "./ui/sidebar": [ + "getSidebarAvailability", + "Sidebar", + "SidebarBaseProps", + "SidebarContentLayoutEvent", + "SidebarContextMenuEvent", + "SidebarItem", + "SidebarItemData", + "SidebarItemProps", + "SidebarProps", + "SidebarRowHeight", + "SidebarSelectionEvent" + ], + "./ui/split-view": [ + "getSplitViewAvailability", + "SidebarSplitView", + "SidebarSplitViewAppearance", + "SidebarSplitViewPaneMetrics", + "SidebarSplitViewProps", + "SidebarSplitViewResizeEvent", + "SidebarSplitViewTitlebarMaterial", + "SplitViewTitleBarOptions", + "SplitViewTitleBarOverlay" + ], + "./ui/swipe": [ + "getSwipeActionsAvailability", + "SwipeActionsProps" + ], + "./ui/symbol": [ + "getSFSymbolAvailability", + "SFSymbol", + "SFSymbolProps", + "SFSymbolScale" + ], + "./ui/tab-view": [ + "getTabViewAvailability", + "TabProps", + "TabViewProps", + "TabViewSelectionEvent" + ], + "./ui/table": [ + "TableBaseProps", + "TableColumn", + "TableContextMenuEvent", + "TableProps", + "TableSelectionMode", + "TableSelectionProps", + "TableSortDirection" + ], + "./ui/text-services": [ + "FindTextOptions", + "FindTextResult", + "getTextServicesAvailability" + ], + "./ui/uniwind": [ + "Button", + "SegmentedControl", + "Select", + "TextInput" + ], + "./updates": [ + "CheckForUpdatesOptions", + "onUpdateEvent", + "Subscription", + "UpdateAvailability", + "UpdateConfiguration", + "UpdateEvent", + "UpdateStatus", + "UpdateUnavailableReason" + ], + "./video": [ + "Availability", + "Subscription", + "VideoContentFit", + "VideoPlayer", + "VideoPlayerOptions", + "VideoViewProps" + ], + "./webview": [ + "WebViewErrorEvent", + "WebViewMessageEvent", + "WebViewNavigation", + "WebViewNavigationRequest", + "WebViewRef", + "WebViewSource" + ], + "./windows": [ + "AsyncRegistration", + "centerWindow", + "CenterWindowOptions", + "closeWindow", + "CommandBar", + "CommandBarOptions", + "CommandBarShowOptions", + "createWindowMessages", + "createWindowsNavigator", + "createWindowState", + "createWindowWorkspaces", + "DesktopWidgetOptions", + "DisplayId", + "DisplayInfo", + "DuplicateWindowOptions", + "EdgeDrawer", + "EdgeDrawerOptions", + "ErrorFallbackProps", + "getDisplays", + "getWindow", + "hideWindow", + "HUD", + "HUDOptions", + "HUDShowOptions", + "listWindows", + "MacOSStartupSplitViewOptions", + "MacOSTitleBarControl", + "MacOSTitleBarOptions", + "MacOSToolbarDisplayMode", + "MacOSToolbarItem", + "MacOSToolbarOptions", + "MacOSToolbarSegment", + "MacOSToolbarState", + "MacOSWindowBlurOptions", + "MacOSWindowBoundsOptions", + "MacOSWindowEvent", + "MacOSWindowOptions", + "maximizeWindow", + "MenuAction", + "minimizeWindow", + "NavigatorOpenOptions", + "NavigatorWindowOptions", + "openWindow", + "OverlayOptions", + "PictureInPicture", + "PictureInPictureOptions", + "Popover", + "PopoverOptions", + "PresenterEndReason", + "PresenterMode", + "PresenterModeOptions", + "PreviewWindow", + "PreviewWindowOptions", + "PrimaryWindowLifecycleOptions", + "ScreenCorner", + "ScreenEdge", + "SetWindowBoundsOptions", + "setWindowFullscreen", + "setWindowOptions", + "showWindow", + "ShowWindowOptions", + "Size", + "SpecialWindowErrorOptions", + "SpecialWindowType", + "UndoEntry", + "UndoMenuLabels", + "UndoStack", + "UndoState", + "unmaximizeWindow", + "usePrimaryWindowLifecycle", + "UsePrimaryWindowLifecycleOptions", + "useWindowFocusEffect", + "useWindowId", + "WindowAppearanceOptions", + "WindowAttachment", + "WindowAttachmentEdge", + "WindowAttachOptions", + "WindowBroadcastListener", + "WindowBroadcastListenerOptions", + "WindowBroadcastOptions", + "WindowChoreographyFeature", + "WindowCloseTransition", + "WindowComponentLoader", + "WindowComponentProps", + "WindowEasing", + "WindowErrorEvent", + "WindowEventMap", + "WindowHookOptions", + "WindowHotkeys", + "WindowId", + "WindowInfo", + "WindowKind", + "WindowListenerOptions", + "WindowMacOS", + "WindowNewTabEvent", + "WindowOpenOptions", + "WindowProvider", + "WindowProviderProps", + "WindowRequestOptions", + "WindowRequestPayload", + "WindowRequestResult", + "WindowsConfig", + "WindowsNavigatorOptions", + "WindowStateOptions", + "WindowTabbingOptions", + "WindowTransition", + "WindowUpdateOptions", + "WindowWorkspacesOptions", + "WindowZoom", + "WindowZoomOptions", + "withWindowProvider", + "WorkspaceMenuLabels" + ], + "./windows/macos": [ + "addMacOSWindowListener", + "AsyncRegistration", + "focusToolbarSearch", + "FocusToolbarSearchOptions", + "MacOSToolbarDisplayMode", + "MacOSToolbarItem", + "MacOSToolbarOptions", + "MacOSToolbarState", + "MacOSWindowBlurOptions", + "MacOSWindowEvent", + "MenuAction", + "setToolbarItemText" + ] + } +} diff --git a/docs/e2e-flows.md b/docs/e2e-flows.md index b55a2170..526f0a48 100644 --- a/docs/e2e-flows.md +++ b/docs/e2e-flows.md @@ -1105,9 +1105,10 @@ the same and links to the runner's page. An audit is a gate step that checks the release artifact itself instead of driving the app. A check registered with `audit: ` is verified by that step, so no flow lists it (coverage reports `audit-flow` if one does). The -gate runs each audit step once, after the build check, and judges every audit check for the target's platform: a +gate runs each audit step once, one at a time, after the build check, and judges every audit check for the target's platform: a `failed` or `unsupported` result, or no result, fails a blocking check. A scoped run (`--flow`) runs no audits. -`bun run e2e:audit [--artifact ] [--package ]` runs only the audits for this machine's target and prints each result. +`bun run e2e:audit [--artifact ] [--package ] [--audit ]…` runs only the audits for this machine's +target (with `--audit`, only those steps) and prints each result. `bundle` (`scripts/e2e/audit/bundle.ts`, registered in `checks/packaging-distribution.yaml`) audits release bundle hygiene (#153): @@ -1152,6 +1153,63 @@ the distribution ZIP, or the folder holding the one `--.zip Spark makes no DMG, so there is no DMG layout check. App translocation at launch is not checked: it needs a notarized app launched from its quarantined location. +#### Developer experience audits + +Three audit steps, registered in `checks/developer-experience.yaml` (#171), check the developer's side of a release. +The `templates` step builds apps, so run it under the native build lock on a shared Mac. + +`api-docs` (`scripts/api-docs.ts`) reads the public surface the way `docs/api-public-surface.json` snapshots it. +`docs/api-undocumented.json` lists the known gaps. `tests/api-docs-coverage.test.ts` (part of `bun run test`) fails on an +undocumented export or page-less subpath that is not listed, and on a listed one that is now documented, so the list only +shrinks. The audit fails while the list is not empty. + +| Check | What it verifies | +|---|---| +| `DX-DOCS-01` | Every public export has a TSDoc comment on its declaration (what an editor shows). | +| `DX-DOCS-02` | No public export, parameter or return type is `any`. | +| `DX-DOCS-03` | Every public subpath has a `docs/*.md` page that imports it (`@legendapp/spark/` in quotes or backticks). | + +`cli` (`scripts/e2e/audit/cli.ts`) runs the SDK's `spark` bin (`packages/desktop/bin/spark.cjs`) headless in a scratch +directory, only with arguments that fail or print help before any side effect. + +| Check | What it verifies | +|---|---| +| `DX-CLI-01` | `spark --help` names every command of the CLI's grammar (`packages/cli/src/cli-arguments.ts`) and `dev`; `-h`, no arguments and each command's `--help` exit 0; `spark dev --help` in the Kitchen Sink adds `expo start`'s options; `spark --version` prints the SDK version. | +| `DX-CLI-02` | An unknown command or option, missing or extra arguments, a group without its subcommand (`spark sdk`), invalid values, conflicting build modes and running outside an app each exit 1 with one line on stderr that names the problem, and no stack trace. | + +`templates` (`scripts/e2e/audit/templates.ts`) packs the SDK (`scripts/pack.ts`), then works in a scratch directory with +`CI=1`, no TTY, no stdin and a scratch `SPARK_HOME`, so the developer's registered SDKs and runners are untouched. It +unregisters every app it built from LaunchServices (`lsregister -u`) and removes the directory. It took 8 to 10 +minutes on an M4 Max: the packages come from the npm registry and the two release builds compile React Native twice. + +| Check | What it verifies | +|---|---| +| `DX-TPL-01` | `spark create --packages ` makes the macOS, Windows and universal templates; each depends on the packed SDK, installs and type-checks (`tsc --noEmit`). | +| `DX-TPL-02` | `spark build` of the macOS template makes a release app with a Hermes bytecode `main.jsbundle` that `codesign --verify --deep --strict` accepts. | +| `DX-REPRO-01` | A second release build, after deleting `.spark/` and the generated `macos/` project, has byte-identical JavaScript (`*.jsbundle`, `*.hbc`, `*.js`). | +| `DX-REPRO-02` | Every other file in the app is byte-identical too. Each difference is listed; for a Mach-O file, whether its `LC_UUID` or other load commands differ. See [Reproducible builds](#reproducible-builds). | + +#### Reproducible builds + +`DX-REPRO-01` and `DX-REPRO-02` build the macOS template twice. `spark build` prepares the project in place (it patches +`node_modules` and writes `macos/`, `ios/`, `react-native.config.js` and configuration), so each build starts from an APFS +clone of the project as `spark create` left it, at the same path. With identical inputs the JavaScript bundle and +`spark-runtime.json` are byte-identical. Built from an already-prepared project instead, `spark-runtime.json`'s +`fingerprint` differs, because the first build changed its inputs. + +The native output is not bit-reproducible, because of Apple's tools. On Xcode 27 (macOS 27, arm64), four files differ. For +each of them, `DX-REPRO-02` checks the parts that should still match: + +| File | Why it differs | What must still match | +|---|---|---| +| The main executable (`Contents/MacOS/`) | The linker puts two symbol pointers in the other order, so some instructions and `__objc_stubs` refer to other slots. `LC_UUID` is a hash of the output, and the code signature is a hash of the file. | Every load command apart from `LC_UUID`, and the symbols (`nm -m`). | +| `Assets.car` | `actool` stamps the build time and gives each rendition a unique name, which also changes its digest. | Every rendition's other attributes (`assetutil --info`). | +| `*.nib` | `ibtool` writes the same objects in another order. | The objects, each with its class and values, with references as the referenced object's class. | +| `_CodeSignature/CodeResources` | It holds the hashes of the files above. | Nothing more: the files it seals are compared themselves. | + +Any other difference, or a difference in what these compare, fails `DX-REPRO-02`. The frameworks' binaries (Hermes and +the rest) and every other resource are byte-identical. + ### Budgets `e2e/budgets.yaml` (schema: `e2e/schema/budgets.schema.json`) holds baselines per diff --git a/e2e/checks/developer-experience.yaml b/e2e/checks/developer-experience.yaml new file mode 100644 index 00000000..fe22d908 --- /dev/null +++ b/e2e/checks/developer-experience.yaml @@ -0,0 +1,53 @@ +# Developer experience (#171): gate audit steps, not flows (docs/e2e-flows.md#audits). +area: developer-experience +prefix: DX +checks: + DX-DOCS-01: + title: Every public export (docs/api-public-surface.json) has a TSDoc comment on its declaration + blocking: true + spec: "#171 Every export documented" + audit: api-docs + DX-DOCS-02: + title: No public export, parameter or return type is any + blocking: true + spec: "#171 Every export typed" + audit: api-docs + DX-DOCS-03: + title: Every public subpath has a docs page that imports it + blocking: true + spec: "#171 Every export documented" + audit: api-docs + DX-CLI-01: + title: spark --help names every command; -h, no arguments and each command's --help exit 0; spark dev --help in an app adds expo start's options; spark --version prints the SDK version + blocking: true + spec: "#171 CLI commands" + audit: cli + DX-CLI-02: + title: CLI misuse (unknown command or option, missing or extra arguments, a group without its subcommand, invalid values, conflicting modes, no app) exits 1 with one line on stderr that names the problem and no stack trace + blocking: true + spec: "#171 CLI commands" + audit: cli + DX-TPL-01: + title: spark create makes each template (macOS, Windows, universal) from the packed SDK, headless (CI=1, no TTY), and it installs and type-checks + platforms: [macos] + blocking: true + spec: "#171 Templates build; headless CI" + audit: templates + DX-TPL-02: + title: spark build of the macOS template from the packed SDK makes a release app headless, with Hermes bytecode and a signature codesign verifies + platforms: [macos] + blocking: true + spec: "#171 Templates build; headless CI" + audit: templates + DX-REPRO-01: + title: Two clean release builds of the macOS template have byte-identical JavaScript bundles + platforms: [macos] + blocking: true + spec: "#171 Reproducible builds" + audit: templates + DX-REPRO-02: + title: Two clean release builds of the macOS template are byte-identical apart from what docs/e2e-flows.md#reproducible-builds explains, and each difference is reported + platforms: [macos] + blocking: true + spec: "#171 Reproducible builds" + audit: templates diff --git a/e2e/schema/checks.schema.json b/e2e/schema/checks.schema.json index 886bad3b..4bcd29dd 100644 --- a/e2e/schema/checks.schema.json +++ b/e2e/schema/checks.schema.json @@ -59,7 +59,10 @@ "audit": { "enum": [ "bundle", - "package" + "package", + "api-docs", + "cli", + "templates" ], "description": "The gate audit step that verifies this check against the release build, instead of a flow (see docs/e2e-flows.md#audits)." } diff --git a/packages/cli/src/cli-arguments.ts b/packages/cli/src/cli-arguments.ts index a1c5d3ff..52183d7e 100644 --- a/packages/cli/src/cli-arguments.ts +++ b/packages/cli/src/cli-arguments.ts @@ -7,9 +7,9 @@ const options = { device: { type: "string" }, project: { type: "string" }, platform: { type: "string" }, packages: { type: "string" }, port: { type: "string" }, runner: { type: "boolean" }, dev: { type: "boolean" }, release: { type: "boolean" }, preview: { type: "boolean" }, - force: { type: "boolean" }, dsym: { type: "string", multiple: true }, "submission-id": { type: "string" }, "ad-hoc": { type: "boolean" }, help: { type: "boolean", short: "h" }, + force: { type: "boolean" }, dsym: { type: "string", multiple: true }, "submission-id": { type: "string" }, "ad-hoc": { type: "boolean" }, help: { type: "boolean", short: "h" }, version: { type: "boolean" }, } as const; -const commands: Record = { +export const commands: Record = { create: { flags: ["example", "package-manager", "universal", "platform", "packages"], arguments: [1, 1] }, "add desktop": { flags: ["project", "packages"], arguments: [0, 0] }, "sdk export": { flags: ["packages", "runtime"], arguments: [1, 1] }, @@ -35,9 +35,10 @@ export function cliArguments(args: string[]) { const grouped = ["sdk", "add", "updates"].includes(positionals[0]); const command = positionals.slice(0, grouped ? 2 : 1).join(" "); if (!command || (grouped && positionals.length === 1 && values.help)) { - if (Object.keys(values).some(key => key !== "help")) throw new Error("Choose a command before specifying options. Run spark --help."); + if (Object.keys(values).some(key => key !== "help" && key !== "version")) throw new Error("Choose a command before specifying options. Run spark --help."); return parsed; } + if (grouped && positionals.length === 1) throw new Error(`Choose a subcommand: ${Object.keys(commands).filter(name => name.startsWith(`${command} `)).map(name => `spark ${name}`).join(", ")}.`); const contract = commands[command]; if (!contract) throw new Error(`Unknown command: ${command}. Run spark --help.`); for (const key of Object.keys(values)) if (key !== "help" && !contract.flags.includes(key)) throw new Error(`--${key} is not an option for spark ${command}`); diff --git a/packages/cli/src/dev-command.ts b/packages/cli/src/dev-command.ts index 01459021..0cc3d605 100644 --- a/packages/cli/src/dev-command.ts +++ b/packages/cli/src/dev-command.ts @@ -2,7 +2,7 @@ import { spawnProcess } from "./process.ts"; import path from "node:path"; import { prepareConfig, selectTarget, supportedPlatforms } from "@legendapp/spark-desktop-config/config.cjs"; import { dev } from "./dev.ts"; -import { findProject } from "./local.ts"; +import { appRoot, findProject } from "./local.ts"; import { hostPlatform } from "./platform.ts"; import { nodeCommand } from "./windows.ts"; @@ -44,7 +44,8 @@ export async function devCommand(args: string[]) { const options = devArguments(args); const start = path.resolve(options.project ?? process.cwd()); if (options.expo.includes("--help") || options.expo.includes("-h")) { - console.log("spark dev options:\n --project Application directory\n --platform Initial launch target; all declared platforms stay available\n --runner-binary Register and use a Spark Runner\n --app-arg Pass a desktop launch argument (repeatable)\n --no-open Wait for a launch key (explicit Expo launch flags still apply)\n\nAll other options belong to expo start:\n"); + console.log("spark dev options:\n --project Application directory\n --platform Initial launch target; all declared platforms stay available\n --runner-binary Register and use a Spark Runner\n --app-arg Pass a desktop launch argument (repeatable)\n --no-open Wait for a launch key (explicit Expo launch flags still apply)\n\nAll other options belong to expo start" + (appRoot(start) ? ":\n" : "; run spark dev --help inside an app to list them.")); + if (!appRoot(start)) return; const child = spawnProcess(nodeCommand(start, "expo", "expo", ["start", "--help"]), { cwd: start, stdin: "inherit", stdout: "inherit", stderr: "inherit" }); process.exitCode = await child.exited; return; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index b5a9e4ac..d5a0ba5c 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -59,7 +59,8 @@ async function main() { process.exit(process.exitCode); } } - if (values.help || !command) { + if (values.version && !command) console.log(VERSION); + else if (values.help || !command) { console.log(`Legend Spark spark create MyApp Create an app (--universal for shared Settings; --platform selects a target) @@ -76,6 +77,7 @@ async function main() { Inside an app: npm run dev, npm run build, npm run package (or pnpm, yarn, bun) Advanced: updates init , credentials, doctor, analyze, open [app], build --dev, build --preview, package --ad-hoc (test package, not notarized) +Version: spark --version Crash reports: symbolicate --dsym (prints the report with symbols and source lines) Windows: dev and build --dev; production builds are not yet supported. SDK transfer: sdk export [--runtime ], sdk import diff --git a/packages/cli/src/local.ts b/packages/cli/src/local.ts index 7e7f141b..838a9849 100644 --- a/packages/cli/src/local.ts +++ b/packages/cli/src/local.ts @@ -21,16 +21,20 @@ export function findFramework(start = import.meta.dirname): string | undefined { } } -export function findProject(start: string): string { - let dir = path.resolve(start); - while (true) { +/** The app that contains `start`, or undefined outside an app. */ +export function appRoot(start: string): string | undefined { + for (let dir = path.resolve(start); ; dir = path.dirname(dir)) { if ((existsSync(path.join(dir, "app.json")) || existsSync(path.join(dir, "desktop.config.json"))) && existsSync(path.join(dir, "package.json"))) return dir; - const parent = path.dirname(dir); - if (parent === dir) throw new Error("No spark app found. Run this command inside your app, or create one with spark create MyApp."); - dir = parent; + if (path.dirname(dir) === dir) return undefined; } } +export function findProject(start: string): string { + const root = appRoot(start); + if (!root) throw new Error("No spark app found. Run this command inside your app, or create one with spark create MyApp."); + return root; +} + export function readRuntime(app: string): Runtime | undefined { try { const windows = existsSync(path.join(app, "spark-runtime.json")); diff --git a/scripts/api-docs.ts b/scripts/api-docs.ts new file mode 100644 index 00000000..887e513c --- /dev/null +++ b/scripts/api-docs.ts @@ -0,0 +1,43 @@ +// API documentation coverage of the public SDK surface (docs/api-public-surface.json): what an editor shows for +// each export, and the docs page for each subpath. tests/api-docs-coverage.test.ts and the gate's api-docs audit use it. +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import ts from "typescript"; +import { sdkProgram } from "./api-surface.ts"; + +const docs = path.resolve(import.meta.dirname, "../docs"); +/** Known gaps, drained as they are documented: tests/api-docs-coverage.test.ts keeps it exact. */ +export const API_DOCS_GAPS = "docs/api-undocumented.json"; + +export type ApiDocs = { + /** Per subpath, the exports without a TSDoc comment on their declaration. */ + undocumented: Record; + /** Per subpath, the runtime exports whose type, a parameter or a return type is `any`. */ + untyped: Record; + /** Subpaths that no docs/*.md page imports (`@legendapp/spark/` in quotes or backticks). */ + pageless: string[]; +}; + +const isAny = (type: ts.Type) => !!(type.flags & ts.TypeFlags.Any); + +export function readApiDocs(): ApiDocs { + const { checker, modules } = sdkProgram(); + const pages = readdirSync(docs, { recursive: true, encoding: "utf8" }).filter(file => file.endsWith(".md")).map(file => readFileSync(path.join(docs, file), "utf8")).join("\n"); + const result: ApiDocs = { undocumented: {}, untyped: {}, pageless: [] }; + const add = (map: Record, subpath: string, name: string) => (map[subpath] ??= []).push(name); + for (const { subpath, source, symbols } of [...modules].sort((a, b) => a.subpath.localeCompare(b.subpath))) { + const specifier = `@legendapp/spark${subpath.slice(1)}`; + if (![`"${specifier}"`, `'${specifier}'`, `\`${specifier}\``].some(quoted => pages.includes(quoted))) result.pageless.push(subpath); + for (const { symbol, target } of [...symbols].sort((a, b) => a.symbol.name.localeCompare(b.symbol.name))) { + if (!ts.displayPartsToString(target.getDocumentationComment(checker)).trim()) add(result.undocumented, subpath, symbol.name); + if (!(target.flags & ts.SymbolFlags.Value)) continue; + const type = checker.getTypeOfSymbolAtLocation(target, source); + const signatures = type.getCallSignatures(); + if (isAny(type) || signatures.some(signature => isAny(signature.getReturnType()) || signature.parameters.some(parameter => isAny(checker.getTypeOfSymbolAtLocation(parameter, source))))) + add(result.untyped, subpath, symbol.name); + } + } + return result; +} + +export const count = (map: Record) => Object.values(map).reduce((total, names) => total + names.length, 0); diff --git a/scripts/api-surface.ts b/scripts/api-surface.ts index fb382a2c..3006b1fe 100644 --- a/scripts/api-surface.ts +++ b/scripts/api-surface.ts @@ -27,23 +27,33 @@ function availabilityFlags(checker: ts.TypeChecker, symbol: ts.Symbol, at: ts.No return args.flatMap(arg => booleans.map(flag => `(${arg}).${flag}`)); } -/** - * The public SDK surface: every subpath in packages/desktop/package.json `exports`, and for TypeScript - * entries every export the type checker sees. docs/api-public-surface.json snapshots the export names. - */ -export function readSdkSurface(): SdkSurface { +/** The SDK's TypeScript entries, type-checked with the repository's tsconfig: per subpath, its exports and what each aliases. */ +export function sdkProgram() { const exports = JSON.parse(readFileSync(path.join(sdk, "package.json"), "utf8")).exports as Record; const entries = Object.entries(exports).filter(([, file]) => file.endsWith(".ts")); const config = ts.readConfigFile(path.join(repo, "tsconfig.json"), ts.sys.readFile); const options = ts.parseJsonConfigFileContent(config.config, ts.sys, repo).options; const program = ts.createProgram(entries.map(([, file]) => path.join(sdk, file)), options); const checker = program.getTypeChecker(); - const surface: SdkSurface = Object.fromEntries(Object.keys(exports).sort().map(subpath => [subpath, { values: [], types: [], flags: {} }])); - for (const [subpath, file] of entries) { + const modules = entries.map(([subpath, file]) => { const source = program.getSourceFile(path.join(sdk, file))!; + const symbols = checker.getExportsOfModule(checker.getSymbolAtLocation(source)!) + .map(symbol => ({ symbol, target: symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol })); + return { subpath, source, symbols }; + }); + return { exports, checker, modules }; +} + +/** + * The public SDK surface: every subpath in packages/desktop/package.json `exports`, and for TypeScript + * entries every export the type checker sees. docs/api-public-surface.json snapshots the export names. + */ +export function readSdkSurface(): SdkSurface { + const { exports, modules, checker } = sdkProgram(); + const surface: SdkSurface = Object.fromEntries(Object.keys(exports).sort().map(subpath => [subpath, { values: [], types: [], flags: {} }])); + for (const { subpath, source, symbols } of modules) { const entry = surface[subpath]!; - for (const symbol of checker.getExportsOfModule(checker.getSymbolAtLocation(source)!)) { - const target = symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol; + for (const { symbol, target } of symbols) { if (!(target.flags & ts.SymbolFlags.Value)) { entry.types.push(symbol.name); continue; } entry.values.push(symbol.name); if (AVAILABILITY.test(symbol.name)) entry.flags[symbol.name] = availabilityFlags(checker, target, source); diff --git a/scripts/e2e/audit/api-docs.ts b/scripts/e2e/audit/api-docs.ts new file mode 100644 index 00000000..e1aca436 --- /dev/null +++ b/scripts/e2e/audit/api-docs.ts @@ -0,0 +1,18 @@ +// The gate's api-docs audit: API documentation coverage of the public SDK surface (docs/e2e-flows.md#audits, #171). +// The same facts as tests/api-docs-coverage.test.ts, which keeps docs/api-undocumented.json exact; this fails while +// any gap remains. +import { API_DOCS_GAPS, count, readApiDocs } from "../../api-docs.ts"; +import type { AuditResult } from "./bundle.ts"; + +const listed = (map: Record) => Object.entries(map).map(([subpath, names]) => `${subpath}: ${names.join(", ")}`); + +export function auditApiDocs(): AuditResult[] { + const { undocumented, untyped, pageless } = readApiDocs(); + const result = (check: string, problems: string[], summary: string, passed: string): AuditResult => + ({ check, status: problems.length ? "failed" : "passed", details: problems.length ? [summary, ...problems] : [passed] }); + return [ + result("DX-DOCS-01", listed(undocumented), `${count(undocumented)} public exports have no TSDoc comment (listed in ${API_DOCS_GAPS})`, "every public export has a TSDoc comment"), + result("DX-DOCS-02", listed(untyped), `${count(untyped)} public exports are typed any`, "no public export, parameter or return type is any"), + result("DX-DOCS-03", pageless, `${pageless.length} public subpaths have no docs page (listed in ${API_DOCS_GAPS})`, "every public subpath has a docs page"), + ]; +} diff --git a/scripts/e2e/audit/cli.ts b/scripts/e2e/audit/cli.ts new file mode 100644 index 00000000..821459bc --- /dev/null +++ b/scripts/e2e/audit/cli.ts @@ -0,0 +1,62 @@ +// The gate's cli audit: the `spark` command line as an app gets it (packages/desktop/bin/spark.cjs, the SDK's bin), +// its help and its errors (docs/e2e-flows.md#audits, #171). Every case runs headless (no TTY, CI=1) with a scratch +// SPARK_HOME, and only cases that fail before any side effect, so nothing is created, built or registered. +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { commands } from "../../../packages/cli/src/cli-arguments.ts"; +import { KITCHEN_SINK } from "../kitchen-sink.ts"; +import type { AuditResult } from "./bundle.ts"; + +const repo = path.resolve(import.meta.dirname, "../../.."); +const bin = path.join(repo, "packages/desktop/bin/spark.cjs"); +const version = JSON.parse(readFileSync(path.join(repo, "packages/desktop/package.json"), "utf8")).version as string; + +/** Each error case: the arguments and what its one-line message must name. */ +const ERRORS: Array<[string[], string]> = [ + [["bogus"], "Unknown command: bogus"], + [["build", "--bogus"], "--bogus"], + [["create"], "spark create"], + [["build", "extra"], "spark build"], + [["sdk"], "spark sdk export"], + [["build", "--platform", "beos"], "Platform must be"], + [["build", "--port", "99999"], "Port must be"], + [["create", "App", "--package-manager", "zz"], "Package manager must be"], + [["create", "App", "--example", "nope"], "--example"], + [["build", "--dev", "--release"], "build mode"], + [["build"], "No spark app found"], + [["dev"], "No spark app found"], +]; + +export function auditCli(): AuditResult[] { + const scratch = mkdtempSync(path.join(os.tmpdir(), "spark-cli-audit-")); + try { + const spark = (args: string[], cwd = scratch) => { + const result = spawnSync("node", [bin, ...args], { cwd, encoding: "utf8", timeout: 60_000, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, CI: "1", SPARK_HOME: path.join(scratch, ".spark") } }); + return { status: result.status, stdout: result.stdout, stderr: result.stderr, label: `spark ${args.join(" ")}`.trim() }; + }; + const help: string[] = [], errors: string[] = []; + const global = spark(["--help"]); + const names = ["dev", ...Object.keys(commands)]; + const missing = names.filter(name => !new RegExp(`(^|[\\s/])${name}\\b`, "m").test(global.stdout)); + if (global.status !== 0 || missing.length) help.push(`spark --help exited ${global.status} and does not name ${missing.join(", ") || "-"}`); + for (const args of [[], ["-h"], ...names.map(name => [...name.split(" "), "--help"])]) { + const run = spark(args); + if (run.status !== 0 || !run.stdout.includes("spark ") || run.stderr.trim()) help.push(`${run.label} exited ${run.status}${run.stderr.trim() ? ` with ${JSON.stringify(run.stderr.trim().split("\n")[0])}` : ""}`); + } + const devHelp = spark(["dev", "--help"], KITCHEN_SINK); + if (devHelp.status !== 0 || !devHelp.stdout.includes("--no-open") || !devHelp.stdout.includes("--port")) help.push(`spark dev --help in an app exited ${devHelp.status} without spark's and expo start's options`); + const printed = spark(["--version"]); + if (printed.status !== 0 || printed.stdout.trim() !== version) help.push(`spark --version exited ${printed.status} and printed ${JSON.stringify(printed.stdout.trim())}, not ${version}`); + for (const [args, names] of ERRORS) { + const run = spark(args), lines = run.stderr.trim().split("\n"); + if (run.status !== 1 || run.stdout.trim() || lines.length !== 1 || !lines[0]!.includes(names)) + errors.push(`${run.label}: exit ${run.status}, stdout ${JSON.stringify(run.stdout.trim().slice(0, 80))}, stderr ${JSON.stringify(run.stderr.trim().slice(0, 300))}; expected exit 1 and one line naming ${JSON.stringify(names)}`); + } + return [ + { check: "DX-CLI-01", status: help.length ? "failed" : "passed", details: help.length ? help : [`spark --help names all ${names.length} commands; -h, no arguments and each command's --help exit 0; spark dev --help in an app adds expo start's options; spark --version prints ${version}`] }, + { check: "DX-CLI-02", status: errors.length ? "failed" : "passed", details: errors.length ? errors : [`${ERRORS.length} misuses (unknown command and option, missing and extra arguments, a group without its subcommand, invalid values, conflicting modes, no app) exit 1 with one line on stderr that names the problem, no stack trace`] }, + ]; + } finally { rmSync(scratch, { recursive: true, force: true }); } +} diff --git a/scripts/e2e/audit/steps.ts b/scripts/e2e/audit/steps.ts index 6913ba70..da6d4ba0 100644 --- a/scripts/e2e/audit/steps.ts +++ b/scripts/e2e/audit/steps.ts @@ -1,6 +1,11 @@ // The gate audit steps by the name a check registers with `audit:` (docs/e2e-flows.md#audits). import type { Audit } from "../format/primitives.ts"; +import { auditApiDocs } from "./api-docs.ts"; import { auditBundle, type AuditOptions, type AuditResult } from "./bundle.ts"; +import { auditCli } from "./cli.ts"; import { auditPackage } from "./package.ts"; +import { auditTemplates } from "./templates.ts"; -export const AUDIT_STEPS: Readonly AuditResult[] | Promise>> = { bundle: auditBundle, package: options => auditPackage(options) }; +export const AUDIT_STEPS: Readonly AuditResult[] | Promise>> = { + bundle: auditBundle, package: options => auditPackage(options), "api-docs": auditApiDocs, cli: auditCli, templates: auditTemplates, +}; diff --git a/scripts/e2e/audit/templates.ts b/scripts/e2e/audit/templates.ts new file mode 100644 index 00000000..83106d64 --- /dev/null +++ b/scripts/e2e/audit/templates.ts @@ -0,0 +1,169 @@ +// The gate's templates audit (docs/e2e-flows.md#audits, #171): `spark create` from the packed SDK, headless, in a +// scratch directory, then two clean release builds of the macOS template compared file by file. Everything runs with +// CI=1, no TTY and no stdin, as on a CI runner, with a scratch SPARK_HOME, so the developer's registered SDKs and +// runners are untouched. The built apps are unregistered from LaunchServices and the scratch directory is removed. +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, openSync, closeSync, readdirSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { readJson, stateFile } from "../../../packages/cli/src/project.ts"; +import { isMachO, type AuditResult } from "./bundle.ts"; + +const repo = path.resolve(import.meta.dirname, "../../.."); +const LSREGISTER = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister"; +const HERMES_MAGIC = Buffer.from([0xc6, 0x1f, 0xbc, 0x03, 0xc1, 0x03, 0x19, 0x1f]); +/** The templates `spark create` offers, by the flag that selects each. */ +const TEMPLATES: Array<[name: string, flags: string[]]> = [["DxMac", ["--platform", "macos"]], ["DxWindows", ["--platform", "windows"]], ["DxUniversal", ["--universal"]]]; +export const TEMPLATE_CHECKS = ["DX-TPL-01", "DX-TPL-02", "DX-REPRO-01", "DX-REPRO-02"] as const; + +class StepError extends Error {} + +/** Every file and link in an app bundle, by its path inside the bundle: a SHA-256 of a file's bytes, a link's target. */ +function digests(app: string, rel = ""): Map { + const files = new Map(); + for (const entry of readdirSync(path.join(app, rel), { withFileTypes: true })) { + const name = path.join(rel, entry.name), abs = path.join(app, name); + if (entry.isSymbolicLink()) files.set(name, `-> ${readlinkSync(abs)}`); + else if (entry.isDirectory()) for (const [key, value] of digests(app, name)) files.set(key, value); + else files.set(name, createHash("sha256").update(readFileSync(abs)).digest("hex")); + } + return files; +} + +const isScript = (name: string) => /\.(jsbundle|hbc|js)$/.test(name); +const output = (argv: string[]) => spawnSync(argv[0]!, argv.slice(1), { encoding: "utf8", maxBuffer: 1 << 28 }).stdout; +/** A Mach-O file's load commands apart from LC_UUID, and its symbols. */ +const machOShape = (file: string) => [output(["otool", "-l", file]).split("\n").slice(1).filter(line => !/^\s*uuid /.test(line)).join("\n"), output(["nm", "-m", file])].join("\n"); +/** An asset catalog's renditions without what actool makes unique per build: its timestamp, rendition names' unique suffixes and digests. */ +const catalogShape = (file: string) => output(["xcrun", "assetutil", "--info", file]).replace(/"Timestamp" : \d+/g, "").replace(/_[0-9A-F]{8}-[0-9A-F-]{27}-\d+-[0-9A-F]+\.png/g, ".png").replace(/"SHA1Digest" : "\w+"/g, ""); +/** + * A compiled nib (NIBArchive) as the sorted list of its objects: each its class and its values, with a reference to + * another object written as that object's class. The same objects archived in another order have the same list. + */ +function nibShape(file: string): string { + const data = readFileSync(file); + if (data.subarray(0, 10).toString("latin1") !== "NIBArchive") return data.toString("hex"); + const header = Array.from({ length: 8 }, (_, index) => data.readUInt32LE(18 + index * 4)); + let at = 0; + const varint = () => { let value = 0, shift = 0, byte; do { byte = data[at++]!; value |= (byte & 0x7f) << shift; shift += 7; } while (!(byte & 0x80)); return value; }; + const list = (count: number, offset: number, read: () => T) => { at = offset; return Array.from({ length: count }, read); }; + const objects = list(header[0]!, header[1]!, () => ({ type: varint(), start: varint(), count: varint() })); + const keys = list(header[2]!, header[3]!, () => { const length = varint(); return data.subarray(at, (at += length)).toString("latin1"); }); + const SIZES: Record = { 0: 1, 1: 2, 2: 4, 3: 8, 4: 0, 5: 0, 6: 4, 7: 8, 9: 0, 10: 4 }; + const values = list(header[4]!, header[5]!, () => { + const key = keys[varint()]!, type = data[at++]!; + const length = type === 8 ? varint() : SIZES[type] ?? 0; + return { key, type, bytes: data.subarray(at, (at += length)) }; + }); + const classes = list(header[6]!, header[7]!, () => { const length = varint(), extra = varint(); at += extra * 4; return data.subarray(at, (at += length)).toString("latin1"); }); + const value = ({ key, type, bytes }: typeof values[number]) => `${key}=${type === 10 ? `@${classes[objects[bytes.readUInt32LE(0)]!.type]}` : `${type}:${bytes.toString("hex")}`}`; + return objects.map(object => `${classes[object.type]}{${values.slice(object.start, object.start + object.count).map(value).sort().join(",")}}`).sort().join("\n"); +} +/** + * Files Apple's tools do not write reproducibly (docs/e2e-flows.md#reproducible-builds), and what of them must still match: + * a difference anywhere else, or in what these compare, fails. + */ +const KNOWN: Array<{ match: (name: string, abs: string) => boolean; why: string; shape: (file: string) => string }> = [ + { match: (name, abs) => !name.includes("/_CodeSignature/") && isMachO({ rel: name, abs, size: statSync(abs).size, kind: "file" }), why: "the linker orders some symbol pointers differently, so LC_UUID and the code signature differ; load commands and symbols match", shape: machOShape }, + { match: name => name.endsWith(".car"), why: "actool stamps the build time and unique rendition names; the renditions match", shape: catalogShape }, + { match: name => name.endsWith(".nib"), why: "ibtool archives the same objects in another order; the objects and their values match", shape: nibShape }, + { match: name => name.endsWith("_CodeSignature/CodeResources"), why: "it seals the files above", shape: () => "" }, +]; + +/** DX-REPRO-01 and DX-REPRO-02 for two release builds of the same app. */ +export function compareBuilds(first: string, second: string): AuditResult[] { + const [a, b] = [digests(first), digests(second)]; + const names = [...new Set([...a.keys(), ...b.keys()])].sort(); + const differing = names.filter(name => a.get(name) !== b.get(name)); + const scripts = names.filter(isScript), jsDiffs = differing.filter(isScript); + const native: string[] = [], unexplained: string[] = []; + for (const name of differing.filter(name => !isScript(name))) { + const [left, right] = [path.join(first, name), path.join(second, name)]; + const known = a.has(name) && b.has(name) && !a.get(name)!.startsWith("-> ") ? KNOWN.find(entry => entry.match(name, left)) : undefined; + if (known && known.shape(left) === known.shape(right)) native.push(`${name}: ${known.why}`); + else unexplained.push(`${name}: ${!a.has(name) ? "only in the second build" : !b.has(name) ? "only in the first build" : known ? `differs beyond what ${known.why.split(";")[0]} explains` : "differs"}`); + } + return [ + { check: "DX-REPRO-01", status: jsDiffs.length || !scripts.length ? "failed" : "passed", details: jsDiffs.length ? jsDiffs.map(name => `${name} differs between two clean release builds`) + : scripts.length ? [`${scripts.join(", ")}: byte-identical across two clean release builds`] : ["no JavaScript in the release build"] }, + { check: "DX-REPRO-02", status: unexplained.length ? "failed" : "passed", details: [`${names.length - differing.length} of ${names.length} files and links are byte-identical across two clean release builds`, ...unexplained, ...native] }, + ]; +} + +export async function auditTemplates(): Promise { + const results = new Map(); + const set = (check: string, status: AuditResult["status"], details: string[]) => results.set(check, { check, status, details }); + const scratch = mkdtempSync(path.join(os.tmpdir(), "spark-templates-audit-")); + const env = { ...process.env, CI: "1", SPARK_HOME: path.join(scratch, ".spark-home") }; + const step = (label: string, argv: string[], cwd: string) => { + const log = path.join(scratch, `${label.replace(/\W+/g, "-")}.log`), fd = openSync(log, "w"); + const started = Date.now(); + try { + const result = spawnSync(argv[0]!, argv.slice(1), { cwd, env, stdio: ["ignore", fd, fd] }); + const seconds = Math.round((Date.now() - started) / 100) / 10; + if (result.status !== 0) throw new StepError(`${label} exited ${result.status ?? result.signal} after ${seconds} s:\n${readFileSync(log, "utf8").trim().split("\n").slice(-15).join("\n")}`); + return seconds; + } finally { closeSync(fd); } + }; + const spark = (app: string, label: string, args: string[]) => step(label, ["node", path.join(app, "node_modules/@legendapp/spark/bin/spark.cjs"), ...args], app); + try { + const timings: string[] = [`pack ${step("spark sdk pack", [process.execPath, path.join(repo, "scripts/pack.ts")], repo)} s`]; + const manifest = path.join(repo, "artifacts/packages/manifest.json"); + // DX-TPL-01: each template from the packed SDK, through the SDK's own `spark` bin in a scratch directory. + const sdkBin = path.join(repo, "packages/desktop/bin/spark.cjs"); + for (const [name, flags] of TEMPLATES) { + const app = path.join(scratch, name); + timings.push(`create ${name} ${step(`spark create ${name}`, ["node", sdkBin, "create", app, "--packages", manifest, ...flags], scratch)} s`); + const config = JSON.parse(readFileSync(path.join(app, "desktop.config.json"), "utf8")); + const dependency = JSON.parse(readFileSync(path.join(app, "package.json"), "utf8")).dependencies["@legendapp/spark"] as string; + if (!dependency.includes(path.join(repo, "artifacts/packages"))) throw new StepError(`${name} depends on @legendapp/spark ${dependency}, not the packed SDK`); + if (config.name !== name) throw new StepError(`${name}: desktop.config.json names the app ${config.name}`); + timings.push(`typecheck ${name} ${step(`tsc ${name}`, ["node", "node_modules/typescript/bin/tsc", "--noEmit"], app)} s`); + } + set("DX-TPL-01", "passed", [`spark create made ${TEMPLATES.map(([name, flags]) => `${name} (${flags.join(" ")})`).join(", ")} from the packed SDK (${path.relative(repo, manifest)}), installed and type-checked, headless`, timings.join(", ")]); + + // DX-TPL-02 and the reproducibility checks: two clean release builds of the macOS template. + // `spark build` prepares the project in place (it patches node_modules and writes the native project and + // configuration), so each build starts from a clone of the project as `spark create` left it, at the same path. + const app = path.join(scratch, "DxMac"), pristine = path.join(scratch, "DxMac.pristine"); + step("clone the project", ["cp", "-c", "-R", app, pristine], scratch); + const builds: string[] = []; + for (const run of [1, 2]) { + if (run === 2) { + rmSync(app, { recursive: true, force: true }); + renameSync(pristine, app); + } + const seconds = spark(app, `spark build ${run}`, ["build"]); + const product = readJson(stateFile(app, "release-build.json")).app as string; + if (!existsSync(path.join(product, "Contents/Info.plist"))) throw new StepError(`spark build ${run} finished without ${product}`); + const copy = path.join(scratch, `release-${run}`, path.basename(product)); + mkdirSync(path.dirname(copy), { recursive: true }); + cpSync(product, copy, { recursive: true, verbatimSymlinks: true }); + builds.push(copy); + if (run === 1) { + const bundle = path.join(copy, "Contents/Resources/main.jsbundle"); + const hermes = existsSync(bundle) && readFileSync(bundle).subarray(0, 8).equals(HERMES_MAGIC); + const verify = spawnSync("codesign", ["--verify", "--deep", "--strict", copy], { encoding: "utf8" }); + if (!hermes || verify.status !== 0) throw new StepError(`the release build ${!hermes ? "has no Hermes bytecode main.jsbundle" : `fails codesign --verify: ${verify.stderr.trim()}`}`); + set("DX-TPL-02", "passed", [`spark build of the macOS template made DxMac.app headless in ${seconds} s (clean): Hermes bytecode main.jsbundle, codesign --verify --deep --strict accepts it`]); + } + } + for (const result of compareBuilds(builds[0]!, builds[1]!)) results.set(result.check, result); + } catch (error) { + const message = error instanceof StepError ? error.message : String(error instanceof Error ? error.stack : error); + for (const check of TEMPLATE_CHECKS) if (!results.has(check)) set(check, "failed", [message]); + } finally { + // Xcode registers what it builds with LaunchServices: unregister every app under the scratch directory. + const registered = () => [...spawnSync(LSREGISTER, ["-dump"], { encoding: "utf8", maxBuffer: 1 << 30 }).stdout.matchAll(/^path:\s+(.+?\.app)\b/gm)] + .map(match => match[1]!).filter(app => app.startsWith(`${realpathSync(scratch)}/`)); + for (const app of new Set(registered())) spawnSync(LSREGISTER, ["-u", app]); + const left = [...new Set(registered())]; + if (left.length) for (const check of TEMPLATE_CHECKS) { + const result = results.get(check); + if (result) results.set(check, { check, status: "failed", details: [...result.details, `still registered with LaunchServices: ${left.join(", ")}`] }); + } + rmSync(scratch, { recursive: true, force: true }); + } + return TEMPLATE_CHECKS.map(check => results.get(check)!); +} diff --git a/scripts/e2e/cli.ts b/scripts/e2e/cli.ts index d42b30e3..8ae44bcc 100644 --- a/scripts/e2e/cli.ts +++ b/scripts/e2e/cli.ts @@ -63,6 +63,6 @@ if (mode === "schema") { ? await gateCommand(args, path.join(repo, "e2e"), { ...hostEnv(), signal: controller.signal }) : await runCommand(args, { backends: BACKENDS, host: detectHost, stdout: line => console.log(line), stderr: line => console.error(line), signal: controller.signal }); } else { - console.error("usage: bun run e2e:validate [paths…] | bun run e2e:lint [paths…] | bun run e2e:coverage [paths…] [--json] | bun run e2e:schema | bun run e2e:run [flows…] [--backend …] [--agent] [--report dir] [--app path] | bun run gate [--target id] [--flow path…] [--agent] [--dry-run] | bun run e2e:audit [--artifact app] | bun run gate:signoff [--list] | bun run e2e:golden approve [--to ]"); + console.error("usage: bun run e2e:validate [paths…] | bun run e2e:lint [paths…] | bun run e2e:coverage [paths…] [--json] | bun run e2e:schema | bun run e2e:run [flows…] [--backend …] [--agent] [--report dir] [--app path] | bun run gate [--target id] [--flow path…] [--agent] [--dry-run] | bun run e2e:audit [--artifact app] [--package path] [--audit name]… | bun run gate:signoff [--list] | bun run e2e:golden approve [--to ]"); process.exitCode = 2; } diff --git a/scripts/e2e/format/primitives.ts b/scripts/e2e/format/primitives.ts index 71d116f4..788d75db 100644 --- a/scripts/e2e/format/primitives.ts +++ b/scripts/e2e/format/primitives.ts @@ -4,7 +4,7 @@ export type Schema = Record; export const CHECK_ID = "^[A-Z][A-Z0-9]*-[A-Z0-9]+-\\d{2}$"; export const PLATFORMS = ["macos", "windows"] as const; /** Gate audit steps: a check registered with `audit: ` is verified by that step against the release build, not by a flow. */ -export const AUDITS = ["bundle", "package"] as const; +export const AUDITS = ["bundle", "package", "api-docs", "cli", "templates"] as const; export type Audit = typeof AUDITS[number]; const ref = (name: string): Schema => ({ $ref: `#/definitions/${name}` }); diff --git a/scripts/e2e/gate/gate.ts b/scripts/e2e/gate/gate.ts index 539a5b9e..eb3e3283 100644 --- a/scripts/e2e/gate/gate.ts +++ b/scripts/e2e/gate/gate.ts @@ -7,7 +7,7 @@ import { collect } from "../files.ts"; import { checkFile } from "../format/lint.ts"; import type { AuditOptions, AuditResult } from "../audit/bundle.ts"; import { checkSignoffs, FlowFormatError, formatDiagnostic, type Signoff } from "../format/parser.ts"; -import type { Audit } from "../format/primitives.ts"; +import { AUDITS, type Audit } from "../format/primitives.ts"; import type { ManualStep } from "../format/signoff.ts"; import type { BackendFactory } from "../runner/backend.ts"; import { ms, realClock, type Clock, type FlowResult } from "../runner/executor.ts"; @@ -150,7 +150,12 @@ function judgeFlows(plan: GatePlan, results: FlowResult[] | undefined, runDir: s export async function judgeAudits(plan: GatePlan, env: GateEnv, exists: boolean, reasons: Reason[], warnings: string[]): Promise { const build = plan.target && plan.manifest.build[plan.target.platform]; const options = build && { app: plan.artifact!, platform: plan.target!.platform, arch: plan.target!.arch, appId: plan.manifest.appId, sizeBudgetMB: build.sizeBudgetMB, symbols: plan.symbols, package: plan.package }; - const results = exists && options ? (await Promise.all([...new Set(plan.audits.map(audit => audit.audit))].map(name => env.audit(name, options)))).flat() : undefined; + // One step at a time: steps that build or time apps must not overlap. + let results: AuditResult[] | undefined; + if (exists && options) { + results = []; + for (const name of new Set(plan.audits.map(audit => audit.audit))) results.push(...await env.audit(name, options)); + } return plan.audits.map(({ check, audit, title, blocking }) => { const result = results?.find(candidate => candidate.check === check); const entry: AuditEntry = { check, audit, title, blocking, status: result?.status ?? "not-run", details: result?.details ?? [] }; @@ -366,17 +371,23 @@ export async function gateCommand(args: string[], root: string, env: GateEnv): P */ export async function auditCommand(args: string[], root: string, env: GateEnv): Promise { const paths: { artifact?: string; package?: string } = {}; + const only: string[] = []; for (let index = 0; index < args.length; index++) { const [flag, inline] = args[index]!.split(/=(.*)/s, 2) as [string, string | undefined]; const value = inline ?? args[++index]; - if ((flag !== "--artifact" && flag !== "--package") || !value) { env.stderr("usage: bun run e2e:audit [--artifact ] [--package ]"); return 2; } - paths[flag.slice(2) as "artifact" | "package"] = path.resolve(value); + if (!["--artifact", "--package", "--audit"].includes(flag) || !value || (flag === "--audit" && !(AUDITS as readonly string[]).includes(value))) { + env.stderr(`usage: bun run e2e:audit [--artifact ] [--package ] [--audit ${AUDITS.join("|")}]…`); + return 2; + } + if (flag === "--audit") only.push(value); + else paths[flag.slice(2) as "artifact" | "package"] = path.resolve(value); } let gatePlan: GatePlan; try { gatePlan = plan(root, env.host(), paths); } catch (error) { if (error instanceof UsageError || error instanceof FlowFormatError) { env.stderr(error.message); return error instanceof UsageError ? 2 : 1; } throw error; } + if (only.length) gatePlan = { ...gatePlan, audits: gatePlan.audits.filter(audit => only.includes(audit.audit)) }; const reasons: Reason[] = [...gatePlan.problems], warnings: string[] = []; if (!gatePlan.audits.length) reasons.push({ check: "audit", message: "no audit checks apply to this target" }); const exists = !!gatePlan.artifact && existsSync(gatePlan.artifact); diff --git a/tests/api-docs-coverage.test.ts b/tests/api-docs-coverage.test.ts new file mode 100644 index 00000000..4fe230d1 --- /dev/null +++ b/tests/api-docs-coverage.test.ts @@ -0,0 +1,26 @@ +import { readFileSync } from "node:fs"; +import { expect, test } from "vitest"; +import { API_DOCS_GAPS, readApiDocs } from "../scripts/api-docs.ts"; + +// Every public export needs a TSDoc comment (what an editor shows), a type other than `any`, and its subpath a docs +// page. docs/api-undocumented.json lists the known gaps: a new undocumented export or page-less subpath fails here, +// and so does a listed one that is now documented, so the list only shrinks. The gate's api-docs audit (DX-DOCS-01, +// DX-DOCS-03) fails while the list is not empty. +const docs = readApiDocs(); +const gaps = JSON.parse(readFileSync(API_DOCS_GAPS, "utf8")) as { pageless: string[]; undocumented: Record }; +const entries = (map: Record) => Object.entries(map).flatMap(([subpath, names]) => names.map(name => `${subpath}#${name}`)); + +test("no public export is typed any", () => { + expect(entries(docs.untyped)).toEqual([]); +}, 20_000); + +test("every public export has a TSDoc comment, apart from the known gaps", () => { + const listed = new Set(entries(gaps.undocumented)), found = entries(docs.undocumented); + expect(found.filter(entry => !listed.has(entry)), "document these exports (a /** … */ comment on the declaration)").toEqual([]); + expect([...listed].filter(entry => !found.includes(entry)), `now documented: remove them from ${API_DOCS_GAPS}`).toEqual([]); +}); + +test("every public subpath has a docs page, apart from the known gaps", () => { + expect(docs.pageless.filter(subpath => !gaps.pageless.includes(subpath)), "add a docs/*.md page that imports these subpaths").toEqual([]); + expect(gaps.pageless.filter(subpath => !docs.pageless.includes(subpath)), `now documented: remove them from ${API_DOCS_GAPS}`).toEqual([]); +}); diff --git a/tests/cli-arguments.test.ts b/tests/cli-arguments.test.ts index e1485887..7c267424 100644 --- a/tests/cli-arguments.test.ts +++ b/tests/cli-arguments.test.ts @@ -11,3 +11,9 @@ test("accepts documented command shapes and rejects incompatible build modes", ( expect(cliArguments(["create", "--help"]).values.help).toBe(true); expect(cliArguments(["sdk", "--help"]).values.help).toBe(true); }); +test("a command group without its subcommand names the subcommands; --version stands alone", () => { + expect(() => cliArguments(["sdk"])).toThrow("Choose a subcommand: spark sdk export, spark sdk import"); + expect(() => cliArguments(["add"])).toThrow("Choose a subcommand: spark add desktop."); + expect(cliArguments(["--version"]).values.version).toBe(true); + expect(() => cliArguments(["build", "--version"])).toThrow("--version is not an option for spark build"); +}); diff --git a/tests/e2e-gate.test.ts b/tests/e2e-gate.test.ts index 0eefbcb3..31a6233b 100644 --- a/tests/e2e-gate.test.ts +++ b/tests/e2e-gate.test.ts @@ -6,7 +6,7 @@ import { describe, expect, test, vi } from "vitest"; import type { SdkSurface } from "../scripts/api-surface.ts"; import type { AuditOptions, AuditResult } from "../scripts/e2e/audit/bundle.ts"; import { checkBudgets, checkFlow, checkGate, checkSignoffs, formatDiagnostic } from "../scripts/e2e/format/parser.ts"; -import { gateCommand, parseGateArgs, runGate, type GateEnv, type GateReport } from "../scripts/e2e/gate/gate.ts"; +import { auditCommand, gateCommand, parseGateArgs, runGate, type GateEnv, type GateReport } from "../scripts/e2e/gate/gate.ts"; import { flowHash } from "../scripts/e2e/gate/manual.ts"; import { plan, type Host } from "../scripts/e2e/gate/plan.ts"; import { mergeDecisions, signoffCommand, type SignoffEnv } from "../scripts/e2e/gate/signoff.ts"; @@ -199,6 +199,26 @@ describe("release gate", () => { expect([scoped.audits, calls]).toEqual([[], []]); }); + test("audit steps run one at a time; e2e:audit --audit runs only the named steps", async () => { + const registry = "area: dx\nprefix: DX\nchecks:\n DX-CLI-01:\n title: Help\n audit: cli\n DX-DOCS-01:\n title: Docs\n audit: api-docs\n"; + const dir = repo({ "e2e/checks/dx.yaml": registry }); + const events: string[] = []; + const audit = async (name: string) => { + events.push(`start ${name}`); + await new Promise(resolve => setTimeout(resolve, 5)); + events.push(`end ${name}`); + return [{ check: name === "cli" ? "DX-CLI-01" : "DX-DOCS-01", status: "passed" as const, details: [] }]; + }; + await gate(dir, { audit }); + expect(events).toEqual(["start cli", "end cli", "start api-docs", "end api-docs"]); + events.length = 0; + const { env: auditEnv, out } = env({ audit }); + expect(await auditCommand(["--audit", "api-docs"], path.join(dir, "e2e"), auditEnv)).toBe(0); + expect(events).toEqual(["start api-docs", "end api-docs"]); + expect(out.filter(line => line.startsWith("PASSED"))).toEqual(["PASSED DX-DOCS-01 Docs"]); + expect(await auditCommand(["--audit", "nope"], path.join(dir, "e2e"), env().env)).toBe(2); + }); + test("uncommitted changes fail: the gate certifies a commit", async () => { expect(reasons(await gate(repo(), { source: () => ({ commit: COMMIT, dirty: true }) }))).toEqual(["source: the working tree has uncommitted changes; the gate certifies a commit"]); }); From bead73f1746003f5ae90df02eead142284d75bf6 Mon Sep 17 00:00:00 2001 From: sethwebster Date: Tue, 13 Oct 2026 04:20:17 -0400 Subject: [PATCH 2/2] DX gate audit: dev loop (#170) dev-loop audit: a real spark dev session of the Kitchen Sink with its dev build in driver mode. Cold start, Fast Refresh (hot update, no reload), native change detection (JS edits need no rebuild, a native config change does) and the syntax error overlay, against M1 8 GB budgets derived from M4 Max measurements. --- docs/e2e-flows.md | 29 +++++++- e2e/checks/developer-experience.yaml | 27 +++++++- e2e/schema/checks.schema.json | 3 +- scripts/e2e/audit/dev-loop.ts | 98 ++++++++++++++++++++++++++++ scripts/e2e/audit/steps.ts | 3 +- scripts/e2e/format/primitives.ts | 2 +- scripts/testing/dev-session.ts | 6 +- 7 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 scripts/e2e/audit/dev-loop.ts diff --git a/docs/e2e-flows.md b/docs/e2e-flows.md index 526f0a48..faeffc97 100644 --- a/docs/e2e-flows.md +++ b/docs/e2e-flows.md @@ -1155,8 +1155,8 @@ app launched from its quarantined location. #### Developer experience audits -Three audit steps, registered in `checks/developer-experience.yaml` (#171), check the developer's side of a release. -The `templates` step builds apps, so run it under the native build lock on a shared Mac. +Four audit steps, registered in `checks/developer-experience.yaml` (#170, #171), check the developer's side of a release. +The `templates` and `dev-loop` steps build and run apps, so run them under the native build lock on a shared Mac. `api-docs` (`scripts/api-docs.ts`) reads the public surface the way `docs/api-public-surface.json` snapshots it. `docs/api-undocumented.json` lists the known gaps. `tests/api-docs-coverage.test.ts` (part of `bun run test`) fails on an @@ -1189,6 +1189,31 @@ minutes on an M4 Max: the packages come from the npm registry and the two releas | `DX-REPRO-01` | A second release build, after deleting `.spark/` and the generated `macos/` project, has byte-identical JavaScript (`*.jsbundle`, `*.hbc`, `*.js`). | | `DX-REPRO-02` | Every other file in the app is byte-identical too. Each difference is listed; for a Mach-O file, whether its `LC_UUID` or other load commands differ. See [Reproducible builds](#reproducible-builds). | +`dev-loop` (`scripts/e2e/audit/dev-loop.ts`) runs `spark dev --clear --no-open --max-workers 2` for the Kitchen Sink and +launches its dev build in driver mode (`scripts/testing/dev-session.ts`; build it first with `cd examples/kitchen-sink && +bun run rebuild:macos`). It edits `screens/devtools/FastRefreshMarker.tsx` and `desktop.config.json` and always restores them. + +| Check | What it verifies | +|---|---| +| `DX-LOOP-01` | From starting `spark dev --clear` (an empty Metro cache) to the dev build's first settled render, within the cold start budget. | +| `DX-LOOP-02` | Three edits each reach the running runtime within the Fast Refresh budget, as hot updates: a global set before them survives (a reload clears it). | +| `DX-LOOP-03` | After the edits the dev build is still compatible (`.spark/session.json`) and the same app process runs: no native rebuild. A native configuration change (the app's `version`) makes `spark dev` report "Native configuration or host changed." with a rebuild offered, within the native change budget; reverting it makes the build compatible again. | +| `DX-LOOP-04` | A syntax error shows React Native's error overlay (LogBox, in the app's window) with the `SyntaxError`; the fix closes it and reaches the app without a reload. | + +##### Dev loop budgets + +The budgets (`DEV_LOOP_BUDGETS` in `scripts/e2e/audit/dev-loop.ts`) are for the reference MacBook Air (M1, 8 GB) and +apply on every Mac. They were not measured on the M1: they are three times what a 16-inch MacBook Pro (M4 Max, Mac16,5, +macOS 27) measured, rounded up. Replace them with values measured on the reference machine. + +| Budget | M4 Max measured | M1 budget | +|---|---|---| +| Cold start (`DX-LOOP-01`) | 10.8 s, 13.2 s | 45 s | +| Fast Refresh, per edit (`DX-LOOP-02`) | 0.3 to 0.6 s | 2 s | +| Native change detected (`DX-LOOP-03`) | 0.8 s | 5 s | + +DevTools in the dev loop has its own checks (`checks/devtools.yaml`). + #### Reproducible builds `DX-REPRO-01` and `DX-REPRO-02` build the macOS template twice. `spark build` prepares the project in place (it patches diff --git a/e2e/checks/developer-experience.yaml b/e2e/checks/developer-experience.yaml index fe22d908..2f1451ab 100644 --- a/e2e/checks/developer-experience.yaml +++ b/e2e/checks/developer-experience.yaml @@ -1,7 +1,32 @@ -# Developer experience (#171): gate audit steps, not flows (docs/e2e-flows.md#audits). +# Developer experience (#170, #171): gate audit steps, not flows (docs/e2e-flows.md#audits). DevTools has its own +# registry (checks/devtools.yaml, epic #267). area: developer-experience prefix: DX checks: + DX-LOOP-01: + title: Cold start of the dev loop, spark dev --clear for the Kitchen Sink to the first settled render of its dev build, is within the reference MacBook Air (M1, 8 GB) budget + platforms: [macos] + blocking: true + spec: "#170 Dev loop" + audit: dev-loop + DX-LOOP-02: + title: Fast Refresh delivers each source edit to the running app as a hot update, with no reload of the JavaScript runtime, within the M1 budget per edit + platforms: [macos] + blocking: true + spec: "#170 Fast Refresh" + audit: dev-loop + DX-LOOP-03: + title: JavaScript edits never require a native rebuild (the dev build stays compatible and the same process keeps running); a native configuration change makes spark dev report that the build needs a rebuild, within the M1 budget, and reverting it makes the build compatible again + platforms: [macos] + blocking: true + spec: "#170 Native change detection requires rebuild" + audit: dev-loop + DX-LOOP-04: + title: A syntax error in an edit shows React Native's error overlay in the running app; fixing it dismisses the overlay and Fast Refresh resumes without a reload + platforms: [macos] + blocking: true + spec: "#170 Error overlay" + audit: dev-loop DX-DOCS-01: title: Every public export (docs/api-public-surface.json) has a TSDoc comment on its declaration blocking: true diff --git a/e2e/schema/checks.schema.json b/e2e/schema/checks.schema.json index 4bcd29dd..d60e2118 100644 --- a/e2e/schema/checks.schema.json +++ b/e2e/schema/checks.schema.json @@ -62,7 +62,8 @@ "package", "api-docs", "cli", - "templates" + "templates", + "dev-loop" ], "description": "The gate audit step that verifies this check against the release build, instead of a flow (see docs/e2e-flows.md#audits)." } diff --git a/scripts/e2e/audit/dev-loop.ts b/scripts/e2e/audit/dev-loop.ts new file mode 100644 index 00000000..4a70205f --- /dev/null +++ b/scripts/e2e/audit/dev-loop.ts @@ -0,0 +1,98 @@ +// The gate's dev-loop audit (docs/e2e-flows.md#audits, #170): a real `spark dev` session of the Kitchen Sink with its +// dev build launched in driver mode (no window, no focus; scripts/testing/dev-session.ts), timed against the budgets of +// the reference MacBook Air (M1, 8 GB). Edits go to two Kitchen Sink files and are always restored. +import { readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { readJson, stateFile } from "../../../packages/cli/src/project.ts"; +import { startDevSession, type Cdp, type DevSession } from "../../testing/dev-session.ts"; +import { KITCHEN_SINK } from "../kitchen-sink.ts"; +import type { AuditResult } from "./bundle.ts"; + +/** + * Budgets for the reference MacBook Air (M1, 8 GB), in milliseconds. They apply on every Mac: a faster one meets them + * more easily. Not measured on the M1: they are three times what an M4 Max (Mac16,5) measured, rounded up (see + * docs/e2e-flows.md#dev-loop-budgets); replace them with values measured on the reference machine. + */ +export const DEV_LOOP_BUDGETS = { coldStartMs: 45_000, fastRefreshMs: 2_000, nativeChangeMs: 5_000 }; +export const DEV_LOOP_CHECKS = ["DX-LOOP-01", "DX-LOOP-02", "DX-LOOP-03", "DX-LOOP-04"] as const; + +const MARKER = path.join(KITCHEN_SINK, "screens/devtools/FastRefreshMarker.tsx"); +const CONFIG = path.join(KITCHEN_SINK, "desktop.config.json"); +const REVISION = 'const REVISION = "1";'; +const seconds = (ms: number) => `${(ms / 1000).toFixed(1)} s`; + +export async function auditDevLoop(): Promise { + const results = new Map(); + const set = (check: string, ok: boolean, details: string[]) => results.set(check, { check, status: ok ? "passed" : "failed", details }); + const marker = readFileSync(MARKER, "utf8"), config = readFileSync(CONFIG, "utf8"); + if (!marker.includes(REVISION)) throw new Error(`${MARKER} no longer declares ${REVISION}`); + const session = () => readJson(stateFile(KITCHEN_SINK, "session.json")) as { compatible: boolean; reason: string; canBuild?: boolean }; + let dev: DevSession | undefined; + try { + // DX-LOOP-01: from starting spark dev with an empty Metro cache to the dev build's first settled render. + const started = performance.now(); + dev = await startDevSession({ prefix: "ks-dev-loop-", args: ["--clear"] }); + const { driver, name } = await dev.launch(); + await driver.startup(300_000); + const coldMs = performance.now() - started; + set("DX-LOOP-01", coldMs <= DEV_LOOP_BUDGETS.coldStartMs, [`spark dev --clear to the first settled render: ${seconds(coldMs)} (budget ${seconds(DEV_LOOP_BUDGETS.coldStartMs)})`]); + + const cdp: Cdp = await dev.connect(await dev.target(`${name} (shared runtime)`)); + await cdp.send("Runtime.enable"); + // A global set now survives a hot update and is gone after a reload of the runtime (the debugger stays attached). + let sentinel = ""; + const arm = async () => { sentinel = `dx-loop-${performance.now()}`; await cdp.evaluate(`globalThis.__dxLoopSentinel = ${JSON.stringify(sentinel)}`); }; + const reloaded = async () => (await cdp.evaluate("globalThis.__dxLoopSentinel")) !== sentinel; + const revision = () => cdp.evaluate("globalThis.ksDevToolsRevision"); + const edit = async (value: string, label: string) => { + const begun = performance.now(); + writeFileSync(MARKER, marker.replace(REVISION, `const REVISION = "${value}";`)); + await dev!.until(async () => (await revision()) === value, label, 60_000); + return performance.now() - begun; + }; + const pid = driver.app.pid; + + // DX-LOOP-02: three edits, each a hot update of the running runtime. + await arm(); + const latencies: number[] = []; + for (const n of [1, 2, 3]) latencies.push(await edit(`dx-${n}`, `Fast Refresh delivers edit ${n}`)); + const hot = !await reloaded(), slowest = Math.max(...latencies); + set("DX-LOOP-02", hot && slowest <= DEV_LOOP_BUDGETS.fastRefreshMs, [`edit to the running app: ${latencies.map(seconds).join(", ")} (budget ${seconds(DEV_LOOP_BUDGETS.fastRefreshMs)} each)`, + hot ? "hot updates: the runtime's globals survived" : "the edits reloaded the JavaScript runtime (its globals are gone) instead of a hot update (#291)"]); + + // DX-LOOP-04: a syntax error shows the overlay; the fix dismisses it and refreshes without a reload. + await arm(); + writeFileSync(MARKER, marker.replace(REVISION, `${REVISION} const broken = ;`)); + const overlay = await dev.until(async () => (await driver.elements()).elements.find(element => /SyntaxError/.test(`${element.text ?? ""} ${element.label ?? ""}`)), "the error overlay shows the syntax error", 30_000); + const fixedMs = await edit("dx-fixed", "Fast Refresh resumes after the fix"); + await dev.until(async () => !(await driver.elements()).elements.some(element => /SyntaxError/.test(`${element.text ?? ""} ${element.label ?? ""}`)), "the error overlay closes", 30_000); + const hotFix = !await reloaded(); + set("DX-LOOP-04", hotFix, [`the overlay showed ${JSON.stringify((overlay.text ?? overlay.label ?? "").split("\n")[0]!.slice(0, 160))}; the fix reached the app in ${seconds(fixedMs)} and closed it`, + hotFix ? "no reload" : "the fix reloaded the JavaScript runtime instead of a hot update (#291)"]); + + // DX-LOOP-03: after the JavaScript edits the dev build is still compatible and the same process runs; a native + // configuration change (the app's version, which Info.plist carries) needs a rebuild until it is reverted. + const afterEdits = session(), samePid = driver.app.pid === pid && driver.app.exitCode === null; + const changed = JSON.parse(config); + changed.version = changed.version === "9.9.9" ? "9.9.8" : "9.9.9"; + const changedAt = performance.now(); + writeFileSync(CONFIG, JSON.stringify(changed, null, 2) + "\n"); + const stale = await dev.until(async () => { const state = session(); return !state.compatible && /Native configuration/.test(state.reason) && state; }, "spark dev reports the native change", 60_000); + const detectMs = performance.now() - changedAt; + writeFileSync(CONFIG, config); + await dev.until(async () => session().compatible, "reverting the change makes the dev build compatible again", 60_000); + set("DX-LOOP-03", afterEdits.compatible && samePid && !!stale.canBuild && detectMs <= DEV_LOOP_BUDGETS.nativeChangeMs, [ + `after ${latencies.length + 2} JavaScript edits: ${afterEdits.compatible ? "the dev build is compatible" : `incompatible (${afterEdits.reason})`}, ${samePid ? "the same app process runs" : "the app process changed"}`, + `a native configuration change: "${stale.reason}"${stale.canBuild ? ", with a rebuild offered" : ", but no rebuild offered"}, in ${seconds(detectMs)} (budget ${seconds(DEV_LOOP_BUDGETS.nativeChangeMs)}); reverting it made the build compatible again`, + ]); + } catch (error) { + dev?.printLogs(); + const message = error instanceof Error ? error.message : String(error); + for (const check of DEV_LOOP_CHECKS) if (!results.has(check)) results.set(check, { check, status: "failed", details: [message] }); + } finally { + writeFileSync(MARKER, marker); + writeFileSync(CONFIG, config); + await dev?.close(); + } + return DEV_LOOP_CHECKS.map(check => results.get(check)!); +} diff --git a/scripts/e2e/audit/steps.ts b/scripts/e2e/audit/steps.ts index da6d4ba0..20543b86 100644 --- a/scripts/e2e/audit/steps.ts +++ b/scripts/e2e/audit/steps.ts @@ -3,9 +3,10 @@ import type { Audit } from "../format/primitives.ts"; import { auditApiDocs } from "./api-docs.ts"; import { auditBundle, type AuditOptions, type AuditResult } from "./bundle.ts"; import { auditCli } from "./cli.ts"; +import { auditDevLoop } from "./dev-loop.ts"; import { auditPackage } from "./package.ts"; import { auditTemplates } from "./templates.ts"; export const AUDIT_STEPS: Readonly AuditResult[] | Promise>> = { - bundle: auditBundle, package: options => auditPackage(options), "api-docs": auditApiDocs, cli: auditCli, templates: auditTemplates, + bundle: auditBundle, package: options => auditPackage(options), "api-docs": auditApiDocs, cli: auditCli, templates: auditTemplates, "dev-loop": auditDevLoop, }; diff --git a/scripts/e2e/format/primitives.ts b/scripts/e2e/format/primitives.ts index 788d75db..d9314ef2 100644 --- a/scripts/e2e/format/primitives.ts +++ b/scripts/e2e/format/primitives.ts @@ -4,7 +4,7 @@ export type Schema = Record; export const CHECK_ID = "^[A-Z][A-Z0-9]*-[A-Z0-9]+-\\d{2}$"; export const PLATFORMS = ["macos", "windows"] as const; /** Gate audit steps: a check registered with `audit: ` is verified by that step against the release build, not by a flow. */ -export const AUDITS = ["bundle", "package", "api-docs", "cli", "templates"] as const; +export const AUDITS = ["bundle", "package", "api-docs", "cli", "templates", "dev-loop"] as const; export type Audit = typeof AUDITS[number]; const ref = (name: string): Schema => ({ $ref: `#/definitions/${name}` }); diff --git a/scripts/testing/dev-session.ts b/scripts/testing/dev-session.ts index eeac22ba..dd366e86 100644 --- a/scripts/testing/dev-session.ts +++ b/scripts/testing/dev-session.ts @@ -18,15 +18,15 @@ export type Message = { id?: number; method?: string; params?: any; result?: any export type Cdp = Awaited>; export type DevSession = Awaited>; -/** `env` reaches spark dev (and React Native's browser launcher); `prefix` names the scratch directory. */ -export async function startDevSession(options: { prefix: string; env?: Record }) { +/** `env` reaches spark dev (and React Native's browser launcher); `args` are more spark dev options; `prefix` names the scratch directory. */ +export async function startDevSession(options: { prefix: string; env?: Record; args?: string[] }) { const work = mkdtempSync(path.join(os.tmpdir(), options.prefix)); const sessionLog = path.join(work, "spark-dev.log"), appLog = path.join(work, "app.log"); const port = await availablePort(); const base = `http://127.0.0.1:${port}`; const app = new KitchenSinkApp(undefined, new URL(base)); const log = openSync(sessionLog, "a"); - const session = spawnProcess([process.execPath, path.join(import.meta.dirname, "../spark.mjs"), "dev", "--platform", "macos", "--no-open", "--port", String(port), "--max-workers", "2"], { + const session = spawnProcess([process.execPath, path.join(import.meta.dirname, "../spark.mjs"), "dev", "--platform", "macos", "--no-open", "--port", String(port), "--max-workers", "2", ...options.args ?? []], { cwd: KITCHEN_SINK, env: { ...process.env, CI: "false", ...options.env }, stdin: "ignore", stdout: log, stderr: log, }); const sockets: WebSocket[] = [];