diff --git a/docs/api-undocumented.json b/docs/api-undocumented.json new file mode 100644 index 00000000..95770b68 --- /dev/null +++ b/docs/api-undocumented.json @@ -0,0 +1,925 @@ +{ + "pageless": [ + "./contracts", + "./ui/classnames", + "./ui/popover", + "./ui/search", + "./ui/sidebar", + "./ui/swipe", + "./ui/symbol", + "./ui/tab-view", + "./webview" + ], + "undocumented": { + "./ai": [ + "AIAvailabilityOptions", + "AICodexInvocationOptions", + "AICommandAvailability", + "AIErrorOutputOptions", + "AIInvocation", + "AIInvocationOptions", + "AIToolId", + "AIToolRunResult", + "buildAIInvocation", + "extractJsonCandidate", + "formatAIErrorOutput", + "getAICommandAvailability", + "parseAIJson", + "runAITool", + "RunAIToolOptions", + "selectPreferredAITool" + ], + "./ai/codex": [ + "CodexAvailability", + "CodexRunOptions", + "CodexRunResult", + "runCodexPrompt" + ], + "./app": [ + "activate", + "AppContext", + "AppEvent", + "AppEventMap", + "AppRuntime", + "AsyncRegistration", + "BeforeQuitOptions", + "getAppContext", + "getAppLanguages", + "hide", + "QuitResult", + "RelaunchOptions", + "Subscription" + ], + "./app-intents": [ + "AppIntentHandlerOptions", + "AppIntentInvocation", + "AppIntentParameter", + "AppIntentParameterType", + "AppIntentValue", + "AsyncRegistration", + "Availability" + ], + "./app/documents": [ + "AsyncRegistration", + "clearRecentDocuments", + "createDocumentSession", + "DocumentAppController", + "DocumentAppControllerOptions", + "DocumentAppControllerState", + "DocumentReloadOptions", + "DocumentSaveOptions", + "DocumentSaveResult", + "DocumentSessionOptions", + "DocumentState", + "getLaunchDocumentPath", + "GetLaunchDocumentPathOptions", + "getRecentDocuments", + "noteRecentDocument", + "OpenRequest", + "openSelectedDocumentPath", + "OpenSelectedDocumentPathOptions", + "RecentDocument", + "Subscription", + "UseDocumentAppControllerOptions", + "useWatchedDocumentReload", + "UseWatchedDocumentReloadOptions" + ], + "./apple-authentication": [ + "AppleAuthenticationScope", + "AppleIDCredential", + "AppleIDCredentialState", + "AppleIDFullName", + "AppleSignInOptions", + "AppleSignInResult", + "Availability" + ], + "./audio": [ + "AudioMetadata", + "AudioPlayer", + "AudioPlayerHookOptions", + "AudioPlayerOptions", + "AudioPlayerState", + "AudioSource", + "createAudioPlayer", + "MediaCommand", + "MediaCommandName", + "MediaSession", + "MediaSessionOptions", + "MediaSessionUpdate", + "sessionOptions", + "sessionSnapshot", + "Subscription", + "validateMetadata", + "validateSession", + "validateVolume", + "validMediaCommand" + ], + "./auth-session": [ + "authorize", + "AuthSession", + "AuthSessionOptions", + "AuthSessionResult", + "callbackMatches", + "CompleteAuthSessionResult", + "createAuthSession", + "digestStringAsync", + "getRandomBytesAsync", + "timeout", + "validateCount", + "validateDigest", + "validateRedirect" + ], + "./biometrics": [ + "Availability", + "BiometricAuthOptions", + "BiometricAuthResult", + "BiometricErrorReason", + "BiometricPromptOptions", + "BiometricSecretOptions", + "BiometricStatus" + ], + "./capture": [ + "AudioRecordingOptions", + "Availability", + "CameraContentFit", + "CameraOptions", + "CameraSession", + "CameraStatus", + "CameraViewProps", + "CaptureDevice", + "CaptureDeviceKind", + "CaptureRecordingOutcome", + "CaptureRecordingSession", + "ScreenCaptureOptions", + "ScreenDisplaySource", + "ScreenRecordingOptions", + "ScreenshotOptions", + "ScreenWindowSource" + ], + "./clipboard": [ + "clearClipboard", + "ClipboardChangeEvent", + "ClipboardChangeOptions", + "ClipboardData", + "ClipboardImage", + "getClipboardFormats", + "GetStringOptions", + "hasStringAsync", + "PromisedClipboardContent", + "PromisedClipboardOptions", + "readClipboard", + "setStringAsync", + "SetStringOptions", + "StringFormat", + "writeClipboard" + ], + "./context-menu": [ + "ContextMenuOptions", + "ContextMenuResult", + "getContextMenuAvailability", + "MenuAction", + "MenuIcon", + "MenuItem" + ], + "./contracts": [ + "AsyncRegistration", + "Availability", + "SparkError", + "Subscription" + ], + "./devtools": [ + "NetworkFrame", + "NetworkInspectorOptions", + "OpenDevToolsOptions", + "ProfilingResult", + "StartProfilingOptions", + "StopProfilingOptions" + ], + "./diagnostics": [ + "CrashReport", + "HangReport", + "HangStackFrame", + "HangWatchdogOptions", + "StartupTiming" + ], + "./dialogs": [ + "confirm", + "ConfirmOptions", + "DialogAccessoryOption", + "FileDialogOptions", + "MessageDialogButton", + "MessageDialogOptions", + "MessageDialogResult", + "openFileDialog", + "OpenFileDialogOptions", + "PromptOptions", + "saveFileDialog", + "SaveFileDialogOptions", + "SaveFileDialogResult", + "showMessage" + ], + "./displays": [ + "AsyncRegistration", + "DisplayColorSpace", + "DisplayDetails", + "DisplayField", + "DisplayId", + "DisplayInfo", + "DisplayInsets", + "DisplayListenerOptions" + ], + "./drag-drop": [ + "DragDropView", + "DragDropViewProps", + "DragEndEvent", + "DragOperation", + "DragOptions", + "DropEvent", + "SpringLoadHighlight", + "SpringLoadHighlightEvent" + ], + "./files": [ + "AsyncRegistration", + "CopyMoveOptions", + "CreateBookmarkOptions", + "CreateDirectoryOptions", + "DirectoryEntry", + "exists", + "FileApplication", + "FileCleanupError", + "FileHandle", + "FileInfo", + "FileMode", + "FileScanBatch", + "FileScanOptions", + "FileScanProgress", + "FileScanResult", + "FileScanSkipEntry", + "getAppIconAvailability", + "getDirectory", + "getDiskSpaceAvailability", + "getFileIconAvailability", + "getFileScanAvailability", + "getFileSystemAvailability", + "getOpenWithAvailability", + "getQuarantineAvailability", + "getQuickLookAvailability", + "getThumbnailAvailability", + "mkdir", + "OpenFileOptions", + "readBytes", + "ReadChunksOptions", + "RemoveOptions", + "scanFiles", + "ScannedFile", + "setExtendedAttribute", + "WatchOptions", + "writeBytes", + "WriteChunksOptions" + ], + "./global-shortcuts": [ + "AsyncRegistration", + "getGlobalShortcutAvailability", + "GlobalShortcutOptions" + ], + "./links": [ + "Availability", + "canOpenURL", + "Subscription", + "URLListener" + ], + "./logging": [ + "DiagnosticsExport", + "ExportDiagnosticsOptions", + "LogFile", + "LogFileOptions", + "LoggerOptions", + "LoggingOptions" + ], + "./menus": [ + "addDeveloperMenuListener", + "AsyncRegistration", + "createMenu", + "DeveloperMenuState", + "getMenuAvailability", + "Menu", + "MenuAction", + "MenuFeature", + "MenuIcon", + "MenuItem", + "MenuLifecycleEvent", + "MenuLocation", + "MenuOptions", + "MenuPlacement", + "MenuRole", + "MenuRootItem", + "MenuState", + "MenuTarget", + "MenuUpdate", + "UseMenuOptions" + ], + "./motion": [ + "AnimateOptions", + "AnimationOptions", + "CubicBezier", + "decay", + "DecayOptions", + "DecaySpec", + "Easing", + "getMotionAvailability", + "KeyframesOptions", + "KeyframesSpec", + "MotionAnimation", + "MotionColorProperty", + "MotionEmitterProps", + "MotionFrameReport", + "MotionMaterialProps", + "MotionNumberProps", + "MotionPathOptions", + "MotionPathValues", + "MotionPresenceProps", + "MotionProperty", + "MotionShaderEffect", + "MotionShaderProps", + "MotionShapeFillRule", + "MotionShapeLineCap", + "MotionShapeLineJoin", + "MotionShapeProps", + "MotionState", + "MotionSymbolProps", + "MotionTextProps", + "MotionValues", + "MotionViewProps", + "ParticleEmitter", + "ParticleOptions", + "ParticlePlayback", + "ScrollLinkOptions", + "SequenceOptions", + "SequenceSpec", + "ShaderEffect", + "SpringPreset", + "SpringSpec", + "StaggerOptions", + "SymbolEffect", + "SymbolEffectDirection", + "SymbolEffectOptions", + "SymbolEffectPlayback", + "TimelineStep", + "TimingOptions", + "TimingSpec" + ], + "./notifications": [ + "cancelAllNotifications", + "dismissAllNotifications", + "getDeliveredNotifications", + "getNotificationAvailability", + "getNotificationPermission", + "getPendingNotifications", + "NotificationAction", + "NotificationContent", + "NotificationPermission", + "NotificationTrigger", + "scheduleNotification", + "ScheduleNotificationOptions", + "ShowNotificationOptions", + "Subscription" + ], + "./passkeys": [ + "Availability", + "PasskeyAssertion", + "PasskeyAssertionOptions", + "PasskeyAssertionResult", + "PasskeyRegistration", + "PasskeyRegistrationOptions", + "PasskeyRegistrationResult", + "PasskeyUserVerification" + ], + "./permissions": [ + "AsyncRegistration", + "Availability", + "PermissionChangeEvent", + "PermissionListenerOptions", + "PermissionOptions", + "PermissionResponse" + ], + "./plugins": [ + "Availability", + "PLUGIN_CAPABILITIES", + "PluginCommand", + "PluginFileHandler", + "PluginHost", + "PluginHostOptions", + "PluginInfo", + "PluginManifest", + "PluginMenuItem", + "PluginState" + ], + "./pointer": [ + "CustomCursor", + "HapticFeedbackOptions", + "PointerViewProps" + ], + "./printing": [ + "ExportPDFOptions", + "ExportPDFResult", + "PageOrientation", + "PageSetupOptions", + "PageSetupResult", + "PrintOptions" + ], + "./privileged-helper": [ + "AdminAuthorizationOptions", + "AdminAuthorizationResult", + "Availability", + "PrivilegedHelperMessageOptions" + ], + "./processes": [ + "getProcessAvailability", + "ProcessExit", + "ProcessHandle", + "ProcessInput", + "ProcessOptions", + "ProcessOutput", + "ProcessResult", + "ProcessTarget", + "resolveCommand", + "runCommand", + "RunCommandOptions", + "spawn" + ], + "./secure-storage": [ + "deleteItemAsync", + "isAvailableAsync", + "setItemAsync" + ], + "./settings": [ + "Json", + "settings", + "SettingsReadOptions", + "SettingsStorage", + "SettingsStore", + "SettingsStoreOptions" + ], + "./settings/observable": [ + "ObservableFile", + "ObservableFileOptions", + "ObservableSettingsField", + "ObservableSettingsFields", + "ObservableSettingsOptions", + "ObservableSettingsValues" + ], + "./settings/window": [ + "CreateSettingsWindowOptionsInput", + "SettingsCard", + "SettingsCardProps", + "SettingsPage", + "SettingsPageProps", + "SettingsRow", + "SettingsRowGroup", + "SettingsRowGroupProps", + "SettingsRowProps", + "SettingsSection", + "SettingsSectionProps", + "SettingsSidebar", + "SettingsSidebarProps", + "SettingsWindow", + "SettingsWindowBaseProps", + "SettingsWindowPage", + "SettingsWindowProps", + "SettingsWindowSelectionProps", + "VirtualizedSettingsWindow", + "VirtualizedSettingsWindowProps" + ], + "./sharing": [ + "AsyncRegistration", + "Availability", + "ListenerOptions", + "ShareOptions", + "SharingFeature", + "SpotlightDeletion", + "SpotlightItem", + "SpotlightOpenEvent", + "Subscription" + ], + "./shortcuts": [ + "Accelerator", + "AsyncRegistration", + "getShortcutAvailability", + "parseAccelerator", + "ShortcutOptions" + ], + "./shortcuts/commands": [ + "createHotkeyRouter", + "createHotkeyStore", + "createShortcutMap", + "formatHotkey", + "getDefaultHotkeyBindings", + "getHotkeyBindingConflicts", + "HotkeyBindingLimitOptions", + "hotkeyBindingListsEqual", + "HotkeyBindingsSettingsContent", + "HotkeyBindingsSettingsPage", + "HotkeyBindingsSettingsPageProps", + "HotkeyBindingState", + "HotkeyCapture", + "HotkeyCaptureProps", + "HotkeyDefinition", + "HotkeyFile", + "hotkeyFileVersion", + "HotkeyHandlerContext", + "HotkeyMatchOptions", + "HotkeyRegistration", + "HotkeyRegistrationOptions", + "HotkeyRegistrationState", + "HotkeyRouter", + "HotkeyScope", + "HotkeyStoreOptions", + "HotkeySuspensionOptions", + "HotkeyValue", + "matchesHotkey", + "normalizeHotkeyBindings", + "normalizeHotkeyFile", + "RoutedHotkeyHandlers", + "serializeHotkeyFile", + "ShortcutCommand", + "ShortcutCommandEvent", + "ShortcutConflict", + "ShortcutConflictParty", + "ShortcutMap", + "ShortcutMapChange", + "ShortcutMapCommandState", + "ShortcutMapHandlers", + "ShortcutMapOptions", + "ShortcutMapRegisterOptions", + "ShortcutMapSnapshot", + "shortcutMapVersion", + "ShortcutPlatform", + "useHotkeySuspension", + "useRoutedHotkeys", + "UseRoutedHotkeysOptions" + ], + "./shortcuts/keyboard": [ + "AsyncRegistration", + "createModifierMask", + "getKeyboardAvailability", + "hasModifier", + "KeyboardEvent", + "KeyboardEventListener", + "KeyboardListenerOptions", + "KeyCodes", + "KeyText" + ], + "./speech": [ + "Availability", + "getSpeakingStatus", + "pauseSpeaking", + "PauseSpeakingOptions", + "resumeSpeaking", + "SpeakingStatus", + "SpeakOptions", + "SpeechRecognitionOptions", + "SpeechRecognitionOutcome", + "SpeechRecognitionSegment", + "SpeechRecognitionSession", + "SpeechRecognizerStatus", + "SpeechTranscript", + "SpeechVoice" + ], + "./sqlite": [ + "Database", + "DatabaseOptions", + "IntegerMode", + "RunResult", + "SqlExecutor", + "SqlRow", + "SqlValue" + ], + "./system": [ + "AsyncRegistration", + "AttentionOptions", + "createDockMenu", + "createTaskbarMenu", + "DockMenuItem", + "DockMenuOptions", + "getLoginItemStatus", + "getSystemInfo", + "LoginItemStatus", + "MemoryPressureLevel", + "NetworkInterfaceType", + "NetworkStatus", + "onSystemEvent", + "preventSleep", + "PreventSleepOptions", + "requestAttention", + "setLaunchAtLogin", + "SystemEventOptions", + "SystemInfo", + "TaskbarMenuItem", + "TaskbarMenuOptions", + "ThermalState" + ], + "./tray": [ + "AsyncRegistration", + "createTray", + "getTrayAvailability", + "MenuAction", + "MenuIcon", + "MenuItem", + "Tray", + "TrayAction", + "TrayOptions", + "TrayUpdate" + ], + "./ui": [ + "Button", + "ButtonAvailabilityOptions", + "ButtonVariant", + "Checkbox", + "CheckboxValue", + "ColorPanelChangeEvent", + "ColorPanelOptions", + "ColorWellProps", + "ColorWellVariant", + "ComboBox", + "ComboBoxProps", + "ControlKind", + "ControlledTextInputProps", + "ControlProps", + "ControlSize", + "DatePickerMode", + "DatePickerProps", + "DatePickerVariant", + "DisclosureTriangle", + "DisclosureTriangleProps", + "FontDescriptor", + "FontPanelChangeEvent", + "FontPanelOptions", + "FontPanelSession", + "FontWeight", + "getControlAvailability", + "getFontPanelAvailability", + "LevelIndicator", + "LevelIndicatorProps", + "NumericControlProps", + "PanelCloseEvent", + "PathControl", + "PrimitiveControlProps", + "PrimitiveKind", + "Progress", + "ProgressProps", + "RadioGroup", + "RadioGroupProps", + "ScreenColorResult", + "SegmentedControl", + "SegmentedControlProps", + "Select", + "SelectOption", + "SelectProps", + "Slider", + "SliderProps", + "Stepper", + "StepperProps", + "Switch", + "SwitchProps", + "TextInput", + "TextInputProps", + "TokenField", + "TokenFieldProps", + "UncontrolledTextInputProps" + ], + "./ui/classnames": [ + "ClassValue", + "cn" + ], + "./ui/glass": [ + "getGlassAvailability", + "GlassStyle", + "GlassViewProps" + ], + "./ui/popover": [ + "getPopoverAvailability", + "PopoverAnchor", + "PopoverProps" + ], + "./ui/search": [ + "getSearchAvailability", + "TextInputSearch", + "TextInputSearchProps", + "TextInputSearchRef" + ], + "./ui/sidebar": [ + "getSidebarAvailability", + "Sidebar", + "SidebarBaseProps", + "SidebarContentLayoutEvent", + "SidebarContextMenuEvent", + "SidebarItem", + "SidebarItemData", + "SidebarItemProps", + "SidebarProps", + "SidebarRowHeight", + "SidebarSelectionEvent" + ], + "./ui/split-view": [ + "getSplitViewAvailability", + "SidebarSplitView", + "SidebarSplitViewAppearance", + "SidebarSplitViewPaneMetrics", + "SidebarSplitViewProps", + "SidebarSplitViewResizeEvent", + "SidebarSplitViewTitlebarMaterial", + "SplitViewTitleBarOptions", + "SplitViewTitleBarOverlay" + ], + "./ui/swipe": [ + "getSwipeActionsAvailability", + "SwipeActionsProps" + ], + "./ui/symbol": [ + "getSFSymbolAvailability", + "SFSymbol", + "SFSymbolProps", + "SFSymbolScale" + ], + "./ui/tab-view": [ + "getTabViewAvailability", + "TabProps", + "TabViewProps", + "TabViewSelectionEvent" + ], + "./ui/table": [ + "TableBaseProps", + "TableColumn", + "TableContextMenuEvent", + "TableProps", + "TableSelectionMode", + "TableSelectionProps", + "TableSortDirection" + ], + "./ui/text-services": [ + "FindTextOptions", + "FindTextResult", + "getTextServicesAvailability" + ], + "./ui/uniwind": [ + "Button", + "SegmentedControl", + "Select", + "TextInput" + ], + "./updates": [ + "CheckForUpdatesOptions", + "onUpdateEvent", + "Subscription", + "UpdateAvailability", + "UpdateConfiguration", + "UpdateEvent", + "UpdateStatus", + "UpdateUnavailableReason" + ], + "./video": [ + "Availability", + "Subscription", + "VideoContentFit", + "VideoPlayer", + "VideoPlayerOptions", + "VideoViewProps" + ], + "./webview": [ + "WebViewErrorEvent", + "WebViewMessageEvent", + "WebViewNavigation", + "WebViewNavigationRequest", + "WebViewRef", + "WebViewSource" + ], + "./windows": [ + "AsyncRegistration", + "centerWindow", + "CenterWindowOptions", + "closeWindow", + "CommandBar", + "CommandBarOptions", + "CommandBarShowOptions", + "createWindowMessages", + "createWindowsNavigator", + "createWindowState", + "createWindowWorkspaces", + "DesktopWidgetOptions", + "DisplayId", + "DisplayInfo", + "DuplicateWindowOptions", + "EdgeDrawer", + "EdgeDrawerOptions", + "ErrorFallbackProps", + "getDisplays", + "getWindow", + "hideWindow", + "HUD", + "HUDOptions", + "HUDShowOptions", + "listWindows", + "MacOSStartupSplitViewOptions", + "MacOSTitleBarControl", + "MacOSTitleBarOptions", + "MacOSToolbarDisplayMode", + "MacOSToolbarItem", + "MacOSToolbarOptions", + "MacOSToolbarSegment", + "MacOSToolbarState", + "MacOSWindowBlurOptions", + "MacOSWindowBoundsOptions", + "MacOSWindowEvent", + "MacOSWindowOptions", + "maximizeWindow", + "MenuAction", + "minimizeWindow", + "NavigatorOpenOptions", + "NavigatorWindowOptions", + "openWindow", + "OverlayOptions", + "PictureInPicture", + "PictureInPictureOptions", + "Popover", + "PopoverOptions", + "PresenterEndReason", + "PresenterMode", + "PresenterModeOptions", + "PreviewWindow", + "PreviewWindowOptions", + "PrimaryWindowLifecycleOptions", + "ScreenCorner", + "ScreenEdge", + "SetWindowBoundsOptions", + "setWindowFullscreen", + "setWindowOptions", + "showWindow", + "ShowWindowOptions", + "Size", + "SpecialWindowErrorOptions", + "SpecialWindowType", + "UndoEntry", + "UndoMenuLabels", + "UndoStack", + "UndoState", + "unmaximizeWindow", + "usePrimaryWindowLifecycle", + "UsePrimaryWindowLifecycleOptions", + "useWindowFocusEffect", + "useWindowId", + "WindowAppearanceOptions", + "WindowAttachment", + "WindowAttachmentEdge", + "WindowAttachOptions", + "WindowBroadcastListener", + "WindowBroadcastListenerOptions", + "WindowBroadcastOptions", + "WindowChoreographyFeature", + "WindowCloseTransition", + "WindowComponentLoader", + "WindowComponentProps", + "WindowEasing", + "WindowErrorEvent", + "WindowEventMap", + "WindowHookOptions", + "WindowHotkeys", + "WindowId", + "WindowInfo", + "WindowKind", + "WindowListenerOptions", + "WindowMacOS", + "WindowNewTabEvent", + "WindowOpenOptions", + "WindowProvider", + "WindowProviderProps", + "WindowRequestOptions", + "WindowRequestPayload", + "WindowRequestResult", + "WindowsConfig", + "WindowsNavigatorOptions", + "WindowStateOptions", + "WindowTabbingOptions", + "WindowTransition", + "WindowUpdateOptions", + "WindowWorkspacesOptions", + "WindowZoom", + "WindowZoomOptions", + "withWindowProvider", + "WorkspaceMenuLabels" + ], + "./windows/macos": [ + "addMacOSWindowListener", + "AsyncRegistration", + "focusToolbarSearch", + "FocusToolbarSearchOptions", + "MacOSToolbarDisplayMode", + "MacOSToolbarItem", + "MacOSToolbarOptions", + "MacOSToolbarState", + "MacOSWindowBlurOptions", + "MacOSWindowEvent", + "MenuAction", + "setToolbarItemText" + ] + } +} diff --git a/docs/e2e-flows.md b/docs/e2e-flows.md index b55a2170..faeffc97 100644 --- a/docs/e2e-flows.md +++ b/docs/e2e-flows.md @@ -1105,9 +1105,10 @@ the same and links to the runner's page. An audit is a gate step that checks the release artifact itself instead of driving the app. A check registered with `audit: ` is verified by that step, so no flow lists it (coverage reports `audit-flow` if one does). The -gate runs each audit step once, after the build check, and judges every audit check for the target's platform: a +gate runs each audit step once, one at a time, after the build check, and judges every audit check for the target's platform: a `failed` or `unsupported` result, or no result, fails a blocking check. A scoped run (`--flow`) runs no audits. -`bun run e2e:audit [--artifact ] [--package ]` runs only the audits for this machine's target and prints each result. +`bun run e2e:audit [--artifact ] [--package ] [--audit ]…` runs only the audits for this machine's +target (with `--audit`, only those steps) and prints each result. `bundle` (`scripts/e2e/audit/bundle.ts`, registered in `checks/packaging-distribution.yaml`) audits release bundle hygiene (#153): @@ -1152,6 +1153,88 @@ the distribution ZIP, or the folder holding the one `--.zip Spark makes no DMG, so there is no DMG layout check. App translocation at launch is not checked: it needs a notarized app launched from its quarantined location. +#### Developer experience audits + +Four audit steps, registered in `checks/developer-experience.yaml` (#170, #171), check the developer's side of a release. +The `templates` and `dev-loop` steps build and run apps, so run them under the native build lock on a shared Mac. + +`api-docs` (`scripts/api-docs.ts`) reads the public surface the way `docs/api-public-surface.json` snapshots it. +`docs/api-undocumented.json` lists the known gaps. `tests/api-docs-coverage.test.ts` (part of `bun run test`) fails on an +undocumented export or page-less subpath that is not listed, and on a listed one that is now documented, so the list only +shrinks. The audit fails while the list is not empty. + +| Check | What it verifies | +|---|---| +| `DX-DOCS-01` | Every public export has a TSDoc comment on its declaration (what an editor shows). | +| `DX-DOCS-02` | No public export, parameter or return type is `any`. | +| `DX-DOCS-03` | Every public subpath has a `docs/*.md` page that imports it (`@legendapp/spark/` in quotes or backticks). | + +`cli` (`scripts/e2e/audit/cli.ts`) runs the SDK's `spark` bin (`packages/desktop/bin/spark.cjs`) headless in a scratch +directory, only with arguments that fail or print help before any side effect. + +| Check | What it verifies | +|---|---| +| `DX-CLI-01` | `spark --help` names every command of the CLI's grammar (`packages/cli/src/cli-arguments.ts`) and `dev`; `-h`, no arguments and each command's `--help` exit 0; `spark dev --help` in the Kitchen Sink adds `expo start`'s options; `spark --version` prints the SDK version. | +| `DX-CLI-02` | An unknown command or option, missing or extra arguments, a group without its subcommand (`spark sdk`), invalid values, conflicting build modes and running outside an app each exit 1 with one line on stderr that names the problem, and no stack trace. | + +`templates` (`scripts/e2e/audit/templates.ts`) packs the SDK (`scripts/pack.ts`), then works in a scratch directory with +`CI=1`, no TTY, no stdin and a scratch `SPARK_HOME`, so the developer's registered SDKs and runners are untouched. It +unregisters every app it built from LaunchServices (`lsregister -u`) and removes the directory. It took 8 to 10 +minutes on an M4 Max: the packages come from the npm registry and the two release builds compile React Native twice. + +| Check | What it verifies | +|---|---| +| `DX-TPL-01` | `spark create --packages ` makes the macOS, Windows and universal templates; each depends on the packed SDK, installs and type-checks (`tsc --noEmit`). | +| `DX-TPL-02` | `spark build` of the macOS template makes a release app with a Hermes bytecode `main.jsbundle` that `codesign --verify --deep --strict` accepts. | +| `DX-REPRO-01` | A second release build, after deleting `.spark/` and the generated `macos/` project, has byte-identical JavaScript (`*.jsbundle`, `*.hbc`, `*.js`). | +| `DX-REPRO-02` | Every other file in the app is byte-identical too. Each difference is listed; for a Mach-O file, whether its `LC_UUID` or other load commands differ. See [Reproducible builds](#reproducible-builds). | + +`dev-loop` (`scripts/e2e/audit/dev-loop.ts`) runs `spark dev --clear --no-open --max-workers 2` for the Kitchen Sink and +launches its dev build in driver mode (`scripts/testing/dev-session.ts`; build it first with `cd examples/kitchen-sink && +bun run rebuild:macos`). It edits `screens/devtools/FastRefreshMarker.tsx` and `desktop.config.json` and always restores them. + +| Check | What it verifies | +|---|---| +| `DX-LOOP-01` | From starting `spark dev --clear` (an empty Metro cache) to the dev build's first settled render, within the cold start budget. | +| `DX-LOOP-02` | Three edits each reach the running runtime within the Fast Refresh budget, as hot updates: a global set before them survives (a reload clears it). | +| `DX-LOOP-03` | After the edits the dev build is still compatible (`.spark/session.json`) and the same app process runs: no native rebuild. A native configuration change (the app's `version`) makes `spark dev` report "Native configuration or host changed." with a rebuild offered, within the native change budget; reverting it makes the build compatible again. | +| `DX-LOOP-04` | A syntax error shows React Native's error overlay (LogBox, in the app's window) with the `SyntaxError`; the fix closes it and reaches the app without a reload. | + +##### Dev loop budgets + +The budgets (`DEV_LOOP_BUDGETS` in `scripts/e2e/audit/dev-loop.ts`) are for the reference MacBook Air (M1, 8 GB) and +apply on every Mac. They were not measured on the M1: they are three times what a 16-inch MacBook Pro (M4 Max, Mac16,5, +macOS 27) measured, rounded up. Replace them with values measured on the reference machine. + +| Budget | M4 Max measured | M1 budget | +|---|---|---| +| Cold start (`DX-LOOP-01`) | 10.8 s, 13.2 s | 45 s | +| Fast Refresh, per edit (`DX-LOOP-02`) | 0.3 to 0.6 s | 2 s | +| Native change detected (`DX-LOOP-03`) | 0.8 s | 5 s | + +DevTools in the dev loop has its own checks (`checks/devtools.yaml`). + +#### Reproducible builds + +`DX-REPRO-01` and `DX-REPRO-02` build the macOS template twice. `spark build` prepares the project in place (it patches +`node_modules` and writes `macos/`, `ios/`, `react-native.config.js` and configuration), so each build starts from an APFS +clone of the project as `spark create` left it, at the same path. With identical inputs the JavaScript bundle and +`spark-runtime.json` are byte-identical. Built from an already-prepared project instead, `spark-runtime.json`'s +`fingerprint` differs, because the first build changed its inputs. + +The native output is not bit-reproducible, because of Apple's tools. On Xcode 27 (macOS 27, arm64), four files differ. For +each of them, `DX-REPRO-02` checks the parts that should still match: + +| File | Why it differs | What must still match | +|---|---|---| +| The main executable (`Contents/MacOS/`) | The linker puts two symbol pointers in the other order, so some instructions and `__objc_stubs` refer to other slots. `LC_UUID` is a hash of the output, and the code signature is a hash of the file. | Every load command apart from `LC_UUID`, and the symbols (`nm -m`). | +| `Assets.car` | `actool` stamps the build time and gives each rendition a unique name, which also changes its digest. | Every rendition's other attributes (`assetutil --info`). | +| `*.nib` | `ibtool` writes the same objects in another order. | The objects, each with its class and values, with references as the referenced object's class. | +| `_CodeSignature/CodeResources` | It holds the hashes of the files above. | Nothing more: the files it seals are compared themselves. | + +Any other difference, or a difference in what these compare, fails `DX-REPRO-02`. The frameworks' binaries (Hermes and +the rest) and every other resource are byte-identical. + ### Budgets `e2e/budgets.yaml` (schema: `e2e/schema/budgets.schema.json`) holds baselines per diff --git a/e2e/checks/developer-experience.yaml b/e2e/checks/developer-experience.yaml new file mode 100644 index 00000000..2f1451ab --- /dev/null +++ b/e2e/checks/developer-experience.yaml @@ -0,0 +1,78 @@ +# Developer experience (#170, #171): gate audit steps, not flows (docs/e2e-flows.md#audits). DevTools has its own +# registry (checks/devtools.yaml, epic #267). +area: developer-experience +prefix: DX +checks: + DX-LOOP-01: + title: Cold start of the dev loop, spark dev --clear for the Kitchen Sink to the first settled render of its dev build, is within the reference MacBook Air (M1, 8 GB) budget + platforms: [macos] + blocking: true + spec: "#170 Dev loop" + audit: dev-loop + DX-LOOP-02: + title: Fast Refresh delivers each source edit to the running app as a hot update, with no reload of the JavaScript runtime, within the M1 budget per edit + platforms: [macos] + blocking: true + spec: "#170 Fast Refresh" + audit: dev-loop + DX-LOOP-03: + title: JavaScript edits never require a native rebuild (the dev build stays compatible and the same process keeps running); a native configuration change makes spark dev report that the build needs a rebuild, within the M1 budget, and reverting it makes the build compatible again + platforms: [macos] + blocking: true + spec: "#170 Native change detection requires rebuild" + audit: dev-loop + DX-LOOP-04: + title: A syntax error in an edit shows React Native's error overlay in the running app; fixing it dismisses the overlay and Fast Refresh resumes without a reload + platforms: [macos] + blocking: true + spec: "#170 Error overlay" + audit: dev-loop + DX-DOCS-01: + title: Every public export (docs/api-public-surface.json) has a TSDoc comment on its declaration + blocking: true + spec: "#171 Every export documented" + audit: api-docs + DX-DOCS-02: + title: No public export, parameter or return type is any + blocking: true + spec: "#171 Every export typed" + audit: api-docs + DX-DOCS-03: + title: Every public subpath has a docs page that imports it + blocking: true + spec: "#171 Every export documented" + audit: api-docs + DX-CLI-01: + title: spark --help names every command; -h, no arguments and each command's --help exit 0; spark dev --help in an app adds expo start's options; spark --version prints the SDK version + blocking: true + spec: "#171 CLI commands" + audit: cli + DX-CLI-02: + title: CLI misuse (unknown command or option, missing or extra arguments, a group without its subcommand, invalid values, conflicting modes, no app) exits 1 with one line on stderr that names the problem and no stack trace + blocking: true + spec: "#171 CLI commands" + audit: cli + DX-TPL-01: + title: spark create makes each template (macOS, Windows, universal) from the packed SDK, headless (CI=1, no TTY), and it installs and type-checks + platforms: [macos] + blocking: true + spec: "#171 Templates build; headless CI" + audit: templates + DX-TPL-02: + title: spark build of the macOS template from the packed SDK makes a release app headless, with Hermes bytecode and a signature codesign verifies + platforms: [macos] + blocking: true + spec: "#171 Templates build; headless CI" + audit: templates + DX-REPRO-01: + title: Two clean release builds of the macOS template have byte-identical JavaScript bundles + platforms: [macos] + blocking: true + spec: "#171 Reproducible builds" + audit: templates + DX-REPRO-02: + title: Two clean release builds of the macOS template are byte-identical apart from what docs/e2e-flows.md#reproducible-builds explains, and each difference is reported + platforms: [macos] + blocking: true + spec: "#171 Reproducible builds" + audit: templates diff --git a/e2e/schema/checks.schema.json b/e2e/schema/checks.schema.json index 886bad3b..d60e2118 100644 --- a/e2e/schema/checks.schema.json +++ b/e2e/schema/checks.schema.json @@ -59,7 +59,11 @@ "audit": { "enum": [ "bundle", - "package" + "package", + "api-docs", + "cli", + "templates", + "dev-loop" ], "description": "The gate audit step that verifies this check against the release build, instead of a flow (see docs/e2e-flows.md#audits)." } diff --git a/packages/cli/src/cli-arguments.ts b/packages/cli/src/cli-arguments.ts index a1c5d3ff..52183d7e 100644 --- a/packages/cli/src/cli-arguments.ts +++ b/packages/cli/src/cli-arguments.ts @@ -7,9 +7,9 @@ const options = { device: { type: "string" }, project: { type: "string" }, platform: { type: "string" }, packages: { type: "string" }, port: { type: "string" }, runner: { type: "boolean" }, dev: { type: "boolean" }, release: { type: "boolean" }, preview: { type: "boolean" }, - force: { type: "boolean" }, dsym: { type: "string", multiple: true }, "submission-id": { type: "string" }, "ad-hoc": { type: "boolean" }, help: { type: "boolean", short: "h" }, + force: { type: "boolean" }, dsym: { type: "string", multiple: true }, "submission-id": { type: "string" }, "ad-hoc": { type: "boolean" }, help: { type: "boolean", short: "h" }, version: { type: "boolean" }, } as const; -const commands: Record = { +export const commands: Record = { create: { flags: ["example", "package-manager", "universal", "platform", "packages"], arguments: [1, 1] }, "add desktop": { flags: ["project", "packages"], arguments: [0, 0] }, "sdk export": { flags: ["packages", "runtime"], arguments: [1, 1] }, @@ -35,9 +35,10 @@ export function cliArguments(args: string[]) { const grouped = ["sdk", "add", "updates"].includes(positionals[0]); const command = positionals.slice(0, grouped ? 2 : 1).join(" "); if (!command || (grouped && positionals.length === 1 && values.help)) { - if (Object.keys(values).some(key => key !== "help")) throw new Error("Choose a command before specifying options. Run spark --help."); + if (Object.keys(values).some(key => key !== "help" && key !== "version")) throw new Error("Choose a command before specifying options. Run spark --help."); return parsed; } + if (grouped && positionals.length === 1) throw new Error(`Choose a subcommand: ${Object.keys(commands).filter(name => name.startsWith(`${command} `)).map(name => `spark ${name}`).join(", ")}.`); const contract = commands[command]; if (!contract) throw new Error(`Unknown command: ${command}. Run spark --help.`); for (const key of Object.keys(values)) if (key !== "help" && !contract.flags.includes(key)) throw new Error(`--${key} is not an option for spark ${command}`); diff --git a/packages/cli/src/dev-command.ts b/packages/cli/src/dev-command.ts index 01459021..0cc3d605 100644 --- a/packages/cli/src/dev-command.ts +++ b/packages/cli/src/dev-command.ts @@ -2,7 +2,7 @@ import { spawnProcess } from "./process.ts"; import path from "node:path"; import { prepareConfig, selectTarget, supportedPlatforms } from "@legendapp/spark-desktop-config/config.cjs"; import { dev } from "./dev.ts"; -import { findProject } from "./local.ts"; +import { appRoot, findProject } from "./local.ts"; import { hostPlatform } from "./platform.ts"; import { nodeCommand } from "./windows.ts"; @@ -44,7 +44,8 @@ export async function devCommand(args: string[]) { const options = devArguments(args); const start = path.resolve(options.project ?? process.cwd()); if (options.expo.includes("--help") || options.expo.includes("-h")) { - console.log("spark dev options:\n --project Application directory\n --platform Initial launch target; all declared platforms stay available\n --runner-binary Register and use a Spark Runner\n --app-arg Pass a desktop launch argument (repeatable)\n --no-open Wait for a launch key (explicit Expo launch flags still apply)\n\nAll other options belong to expo start:\n"); + console.log("spark dev options:\n --project Application directory\n --platform Initial launch target; all declared platforms stay available\n --runner-binary Register and use a Spark Runner\n --app-arg Pass a desktop launch argument (repeatable)\n --no-open Wait for a launch key (explicit Expo launch flags still apply)\n\nAll other options belong to expo start" + (appRoot(start) ? ":\n" : "; run spark dev --help inside an app to list them.")); + if (!appRoot(start)) return; const child = spawnProcess(nodeCommand(start, "expo", "expo", ["start", "--help"]), { cwd: start, stdin: "inherit", stdout: "inherit", stderr: "inherit" }); process.exitCode = await child.exited; return; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index b5a9e4ac..d5a0ba5c 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -59,7 +59,8 @@ async function main() { process.exit(process.exitCode); } } - if (values.help || !command) { + if (values.version && !command) console.log(VERSION); + else if (values.help || !command) { console.log(`Legend Spark spark create MyApp Create an app (--universal for shared Settings; --platform selects a target) @@ -76,6 +77,7 @@ async function main() { Inside an app: npm run dev, npm run build, npm run package (or pnpm, yarn, bun) Advanced: updates init , credentials, doctor, analyze, open [app], build --dev, build --preview, package --ad-hoc (test package, not notarized) +Version: spark --version Crash reports: symbolicate --dsym (prints the report with symbols and source lines) Windows: dev and build --dev; production builds are not yet supported. SDK transfer: sdk export [--runtime ], sdk import diff --git a/packages/cli/src/local.ts b/packages/cli/src/local.ts index 7e7f141b..838a9849 100644 --- a/packages/cli/src/local.ts +++ b/packages/cli/src/local.ts @@ -21,16 +21,20 @@ export function findFramework(start = import.meta.dirname): string | undefined { } } -export function findProject(start: string): string { - let dir = path.resolve(start); - while (true) { +/** The app that contains `start`, or undefined outside an app. */ +export function appRoot(start: string): string | undefined { + for (let dir = path.resolve(start); ; dir = path.dirname(dir)) { if ((existsSync(path.join(dir, "app.json")) || existsSync(path.join(dir, "desktop.config.json"))) && existsSync(path.join(dir, "package.json"))) return dir; - const parent = path.dirname(dir); - if (parent === dir) throw new Error("No spark app found. Run this command inside your app, or create one with spark create MyApp."); - dir = parent; + if (path.dirname(dir) === dir) return undefined; } } +export function findProject(start: string): string { + const root = appRoot(start); + if (!root) throw new Error("No spark app found. Run this command inside your app, or create one with spark create MyApp."); + return root; +} + export function readRuntime(app: string): Runtime | undefined { try { const windows = existsSync(path.join(app, "spark-runtime.json")); diff --git a/scripts/api-docs.ts b/scripts/api-docs.ts new file mode 100644 index 00000000..887e513c --- /dev/null +++ b/scripts/api-docs.ts @@ -0,0 +1,43 @@ +// API documentation coverage of the public SDK surface (docs/api-public-surface.json): what an editor shows for +// each export, and the docs page for each subpath. tests/api-docs-coverage.test.ts and the gate's api-docs audit use it. +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import ts from "typescript"; +import { sdkProgram } from "./api-surface.ts"; + +const docs = path.resolve(import.meta.dirname, "../docs"); +/** Known gaps, drained as they are documented: tests/api-docs-coverage.test.ts keeps it exact. */ +export const API_DOCS_GAPS = "docs/api-undocumented.json"; + +export type ApiDocs = { + /** Per subpath, the exports without a TSDoc comment on their declaration. */ + undocumented: Record; + /** Per subpath, the runtime exports whose type, a parameter or a return type is `any`. */ + untyped: Record; + /** Subpaths that no docs/*.md page imports (`@legendapp/spark/` in quotes or backticks). */ + pageless: string[]; +}; + +const isAny = (type: ts.Type) => !!(type.flags & ts.TypeFlags.Any); + +export function readApiDocs(): ApiDocs { + const { checker, modules } = sdkProgram(); + const pages = readdirSync(docs, { recursive: true, encoding: "utf8" }).filter(file => file.endsWith(".md")).map(file => readFileSync(path.join(docs, file), "utf8")).join("\n"); + const result: ApiDocs = { undocumented: {}, untyped: {}, pageless: [] }; + const add = (map: Record, subpath: string, name: string) => (map[subpath] ??= []).push(name); + for (const { subpath, source, symbols } of [...modules].sort((a, b) => a.subpath.localeCompare(b.subpath))) { + const specifier = `@legendapp/spark${subpath.slice(1)}`; + if (![`"${specifier}"`, `'${specifier}'`, `\`${specifier}\``].some(quoted => pages.includes(quoted))) result.pageless.push(subpath); + for (const { symbol, target } of [...symbols].sort((a, b) => a.symbol.name.localeCompare(b.symbol.name))) { + if (!ts.displayPartsToString(target.getDocumentationComment(checker)).trim()) add(result.undocumented, subpath, symbol.name); + if (!(target.flags & ts.SymbolFlags.Value)) continue; + const type = checker.getTypeOfSymbolAtLocation(target, source); + const signatures = type.getCallSignatures(); + if (isAny(type) || signatures.some(signature => isAny(signature.getReturnType()) || signature.parameters.some(parameter => isAny(checker.getTypeOfSymbolAtLocation(parameter, source))))) + add(result.untyped, subpath, symbol.name); + } + } + return result; +} + +export const count = (map: Record) => Object.values(map).reduce((total, names) => total + names.length, 0); diff --git a/scripts/api-surface.ts b/scripts/api-surface.ts index fb382a2c..3006b1fe 100644 --- a/scripts/api-surface.ts +++ b/scripts/api-surface.ts @@ -27,23 +27,33 @@ function availabilityFlags(checker: ts.TypeChecker, symbol: ts.Symbol, at: ts.No return args.flatMap(arg => booleans.map(flag => `(${arg}).${flag}`)); } -/** - * The public SDK surface: every subpath in packages/desktop/package.json `exports`, and for TypeScript - * entries every export the type checker sees. docs/api-public-surface.json snapshots the export names. - */ -export function readSdkSurface(): SdkSurface { +/** The SDK's TypeScript entries, type-checked with the repository's tsconfig: per subpath, its exports and what each aliases. */ +export function sdkProgram() { const exports = JSON.parse(readFileSync(path.join(sdk, "package.json"), "utf8")).exports as Record; const entries = Object.entries(exports).filter(([, file]) => file.endsWith(".ts")); const config = ts.readConfigFile(path.join(repo, "tsconfig.json"), ts.sys.readFile); const options = ts.parseJsonConfigFileContent(config.config, ts.sys, repo).options; const program = ts.createProgram(entries.map(([, file]) => path.join(sdk, file)), options); const checker = program.getTypeChecker(); - const surface: SdkSurface = Object.fromEntries(Object.keys(exports).sort().map(subpath => [subpath, { values: [], types: [], flags: {} }])); - for (const [subpath, file] of entries) { + const modules = entries.map(([subpath, file]) => { const source = program.getSourceFile(path.join(sdk, file))!; + const symbols = checker.getExportsOfModule(checker.getSymbolAtLocation(source)!) + .map(symbol => ({ symbol, target: symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol })); + return { subpath, source, symbols }; + }); + return { exports, checker, modules }; +} + +/** + * The public SDK surface: every subpath in packages/desktop/package.json `exports`, and for TypeScript + * entries every export the type checker sees. docs/api-public-surface.json snapshots the export names. + */ +export function readSdkSurface(): SdkSurface { + const { exports, modules, checker } = sdkProgram(); + const surface: SdkSurface = Object.fromEntries(Object.keys(exports).sort().map(subpath => [subpath, { values: [], types: [], flags: {} }])); + for (const { subpath, source, symbols } of modules) { const entry = surface[subpath]!; - for (const symbol of checker.getExportsOfModule(checker.getSymbolAtLocation(source)!)) { - const target = symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol; + for (const { symbol, target } of symbols) { if (!(target.flags & ts.SymbolFlags.Value)) { entry.types.push(symbol.name); continue; } entry.values.push(symbol.name); if (AVAILABILITY.test(symbol.name)) entry.flags[symbol.name] = availabilityFlags(checker, target, source); diff --git a/scripts/e2e/audit/api-docs.ts b/scripts/e2e/audit/api-docs.ts new file mode 100644 index 00000000..e1aca436 --- /dev/null +++ b/scripts/e2e/audit/api-docs.ts @@ -0,0 +1,18 @@ +// The gate's api-docs audit: API documentation coverage of the public SDK surface (docs/e2e-flows.md#audits, #171). +// The same facts as tests/api-docs-coverage.test.ts, which keeps docs/api-undocumented.json exact; this fails while +// any gap remains. +import { API_DOCS_GAPS, count, readApiDocs } from "../../api-docs.ts"; +import type { AuditResult } from "./bundle.ts"; + +const listed = (map: Record) => Object.entries(map).map(([subpath, names]) => `${subpath}: ${names.join(", ")}`); + +export function auditApiDocs(): AuditResult[] { + const { undocumented, untyped, pageless } = readApiDocs(); + const result = (check: string, problems: string[], summary: string, passed: string): AuditResult => + ({ check, status: problems.length ? "failed" : "passed", details: problems.length ? [summary, ...problems] : [passed] }); + return [ + result("DX-DOCS-01", listed(undocumented), `${count(undocumented)} public exports have no TSDoc comment (listed in ${API_DOCS_GAPS})`, "every public export has a TSDoc comment"), + result("DX-DOCS-02", listed(untyped), `${count(untyped)} public exports are typed any`, "no public export, parameter or return type is any"), + result("DX-DOCS-03", pageless, `${pageless.length} public subpaths have no docs page (listed in ${API_DOCS_GAPS})`, "every public subpath has a docs page"), + ]; +} diff --git a/scripts/e2e/audit/cli.ts b/scripts/e2e/audit/cli.ts new file mode 100644 index 00000000..821459bc --- /dev/null +++ b/scripts/e2e/audit/cli.ts @@ -0,0 +1,62 @@ +// The gate's cli audit: the `spark` command line as an app gets it (packages/desktop/bin/spark.cjs, the SDK's bin), +// its help and its errors (docs/e2e-flows.md#audits, #171). Every case runs headless (no TTY, CI=1) with a scratch +// SPARK_HOME, and only cases that fail before any side effect, so nothing is created, built or registered. +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { commands } from "../../../packages/cli/src/cli-arguments.ts"; +import { KITCHEN_SINK } from "../kitchen-sink.ts"; +import type { AuditResult } from "./bundle.ts"; + +const repo = path.resolve(import.meta.dirname, "../../.."); +const bin = path.join(repo, "packages/desktop/bin/spark.cjs"); +const version = JSON.parse(readFileSync(path.join(repo, "packages/desktop/package.json"), "utf8")).version as string; + +/** Each error case: the arguments and what its one-line message must name. */ +const ERRORS: Array<[string[], string]> = [ + [["bogus"], "Unknown command: bogus"], + [["build", "--bogus"], "--bogus"], + [["create"], "spark create"], + [["build", "extra"], "spark build"], + [["sdk"], "spark sdk export"], + [["build", "--platform", "beos"], "Platform must be"], + [["build", "--port", "99999"], "Port must be"], + [["create", "App", "--package-manager", "zz"], "Package manager must be"], + [["create", "App", "--example", "nope"], "--example"], + [["build", "--dev", "--release"], "build mode"], + [["build"], "No spark app found"], + [["dev"], "No spark app found"], +]; + +export function auditCli(): AuditResult[] { + const scratch = mkdtempSync(path.join(os.tmpdir(), "spark-cli-audit-")); + try { + const spark = (args: string[], cwd = scratch) => { + const result = spawnSync("node", [bin, ...args], { cwd, encoding: "utf8", timeout: 60_000, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, CI: "1", SPARK_HOME: path.join(scratch, ".spark") } }); + return { status: result.status, stdout: result.stdout, stderr: result.stderr, label: `spark ${args.join(" ")}`.trim() }; + }; + const help: string[] = [], errors: string[] = []; + const global = spark(["--help"]); + const names = ["dev", ...Object.keys(commands)]; + const missing = names.filter(name => !new RegExp(`(^|[\\s/])${name}\\b`, "m").test(global.stdout)); + if (global.status !== 0 || missing.length) help.push(`spark --help exited ${global.status} and does not name ${missing.join(", ") || "-"}`); + for (const args of [[], ["-h"], ...names.map(name => [...name.split(" "), "--help"])]) { + const run = spark(args); + if (run.status !== 0 || !run.stdout.includes("spark ") || run.stderr.trim()) help.push(`${run.label} exited ${run.status}${run.stderr.trim() ? ` with ${JSON.stringify(run.stderr.trim().split("\n")[0])}` : ""}`); + } + const devHelp = spark(["dev", "--help"], KITCHEN_SINK); + if (devHelp.status !== 0 || !devHelp.stdout.includes("--no-open") || !devHelp.stdout.includes("--port")) help.push(`spark dev --help in an app exited ${devHelp.status} without spark's and expo start's options`); + const printed = spark(["--version"]); + if (printed.status !== 0 || printed.stdout.trim() !== version) help.push(`spark --version exited ${printed.status} and printed ${JSON.stringify(printed.stdout.trim())}, not ${version}`); + for (const [args, names] of ERRORS) { + const run = spark(args), lines = run.stderr.trim().split("\n"); + if (run.status !== 1 || run.stdout.trim() || lines.length !== 1 || !lines[0]!.includes(names)) + errors.push(`${run.label}: exit ${run.status}, stdout ${JSON.stringify(run.stdout.trim().slice(0, 80))}, stderr ${JSON.stringify(run.stderr.trim().slice(0, 300))}; expected exit 1 and one line naming ${JSON.stringify(names)}`); + } + return [ + { check: "DX-CLI-01", status: help.length ? "failed" : "passed", details: help.length ? help : [`spark --help names all ${names.length} commands; -h, no arguments and each command's --help exit 0; spark dev --help in an app adds expo start's options; spark --version prints ${version}`] }, + { check: "DX-CLI-02", status: errors.length ? "failed" : "passed", details: errors.length ? errors : [`${ERRORS.length} misuses (unknown command and option, missing and extra arguments, a group without its subcommand, invalid values, conflicting modes, no app) exit 1 with one line on stderr that names the problem, no stack trace`] }, + ]; + } finally { rmSync(scratch, { recursive: true, force: true }); } +} diff --git a/scripts/e2e/audit/dev-loop.ts b/scripts/e2e/audit/dev-loop.ts new file mode 100644 index 00000000..4a70205f --- /dev/null +++ b/scripts/e2e/audit/dev-loop.ts @@ -0,0 +1,98 @@ +// The gate's dev-loop audit (docs/e2e-flows.md#audits, #170): a real `spark dev` session of the Kitchen Sink with its +// dev build launched in driver mode (no window, no focus; scripts/testing/dev-session.ts), timed against the budgets of +// the reference MacBook Air (M1, 8 GB). Edits go to two Kitchen Sink files and are always restored. +import { readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { readJson, stateFile } from "../../../packages/cli/src/project.ts"; +import { startDevSession, type Cdp, type DevSession } from "../../testing/dev-session.ts"; +import { KITCHEN_SINK } from "../kitchen-sink.ts"; +import type { AuditResult } from "./bundle.ts"; + +/** + * Budgets for the reference MacBook Air (M1, 8 GB), in milliseconds. They apply on every Mac: a faster one meets them + * more easily. Not measured on the M1: they are three times what an M4 Max (Mac16,5) measured, rounded up (see + * docs/e2e-flows.md#dev-loop-budgets); replace them with values measured on the reference machine. + */ +export const DEV_LOOP_BUDGETS = { coldStartMs: 45_000, fastRefreshMs: 2_000, nativeChangeMs: 5_000 }; +export const DEV_LOOP_CHECKS = ["DX-LOOP-01", "DX-LOOP-02", "DX-LOOP-03", "DX-LOOP-04"] as const; + +const MARKER = path.join(KITCHEN_SINK, "screens/devtools/FastRefreshMarker.tsx"); +const CONFIG = path.join(KITCHEN_SINK, "desktop.config.json"); +const REVISION = 'const REVISION = "1";'; +const seconds = (ms: number) => `${(ms / 1000).toFixed(1)} s`; + +export async function auditDevLoop(): Promise { + const results = new Map(); + const set = (check: string, ok: boolean, details: string[]) => results.set(check, { check, status: ok ? "passed" : "failed", details }); + const marker = readFileSync(MARKER, "utf8"), config = readFileSync(CONFIG, "utf8"); + if (!marker.includes(REVISION)) throw new Error(`${MARKER} no longer declares ${REVISION}`); + const session = () => readJson(stateFile(KITCHEN_SINK, "session.json")) as { compatible: boolean; reason: string; canBuild?: boolean }; + let dev: DevSession | undefined; + try { + // DX-LOOP-01: from starting spark dev with an empty Metro cache to the dev build's first settled render. + const started = performance.now(); + dev = await startDevSession({ prefix: "ks-dev-loop-", args: ["--clear"] }); + const { driver, name } = await dev.launch(); + await driver.startup(300_000); + const coldMs = performance.now() - started; + set("DX-LOOP-01", coldMs <= DEV_LOOP_BUDGETS.coldStartMs, [`spark dev --clear to the first settled render: ${seconds(coldMs)} (budget ${seconds(DEV_LOOP_BUDGETS.coldStartMs)})`]); + + const cdp: Cdp = await dev.connect(await dev.target(`${name} (shared runtime)`)); + await cdp.send("Runtime.enable"); + // A global set now survives a hot update and is gone after a reload of the runtime (the debugger stays attached). + let sentinel = ""; + const arm = async () => { sentinel = `dx-loop-${performance.now()}`; await cdp.evaluate(`globalThis.__dxLoopSentinel = ${JSON.stringify(sentinel)}`); }; + const reloaded = async () => (await cdp.evaluate("globalThis.__dxLoopSentinel")) !== sentinel; + const revision = () => cdp.evaluate("globalThis.ksDevToolsRevision"); + const edit = async (value: string, label: string) => { + const begun = performance.now(); + writeFileSync(MARKER, marker.replace(REVISION, `const REVISION = "${value}";`)); + await dev!.until(async () => (await revision()) === value, label, 60_000); + return performance.now() - begun; + }; + const pid = driver.app.pid; + + // DX-LOOP-02: three edits, each a hot update of the running runtime. + await arm(); + const latencies: number[] = []; + for (const n of [1, 2, 3]) latencies.push(await edit(`dx-${n}`, `Fast Refresh delivers edit ${n}`)); + const hot = !await reloaded(), slowest = Math.max(...latencies); + set("DX-LOOP-02", hot && slowest <= DEV_LOOP_BUDGETS.fastRefreshMs, [`edit to the running app: ${latencies.map(seconds).join(", ")} (budget ${seconds(DEV_LOOP_BUDGETS.fastRefreshMs)} each)`, + hot ? "hot updates: the runtime's globals survived" : "the edits reloaded the JavaScript runtime (its globals are gone) instead of a hot update (#291)"]); + + // DX-LOOP-04: a syntax error shows the overlay; the fix dismisses it and refreshes without a reload. + await arm(); + writeFileSync(MARKER, marker.replace(REVISION, `${REVISION} const broken = ;`)); + const overlay = await dev.until(async () => (await driver.elements()).elements.find(element => /SyntaxError/.test(`${element.text ?? ""} ${element.label ?? ""}`)), "the error overlay shows the syntax error", 30_000); + const fixedMs = await edit("dx-fixed", "Fast Refresh resumes after the fix"); + await dev.until(async () => !(await driver.elements()).elements.some(element => /SyntaxError/.test(`${element.text ?? ""} ${element.label ?? ""}`)), "the error overlay closes", 30_000); + const hotFix = !await reloaded(); + set("DX-LOOP-04", hotFix, [`the overlay showed ${JSON.stringify((overlay.text ?? overlay.label ?? "").split("\n")[0]!.slice(0, 160))}; the fix reached the app in ${seconds(fixedMs)} and closed it`, + hotFix ? "no reload" : "the fix reloaded the JavaScript runtime instead of a hot update (#291)"]); + + // DX-LOOP-03: after the JavaScript edits the dev build is still compatible and the same process runs; a native + // configuration change (the app's version, which Info.plist carries) needs a rebuild until it is reverted. + const afterEdits = session(), samePid = driver.app.pid === pid && driver.app.exitCode === null; + const changed = JSON.parse(config); + changed.version = changed.version === "9.9.9" ? "9.9.8" : "9.9.9"; + const changedAt = performance.now(); + writeFileSync(CONFIG, JSON.stringify(changed, null, 2) + "\n"); + const stale = await dev.until(async () => { const state = session(); return !state.compatible && /Native configuration/.test(state.reason) && state; }, "spark dev reports the native change", 60_000); + const detectMs = performance.now() - changedAt; + writeFileSync(CONFIG, config); + await dev.until(async () => session().compatible, "reverting the change makes the dev build compatible again", 60_000); + set("DX-LOOP-03", afterEdits.compatible && samePid && !!stale.canBuild && detectMs <= DEV_LOOP_BUDGETS.nativeChangeMs, [ + `after ${latencies.length + 2} JavaScript edits: ${afterEdits.compatible ? "the dev build is compatible" : `incompatible (${afterEdits.reason})`}, ${samePid ? "the same app process runs" : "the app process changed"}`, + `a native configuration change: "${stale.reason}"${stale.canBuild ? ", with a rebuild offered" : ", but no rebuild offered"}, in ${seconds(detectMs)} (budget ${seconds(DEV_LOOP_BUDGETS.nativeChangeMs)}); reverting it made the build compatible again`, + ]); + } catch (error) { + dev?.printLogs(); + const message = error instanceof Error ? error.message : String(error); + for (const check of DEV_LOOP_CHECKS) if (!results.has(check)) results.set(check, { check, status: "failed", details: [message] }); + } finally { + writeFileSync(MARKER, marker); + writeFileSync(CONFIG, config); + await dev?.close(); + } + return DEV_LOOP_CHECKS.map(check => results.get(check)!); +} diff --git a/scripts/e2e/audit/steps.ts b/scripts/e2e/audit/steps.ts index 6913ba70..20543b86 100644 --- a/scripts/e2e/audit/steps.ts +++ b/scripts/e2e/audit/steps.ts @@ -1,6 +1,12 @@ // The gate audit steps by the name a check registers with `audit:` (docs/e2e-flows.md#audits). import type { Audit } from "../format/primitives.ts"; +import { auditApiDocs } from "./api-docs.ts"; import { auditBundle, type AuditOptions, type AuditResult } from "./bundle.ts"; +import { auditCli } from "./cli.ts"; +import { auditDevLoop } from "./dev-loop.ts"; import { auditPackage } from "./package.ts"; +import { auditTemplates } from "./templates.ts"; -export const AUDIT_STEPS: Readonly AuditResult[] | Promise>> = { bundle: auditBundle, package: options => auditPackage(options) }; +export const AUDIT_STEPS: Readonly AuditResult[] | Promise>> = { + bundle: auditBundle, package: options => auditPackage(options), "api-docs": auditApiDocs, cli: auditCli, templates: auditTemplates, "dev-loop": auditDevLoop, +}; diff --git a/scripts/e2e/audit/templates.ts b/scripts/e2e/audit/templates.ts new file mode 100644 index 00000000..83106d64 --- /dev/null +++ b/scripts/e2e/audit/templates.ts @@ -0,0 +1,169 @@ +// The gate's templates audit (docs/e2e-flows.md#audits, #171): `spark create` from the packed SDK, headless, in a +// scratch directory, then two clean release builds of the macOS template compared file by file. Everything runs with +// CI=1, no TTY and no stdin, as on a CI runner, with a scratch SPARK_HOME, so the developer's registered SDKs and +// runners are untouched. The built apps are unregistered from LaunchServices and the scratch directory is removed. +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, openSync, closeSync, readdirSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { readJson, stateFile } from "../../../packages/cli/src/project.ts"; +import { isMachO, type AuditResult } from "./bundle.ts"; + +const repo = path.resolve(import.meta.dirname, "../../.."); +const LSREGISTER = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister"; +const HERMES_MAGIC = Buffer.from([0xc6, 0x1f, 0xbc, 0x03, 0xc1, 0x03, 0x19, 0x1f]); +/** The templates `spark create` offers, by the flag that selects each. */ +const TEMPLATES: Array<[name: string, flags: string[]]> = [["DxMac", ["--platform", "macos"]], ["DxWindows", ["--platform", "windows"]], ["DxUniversal", ["--universal"]]]; +export const TEMPLATE_CHECKS = ["DX-TPL-01", "DX-TPL-02", "DX-REPRO-01", "DX-REPRO-02"] as const; + +class StepError extends Error {} + +/** Every file and link in an app bundle, by its path inside the bundle: a SHA-256 of a file's bytes, a link's target. */ +function digests(app: string, rel = ""): Map { + const files = new Map(); + for (const entry of readdirSync(path.join(app, rel), { withFileTypes: true })) { + const name = path.join(rel, entry.name), abs = path.join(app, name); + if (entry.isSymbolicLink()) files.set(name, `-> ${readlinkSync(abs)}`); + else if (entry.isDirectory()) for (const [key, value] of digests(app, name)) files.set(key, value); + else files.set(name, createHash("sha256").update(readFileSync(abs)).digest("hex")); + } + return files; +} + +const isScript = (name: string) => /\.(jsbundle|hbc|js)$/.test(name); +const output = (argv: string[]) => spawnSync(argv[0]!, argv.slice(1), { encoding: "utf8", maxBuffer: 1 << 28 }).stdout; +/** A Mach-O file's load commands apart from LC_UUID, and its symbols. */ +const machOShape = (file: string) => [output(["otool", "-l", file]).split("\n").slice(1).filter(line => !/^\s*uuid /.test(line)).join("\n"), output(["nm", "-m", file])].join("\n"); +/** An asset catalog's renditions without what actool makes unique per build: its timestamp, rendition names' unique suffixes and digests. */ +const catalogShape = (file: string) => output(["xcrun", "assetutil", "--info", file]).replace(/"Timestamp" : \d+/g, "").replace(/_[0-9A-F]{8}-[0-9A-F-]{27}-\d+-[0-9A-F]+\.png/g, ".png").replace(/"SHA1Digest" : "\w+"/g, ""); +/** + * A compiled nib (NIBArchive) as the sorted list of its objects: each its class and its values, with a reference to + * another object written as that object's class. The same objects archived in another order have the same list. + */ +function nibShape(file: string): string { + const data = readFileSync(file); + if (data.subarray(0, 10).toString("latin1") !== "NIBArchive") return data.toString("hex"); + const header = Array.from({ length: 8 }, (_, index) => data.readUInt32LE(18 + index * 4)); + let at = 0; + const varint = () => { let value = 0, shift = 0, byte; do { byte = data[at++]!; value |= (byte & 0x7f) << shift; shift += 7; } while (!(byte & 0x80)); return value; }; + const list = (count: number, offset: number, read: () => T) => { at = offset; return Array.from({ length: count }, read); }; + const objects = list(header[0]!, header[1]!, () => ({ type: varint(), start: varint(), count: varint() })); + const keys = list(header[2]!, header[3]!, () => { const length = varint(); return data.subarray(at, (at += length)).toString("latin1"); }); + const SIZES: Record = { 0: 1, 1: 2, 2: 4, 3: 8, 4: 0, 5: 0, 6: 4, 7: 8, 9: 0, 10: 4 }; + const values = list(header[4]!, header[5]!, () => { + const key = keys[varint()]!, type = data[at++]!; + const length = type === 8 ? varint() : SIZES[type] ?? 0; + return { key, type, bytes: data.subarray(at, (at += length)) }; + }); + const classes = list(header[6]!, header[7]!, () => { const length = varint(), extra = varint(); at += extra * 4; return data.subarray(at, (at += length)).toString("latin1"); }); + const value = ({ key, type, bytes }: typeof values[number]) => `${key}=${type === 10 ? `@${classes[objects[bytes.readUInt32LE(0)]!.type]}` : `${type}:${bytes.toString("hex")}`}`; + return objects.map(object => `${classes[object.type]}{${values.slice(object.start, object.start + object.count).map(value).sort().join(",")}}`).sort().join("\n"); +} +/** + * Files Apple's tools do not write reproducibly (docs/e2e-flows.md#reproducible-builds), and what of them must still match: + * a difference anywhere else, or in what these compare, fails. + */ +const KNOWN: Array<{ match: (name: string, abs: string) => boolean; why: string; shape: (file: string) => string }> = [ + { match: (name, abs) => !name.includes("/_CodeSignature/") && isMachO({ rel: name, abs, size: statSync(abs).size, kind: "file" }), why: "the linker orders some symbol pointers differently, so LC_UUID and the code signature differ; load commands and symbols match", shape: machOShape }, + { match: name => name.endsWith(".car"), why: "actool stamps the build time and unique rendition names; the renditions match", shape: catalogShape }, + { match: name => name.endsWith(".nib"), why: "ibtool archives the same objects in another order; the objects and their values match", shape: nibShape }, + { match: name => name.endsWith("_CodeSignature/CodeResources"), why: "it seals the files above", shape: () => "" }, +]; + +/** DX-REPRO-01 and DX-REPRO-02 for two release builds of the same app. */ +export function compareBuilds(first: string, second: string): AuditResult[] { + const [a, b] = [digests(first), digests(second)]; + const names = [...new Set([...a.keys(), ...b.keys()])].sort(); + const differing = names.filter(name => a.get(name) !== b.get(name)); + const scripts = names.filter(isScript), jsDiffs = differing.filter(isScript); + const native: string[] = [], unexplained: string[] = []; + for (const name of differing.filter(name => !isScript(name))) { + const [left, right] = [path.join(first, name), path.join(second, name)]; + const known = a.has(name) && b.has(name) && !a.get(name)!.startsWith("-> ") ? KNOWN.find(entry => entry.match(name, left)) : undefined; + if (known && known.shape(left) === known.shape(right)) native.push(`${name}: ${known.why}`); + else unexplained.push(`${name}: ${!a.has(name) ? "only in the second build" : !b.has(name) ? "only in the first build" : known ? `differs beyond what ${known.why.split(";")[0]} explains` : "differs"}`); + } + return [ + { check: "DX-REPRO-01", status: jsDiffs.length || !scripts.length ? "failed" : "passed", details: jsDiffs.length ? jsDiffs.map(name => `${name} differs between two clean release builds`) + : scripts.length ? [`${scripts.join(", ")}: byte-identical across two clean release builds`] : ["no JavaScript in the release build"] }, + { check: "DX-REPRO-02", status: unexplained.length ? "failed" : "passed", details: [`${names.length - differing.length} of ${names.length} files and links are byte-identical across two clean release builds`, ...unexplained, ...native] }, + ]; +} + +export async function auditTemplates(): Promise { + const results = new Map(); + const set = (check: string, status: AuditResult["status"], details: string[]) => results.set(check, { check, status, details }); + const scratch = mkdtempSync(path.join(os.tmpdir(), "spark-templates-audit-")); + const env = { ...process.env, CI: "1", SPARK_HOME: path.join(scratch, ".spark-home") }; + const step = (label: string, argv: string[], cwd: string) => { + const log = path.join(scratch, `${label.replace(/\W+/g, "-")}.log`), fd = openSync(log, "w"); + const started = Date.now(); + try { + const result = spawnSync(argv[0]!, argv.slice(1), { cwd, env, stdio: ["ignore", fd, fd] }); + const seconds = Math.round((Date.now() - started) / 100) / 10; + if (result.status !== 0) throw new StepError(`${label} exited ${result.status ?? result.signal} after ${seconds} s:\n${readFileSync(log, "utf8").trim().split("\n").slice(-15).join("\n")}`); + return seconds; + } finally { closeSync(fd); } + }; + const spark = (app: string, label: string, args: string[]) => step(label, ["node", path.join(app, "node_modules/@legendapp/spark/bin/spark.cjs"), ...args], app); + try { + const timings: string[] = [`pack ${step("spark sdk pack", [process.execPath, path.join(repo, "scripts/pack.ts")], repo)} s`]; + const manifest = path.join(repo, "artifacts/packages/manifest.json"); + // DX-TPL-01: each template from the packed SDK, through the SDK's own `spark` bin in a scratch directory. + const sdkBin = path.join(repo, "packages/desktop/bin/spark.cjs"); + for (const [name, flags] of TEMPLATES) { + const app = path.join(scratch, name); + timings.push(`create ${name} ${step(`spark create ${name}`, ["node", sdkBin, "create", app, "--packages", manifest, ...flags], scratch)} s`); + const config = JSON.parse(readFileSync(path.join(app, "desktop.config.json"), "utf8")); + const dependency = JSON.parse(readFileSync(path.join(app, "package.json"), "utf8")).dependencies["@legendapp/spark"] as string; + if (!dependency.includes(path.join(repo, "artifacts/packages"))) throw new StepError(`${name} depends on @legendapp/spark ${dependency}, not the packed SDK`); + if (config.name !== name) throw new StepError(`${name}: desktop.config.json names the app ${config.name}`); + timings.push(`typecheck ${name} ${step(`tsc ${name}`, ["node", "node_modules/typescript/bin/tsc", "--noEmit"], app)} s`); + } + set("DX-TPL-01", "passed", [`spark create made ${TEMPLATES.map(([name, flags]) => `${name} (${flags.join(" ")})`).join(", ")} from the packed SDK (${path.relative(repo, manifest)}), installed and type-checked, headless`, timings.join(", ")]); + + // DX-TPL-02 and the reproducibility checks: two clean release builds of the macOS template. + // `spark build` prepares the project in place (it patches node_modules and writes the native project and + // configuration), so each build starts from a clone of the project as `spark create` left it, at the same path. + const app = path.join(scratch, "DxMac"), pristine = path.join(scratch, "DxMac.pristine"); + step("clone the project", ["cp", "-c", "-R", app, pristine], scratch); + const builds: string[] = []; + for (const run of [1, 2]) { + if (run === 2) { + rmSync(app, { recursive: true, force: true }); + renameSync(pristine, app); + } + const seconds = spark(app, `spark build ${run}`, ["build"]); + const product = readJson(stateFile(app, "release-build.json")).app as string; + if (!existsSync(path.join(product, "Contents/Info.plist"))) throw new StepError(`spark build ${run} finished without ${product}`); + const copy = path.join(scratch, `release-${run}`, path.basename(product)); + mkdirSync(path.dirname(copy), { recursive: true }); + cpSync(product, copy, { recursive: true, verbatimSymlinks: true }); + builds.push(copy); + if (run === 1) { + const bundle = path.join(copy, "Contents/Resources/main.jsbundle"); + const hermes = existsSync(bundle) && readFileSync(bundle).subarray(0, 8).equals(HERMES_MAGIC); + const verify = spawnSync("codesign", ["--verify", "--deep", "--strict", copy], { encoding: "utf8" }); + if (!hermes || verify.status !== 0) throw new StepError(`the release build ${!hermes ? "has no Hermes bytecode main.jsbundle" : `fails codesign --verify: ${verify.stderr.trim()}`}`); + set("DX-TPL-02", "passed", [`spark build of the macOS template made DxMac.app headless in ${seconds} s (clean): Hermes bytecode main.jsbundle, codesign --verify --deep --strict accepts it`]); + } + } + for (const result of compareBuilds(builds[0]!, builds[1]!)) results.set(result.check, result); + } catch (error) { + const message = error instanceof StepError ? error.message : String(error instanceof Error ? error.stack : error); + for (const check of TEMPLATE_CHECKS) if (!results.has(check)) set(check, "failed", [message]); + } finally { + // Xcode registers what it builds with LaunchServices: unregister every app under the scratch directory. + const registered = () => [...spawnSync(LSREGISTER, ["-dump"], { encoding: "utf8", maxBuffer: 1 << 30 }).stdout.matchAll(/^path:\s+(.+?\.app)\b/gm)] + .map(match => match[1]!).filter(app => app.startsWith(`${realpathSync(scratch)}/`)); + for (const app of new Set(registered())) spawnSync(LSREGISTER, ["-u", app]); + const left = [...new Set(registered())]; + if (left.length) for (const check of TEMPLATE_CHECKS) { + const result = results.get(check); + if (result) results.set(check, { check, status: "failed", details: [...result.details, `still registered with LaunchServices: ${left.join(", ")}`] }); + } + rmSync(scratch, { recursive: true, force: true }); + } + return TEMPLATE_CHECKS.map(check => results.get(check)!); +} diff --git a/scripts/e2e/cli.ts b/scripts/e2e/cli.ts index d42b30e3..8ae44bcc 100644 --- a/scripts/e2e/cli.ts +++ b/scripts/e2e/cli.ts @@ -63,6 +63,6 @@ if (mode === "schema") { ? await gateCommand(args, path.join(repo, "e2e"), { ...hostEnv(), signal: controller.signal }) : await runCommand(args, { backends: BACKENDS, host: detectHost, stdout: line => console.log(line), stderr: line => console.error(line), signal: controller.signal }); } else { - console.error("usage: bun run e2e:validate [paths…] | bun run e2e:lint [paths…] | bun run e2e:coverage [paths…] [--json] | bun run e2e:schema | bun run e2e:run [flows…] [--backend …] [--agent] [--report dir] [--app path] | bun run gate [--target id] [--flow path…] [--agent] [--dry-run] | bun run e2e:audit [--artifact app] | bun run gate:signoff [--list] | bun run e2e:golden approve [--to ]"); + console.error("usage: bun run e2e:validate [paths…] | bun run e2e:lint [paths…] | bun run e2e:coverage [paths…] [--json] | bun run e2e:schema | bun run e2e:run [flows…] [--backend …] [--agent] [--report dir] [--app path] | bun run gate [--target id] [--flow path…] [--agent] [--dry-run] | bun run e2e:audit [--artifact app] [--package path] [--audit name]… | bun run gate:signoff [--list] | bun run e2e:golden approve [--to ]"); process.exitCode = 2; } diff --git a/scripts/e2e/format/primitives.ts b/scripts/e2e/format/primitives.ts index 71d116f4..d9314ef2 100644 --- a/scripts/e2e/format/primitives.ts +++ b/scripts/e2e/format/primitives.ts @@ -4,7 +4,7 @@ export type Schema = Record; export const CHECK_ID = "^[A-Z][A-Z0-9]*-[A-Z0-9]+-\\d{2}$"; export const PLATFORMS = ["macos", "windows"] as const; /** Gate audit steps: a check registered with `audit: ` is verified by that step against the release build, not by a flow. */ -export const AUDITS = ["bundle", "package"] as const; +export const AUDITS = ["bundle", "package", "api-docs", "cli", "templates", "dev-loop"] as const; export type Audit = typeof AUDITS[number]; const ref = (name: string): Schema => ({ $ref: `#/definitions/${name}` }); diff --git a/scripts/e2e/gate/gate.ts b/scripts/e2e/gate/gate.ts index 539a5b9e..eb3e3283 100644 --- a/scripts/e2e/gate/gate.ts +++ b/scripts/e2e/gate/gate.ts @@ -7,7 +7,7 @@ import { collect } from "../files.ts"; import { checkFile } from "../format/lint.ts"; import type { AuditOptions, AuditResult } from "../audit/bundle.ts"; import { checkSignoffs, FlowFormatError, formatDiagnostic, type Signoff } from "../format/parser.ts"; -import type { Audit } from "../format/primitives.ts"; +import { AUDITS, type Audit } from "../format/primitives.ts"; import type { ManualStep } from "../format/signoff.ts"; import type { BackendFactory } from "../runner/backend.ts"; import { ms, realClock, type Clock, type FlowResult } from "../runner/executor.ts"; @@ -150,7 +150,12 @@ function judgeFlows(plan: GatePlan, results: FlowResult[] | undefined, runDir: s export async function judgeAudits(plan: GatePlan, env: GateEnv, exists: boolean, reasons: Reason[], warnings: string[]): Promise { const build = plan.target && plan.manifest.build[plan.target.platform]; const options = build && { app: plan.artifact!, platform: plan.target!.platform, arch: plan.target!.arch, appId: plan.manifest.appId, sizeBudgetMB: build.sizeBudgetMB, symbols: plan.symbols, package: plan.package }; - const results = exists && options ? (await Promise.all([...new Set(plan.audits.map(audit => audit.audit))].map(name => env.audit(name, options)))).flat() : undefined; + // One step at a time: steps that build or time apps must not overlap. + let results: AuditResult[] | undefined; + if (exists && options) { + results = []; + for (const name of new Set(plan.audits.map(audit => audit.audit))) results.push(...await env.audit(name, options)); + } return plan.audits.map(({ check, audit, title, blocking }) => { const result = results?.find(candidate => candidate.check === check); const entry: AuditEntry = { check, audit, title, blocking, status: result?.status ?? "not-run", details: result?.details ?? [] }; @@ -366,17 +371,23 @@ export async function gateCommand(args: string[], root: string, env: GateEnv): P */ export async function auditCommand(args: string[], root: string, env: GateEnv): Promise { const paths: { artifact?: string; package?: string } = {}; + const only: string[] = []; for (let index = 0; index < args.length; index++) { const [flag, inline] = args[index]!.split(/=(.*)/s, 2) as [string, string | undefined]; const value = inline ?? args[++index]; - if ((flag !== "--artifact" && flag !== "--package") || !value) { env.stderr("usage: bun run e2e:audit [--artifact ] [--package ]"); return 2; } - paths[flag.slice(2) as "artifact" | "package"] = path.resolve(value); + if (!["--artifact", "--package", "--audit"].includes(flag) || !value || (flag === "--audit" && !(AUDITS as readonly string[]).includes(value))) { + env.stderr(`usage: bun run e2e:audit [--artifact ] [--package ] [--audit ${AUDITS.join("|")}]…`); + return 2; + } + if (flag === "--audit") only.push(value); + else paths[flag.slice(2) as "artifact" | "package"] = path.resolve(value); } let gatePlan: GatePlan; try { gatePlan = plan(root, env.host(), paths); } catch (error) { if (error instanceof UsageError || error instanceof FlowFormatError) { env.stderr(error.message); return error instanceof UsageError ? 2 : 1; } throw error; } + if (only.length) gatePlan = { ...gatePlan, audits: gatePlan.audits.filter(audit => only.includes(audit.audit)) }; const reasons: Reason[] = [...gatePlan.problems], warnings: string[] = []; if (!gatePlan.audits.length) reasons.push({ check: "audit", message: "no audit checks apply to this target" }); const exists = !!gatePlan.artifact && existsSync(gatePlan.artifact); diff --git a/scripts/testing/dev-session.ts b/scripts/testing/dev-session.ts index eeac22ba..dd366e86 100644 --- a/scripts/testing/dev-session.ts +++ b/scripts/testing/dev-session.ts @@ -18,15 +18,15 @@ export type Message = { id?: number; method?: string; params?: any; result?: any export type Cdp = Awaited>; export type DevSession = Awaited>; -/** `env` reaches spark dev (and React Native's browser launcher); `prefix` names the scratch directory. */ -export async function startDevSession(options: { prefix: string; env?: Record }) { +/** `env` reaches spark dev (and React Native's browser launcher); `args` are more spark dev options; `prefix` names the scratch directory. */ +export async function startDevSession(options: { prefix: string; env?: Record; args?: string[] }) { const work = mkdtempSync(path.join(os.tmpdir(), options.prefix)); const sessionLog = path.join(work, "spark-dev.log"), appLog = path.join(work, "app.log"); const port = await availablePort(); const base = `http://127.0.0.1:${port}`; const app = new KitchenSinkApp(undefined, new URL(base)); const log = openSync(sessionLog, "a"); - const session = spawnProcess([process.execPath, path.join(import.meta.dirname, "../spark.mjs"), "dev", "--platform", "macos", "--no-open", "--port", String(port), "--max-workers", "2"], { + const session = spawnProcess([process.execPath, path.join(import.meta.dirname, "../spark.mjs"), "dev", "--platform", "macos", "--no-open", "--port", String(port), "--max-workers", "2", ...options.args ?? []], { cwd: KITCHEN_SINK, env: { ...process.env, CI: "false", ...options.env }, stdin: "ignore", stdout: log, stderr: log, }); const sockets: WebSocket[] = []; diff --git a/tests/api-docs-coverage.test.ts b/tests/api-docs-coverage.test.ts new file mode 100644 index 00000000..4fe230d1 --- /dev/null +++ b/tests/api-docs-coverage.test.ts @@ -0,0 +1,26 @@ +import { readFileSync } from "node:fs"; +import { expect, test } from "vitest"; +import { API_DOCS_GAPS, readApiDocs } from "../scripts/api-docs.ts"; + +// Every public export needs a TSDoc comment (what an editor shows), a type other than `any`, and its subpath a docs +// page. docs/api-undocumented.json lists the known gaps: a new undocumented export or page-less subpath fails here, +// and so does a listed one that is now documented, so the list only shrinks. The gate's api-docs audit (DX-DOCS-01, +// DX-DOCS-03) fails while the list is not empty. +const docs = readApiDocs(); +const gaps = JSON.parse(readFileSync(API_DOCS_GAPS, "utf8")) as { pageless: string[]; undocumented: Record }; +const entries = (map: Record) => Object.entries(map).flatMap(([subpath, names]) => names.map(name => `${subpath}#${name}`)); + +test("no public export is typed any", () => { + expect(entries(docs.untyped)).toEqual([]); +}, 20_000); + +test("every public export has a TSDoc comment, apart from the known gaps", () => { + const listed = new Set(entries(gaps.undocumented)), found = entries(docs.undocumented); + expect(found.filter(entry => !listed.has(entry)), "document these exports (a /** … */ comment on the declaration)").toEqual([]); + expect([...listed].filter(entry => !found.includes(entry)), `now documented: remove them from ${API_DOCS_GAPS}`).toEqual([]); +}); + +test("every public subpath has a docs page, apart from the known gaps", () => { + expect(docs.pageless.filter(subpath => !gaps.pageless.includes(subpath)), "add a docs/*.md page that imports these subpaths").toEqual([]); + expect(gaps.pageless.filter(subpath => !docs.pageless.includes(subpath)), `now documented: remove them from ${API_DOCS_GAPS}`).toEqual([]); +}); diff --git a/tests/cli-arguments.test.ts b/tests/cli-arguments.test.ts index e1485887..7c267424 100644 --- a/tests/cli-arguments.test.ts +++ b/tests/cli-arguments.test.ts @@ -11,3 +11,9 @@ test("accepts documented command shapes and rejects incompatible build modes", ( expect(cliArguments(["create", "--help"]).values.help).toBe(true); expect(cliArguments(["sdk", "--help"]).values.help).toBe(true); }); +test("a command group without its subcommand names the subcommands; --version stands alone", () => { + expect(() => cliArguments(["sdk"])).toThrow("Choose a subcommand: spark sdk export, spark sdk import"); + expect(() => cliArguments(["add"])).toThrow("Choose a subcommand: spark add desktop."); + expect(cliArguments(["--version"]).values.version).toBe(true); + expect(() => cliArguments(["build", "--version"])).toThrow("--version is not an option for spark build"); +}); diff --git a/tests/e2e-gate.test.ts b/tests/e2e-gate.test.ts index 0eefbcb3..31a6233b 100644 --- a/tests/e2e-gate.test.ts +++ b/tests/e2e-gate.test.ts @@ -6,7 +6,7 @@ import { describe, expect, test, vi } from "vitest"; import type { SdkSurface } from "../scripts/api-surface.ts"; import type { AuditOptions, AuditResult } from "../scripts/e2e/audit/bundle.ts"; import { checkBudgets, checkFlow, checkGate, checkSignoffs, formatDiagnostic } from "../scripts/e2e/format/parser.ts"; -import { gateCommand, parseGateArgs, runGate, type GateEnv, type GateReport } from "../scripts/e2e/gate/gate.ts"; +import { auditCommand, gateCommand, parseGateArgs, runGate, type GateEnv, type GateReport } from "../scripts/e2e/gate/gate.ts"; import { flowHash } from "../scripts/e2e/gate/manual.ts"; import { plan, type Host } from "../scripts/e2e/gate/plan.ts"; import { mergeDecisions, signoffCommand, type SignoffEnv } from "../scripts/e2e/gate/signoff.ts"; @@ -199,6 +199,26 @@ describe("release gate", () => { expect([scoped.audits, calls]).toEqual([[], []]); }); + test("audit steps run one at a time; e2e:audit --audit runs only the named steps", async () => { + const registry = "area: dx\nprefix: DX\nchecks:\n DX-CLI-01:\n title: Help\n audit: cli\n DX-DOCS-01:\n title: Docs\n audit: api-docs\n"; + const dir = repo({ "e2e/checks/dx.yaml": registry }); + const events: string[] = []; + const audit = async (name: string) => { + events.push(`start ${name}`); + await new Promise(resolve => setTimeout(resolve, 5)); + events.push(`end ${name}`); + return [{ check: name === "cli" ? "DX-CLI-01" : "DX-DOCS-01", status: "passed" as const, details: [] }]; + }; + await gate(dir, { audit }); + expect(events).toEqual(["start cli", "end cli", "start api-docs", "end api-docs"]); + events.length = 0; + const { env: auditEnv, out } = env({ audit }); + expect(await auditCommand(["--audit", "api-docs"], path.join(dir, "e2e"), auditEnv)).toBe(0); + expect(events).toEqual(["start api-docs", "end api-docs"]); + expect(out.filter(line => line.startsWith("PASSED"))).toEqual(["PASSED DX-DOCS-01 Docs"]); + expect(await auditCommand(["--audit", "nope"], path.join(dir, "e2e"), env().env)).toBe(2); + }); + test("uncommitted changes fail: the gate certifies a commit", async () => { expect(reasons(await gate(repo(), { source: () => ({ commit: COMMIT, dirty: true }) }))).toEqual(["source: the working tree has uncommitted changes; the gate certifies a commit"]); });