Some tweaks to the doc building. Doc submodules...

* docs.py:
  - remove/replace some undefined functions
  - blacken file
  - start to understand structure of modules under builtins
* `buiiltin/*/__init__.py`: start adding documentation
* test.py:
  - add `-x` shorthand for stop on failure
  - blacken

Author: rocky

mathics/builtin/__init__.py

@@ -85,26 +85,37 @@ def get_module_doc(module):
     return title, text
 
 def import_builtins(module_names: List[str], submodule_name=None) -> None:
-    for module_name in module_names:
-        import_name = (
-            f"mathics.builtin.{submodule_name}.{module_name}"
-            if submodule_name
-            else f"mathics.builtin.{module_name}"
-        )
+    """
+    Imports the list of Mathics Built-in modules so that inside
+    Mathics we have these Builtin Functions, like Plus[], List[] are defined.
+
+    """
+    def import_module(module_name: str, import_name: str):
         try:
             module = importlib.import_module(import_name)
         except Exception as e:
             print(e)
             print(f"    Not able to load {module_name}. Check your installation.")
             print(f"    mathics.builtin loads from {__file__[:-11]}")
-            continue
+            return None
 
         if __version__ != module.__version__:
             print(
-                f"Version {module.__version__} in the module do not match with {__version__}"
+                f"Version {module.__version__} in the module does not match top-level Mathics version {__version__}"
             )
+        if module:
+            modules.append(module)
 
-        modules.append(module)
+    if submodule_name:
+        import_module(submodule_name, f"mathics.builtin.{submodule_name}")
+
+    for module_name in module_names:
+        import_name = (
+            f"mathics.builtin.{submodule_name}.{module_name}"
+            if submodule_name
+            else f"mathics.builtin.{module_name}"
+        )
+        import_module(module_name, import_name)
 
 
 def is_builtin(var):

mathics/builtin/drawing/__init__.py

@@ -1,3 +1,25 @@
 """
 Graphics, Drawing, and Images
+
+Functions like 'Plot' and 'ListPlot' can be used to draw graphs of functions and data.
+
+Graphics is implemented as a collection of <i>graphics primitives</i>. Primatives are objects like 'Point', 'Line', and 'Polygon' and become elements of a <i>graphics object</i>.
+
+A graphics object can have directives as well such as 'RGBColor', and 'Thickness'.
+
+There are several kinds of graphics objects; each kind has a head which identifies its type.
+
+>> ListPlot[ Table[Prime[n], {n, 20} ]]
+ = -Graphics-
+>> Head[%]
+ = Graphics
+>> Graphics3D[Sphere[]]
+ = -Graphics3D-
+>> Head[%]
+ = Graphics3D
+>>
+
+
 """
+
+from mathics.version import __version__  # noqa used in loading to check consistency.

mathics/builtin/files_io/__init__.py

@@ -1,3 +1,5 @@
 """
 Input/Output, Files, and Filesystem
 """
+
+from mathics.version import __version__  # noqa used in loading to check consistency.

mathics/builtin/numbers/__init__.py

@@ -1,3 +1,5 @@
 """
 Integer and Number-Theoretical Functions
 """
+
+from mathics.version import __version__  # noqa used in loading to check consistency.

mathics/builtin/specialfns/__init__.py

@@ -1,3 +1,14 @@
 """
 Special Functions
+
+There are a number of functions found in mathematical physics and found in standard handbooks.
+
+One thing to note is that the technical literature often contains several conflicting definitions. So beware and check for conformance with the Mathics documentation.
+
+A number of special functions can be evaluated for arbitrary complex values of their arguments. However defining relations may apply only for some special choices of arguments. Here, the full function corresponds to an extension or "analytic continuation" of the defining relation.
+
+For example, integral representations of functions are only valid when the integral exists, but the functions can usually be defined b by analytic continuation.
+
 """
+
+from mathics.version import __version__  # noqa used in loading to check consistency.

mathics/doc/doc.py

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
+from html import escape as html_escape
 import re
-from os import getenv, listdir, path
+from os import getenv, listdir
 import pickle
 import importlib
+import pkgutil
 
 from mathics import settings
 
@@ -109,6 +111,15 @@
 except IOError:
     xml_data = {}
 
+def get_submodule_names(object):
+    modpkgs = []
+    if hasattr(object, '__path__'):
+        for importer, modname, ispkg in pkgutil.iter_modules(object.__path__):
+            modpkgs.append(modname)
+        modpkgs.sort()
+    return modpkgs
+
+
 
 def filter_comments(doc):
     return "\n".join(
@@ -241,21 +252,23 @@ def repl_list(match):
         [
             ("$", r"\$"),
             ("\u03c0", r"$\pi$"),
-            ("≥", r"$\ge$"),
-            ("≤", r"$\le$"),
-            ("≠", r"$\ne$"),
-            ("ç", r"\c{c}"),
-            ("é", r"\'e"),
-            ("ê", r"\^e"),
-            ("ñ", r"\~n"),
-            ("∫", r"\int"),
-            ("", r"d"),
+            (" ", r"$\ge$"),
+            (" ", r"$\le$"),
+            (" ", r"$\ne$"),
+            ("\u00e7", r"\c{c}"),
+            ("\u00e9", r"\'e"),
+            ("\u00ea", r"\^e"),
+            ("\00f1", r"\~n"),
+            ("\u222b", r"\int"),
+            ("\uf74c", r"d"),
         ],
     )
 
     def repl_char(match):
         char = match.group(1)
-        return {"^": "$^\wedge$",}[char]
+        return {
+            "^": "$^\wedge$",
+        }[char]
 
     text = LATEX_CHAR_RE.sub(repl_char, text)
 
@@ -474,6 +487,7 @@ def post_sub(text, post_substitutions):
     return text
 
 
+# FIXME: can we replace this with Python 3's html.escape ?
 def escape_html(text, verbatim_mode=False, counters=None, single_line=False):
     def repl_python(match):
         return (
@@ -614,7 +628,8 @@ def repl_subsection(match):
         text = text.replace("\\" + key, xml)
 
     if not single_line:
-        text = linebreaks(text)
+        # FIXME: linebreaks() is not defined
+        # text = linebreaks(text)
         text = text.replace("<br />", "\n").replace("<br>", "<br />")
 
     text = post_sub(text, post_substitutions)
@@ -654,7 +669,7 @@ def get_prev_next(self):
         return prev, next
 
     def get_title_html(self):
-        return mark_safe(escape_html(self.title, single_line=True))
+        return escape_html(self.title, single_line=True)
 
 
 class Documentation(DocElement):
@@ -788,21 +803,36 @@ def __init__(self):
                 title, text = get_module_doc(module)
                 chapter = DocChapter(builtin_part, title, Doc(text))
                 builtins = builtins_by_module[module.__name__]
-                for instance in builtins:
+
+                if module.__file__.endswith("__init__.py"):
+                    section_names = get_submodule_names(module)
+                else:
+                    section_names = builtins
+
+                for instance in section_names:
                     installed = True
                     for package in getattr(instance, "requires", []):
                         try:
                             importlib.import_module(package)
                         except ImportError:
                             installed = False
                             break
-                    section = DocSection(
-                        chapter,
-                        strip_system_prefix(instance.get_name()),
-                        instance.__doc__ or "",
-                        operator=instance.get_operator(),
-                        installed=installed,
-                    )
+                    if isinstance(instance, str):
+                        section = DocSection(
+                            chapter,
+                            instance,
+                            "",
+                            None,
+                            installed=installed,
+                        )
+                    else:
+                        section = DocSection(
+                            chapter,
+                            strip_system_prefix(instance.get_name()),
+                            instance.__doc__ or "",
+                            operator=instance.get_operator(),
+                            installed=installed,
+                        )
                     chapter.sections.append(section)
                 builtin_part.chapters.append(chapter)
             self.parts.append(builtin_part)
@@ -1167,7 +1197,7 @@ def latex(self, output):
 
     def html(self):
         counters = {}
-        return mark_safe(
+        return html_escape(
             "\n".join(
                 item.html(counters) for item in self.items if not item.is_private()
             )

mathics/test.py

@@ -15,12 +15,10 @@
 from mathics.core.parser import MathicsSingleLineFeeder
 from mathics.builtin import builtins_dict
 
-builtins = builtins_dict()
-
 from mathics import version_string
 from mathics import settings
 
-
+builtins = builtins_dict()
 
 
 class TestOutput(Output):
@@ -47,6 +45,7 @@ def print_and_log(*args):
     if logfile:
         logfile.write(string)
 
+
 def compare(result, wanted):
     if result == wanted:
         return True
@@ -93,16 +92,15 @@ def fail(why):
         time_parsing = datetime.now()
         query = evaluation.parse_feeder(feeder)
         if check_partial_enlapsed_time:
-            print("   parsing took", datetime.now()-time_parsing)
+            print("   parsing took", datetime.now() - time_parsing)
         if query is None:
             # parsed expression is None
             result = None
             out = evaluation.out
         else:
-            time_evaluating = datetime.now()
             result = evaluation.evaluate(query)
             if check_partial_enlapsed_time:
-                print("   evaluation took", datetime.now()-time_parsing)
+                print("   evaluation took", datetime.now() - time_parsing)
             out = result.out
             result = result.result
     except Exception as exc:
@@ -114,7 +112,7 @@ def fail(why):
     time_comparing = datetime.now()
     comparison_result = compare(result, wanted)
     if check_partial_enlapsed_time:
-        print("   comparison took ", datetime.now()-time_comparing)
+        print("   comparison took ", datetime.now() - time_comparing)
     if not comparison_result:
         print("result =!=wanted")
         fail_msg = "Result: %s\nWanted: %s" % (result, wanted)
@@ -132,7 +130,7 @@ def fail(why):
                 output_ok = False
                 break
     if check_partial_enlapsed_time:
-        print("   comparing messages took ", datetime.now()-time_comparing)
+        print("   comparing messages took ", datetime.now() - time_comparing)
     if not output_ok:
         return fail(
             "Output:\n%s\nWanted:\n%s"
@@ -199,7 +197,6 @@ def test_section(sections: set, quiet=False, stop_on_failure=False):
     sections |= {"$" + s for s in sections}
     for tests in documentation.get_tests():
         if tests.section in sections:
-            found = True
             for test in tests.tests:
                 if test.ignore:
                     continue
@@ -366,11 +363,19 @@ def main():
         "--version", "-v", action="version", version="%(prog)s " + mathics.__version__
     )
     parser.add_argument(
-        "--sections", "-s", dest="section", metavar="SECTION", help="only test SECTION(s). "
-        "You can list multiple sections by adding a comma (and no space) in between section names."
+        "--sections",
+        "-s",
+        dest="section",
+        metavar="SECTION",
+        help="only test SECTION(s). "
+        "You can list multiple sections by adding a comma (and no space) in between section names.",
     )
     parser.add_argument(
-        "--logfile", "-f", dest="logfilename", metavar="LOGFILENAME", help="stores the output in [logfilename]. "
+        "--logfile",
+        "-f",
+        dest="logfilename",
+        metavar="LOGFILENAME",
+        help="stores the output in [logfilename]. ",
     )
     parser.add_argument(
         "--pymathics",
@@ -418,7 +423,7 @@ def main():
         help="create documentation even if there is a test failure",
     )
     parser.add_argument(
-        "--stop-on-failure", action="store_true", help="stop on failure"
+        "--stop-on-failure", "-x", action="store_true", help="stop on failure"
     )
     parser.add_argument(
         "--skip",
@@ -443,7 +448,7 @@ def main():
     # If a test for a specific section is called
     # just test it
     if args.logfilename:
-        logfile = open(args.logfilename,"wt")
+        logfile = open(args.logfilename, "wt")
 
     if args.section:
         sections = set(args.section.split(","))