improvement: marimo export ipynb with outputs (#3048)

## 📝 Summary

add `--include-outputs` for marimo export. The goal for this is to
export to ipynb so they can be committed to git and viewable in GitHub.
GitHub doesn't allow scripts/styles that are remote (they get
sanitized), so web-components or widgets (including anywidgets) do not
get shown.

Instead, we will output markdown as that will be best rendered in
GitHub's notebook renderer.

e.g. 
```bash
# with outputs
marimo export ipynb notebook.py -o notebook.ipynb --include-outputs
# without outputs (default)
marimo export ipynb notebook.py -o notebook.ipynb --no-include-outputs
```

Author: mscolnick

.github/workflows/test_be.yaml

@@ -89,10 +89,10 @@ jobs:
       - name: Test with minimal dependencies
         if: ${{ matrix.dependencies == 'core' }}
         run: |
-          hatch run +py=${{ matrix.python-version }} test:test -v tests/ -k "not test_cli"
+          hatch run +py=${{ matrix.python-version }} test:test -v tests/ -k "not test_cli" --durations=10
 
       # Test with optional dependencies
       - name: Test with optional dependencies
         if: ${{ matrix.dependencies == 'core,optional' }}
         run: |
-          hatch run +py=${{ matrix.python-version }} test-optional:test -v tests/ -k "not test_cli"
+          hatch run +py=${{ matrix.python-version }} test-optional:test -v tests/ -k "not test_cli" --durations=10

marimo/_cli/export/commands.py

@@ -14,6 +14,7 @@
     export_as_md,
     export_as_script,
     run_app_then_export_as_html,
+    run_app_then_export_as_ipynb,
 )
 from marimo._server.utils import asyncio_run
 from marimo._utils.file_watcher import FileWatcher
@@ -295,18 +296,32 @@ def export_callback(file_path: MarimoPath) -> str:
     will be printed to stdout.
     """,
 )
+@click.option(
+    "--include-outputs/--no-include-outputs",
+    default=False,
+    show_default=True,
+    type=bool,
+    help="Run the notebook and include outputs in the exported ipynb file.",
+)
 @click.argument("name", required=True)
 def ipynb(
     name: str,
     output: str,
     watch: bool,
     sort: Literal["top-down", "topological"],
+    include_outputs: bool,
 ) -> None:
     """
     Export a marimo notebook as a Jupyter notebook in topological order.
     """
 
     def export_callback(file_path: MarimoPath) -> str:
+        if include_outputs:
+            return asyncio_run(
+                run_app_then_export_as_ipynb(
+                    file_path, sort_mode=sort, cli_args={}
+                )
+            )[0]
         return export_as_ipynb(file_path, sort_mode=sort)[0]
 
     DependencyManager.nbformat.require(

marimo/_config/manager.py

@@ -25,7 +25,7 @@
 class UserConfigManager:
     def __init__(self, config_path: Optional[str] = None) -> None:
         self._config_path = config_path
-        self.config = load_config()
+        self._config = load_config()
 
     def save_config(
         self, config: MarimoConfig | PartialMarimoConfig
@@ -37,13 +37,13 @@ def save_config(
         # Remove the secret placeholders from the incoming config
         config = remove_secret_placeholders(config)
         # Merge the current config with the new config
-        merged = merge_config(self.config, config)
+        merged = merge_config(self._config, config)
 
         with open(config_path, "w", encoding="utf-8") as f:
             tomlkit.dump(merged, f)
 
-        self.config = merge_default_config(merged)
-        return self.config
+        self._config = merge_default_config(merged)
+        return self._config
 
     def save_config_if_missing(self) -> None:
         try:
@@ -55,8 +55,21 @@ def save_config_if_missing(self) -> None:
 
     def get_config(self, hide_secrets: bool = True) -> MarimoConfig:
         if hide_secrets:
-            return mask_secrets(self.config)
-        return self.config
+            return mask_secrets(self._config)
+        return self._config
 
     def get_config_path(self) -> str:
         return get_or_create_config_path()
+
+
+class UserConfigManagerWithOverride(UserConfigManager):
+    def __init__(
+        self, delegate: UserConfigManager, override_config: PartialMarimoConfig
+    ) -> None:
+        self.delegate = delegate
+        self.override_config = override_config
+
+    def get_config(self, hide_secrets: bool = True) -> MarimoConfig:
+        return merge_config(
+            self.delegate.get_config(hide_secrets), self.override_config
+        )

marimo/_output/hypertext.py

@@ -1,8 +1,10 @@
 # Copyright 2024 Marimo. All rights reserved.
 from __future__ import annotations
 
+import os
 import weakref
-from typing import TYPE_CHECKING, Any, Literal, Optional, final
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Iterator, Literal, Optional, cast, final
 
 from marimo._messaging.mimetypes import KnownMimeType
 from marimo._output.mime import MIME
@@ -68,8 +70,6 @@ class Html(MIME):
     - `right`: right-justify this element in the output area
     """
 
-    _text: str
-
     def __init__(self, text: str) -> None:
         """Initialize the HTML element.
 
@@ -116,6 +116,21 @@ def text(self) -> str:
 
     @final
     def _mime_(self) -> tuple[KnownMimeType, str]:
+        no_js = os.getenv("MARIMO_NO_JS", "false").lower() == "true"
+        if no_js and hasattr(self, "_repr_png_"):
+            return (
+                "image/png",
+                cast(
+                    str, cast(Any, self)._repr_png_().decode()
+                ),  # ignore[no-untyped-call]
+            )
+        if no_js and hasattr(self, "_repr_markdown_"):
+            return (
+                "text/markdown",
+                cast(
+                    str, cast(Any, self)._repr_markdown_()
+                ),  # ignore[no-untyped-call]
+            )
         return ("text/html", self.text)
 
     def __format__(self, spec: str) -> str:
@@ -264,3 +279,21 @@ def _repr_html_(self) -> str:
 def _js(text: str) -> Html:
     # TODO: interpolation of Python values to javascript
     return Html("<script>" + text + "</script>")
+
+
+@contextmanager
+def patch_html_for_non_interactive_output() -> Iterator[None]:
+    """
+    Patch Html to return text/markdown for simpler non-interactive outputs,
+    that can be rendered without JS/CSS (just as in the GitHub viewer).
+    """
+    # HACK: we must set MARIMO_NO_JS since the rendering may happen in another
+    # thread
+    # This won't work when we are running a marimo server and are auto-exporting
+    # with this enabled.
+    old_no_js = os.getenv("MARIMO_NO_JS", "false")
+    try:
+        os.environ["MARIMO_NO_JS"] = "true"
+        yield
+    finally:
+        os.environ["MARIMO_NO_JS"] = old_no_js

marimo/_plugins/ui/_core/ui_element.py

@@ -10,6 +10,7 @@
 import uuid
 import weakref
 from dataclasses import dataclass, fields
+from html import escape
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -528,3 +529,8 @@ def __bool__(self) -> bool:
             "probably want to call `.value` instead."
         )
         return True
+
+    def _repr_markdown_(self) -> str:
+        # When rendering to markdown, remove the marimo-ui-element tag
+        # and render the inner-text escaped.
+        return escape(self._inner_text)

marimo/_plugins/ui/_impl/table.py

@@ -590,5 +590,15 @@ def clamp_rows_and_columns(manager: TableManager[Any]) -> JSONType:
             total_rows=result.get_num_rows(force=True) or 0,
         )
 
+    def _repr_markdown_(self) -> str:
+        """
+        Return a markdown representation of the table.
+        Useful for rendering in the GitHub viewer.
+        """
+        df = self.data
+        if hasattr(df, "_repr_html_"):
+            return df._repr_html_()  # type: ignore[attr-defined,no-any-return]
+        return str(df)
+
     def __hash__(self) -> int:
         return id(self)

marimo/_server/export/__init__.py

@@ -4,9 +4,13 @@
 import asyncio
 from typing import Callable, Literal
 
-from marimo._config.manager import UserConfigManager
+from marimo._config.manager import (
+    UserConfigManager,
+    UserConfigManagerWithOverride,
+)
 from marimo._messaging.ops import MessageOperation
 from marimo._messaging.types import KernelMessage
+from marimo._output.hypertext import patch_html_for_non_interactive_output
 from marimo._runtime.requests import AppMetadata, SerializedCLIArgs
 from marimo._server.export.exporter import Exporter
 from marimo._server.file_manager import AppFileManager
@@ -52,6 +56,24 @@ def export_as_ipynb(
     return Exporter().export_as_ipynb(file_manager, sort_mode=sort_mode)
 
 
+async def run_app_then_export_as_ipynb(
+    path: MarimoPath,
+    sort_mode: Literal["top-down", "topological"],
+    cli_args: SerializedCLIArgs,
+) -> tuple[str, str]:
+    file_router = AppFileRouter.from_filename(path)
+    file_key = file_router.get_unique_file_key()
+    assert file_key is not None
+    file_manager = file_router.get_file_manager(file_key)
+
+    with patch_html_for_non_interactive_output():
+        session_view = await run_app_until_completion(file_manager, cli_args)
+
+    return Exporter().export_as_ipynb(
+        file_manager, sort_mode=sort_mode, session_view=session_view
+    )
+
+
 async def run_app_then_export_as_html(
     path: MarimoPath,
     include_code: bool,
@@ -125,7 +147,16 @@ def write_operation(self, op: MessageOperation) -> None:
         def connection_state(self) -> ConnectionState:
             return ConnectionState.OPEN
 
-    config = UserConfigManager()
+    config_manager = UserConfigManagerWithOverride(
+        UserConfigManager(),
+        {
+            "runtime": {
+                "on_cell_change": "autorun",
+                "auto_instantiate": True,
+                "auto_reload": "off",
+            }
+        },
+    )
 
     # Create a session
     session = Session.create(
@@ -140,7 +171,7 @@ def connection_state(self) -> ConnectionState:
             cli_args=cli_args,
         ),
         app_file_manager=file_manager,
-        user_config_manager=config,
+        user_config_manager=config_manager,
         virtual_files_supported=False,
         redirect_console_to_browser=False,
     )