aryn-ai · MarkLindblad · Jul 21, 2025 · Jul 6, 2025 · Jul 6, 2025 · Jul 6, 2025
diff --git a/lib/sycamore/poetry.lock b/lib/sycamore/poetry.lock
diff --git a/lib/sycamore/pyproject.toml b/lib/sycamore/pyproject.toml
@@ -97,6 +97,7 @@ nltk = { version = "^3.9.0", optional = true }
 anthropic = { version = "^0.42.0", optional = true }
 google-genai = {version = "^1.14.0", optional = true}
 tiktoken = "^0.8.0"
+msgpack = "^1.1.1"
 
 [tool.poetry.group.test.dependencies]
 flake8 = "4.0.1"

diff --git a/lib/sycamore/sycamore/connectors/aryn/ArynWriter.py b/lib/sycamore/sycamore/connectors/aryn/ArynWriter.py
@@ -58,7 +58,7 @@ def write_many_records(self, records: list["BaseDBWriter.Record"], target_params
         for record in records:
             assert isinstance(record, ArynWriterRecord)
             doc = record.doc
-            files: Mapping = {"doc": doc.serialize()}
+            files: Mapping = {"doc": doc.web_serialize()}
             sess.post(
                 url=f"{self.aryn_url}/docsets/write",
                 params={"docset_id": docset_id, "update_schema": update_schema},

diff --git a/lib/sycamore/sycamore/data/document.py b/lib/sycamore/sycamore/data/document.py
@@ -1,10 +1,19 @@
 from collections import UserDict
 import json
-from typing import Any, Optional
+from typing import Any, Optional, BinaryIO
+import struct
+
+import msgpack
 
 from sycamore.data import BoundingBox, Element
 from sycamore.data.element import create_element
 from sycamore.data.docid import mkdocid, nanoid36
+from sycamore.decorators import experimental
+
+DOCUMENT_WEB_SERIALIZATION_MAGIC = b"ArynSDoc"
+DOCUMENT_WEB_SERIALIZATION_VERSION_MAJOR = 0
+DOCUMENT_WEB_SERIALIZATION_VERSION_MINOR = 1
+DOCUMENT_WEB_SERIALIZATION_HEADER_FORMAT = "!8s2H4x"
 
 
 class DocumentSource:
@@ -202,6 +211,72 @@ def deserialize(raw: bytes) -> "Document":
         else:
             return Document(data)
 
+    @experimental
+    def web_serialize(self, stream: BinaryIO) -> None:
+        kind = type(self)
+        if kind != Document:  # MetadataDocument, HierarchicalDocument, SummaryDocument are not yet supported
+            raise NotImplementedError(f"web_serialize cannot yet handle type '{kind.__name__}'")
+
+        stream.write(
+            struct.pack(
+                DOCUMENT_WEB_SERIALIZATION_HEADER_FORMAT,
+                DOCUMENT_WEB_SERIALIZATION_MAGIC,
+                DOCUMENT_WEB_SERIALIZATION_VERSION_MAJOR,
+                DOCUMENT_WEB_SERIALIZATION_VERSION_MINOR,
+            )
+        )
+
+        elementless_data = self.data.copy()  # Shallow copy
+        del elementless_data["elements"]
+
+        packed_elementless_data = msgpack.packb(elementless_data)
+        if not packed_elementless_data:
+            raise RuntimeError("Failed to serialize document")
+        stream.write(packed_elementless_data)
+
+        for element in self.elements:
+            element.web_serialize(stream)
+        msgpack.pack("_TERMINATOR", stream)
+
+    @experimental
+    @staticmethod
+    def web_deserialize(stream: BinaryIO) -> "Document":
+        def read_header(stream: BinaryIO):
+            header_size = struct.calcsize(DOCUMENT_WEB_SERIALIZATION_HEADER_FORMAT)
+            data = bytearray()
+            got = 0
+            while got < header_size:
+                to_add = stream.read(header_size - got)
+                if not to_add:
+                    raise RuntimeError("Failed to read document header")
+                data.extend(to_add)
+                got += len(to_add)
+            return data
+
+        header = read_header(stream)
+        magic_bytes, version_major, version_minor = struct.unpack(DOCUMENT_WEB_SERIALIZATION_HEADER_FORMAT, header)
+        if magic_bytes != DOCUMENT_WEB_SERIALIZATION_MAGIC:
+            raise RuntimeError("Input does not appear to be an Aryn serialized document (Bad magic number).")
+        if (
+            version_major != DOCUMENT_WEB_SERIALIZATION_VERSION_MAJOR
+            or version_minor != DOCUMENT_WEB_SERIALIZATION_VERSION_MINOR
+        ):
+            raise RuntimeError(f"Unsupported serialization version: {version_major}.{version_minor}")
+
+        unpacker = msgpack.Unpacker(stream)
+        elementless_data = next(unpacker)
+        doc = Document(elementless_data)
+        elements = doc.elements
+        saw_terminator = False
+        for obj in unpacker:
+            if obj == "_TERMINATOR":
+                saw_terminator = True
+                break
+            elements.append(Element.web_deserialize(obj))
+        if not saw_terminator:
+            raise RuntimeError("Premature end of serialized document stream.")
+        return doc
+
     @staticmethod
     def from_row(row: dict[str, bytes]) -> "Document":
         """Unserialize a Ray row back into a Document."""

diff --git a/lib/sycamore/sycamore/data/element.py b/lib/sycamore/sycamore/data/element.py
@@ -1,12 +1,14 @@
 from collections import UserDict
 from io import BytesIO
 import json
-from typing import Any, Optional
+from typing import Any, Optional, BinaryIO
 
 from PIL import Image
+import msgpack
 
 from sycamore.data.bbox import BoundingBox
 from sycamore.data.table import Table
+from sycamore.decorators import experimental
 
 
 class Element(UserDict):
@@ -125,6 +127,18 @@ def __gt__(self, other) -> bool:
         oidx = -1 if other.element_index is None else other.element_index
         return sidx > oidx
 
+    @experimental
+    def web_serialize(self, file: BinaryIO) -> None:
+        if bits := msgpack.packb(self.data):
+            file.write(bits)
+        else:
+            raise RuntimeError("Failed to serialize element")
+
+    @experimental
+    @staticmethod
+    def web_deserialize(obj: dict[str, Any]) -> "Element":
+        return create_element(**obj)
+
 
 class ImageElement(Element):
     def __init__(
@@ -257,6 +271,17 @@ def text_representation(self) -> Optional[str]:
     def text_representation(self, text_representation: str) -> None:
         self.data["text_representation"] = text_representation
 
+    @experimental
+    def web_serialize(self, file: BinaryIO) -> None:
+        payload = self.data
+        if table := payload.get("table"):
+            payload = payload.copy()
+            payload["table"] = table.to_dict()
+        if bits := msgpack.packb(payload):
+            file.write(bits)
+        else:
+            raise ValueError("Failed to serialize element")
+
 
 def create_element(element_index: Optional[int] = None, **kwargs) -> Element:
     element: Element