aryn-ai · HenryL27 · Oct 17, 2024 · Oct 17, 2024 · Oct 17, 2024
diff --git a/lib/sycamore/sycamore/tests/integration/transforms/test_embed.py b/lib/sycamore/sycamore/tests/integration/transforms/test_embed.py
@@ -20,6 +20,7 @@
         " and Lincoln ended up running the store by himself.[60] Although the economy was booming, the business"
         " struggled and went into debt, causing Lincoln to sell his share."
     ),
+    (""),
 ]
 
 
@@ -41,8 +42,9 @@ def check_embedder(embedder: Embedder, expected_dim: int):
     assert len(new_docs) == len(docs)
 
     for doc in new_docs:
-        assert doc.embedding is not None
-        assert len(doc.embedding) == expected_dim
+        if doc.text_representation != "":
+            assert doc.embedding is not None
+            assert len(doc.embedding) == expected_dim
 
 
 def test_openai_embedding():

diff --git a/lib/sycamore/sycamore/transforms/embed.py b/lib/sycamore/sycamore/transforms/embed.py
@@ -26,6 +26,10 @@ def _pre_process_document(document: Document) -> str:
     return document.text_representation if document.text_representation is not None else ""
 
 
+def _text_representation_is_empty(doc: Document) -> bool:
+    return doc.text_representation is None or doc.text_representation.strip() == ""
+
+
 class Embedder(ABC):
     def __init__(
         self,
@@ -188,14 +192,14 @@ def generate_embeddings(self, doc_batch: list[Document]) -> list[Document]:
             text_to_embed = [
                 self.pre_process_document(doc).replace("\n", " ")
                 for doc in batch
-                if doc.text_representation is not None
+                if not _text_representation_is_empty(doc)
             ]
 
             embeddings = self._client.embeddings.create(model=self.model_name, input=text_to_embed).data
 
             i = 0
             for doc in batch:
-                if doc.text_representation is not None:
+                if not _text_representation_is_empty(doc):
                     doc.embedding = embeddings[i].embedding
                     i += 1
 
@@ -272,7 +276,7 @@ def generate_embeddings(self, doc_batch: list[Document]) -> list[Document]:
         client = boto3.client("bedrock-runtime")
 
         for doc in doc_batch:
-            if doc.text_representation is not None:
+            if not _text_representation_is_empty(doc):
                 doc.embedding = self._generate_embedding(client, self.pre_process_document(doc))
         return doc_batch