From a617f8655a11450aefe379e6b62d987cb1c25bb6 Mon Sep 17 00:00:00 2001 From: mplawner Date: Fri, 23 Jun 2023 21:14:13 -0400 Subject: [PATCH 1/4] Update filetypes.py Added mbox format --- collector/scripts/watch/filetypes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/collector/scripts/watch/filetypes.py b/collector/scripts/watch/filetypes.py index fc93c5b8455..8339575cbe2 100644 --- a/collector/scripts/watch/filetypes.py +++ b/collector/scripts/watch/filetypes.py @@ -2,6 +2,7 @@ from .convert.as_markdown import as_markdown from .convert.as_pdf import as_pdf from .convert.as_docx import as_docx, as_odt +from .convert.as_mbox import as_mbox # make sure to create this function FILETYPES = { '.txt': as_text, @@ -9,6 +10,7 @@ '.pdf': as_pdf, '.docx': as_docx, '.odt': as_odt, + '.mbox': as_mbox, # add mbox file handling } ACCEPTED_MIMES = { @@ -16,4 +18,5 @@ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], 'application/vnd.oasis.opendocument.text': ['.odt'], 'application/pdf': ['.pdf'], -} \ No newline at end of file + 'application/mbox': ['.mbox'], # add mbox MIME type +} From f82d01d3a86b310fa303e8a0ad524707513c5f2c Mon Sep 17 00:00:00 2001 From: mplawner Date: Fri, 23 Jun 2023 21:16:08 -0400 Subject: [PATCH 2/4] Created new file Added support for mbox files as used by many email services, including Google Takeout's Gmail archive. --- collector/scripts/watch/convert/as_mbox.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 collector/scripts/watch/convert/as_mbox.py diff --git a/collector/scripts/watch/convert/as_mbox.py b/collector/scripts/watch/convert/as_mbox.py new file mode 100644 index 00000000000..7078ec57356 --- /dev/null +++ b/collector/scripts/watch/convert/as_mbox.py @@ -0,0 +1,57 @@ +import os +import datetime # Add this line +import email.utils +from mailbox import mbox +from slugify import slugify +from ..utils import guid, file_creation_time, write_to_server_documents, move_source +from ...utils import tokenize +from bs4 import BeautifulSoup + +# Process all mbox-related documents. +def as_mbox(**kwargs): + parent_dir = kwargs.get('directory', 'hotdir') + filename = kwargs.get('filename') + ext = kwargs.get('ext', '.mbox') + remove = kwargs.get('remove_on_complete', False) + fullpath = f"{parent_dir}/{filename}{ext}" + + print(f"-- Working {fullpath} --") + box = mbox(fullpath) + + for message in box: + content = "" + if message.is_multipart(): + for part in message.get_payload(): + if part.get_content_type() == 'text/plain': + content = part.get_payload() + elif part.get_content_type() == 'text/html': + soup = BeautifulSoup(part.get_payload(), 'html.parser') + content = soup.get_text() + else: + content = message.get_payload() + + date_tuple = email.utils.parsedate_tz(message['Date']) + if date_tuple: + local_date = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) + date_sent = local_date.strftime("%a, %d %b %Y %H:%M:%S") + else: + date_sent = None + + data = { + 'id': guid(), + 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{slugify(filename)}-{guid()}{ext}"), + 'title': f"{filename}{ext}", + 'description': "a custom file uploaded by the user.", + 'published': file_creation_time(fullpath), + 'sender': message['From'], + 'recipient': message['To'], + 'subject': message['Subject'], + 'date_sent': date_sent, + 'wordCount': len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + + write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}") + move_source(parent_dir, f"{filename}{ext}") + print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n") From 3a402dc8eec16ff3f7357c47800b3ff9d1657aed Mon Sep 17 00:00:00 2001 From: mplawner Date: Fri, 23 Jun 2023 21:19:43 -0400 Subject: [PATCH 3/4] Update filetypes.py --- collector/scripts/watch/filetypes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/collector/scripts/watch/filetypes.py b/collector/scripts/watch/filetypes.py index 8339575cbe2..809c8207bfe 100644 --- a/collector/scripts/watch/filetypes.py +++ b/collector/scripts/watch/filetypes.py @@ -2,7 +2,7 @@ from .convert.as_markdown import as_markdown from .convert.as_pdf import as_pdf from .convert.as_docx import as_docx, as_odt -from .convert.as_mbox import as_mbox # make sure to create this function +from .convert.as_mbox import as_mbox FILETYPES = { '.txt': as_text, @@ -10,7 +10,7 @@ '.pdf': as_pdf, '.docx': as_docx, '.odt': as_odt, - '.mbox': as_mbox, # add mbox file handling + '.mbox': as_mbox, } ACCEPTED_MIMES = { @@ -18,5 +18,5 @@ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], 'application/vnd.oasis.opendocument.text': ['.odt'], 'application/pdf': ['.pdf'], - 'application/mbox': ['.mbox'], # add mbox MIME type + 'application/mbox': ['.mbox'], } From 986c90bf99655e57c6b398dbf5c8ae56d45531d4 Mon Sep 17 00:00:00 2001 From: mplawner Date: Fri, 23 Jun 2023 21:21:08 -0400 Subject: [PATCH 4/4] Update as_mbox.py --- collector/scripts/watch/convert/as_mbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/scripts/watch/convert/as_mbox.py b/collector/scripts/watch/convert/as_mbox.py index 7078ec57356..0fa17985475 100644 --- a/collector/scripts/watch/convert/as_mbox.py +++ b/collector/scripts/watch/convert/as_mbox.py @@ -1,5 +1,5 @@ import os -import datetime # Add this line +import datetime import email.utils from mailbox import mbox from slugify import slugify @@ -53,5 +53,5 @@ def as_mbox(**kwargs): } write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}") - move_source(parent_dir, f"{filename}{ext}") + move_source(parent_dir, f"{filename}{ext}", remove=remove) print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")