ktims/actual-imap-poll
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add imap_ssl support, some style fixes

Browse Source
This commit is contained in:
ktims committed 2026-10-18 14:18:05 -07:00
1 parent d414d0cf09
commit 43d9a69707
2 files changed
+54 -42
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

actual_imap_poll/parsers.py
+28 -20
View File
@@ -1,17 +1,19 @@
import logging
import re import re
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Sequence from collections.abc import Sequence
from datetime import date, datetime from datetime import date, datetime
from decimal import Decimal from decimal import Decimal
from email.message import EmailMessage, Message from email.message import EmailMessage, Message
from logging import debug, info, warning from typing import Any
from typing import Any, Optional
from uuid import UUID from uuid import UUID
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from .model import Transaction from .model import Transaction
log = logging.getLogger("parse")
def parse_email_time(s: str) -> datetime: def parse_email_time(s: str) -> datetime:
return datetime.strptime(s, "%a, %d %b %Y %H:%M:%S %z") return datetime.strptime(s, "%a, %d %b %Y %H:%M:%S %z")
@@ -29,10 +31,10 @@ class TransactionParser(ABC):
Returns: Returns:
bool: True if the message matches the parser's criteria, False otherwise. bool: True if the message matches the parser's criteria, False otherwise.
""" """
pass ...
@abstractmethod @abstractmethod
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
""" """
Extracts transaction details from the given email message. Extracts transaction details from the given email message.
@@ -45,7 +47,7 @@ class TransactionParser(ABC):
Raises: Raises:
TransactionParsingFailed: If the message cannot be parsed successfully. TransactionParsingFailed: If the message cannot be parsed successfully.
""" """
pass ...
@staticmethod @staticmethod
def get_content( def get_content(
@@ -90,7 +92,7 @@ class RogersBankParser(TransactionParser):
and msg["Subject"] == "Purchase amount alert" and msg["Subject"] == "Purchase amount alert"
) )
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
content = self.strip_html(msg) content = self.strip_html(msg)
matches = self.EXTRACT_RE.search(content) matches = self.EXTRACT_RE.search(content)
if matches is None: if matches is None:
@@ -128,7 +130,7 @@ class MBNAParser(TransactionParser):
and msg["Subject"] == "MBNA - Transaction Alert" and msg["Subject"] == "MBNA - Transaction Alert"
) )
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
content = self.get_content(msg) content = self.get_content(msg)
matches = self.EXTRACT_RE.search(content) matches = self.EXTRACT_RE.search(content)
if matches is None: if matches is None:
@@ -158,7 +160,7 @@ class BMOParser(TransactionParser):
def match(self, msg: Message) -> bool: def match(self, msg: Message) -> bool:
return msg["From"] == "bmoalerts@bmo.com" return msg["From"] == "bmoalerts@bmo.com"
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
content = self.strip_html(msg) content = self.strip_html(msg)
matches = self.EXTRACT_RE.search(content) matches = self.EXTRACT_RE.search(content)
if matches is None: if matches is None:
@@ -170,7 +172,9 @@ class BMOParser(TransactionParser):
date = parse_email_time(msg["Date"]).date() date = parse_email_time(msg["Date"]).date()
account_ref = int(matches[3]) account_ref = int(matches[3])
if account_ref not in self._account_map: if account_ref not in self._account_map:
warning("Account %s not in account map, skipping transaction", account_ref) log.warning(
"Account %s not in account map, skipping transaction", account_ref
)
return None return None
account_id = self._account_map[account_ref] account_id = self._account_map[account_ref]
return Transaction( return Transaction(
@@ -212,7 +216,9 @@ class CIBCParser(TransactionParser):
date = parse_email_time(msg["Date"]).date() date = parse_email_time(msg["Date"]).date()
if account_ref not in self._account_map: if account_ref not in self._account_map:
warning("Account %s not in account map, skipping transaction", account_ref) log.warning(
"Account %s not in account map, skipping transaction", account_ref
)
return None return None
account_id = self._account_map[account_ref] account_id = self._account_map[account_ref]
@@ -225,7 +231,7 @@ class CIBCParser(TransactionParser):
imported_id=msg["Message-ID"], imported_id=msg["Message-ID"],
) )
def extract_purchase(self, msg: EmailMessage) -> Optional[Transaction]: def extract_purchase(self, msg: EmailMessage) -> Transaction | None:
content = self.get_content(msg) content = self.get_content(msg)
matches = self.PURCHASE_EXTRACT_RE.search(content) matches = self.PURCHASE_EXTRACT_RE.search(content)
@@ -237,7 +243,9 @@ class CIBCParser(TransactionParser):
account_ref = int(matches["account"]) account_ref = int(matches["account"])
if account_ref not in self._account_map: if account_ref not in self._account_map:
warning("Account %s not in account map, skipping transaction", account_ref) log.warning(
"Account %s not in account map, skipping transaction", account_ref
)
return None return None
account_id = self._account_map[account_ref] account_id = self._account_map[account_ref]
@@ -250,7 +258,7 @@ class CIBCParser(TransactionParser):
imported_id=msg["Message-ID"], imported_id=msg["Message-ID"],
) )
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
match msg["Subject"]: match msg["Subject"]:
case "New payment to your credit card": case "New payment to your credit card":
return self.extract_payment(msg) return self.extract_payment(msg)
@@ -271,7 +279,7 @@ class ScotiaBankParser(TransactionParser):
def match(self, msg: Message) -> bool: def match(self, msg: Message) -> bool:
return msg["From"] == "Scotia InfoAlerts <infoalerts@scotiabank.com>" return msg["From"] == "Scotia InfoAlerts <infoalerts@scotiabank.com>"
def extract_transaction(self, msg: EmailMessage) -> Optional[Transaction]: def extract_transaction(self, msg: EmailMessage) -> Transaction | None:
content = self.get_content(msg) content = self.get_content(msg)
matches = self.PAYMENT_EXTRACT_RE.search(content) matches = self.PAYMENT_EXTRACT_RE.search(content)
@@ -282,7 +290,7 @@ class ScotiaBankParser(TransactionParser):
date = parse_email_time(msg["Date"]).date() date = parse_email_time(msg["Date"]).date()
if matches["account"] not in self._account_map: if matches["account"] not in self._account_map:
warning( log.warning(
"Account %s not in account map, skipping transaction", "Account %s not in account map, skipping transaction",
matches["account"], matches["account"],
) )
@@ -300,14 +308,14 @@ class ScotiaBankParser(TransactionParser):
imported_id=msg["Message-ID"], imported_id=msg["Message-ID"],
) )
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
match msg["Subject"]: match msg["Subject"]:
case ( case (
"Authorization on your credit account" "Authorization on your credit account"
| "Authorization without credit card present" | "Authorization without credit card present"
): ):
return self.extract_transaction(msg) return self.extract_transaction(msg)
info("Subject `%s` didn't match any extractors", msg["Subject"]) log.info("Subject `%s` didn't match any extractors", msg["Subject"])
return None return None
@@ -325,11 +333,11 @@ class NeoFinancialParser(TransactionParser):
"Subject" "Subject"
].startswith("You earned") ].startswith("You earned")
def extract_purchase(self, msg: EmailMessage) -> Optional[Transaction]: def extract_purchase(self, msg: EmailMessage) -> Transaction | None:
content = self.get_content(msg, preferencelist=["plain"]) content = self.get_content(msg, preferencelist=["plain"])
matches = self.PURCHASE_EXTRACT_RE.search(content) matches = self.PURCHASE_EXTRACT_RE.search(content)
if matches is None: if matches is None:
debug(f"No match on\n\n{content}") log.debug(f"No match on\n\n{content}")
raise TransactionParsingFailed("no matches for purchase extraction RE") raise TransactionParsingFailed("no matches for purchase extraction RE")
amount = Decimal(matches["amount"].replace(",", "")) * -1 amount = Decimal(matches["amount"].replace(",", "")) * -1
@@ -344,7 +352,7 @@ class NeoFinancialParser(TransactionParser):
imported_id=msg["Message-ID"], imported_id=msg["Message-ID"],
) )
def extract(self, msg: EmailMessage) -> Optional[Transaction]: def extract(self, msg: EmailMessage) -> Transaction | None:
if msg["Subject"].startswith("You earned"): if msg["Subject"].startswith("You earned"):
return self.extract_purchase(msg) return self.extract_purchase(msg)
return None return None
actual_imap_poll/watcher.py
+26 -22
View File
@@ -1,4 +1,3 @@
#!/usr/bin/env python
import asyncio import asyncio
import email.policy import email.policy
import importlib.metadata import importlib.metadata
@@ -6,8 +5,7 @@ import importlib.util
import logging import logging
import os import os
import ssl import ssl
from imaplib import IMAP4 from imaplib import IMAP4, IMAP4_SSL
from logging import debug, error, info, warning
from os import getenv from os import getenv
from pydantic import Field, SecretStr from pydantic import Field, SecretStr
@@ -20,6 +18,8 @@ CONFIG_PATH = getenv("CONFIG_PATH", "/data/config.py")
ACTUAL_PATH = "./cli.js" ACTUAL_PATH = "./cli.js"
TIMEOUT = 30 TIMEOUT = 30
log = logging.getLogger("watch")
async def ticker(interval: float): async def ticker(interval: float):
while True: while True:
@@ -44,6 +44,7 @@ class AppConfig(BaseSettings):
imap_mailbox: str = Field(default="INBOX") imap_mailbox: str = Field(default="INBOX")
imap_interval: int = Field(default=300) imap_interval: int = Field(default=300)
imap_starttls: bool = Field(default=True) imap_starttls: bool = Field(default=True)
imap_tls: bool = Field(default_factory=lambda data: data["imap_port"] == 993)
model_config = {"env_prefix": ""} model_config = {"env_prefix": ""}
@@ -63,7 +64,7 @@ class App:
+ f' -d "{t.date}"' + f' -d "{t.date}"'
+ (f' -n "{t.notes}"' if t.notes else "") + (f' -n "{t.notes}"' if t.notes else "")
) )
debug("Actual command: %s", cmd) log.debug("Actual command: %s", cmd)
proc = await asyncio.create_subprocess_shell( proc = await asyncio.create_subprocess_shell(
cmd=cmd, cmd=cmd,
env=os.environ, env=os.environ,
@@ -72,12 +73,12 @@ class App:
) )
stdout, _stderr = await proc.communicate() stdout, _stderr = await proc.communicate()
if proc.returncode != 0: if proc.returncode != 0:
error("Submitting to actual failed: %s", stdout) log.error("Submitting to actual failed: %s", stdout)
async def process_message(self, msg_b: bytes): async def process_message(self, msg_b: bytes):
debug("parsing message") log.debug("parsing message")
msg = email.message_from_bytes(msg_b, policy=email.policy.default) msg = email.message_from_bytes(msg_b, policy=email.policy.default)
info( log.info(
"Found message from %s to %s subject %s", "Found message from %s to %s subject %s",
msg.get("From", "<unknown>"), msg.get("From", "<unknown>"),
msg.get("To", "<unknown>"), msg.get("To", "<unknown>"),
@@ -85,24 +86,27 @@ class App:
) )
for parser in self._parsers: for parser in self._parsers:
debug("Running parser %s", type(parser).__name__) log.debug("Running parser %s", type(parser).__name__)
try: try:
if parser.match(msg): if parser.match(msg):
info("Parser %s claimed message", type(parser).__name__) log.info("Parser %s claimed message", type(parser).__name__)
trans = parser.extract(msg) trans = parser.extract(msg)
if trans is not None: if trans is not None:
info("Submitting transaction to Actual: %s", trans) log.info("Submitting transaction to Actual: %s", trans)
await self.submit_transaction(trans) await self.submit_transaction(trans)
else: else:
warning("Parser %s returned None", type(parser).__name__) log.warning("Parser %s returned None", type(parser).__name__)
except TransactionParsingFailed as e: except TransactionParsingFailed as e:
warning("Unable to parse message %s", e) log.warning("Unable to parse message %s", e)
except Exception as e: except Exception as e: # noqa: BLE001 - trap all errors in the parsers
warning("Unexpected exception %s", e) log.warning("Unexpected exception %s", e)
async def poll_imap(self): async def poll_imap(self):
info("polling mailbox") log.info("polling mailbox")
with IMAP4(self._config.imap_server, self._config.imap_port, TIMEOUT) as M: imap_class = IMAP4_SSL if self._config.imap_tls else IMAP4
with imap_class(
self._config.imap_server, self._config.imap_port, timeout=TIMEOUT
) as M:
if self._config.imap_starttls: if self._config.imap_starttls:
context = ssl.create_default_context() context = ssl.create_default_context()
M.starttls(context) M.starttls(context)
@@ -111,16 +115,16 @@ class App:
_status, m_set = M.search(None, "UNSEEN") _status, m_set = M.search(None, "UNSEEN")
for msg_id in m_set[0].split(): for msg_id in m_set[0].split():
debug("Retrieving msg id %s", msg_id) log.debug("Retrieving msg id %s", msg_id)
_status, msg = M.fetch(msg_id, "(RFC822)") _status, msg = M.fetch(msg_id, "(RFC822)")
if _status != "OK" or msg[0] is None: if _status != "OK" or msg[0] is None:
error("Unable to fetch message %s", msg_id) log.error("Unable to fetch message %s", msg_id)
continue continue
msg_body = msg[0][1] msg_body = msg[0][1]
if isinstance(msg_body, int): if isinstance(msg_body, int):
error("Unable to fetch message %s", msg_id) log.error("Unable to fetch message %s", msg_id)
continue continue
debug("Processing message %s", msg_id) log.debug("Processing message %s", msg_id)
await self.process_message(msg_body) await self.process_message(msg_body)
async def run(self): async def run(self):
@@ -130,11 +134,11 @@ class App:
def main(): def main():
logging.basicConfig(level=getenv("LOG_LEVEL", "INFO")) logging.basicConfig(level=getenv("LOG_LEVEL", "INFO"))
logging.info( log.info(
"actual-imap-parser %s starting up", "actual-imap-parser %s starting up",
importlib.metadata.version("actual-imap-poll"), importlib.metadata.version("actual-imap-poll"),
) )
app_config = AppConfig() app_config = AppConfig()
logging.debug("Config: %s", app_config) log.debug("Config: %s", app_config)
app = App(app_config, CONFIG_PATH) app = App(app_config, CONFIG_PATH)
asyncio.run(app.run()) asyncio.run(app.run())