From 33cd7bf8f685d2cd966d87b3b76754e7fcc49883 Mon Sep 17 00:00:00 2001 From: Gabriel Augendre Date: Tue, 31 Mar 2020 18:32:54 +0200 Subject: [PATCH] Add LCL processor and extract common OFX logic --- ofx_processor/processors/bpvf.py | 29 +------- ofx_processor/processors/ce.py | 12 ++- ofx_processor/processors/lcl.py | 43 +++++++++++ ofx_processor/processors/revolut.py | 9 +-- ofx_processor/utils/base_ofx.py | 36 +++++++++ .../utils/{processor.py => base_processor.py} | 6 +- ofx_processor/utils/utils.py | 6 +- tests/samples/config.ini | 3 + tests/samples/lcl.ofx | 60 +++++++++++++++ tests/samples/lcl_as_downloaded.ofx | 61 ++++++++++++++++ tests/samples/lcl_expected.json | 9 +++ tests/samples/lcl_malformed.ofx | 52 +++++++++++++ tests/samples/lcl_transactions.json | 12 +++ tests/test_bpvf_processor.py | 31 ++------ tests/test_end_to_end.py | 18 +++++ tests/test_lcl_processor.py | 73 +++++++++++++++++++ tests/utils.py | 19 +++++ 17 files changed, 416 insertions(+), 63 deletions(-) create mode 100644 ofx_processor/processors/lcl.py create mode 100644 ofx_processor/utils/base_ofx.py rename ofx_processor/utils/{processor.py => base_processor.py} (96%) create mode 100644 tests/samples/lcl.ofx create mode 100644 tests/samples/lcl_as_downloaded.ofx create mode 100644 tests/samples/lcl_expected.json create mode 100644 tests/samples/lcl_malformed.ofx create mode 100644 tests/samples/lcl_transactions.json create mode 100644 tests/test_lcl_processor.py create mode 100644 tests/utils.py diff --git a/ofx_processor/processors/bpvf.py b/ofx_processor/processors/bpvf.py index 16dd90f..2b403a3 100644 --- a/ofx_processor/processors/bpvf.py +++ b/ofx_processor/processors/bpvf.py @@ -1,23 +1,11 @@ import re -import sys import click -from ofxtools import OFXTree -from ofxtools.header import OFXHeaderError -from ofx_processor.utils.processor import Processor, Line +from ofx_processor.utils.base_ofx import OfxBaseLine, OfxBaseProcessor -class BpvfLine(Line): - def __init__(self, data=None): - super(BpvfLine, self).__init__(data) - - def get_date(self): - return self.data.dtposted.isoformat().split("T")[0] - - def get_amount(self): - return int(self.data.trnamt * 1000) - +class BpvfLine(OfxBaseLine): def get_memo(self): return self._process_name_and_memo(self.data.name, self.data.memo)[1] @@ -41,21 +29,10 @@ class BpvfLine(Line): return name, memo -class BpvfProcessor(Processor): +class BpvfProcessor(OfxBaseProcessor): line_class = BpvfLine account_name = "bpvf" - def parse_file(self): - parser = OFXTree() - try: - parser.parse(self.filename) - except (FileNotFoundError, OFXHeaderError): - click.secho("Couldn't open or parse ofx file", fg="red") - sys.exit(1) - - ofx = parser.convert() - return ofx.statements[0].transactions - @staticmethod @click.command("bpvf") @click.argument("ofx_filename") diff --git a/ofx_processor/processors/ce.py b/ofx_processor/processors/ce.py index bba8552..39daf8c 100644 --- a/ofx_processor/processors/ce.py +++ b/ofx_processor/processors/ce.py @@ -2,10 +2,16 @@ import re import click -from ofx_processor.processors.bpvf import BpvfProcessor, BpvfLine +from ofx_processor.utils.base_ofx import OfxBaseProcessor, OfxBaseLine -class CeLine(BpvfLine): +class CeLine(OfxBaseLine): + def get_memo(self): + return self._process_name_and_memo(self.data.name, self.data.memo)[1] + + def get_payee(self): + return self._process_name_and_memo(self.data.name, self.data.memo)[0] + @staticmethod def _process_name_and_memo(name: str, memo: str): name = name.strip() @@ -20,7 +26,7 @@ class CeLine(BpvfLine): return res_name, res_memo -class CeProcessor(BpvfProcessor): +class CeProcessor(OfxBaseProcessor): account_name = "ce" line_class = CeLine diff --git a/ofx_processor/processors/lcl.py b/ofx_processor/processors/lcl.py new file mode 100644 index 0000000..d6bde92 --- /dev/null +++ b/ofx_processor/processors/lcl.py @@ -0,0 +1,43 @@ +import sys + +import click + +from ofx_processor.utils.base_ofx import OfxBaseLine, OfxBaseProcessor + + +class LclLine(OfxBaseLine): + pass + + +class LclProcessor(OfxBaseProcessor): + line_class = LclLine + account_name = "lcl" + + def parse_file(self): + # The first line of this file needs to be removed. + # It contains something that is not part of the header of an OFX file. + try: + with open(self.filename, "r") as user_file: + data = user_file.read().splitlines(True) + except FileNotFoundError: + click.secho("Couldn't find ofx file", fg="red") + sys.exit(1) + + if "Content-Type:" in data[0]: + with open(self.filename, "w") as temp_file: + temp_file.writelines(data[1:]) + + transactions = super(LclProcessor, self).parse_file() + + if "Content-Type:" in data[0]: + with open(self.filename, "w") as temp_file: + temp_file.writelines(data) + + return transactions + + @staticmethod + @click.command("lcl") + @click.argument("ofx_filename") + def main(ofx_filename): + """Import LCL bank statement (OFX file).""" + LclProcessor(ofx_filename).push_to_ynab() diff --git a/ofx_processor/processors/revolut.py b/ofx_processor/processors/revolut.py index 4387886..20e30f4 100644 --- a/ofx_processor/processors/revolut.py +++ b/ofx_processor/processors/revolut.py @@ -4,7 +4,7 @@ import sys import click import dateparser -from ofx_processor.utils.processor import Processor, Line +from ofx_processor.utils.base_processor import BaseProcessor, BaseLine def _amount_str_to_float(amount): @@ -13,10 +13,7 @@ def _amount_str_to_float(amount): return "" -class RevolutLine(Line): - def __init__(self, data: dict = None): - super(RevolutLine, self).__init__(data) - +class RevolutLine(BaseLine): def _process_inflow(self): return _amount_str_to_float(self.data.get("Paid In (EUR)")) @@ -48,7 +45,7 @@ class RevolutLine(Line): return self.data.get("Reference") -class RevolutProcessor(Processor): +class RevolutProcessor(BaseProcessor): line_class = RevolutLine account_name = "revolut" diff --git a/ofx_processor/utils/base_ofx.py b/ofx_processor/utils/base_ofx.py new file mode 100644 index 0000000..800dba8 --- /dev/null +++ b/ofx_processor/utils/base_ofx.py @@ -0,0 +1,36 @@ +import sys + +import click +from ofxtools import OFXTree +from ofxtools.header import OFXHeaderError + +from ofx_processor.utils.base_processor import BaseLine, BaseProcessor + + +class OfxBaseLine(BaseLine): + def get_date(self): + return self.data.dtposted.isoformat().split("T")[0] + + def get_amount(self): + return int(self.data.trnamt * 1000) + + def get_memo(self): + return self.data.memo + + def get_payee(self): + return self.data.name + + +class OfxBaseProcessor(BaseProcessor): + line_class = OfxBaseLine + + def parse_file(self): + parser = OFXTree() + try: + parser.parse(self.filename) + except (FileNotFoundError, OFXHeaderError): + click.secho("Couldn't open or parse ofx file", fg="red") + sys.exit(1) + + ofx = parser.convert() + return ofx.statements[0].transactions diff --git a/ofx_processor/utils/processor.py b/ofx_processor/utils/base_processor.py similarity index 96% rename from ofx_processor/utils/processor.py rename to ofx_processor/utils/base_processor.py index c0249ce..4b55606 100644 --- a/ofx_processor/utils/processor.py +++ b/ofx_processor/utils/base_processor.py @@ -5,7 +5,7 @@ import click from ofx_processor.utils import ynab -class Line: +class BaseLine: data = None def __init__(self, data=None): @@ -42,8 +42,8 @@ class Line: return ynab_transaction -class Processor: - line_class = Line +class BaseProcessor: + line_class = BaseLine account_name = None def __init__(self, filename): diff --git a/ofx_processor/utils/utils.py b/ofx_processor/utils/utils.py index d103e7e..e6b72b8 100644 --- a/ofx_processor/utils/utils.py +++ b/ofx_processor/utils/utils.py @@ -23,7 +23,11 @@ def discover_processors(cli: click.Group): for module in pkgutil.iter_modules(processors.__path__, prefix): module = importlib.import_module(module.name) for item in dir(module): - if item.endswith("Processor") and item != "Processor": + if ( + item.endswith("Processor") + and item != "Processor" + and "Base" not in item + ): cls = getattr(module, item) cli.add_command(cls.main) diff --git a/tests/samples/config.ini b/tests/samples/config.ini index 69847f3..aafe56d 100644 --- a/tests/samples/config.ini +++ b/tests/samples/config.ini @@ -10,3 +10,6 @@ account = [ce] account = + +[lcl] +account = diff --git a/tests/samples/lcl.ofx b/tests/samples/lcl.ofx new file mode 100644 index 0000000..723cf6a --- /dev/null +++ b/tests/samples/lcl.ofx @@ -0,0 +1,60 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + + + + + 0 +INFO + +20200331170000 +FRA +20200331170000 +20200331170000 + + + + +00 + +0 +INFO + + +EUR + +30002 +01047 +059235W +CHECKING + + +20200214120000 +20200330120000 + +SRVCHG +20200312 ++1000.00 +348 120320 100000 +VIR INST M PAYEE 1 + + + ++1000.00 +20200330120000 + + ++1000.00 +20200330120000 + + + + + diff --git a/tests/samples/lcl_as_downloaded.ofx b/tests/samples/lcl_as_downloaded.ofx new file mode 100644 index 0000000..f6f3627 --- /dev/null +++ b/tests/samples/lcl_as_downloaded.ofx @@ -0,0 +1,61 @@ +Content-Type: application/x-ofx +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + + + + + 0 +INFO + +20200331170000 +FRA +20200331170000 +20200331170000 + + + + +00 + +0 +INFO + + +EUR + +30002 +01047 +059235W +CHECKING + + +20200214120000 +20200330120000 + +SRVCHG +20200312 ++1000.00 +348 120320 100000 +VIR INST M PAYEE 1 + + + ++1000.00 +20200330120000 + + ++1000.00 +20200330120000 + + + + + diff --git a/tests/samples/lcl_expected.json b/tests/samples/lcl_expected.json new file mode 100644 index 0000000..466af23 --- /dev/null +++ b/tests/samples/lcl_expected.json @@ -0,0 +1,9 @@ +[ + { + "date": "2020-03-12", + "amount": 1000000, + "payee_name": "VIR INST M PAYEE 1", + "memo": null, + "import_id": "YNAB:1000000:2020-03-12:1" + } +] \ No newline at end of file diff --git a/tests/samples/lcl_malformed.ofx b/tests/samples/lcl_malformed.ofx new file mode 100644 index 0000000..a51fde1 --- /dev/null +++ b/tests/samples/lcl_malformed.ofx @@ -0,0 +1,52 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +INFO + +20200331170000 +FRA +20200331170000 +20200331170000 + + + + +00 + +0 +INFO + + +EUR + +30002 +01047 +059235W +CHECKING + + +20200214120000 +20200330120000 + +SRVCHG +20200312 ++1000.00 +348 120320 100000 +VIR INST M PAYEE 1 + + + ++1000.00 +20200330120000 + + ++1000.00 +20200330120000 + + + + + diff --git a/tests/samples/lcl_transactions.json b/tests/samples/lcl_transactions.json new file mode 100644 index 0000000..041fcc8 --- /dev/null +++ b/tests/samples/lcl_transactions.json @@ -0,0 +1,12 @@ +{ + "transactions": [ + { + "date": "2020-03-12", + "amount": 1000000, + "payee_name": "VIR INST M PAYEE 1", + "memo": null, + "import_id": "YNAB:1000000:2020-03-12:1", + "account_id": "" + } + ] +} \ No newline at end of file diff --git a/tests/test_bpvf_processor.py b/tests/test_bpvf_processor.py index 888aecc..ab82dc5 100644 --- a/tests/test_bpvf_processor.py +++ b/tests/test_bpvf_processor.py @@ -3,31 +3,14 @@ import json import unittest from ofx_processor.processors.bpvf import BpvfLine, BpvfProcessor - - -class BpvfTransaction: - """ - Mimick what is retrieved via ofxtools when parsing the file - """ - - def __init__( - self, - name: str = "", - memo: str = "", - dtposted: datetime.datetime = None, - trnamt: float = 0, - ): - self.dtposted = dtposted - self.memo = memo - self.trnamt = trnamt - self.name = name +from tests.utils import OfxTransaction class BpvfLineTestCase(unittest.TestCase): def test_process_name_and_memo_no_change(self): name = "business" memo = "2020-01-17" - transaction = BpvfTransaction(name=name, memo=memo) + transaction = OfxTransaction(name=name, memo=memo) line = BpvfLine(transaction) result_name = line.get_payee() @@ -38,7 +21,7 @@ class BpvfLineTestCase(unittest.TestCase): def test_process_name_and_memo_change_required_with_conversion(self): name = "150120 CB****5874" memo = "GUY AND SONS FR LYON 0,90EUR 1 EURO = 1,000000" - transaction = BpvfTransaction(name=name, memo=memo) + transaction = OfxTransaction(name=name, memo=memo) expected_name = "GUY AND SONS FR LYON" expected_memo = "150120 CB****5874 0,90EUR 1 EURO = 1,000000" @@ -52,7 +35,7 @@ class BpvfLineTestCase(unittest.TestCase): def test_process_name_and_memo_change_required_no_conversion(self): name = "150120 CB****5874" memo = "Dott 75PARIS" - transaction = BpvfTransaction(name=name, memo=memo) + transaction = OfxTransaction(name=name, memo=memo) expected_name = "Dott 75PARIS" expected_memo = "150120 CB****5874" @@ -64,21 +47,21 @@ class BpvfLineTestCase(unittest.TestCase): self.assertEqual(result_memo, expected_memo) def test_get_date(self): - transaction = BpvfTransaction(dtposted=datetime.datetime(2020, 1, 23, 1, 2, 3)) + transaction = OfxTransaction(dtposted=datetime.datetime(2020, 1, 23, 1, 2, 3)) expected_date = "2020-01-23" result_date = BpvfLine(transaction).get_date() self.assertEqual(result_date, expected_date) def test_get_amount_positive(self): - transaction = BpvfTransaction(trnamt=52.2) + transaction = OfxTransaction(trnamt=52.2) expected_amount = 52200 result_amount = BpvfLine(transaction).get_amount() self.assertEqual(result_amount, expected_amount) def test_get_amount_negative(self): - transaction = BpvfTransaction(trnamt=-52.2) + transaction = OfxTransaction(trnamt=-52.2) expected_amount = -52200 result_amount = BpvfLine(transaction).get_amount() diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index ac53ddc..05cc305 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -8,6 +8,7 @@ from click.testing import CliRunner from ofx_processor.processors.bpvf import BpvfProcessor from ofx_processor.processors.ce import CeProcessor +from ofx_processor.processors.lcl import LclProcessor from ofx_processor.processors.revolut import RevolutProcessor from ofx_processor.utils import utils from ofx_processor.utils import ynab @@ -25,6 +26,7 @@ class UtilsTestCase(unittest.TestCase): ce_main = CeProcessor.main bpvf_main = BpvfProcessor.main revolut_main = RevolutProcessor.main + lcl_main = LclProcessor.main runner = CliRunner() with mock.patch("click.core.Group.add_command") as add_command: from ofx_processor.main import cli @@ -34,6 +36,7 @@ class UtilsTestCase(unittest.TestCase): call(ce_main), call(bpvf_main), call(revolut_main), + call(lcl_main), call(config, name="config"), ] add_command.assert_has_calls(calls, any_order=True) @@ -264,3 +267,18 @@ class DataTestCase(unittest.TestCase): post.assert_called_once_with( expected_url, json=expected_data, headers=expected_headers ) + + @mock.patch("requests.post") + def test_lcl_sends_to_ynab(self, post): + with open("tests/samples/lcl_transactions.json", encoding="utf-8") as f: + expected_data = json.load(f) + + expected_headers = {"Authorization": f"Bearer "} + expected_url = f"{ynab.BASE_URL}/budgets//transactions" + + runner = CliRunner() + runner.invoke(self.cli, ["lcl", "tests/samples/lcl.ofx"]) + + post.assert_called_once_with( + expected_url, json=expected_data, headers=expected_headers + ) diff --git a/tests/test_lcl_processor.py b/tests/test_lcl_processor.py new file mode 100644 index 0000000..86bad73 --- /dev/null +++ b/tests/test_lcl_processor.py @@ -0,0 +1,73 @@ +import datetime +import json +import unittest + +from ofx_processor.processors.lcl import LclLine, LclProcessor +from tests.utils import OfxTransaction + + +class LclLineTestCase(unittest.TestCase): + def test_get_name(self): + name = "VIR INST" + transaction = OfxTransaction(name=name) + + result_name = LclLine(transaction).get_payee() + self.assertEqual(result_name, name) + + def test_get_memo(self): + memo = "VIR INST" + transaction = OfxTransaction(memo=memo) + + result_memo = LclLine(transaction).get_memo() + self.assertEqual(result_memo, memo) + + def test_get_date(self): + transaction = OfxTransaction(dtposted=datetime.datetime(2020, 1, 23, 1, 2, 3)) + expected_date = "2020-01-23" + + result_date = LclLine(transaction).get_date() + self.assertEqual(result_date, expected_date) + + def test_get_amount_positive(self): + transaction = OfxTransaction(trnamt=52.2) + expected_amount = 52200 + + result_amount = LclLine(transaction).get_amount() + self.assertEqual(result_amount, expected_amount) + + def test_get_amount_negative(self): + transaction = OfxTransaction(trnamt=-52.2) + expected_amount = -52200 + + result_amount = LclLine(transaction).get_amount() + self.assertEqual(result_amount, expected_amount) + + +class LclProcessorTestCase(unittest.TestCase): + def test_file_not_found(self): + with self.assertRaises(SystemExit): + LclProcessor("filenotfound.ofx").get_transactions() + + def test_file(self): + transactions = LclProcessor("tests/samples/lcl.ofx").get_transactions() + with open("tests/samples/lcl_expected.json") as f: + expected_transactions = json.load(f) + + self.assertListEqual(transactions, expected_transactions) + + def test_file_as_downloaded(self): + transactions = LclProcessor( + "tests/samples/lcl_as_downloaded.ofx" + ).get_transactions() + with open("tests/samples/lcl_expected.json") as f: + expected_transactions = json.load(f) + + self.assertListEqual(transactions, expected_transactions) + + def test_file_malformed(self): + with self.assertRaises(SystemExit): + LclProcessor("tests/samples/lcl_malformed.ofx") + + +if __name__ == "__main__": + unittest.main() # pragma: nocover diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..52ff112 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,19 @@ +import datetime + + +class OfxTransaction: + """ + Mimick what is retrieved via ofxtools when parsing the file + """ + + def __init__( + self, + name: str = "", + memo: str = "", + dtposted: datetime.datetime = None, + trnamt: float = 0, + ): + self.dtposted = dtposted + self.memo = memo + self.trnamt = trnamt + self.name = name