From 6bc72468deb1dddd5ef713283ea598dc36ac6ca3 Mon Sep 17 00:00:00 2001 From: Gabriel Augendre Date: Sat, 8 Feb 2020 19:07:05 +0100 Subject: [PATCH] Add YNAB import --- ofx_processor/bpvf_processor/main.py | 36 +++++++++++- ofx_processor/revolut_processor/main.py | 63 ++++++++++++++------ ofx_processor/utils/__init__.py | 0 ofx_processor/utils/ynab.py | 54 ++++++++++++++++++ poetry.lock | 76 ++++++++++++++++++++++++- pyproject.toml | 3 +- 6 files changed, 212 insertions(+), 20 deletions(-) create mode 100644 ofx_processor/utils/__init__.py create mode 100644 ofx_processor/utils/ynab.py diff --git a/ofx_processor/bpvf_processor/main.py b/ofx_processor/bpvf_processor/main.py index e5567bb..115800e 100644 --- a/ofx_processor/bpvf_processor/main.py +++ b/ofx_processor/bpvf_processor/main.py @@ -1,12 +1,15 @@ import os import re import sys +from collections import defaultdict from xml.etree import ElementTree import click from ofxtools.Parser import OFXTree from ofxtools.header import make_header +from ofx_processor.utils import ynab + def _process_name_and_memo(name, memo): if "CB****" in name: @@ -30,7 +33,13 @@ def process_name_and_memo(transaction): @click.command() @click.argument("ofx_filename") -def cli(ofx_filename): +@click.option( + "--ynab/--file-only", + "push_to_ynab", + default=False, + help="Push data directly to YNAB instead of just writing a file.", +) +def cli(ofx_filename, push_to_ynab): parser = OFXTree() try: parser.parse(ofx_filename) @@ -44,6 +53,9 @@ def cli(ofx_filename): click.secho("Couldn't parse ofx file", fg="red") sys.exit(1) + ynab_transactions = [] + transaction_ids = defaultdict(int) + for transaction in ofx.statements[0].transactions: transaction.name, transaction.memo, edited = process_name_and_memo(transaction) @@ -55,15 +67,35 @@ def cli(ofx_filename): fg="blue", ) + date = transaction.dtposted.isoformat().split("T")[0] + amount = int(transaction.trnamt * 1000) + import_id = f"YNAB:{amount}:{date}" + transaction_ids[import_id] += 1 + occurrence = transaction_ids[import_id] + import_id = f"{import_id}:{occurrence}" + + ynab_transactions.append( + { + "date": date, + "amount": amount, + "payee_name": transaction.name, + "memo": transaction.memo, + "import_id": import_id, + } + ) + click.secho(f"Processed {len(ynab_transactions)} transactions total.", fg="blue") + header = str(make_header(version=102)) root = ofx.to_etree() data = ElementTree.tostring(root).decode() - processed_file = os.path.join(os.path.dirname(ofx_filename), "processed.ofx") with open(processed_file, "w") as f: f.write(header + data) click.secho("{} written".format(processed_file), fg="green") + if push_to_ynab: + ynab.push_transactions(ynab_transactions, "bpvf") + if __name__ == "__main__": cli() diff --git a/ofx_processor/revolut_processor/main.py b/ofx_processor/revolut_processor/main.py index aab473b..5b239c4 100644 --- a/ofx_processor/revolut_processor/main.py +++ b/ofx_processor/revolut_processor/main.py @@ -1,5 +1,6 @@ import csv import os +from collections import defaultdict import click import dateparser @@ -34,34 +35,64 @@ def process_outflow(line): @click.command() @click.argument("csv_filename") -def cli(csv_filename): +@click.option( + "--ynab/--file-only", + default=False, + help="Push data directly to YNAB instead of just writing a file.", +) +def cli(csv_filename, ynab): formatted_data = [] + ynab_transactions = [] + transaction_ids = defaultdict(int) with open(csv_filename) as f: reader = csv.DictReader(f, delimiter=";") for line in reader: + date = process_date(line) + payee = line["Reference"] + memo = process_memo(line) + outflow = process_outflow(line) + inflow = process_inflow(line) formatted_data.append( { - "Date": process_date(line), - "Payee": line["Reference"], - "Memo": process_memo(line), - "Outflow": process_outflow(line), - "Inflow": process_inflow(line), + "Date": date, + "Payee": payee, + "Memo": memo, + "Outflow": outflow, + "Inflow": inflow, + } + ) + amount = outflow if outflow else inflow + amount *= 1000 + import_id = f"YNAB:{amount}:{date}" + transaction_ids[import_id] += 1 + occurrence = transaction_ids[import_id] + import_id = f"{import_id}:{occurrence}" + ynab_transactions.append( + { + "date": date, + "amount": amount, + "payee_name": payee, + "memo": memo, + "import_id": import_id, } ) - if not formatted_data: + if formatted_data: + processed_file = os.path.join(os.path.dirname(csv_filename), "processed.csv") + with open(processed_file, "w") as f: + writer = csv.DictWriter( + f, delimiter=",", quotechar='"', fieldnames=formatted_data[0].keys() + ) + writer.writeheader() + writer.writerows(formatted_data) + + click.secho("{} written".format(processed_file), fg="green") + else: click.secho("Nothing to write.") - processed_file = os.path.join(os.path.dirname(csv_filename), "processed.csv") - with open(processed_file, "w") as f: - writer = csv.DictWriter( - f, delimiter=",", quotechar='"', fieldnames=formatted_data[0].keys() - ) - writer.writeheader() - writer.writerows(formatted_data) - - click.secho("{} written".format(processed_file), fg="green") + if ynab and ynab_transactions: + ynab.push_transactions(ynab_transactions, "revolut") if __name__ == "__main__": diff --git a/ofx_processor/utils/__init__.py b/ofx_processor/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ofx_processor/utils/ynab.py b/ofx_processor/utils/ynab.py new file mode 100644 index 0000000..8e7c970 --- /dev/null +++ b/ofx_processor/utils/ynab.py @@ -0,0 +1,54 @@ +import configparser +import os + +import click +import requests + +BASE_URL = "https://api.youneedabudget.com/v1" +DEFAULT_CONFIG_DIR = click.get_app_dir("ofx_processor") +DEFAULT_CONFIG_FILENAME = "config.ini" + + +def get_default_config(): + default_config = configparser.ConfigParser() + default_config["DEFAULT"] = { + "token": "", + "budget": "", + } + default_config["bpvf"] = {"account": ""} + default_config["revolut"] = {"account": ""} + + return default_config + + +def push_transactions(transactions, account): + config = configparser.ConfigParser() + config_file = os.path.join(DEFAULT_CONFIG_DIR, DEFAULT_CONFIG_FILENAME) + if not os.path.isfile(config_file): + os.makedirs(DEFAULT_CONFIG_DIR, exist_ok=True) + config = get_default_config() + with open(config_file, "w") as file_: + config.write(file_) + click.secho("Editing config file...") + click.pause() + click.edit(filename=config_file) + + config.read(config_file) + section = config[account] + budget_id = section["budget"] + url = f"{BASE_URL}/budgets/{budget_id}/transactions" + for transaction in transactions: + transaction["account_id"] = section["account"] + data = {"transactions": transactions} + token = section["token"] + headers = {"Authorization": f"Bearer {token}"} + res = requests.post(url, json=data, headers=headers) + res.raise_for_status() + data = res.json()["data"] + created = data["transactions"] + duplicates = data["duplicate_import_ids"] + click.secho(f"{len(created)} transactions created in YNAB.", fg="green", bold=True) + if duplicates: + click.secho( + f"{len(duplicates)} transactions ignored (duplicates).", fg="yellow" + ) diff --git a/poetry.lock b/poetry.lock index 8583f5f..8e81869 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,6 +49,22 @@ typed-ast = ">=1.4.0" [package.extras] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +[[package]] +category = "main" +description = "Python package for providing Mozilla's CA Bundle." +name = "certifi" +optional = false +python-versions = "*" +version = "2019.11.28" + +[[package]] +category = "main" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" +optional = false +python-versions = "*" +version = "3.0.4" + [[package]] category = "main" description = "Composable command line interface toolkit" @@ -80,6 +96,14 @@ pytz = "*" regex = "*" tzlocal = "*" +[[package]] +category = "main" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.8" + [[package]] category = "dev" description = "Read metadata from Python packages" @@ -216,6 +240,24 @@ optional = false python-versions = "*" version = "2020.1.8" +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.22.0" + +[package.dependencies] +certifi = ">=2017.4.17" +chardet = ">=3.0.2,<3.1.0" +idna = ">=2.5,<2.9" +urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + [[package]] category = "main" description = "Python 2 and 3 compatibility utilities" @@ -251,6 +293,18 @@ version = "2.0.0" [package.dependencies] pytz = "*" +[[package]] +category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +name = "urllib3" +optional = false +python-versions = "*" +version = "1.22" + +[package.extras] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] + [[package]] category = "dev" description = "Measures number of Terminal column cells of wide-character codes" @@ -276,7 +330,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pathlib2", "contextlib2", "unittest2"] [metadata] -content-hash = "be08d3690daf99f4cf8210564a85c7346366473c6bfc2f098dcdf3108883c20b" +content-hash = "0e7a5824506e425f34a1b69d9cd6715b1ab11ef940b18fef10abc8c429cd638a" python-versions = ">=3.7" [metadata.files] @@ -296,6 +350,14 @@ black = [ {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, ] +certifi = [ + {file = "certifi-2019.11.28-py2.py3-none-any.whl", hash = "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"}, + {file = "certifi-2019.11.28.tar.gz", hash = "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"}, +] +chardet = [ + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, +] click = [ {file = "Click-7.0-py2.py3-none-any.whl", hash = "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"}, {file = "Click-7.0.tar.gz", hash = "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"}, @@ -308,6 +370,10 @@ dateparser = [ {file = "dateparser-0.7.2-py2.py3-none-any.whl", hash = "sha256:983d84b5e3861cb0aa240cad07f12899bb10b62328aae188b9007e04ce37d665"}, {file = "dateparser-0.7.2.tar.gz", hash = "sha256:e1eac8ef28de69a554d5fcdb60b172d526d61924b1a40afbbb08df459a36006b"}, ] +idna = [ + {file = "idna-2.8-py2.py3-none-any.whl", hash = "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"}, + {file = "idna-2.8.tar.gz", hash = "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407"}, +] importlib-metadata = [ {file = "importlib_metadata-1.4.0-py2.py3-none-any.whl", hash = "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359"}, {file = "importlib_metadata-1.4.0.tar.gz", hash = "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8"}, @@ -374,6 +440,10 @@ regex = [ {file = "regex-2020.1.8-cp38-cp38-win_amd64.whl", hash = "sha256:e7c7661f7276507bce416eaae22040fd91ca471b5b33c13f8ff21137ed6f248c"}, {file = "regex-2020.1.8.tar.gz", hash = "sha256:d0f424328f9822b0323b3b6f2e4b9c90960b24743d220763c7f07071e0778351"}, ] +requests = [ + {file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"}, + {file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"}, +] six = [ {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, @@ -410,6 +480,10 @@ tzlocal = [ {file = "tzlocal-2.0.0-py2.py3-none-any.whl", hash = "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048"}, {file = "tzlocal-2.0.0.tar.gz", hash = "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"}, ] +urllib3 = [ + {file = "urllib3-1.22-py2.py3-none-any.whl", hash = "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b"}, + {file = "urllib3-1.22.tar.gz", hash = "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"}, +] wcwidth = [ {file = "wcwidth-0.1.8-py2.py3-none-any.whl", hash = "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603"}, {file = "wcwidth-0.1.8.tar.gz", hash = "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"}, diff --git a/pyproject.toml b/pyproject.toml index b78673b..86dec35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ofx-processor" -version = "0.2.2" +version = "0.3.4" description = "Personal ofx processor" readme = "README.md" license = "GPL-3.0-or-later" @@ -15,6 +15,7 @@ python = ">=3.7" ofxtools = "^0.8.20" click = "^7.0" dateparser = "^0.7.2" +requests = "^2.22.0" [tool.poetry.dev-dependencies] pytest = "^5.2"