Skip to content

Commit 7de532d

Browse files
rlanrlan
committed
v0.6.0
1 parent 94770be commit 7de532d

12 files changed

+133
-61
lines changed

‎docs/reference/changelog.md

+5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Changelog
22

3+
0.6.0
4+
5+
* All currency values use decimal.Decimal.
6+
* Handle Rakuten ETC card charges in Rakuten Card.
7+
38
0.5.0
49

510
* Add support for SBI Sumishin Net Bank.

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "beancount-multitool"
3-
version = "0.5.0"
3+
version = "0.6.0"
44
description = "A CLI tool that converts financial data to Beancount files"
55
authors = ["Rick Lan <[email protected]>"]
66
license = "MIT"

‎src/beancount_multitool/JABank.py

+26-8
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
from datetime import datetime
2-
import pandas as pd
2+
from decimal import Decimal
33
from pathlib import Path
4-
import uuid
5-
import sys
4+
5+
import pandas as pd
66

77
from .Institution import Institution
88
from .MappingDatabase import MappingDatabase
@@ -43,7 +43,17 @@ def read_transaction(self, file_name: str, year: int) -> pd.DataFrame:
4343
pd.DataFrame
4444
A dataframe after pre-processing.
4545
"""
46-
df = pd.read_csv(file_name, encoding="shift_jis_2004")
46+
converters = {
47+
"明細区分": str,
48+
"取扱日付": str,
49+
"起算日": str,
50+
"お支払金額": str,
51+
"お預り金額": str,
52+
"取引区分": str,
53+
"残高": str,
54+
"摘要": str,
55+
}
56+
df = pd.read_csv(file_name, encoding="shift_jis_2004", converters=converters)
4757
print(f"Found {len(df.index)} transactions in {file_name}")
4858

4959
# Rename column names to English
@@ -66,13 +76,21 @@ def read_transaction(self, file_name: str, year: int) -> pd.DataFrame:
6676
str(year) + "." + df["Handling Date"], format="%Y.%m月%d日"
6777
)
6878

69-
cols = ["Debit", "Credit"]
79+
cols = ["Debit", "Credit", "Balance"]
7080
df[cols] = df[cols].replace({"\¥": "", ",": ""}, regex=True)
71-
df.fillna({"Debit": 0, "Credit": 0}, inplace=True)
72-
# Convert float to int
73-
df[cols] = df[cols].astype(int)
81+
df.replace(
82+
to_replace="",
83+
value={"Debit": "0", "Credit": "0", "Balance": "0"},
84+
inplace=True,
85+
)
86+
87+
df["Debit"] = df["Debit"].apply(Decimal)
88+
df["Credit"] = df["Credit"].apply(Decimal)
89+
df["Balance"] = df["Balance"].apply(Decimal)
90+
7491
df["amount"] = df["Credit"] - df["Debit"]
7592
df["memo"] = df["Transaction Classification"] + df["Description"]
93+
7694
# print(df.dtypes) # debug
7795
# print(df) # debug
7896
return df

‎src/beancount_multitool/RakutenBank.py

+12-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import pandas as pd
1+
from decimal import Decimal
22
from pathlib import Path
3-
import uuid
4-
import sys
3+
4+
import pandas as pd
55

66
from .Institution import Institution
77
from .MappingDatabase import MappingDatabase
@@ -42,7 +42,12 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
4242
pd.DataFrame
4343
A dataframe after pre-processing.
4444
"""
45-
df = pd.read_csv(file_name, encoding="shiftjis")
45+
converters = {
46+
"取引日": pd.to_datetime,
47+
"入出金(円)": str,
48+
"取引後残高(円)": str,
49+
}
50+
df = pd.read_csv(file_name, encoding="shift_jis", converters=converters)
4651
print(f"Found {len(df.index)} transactions in {file_name}")
4752

4853
# Rename column names to English.
@@ -56,8 +61,9 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
5661
}
5762
df.rename(columns=column_names, inplace=True)
5863

59-
# Convert date column to a datetime object
60-
df["date"] = pd.to_datetime(df["date"], format="%Y%m%d")
64+
df["amount"] = df["amount"].apply(Decimal)
65+
df["Balance"] = df["Balance"].apply(Decimal)
66+
6167
# print(df.dtypes) # debug
6268
# print(df) # debug
6369
return df

‎src/beancount_multitool/RakutenCard.py

+29-13
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
import pandas as pd
1+
from decimal import Decimal
22
from pathlib import Path
33

4+
import pandas as pd
5+
46
from .Institution import Institution
57
from .MappingDatabase import MappingDatabase
68
from .read_config import read_config
@@ -37,7 +39,16 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
3739
pd.DataFrame
3840
A dataframe after pre-processing.
3941
"""
40-
df = pd.read_csv(file_name)
42+
converters = {
43+
"利用日": pd.to_datetime,
44+
"利用店名・商品名": str,
45+
"利用者": str,
46+
# "支払方法": "Payment method",
47+
# "利用金額": "Amount",
48+
# "支払手数料": "Commission paid",
49+
"支払総額": str,
50+
}
51+
df = pd.read_csv(file_name, converters=converters)
4152
print(f"Found {len(df.index)} transactions in {file_name}")
4253

4354
# Rename column names to English.
@@ -55,20 +66,25 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
5566
}
5667
df.rename(columns=column_names, inplace=True)
5768

58-
df["date"] = pd.to_datetime(df["date"], format="%Y/%m/%d")
59-
60-
# Remove rows with empty 支払総額 cell.
61-
# These are extra info such as name of ETC gate or currency exchange rate.
62-
# TODO record as metadata
63-
extra = df.loc[pd.isnull(df["amount"])]
64-
df.drop(extra.index, inplace=True)
65-
# Convert to int type because currency is JPY
66-
df = df.astype({"amount": int})
69+
# ETC transaction has a second row.
70+
# Update firsts to be concatenation of the two memos.
71+
# Then delete the seconds.
72+
etc_index = df.loc[df["user"] == "ETC"].index
73+
df.loc[etc_index, "memo"] = (
74+
df.loc[etc_index, "memo"].values
75+
+ " "
76+
+ df.loc[etc_index + 1, "memo"].values
77+
)
78+
df.drop(df.loc[df["user"] == ""].index, inplace=True)
6779

6880
# Remove rows with zero 支払総額. These are refunds.
81+
# Also currency exchange rate(?)
6982
# TODO record as metadata.
70-
refund = df.loc[df["amount"] == 0]
71-
df.drop(refund.index, inplace=True)
83+
# refund = df.loc[df["amount"] == 0]
84+
# df.drop(refund.index, inplace=True)
85+
86+
# TODO this will fail if the refund rows are not removed
87+
df["amount"] = df["amount"].apply(Decimal)
7288

7389
# Reverse row order because the oldest transaction is on the bottom
7490
# Note: the index column is also reversed

‎src/beancount_multitool/ShinseiBank.py

+31-10
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import pandas as pd
1+
from decimal import Decimal
22
from pathlib import Path
3-
import uuid
4-
import sys
3+
4+
import pandas as pd
55

66
from .Institution import Institution
77
from .MappingDatabase import MappingDatabase
@@ -42,7 +42,17 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
4242
pd.DataFrame
4343
A dataframe after pre-processing.
4444
"""
45-
df = pd.read_csv(file_name)
45+
converters = {
46+
"取引日": pd.to_datetime,
47+
"出金金額": str,
48+
"入金金額": str,
49+
"残高": str,
50+
"Value Date": pd.to_datetime,
51+
"Debit": str,
52+
"Credit": str,
53+
"Balance": str,
54+
}
55+
df = pd.read_csv(file_name, converters=converters)
4656
print(f"Found {len(df.index)} transactions in {file_name}")
4757

4858
# Rename column names to English.
@@ -64,13 +74,24 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
6474
}
6575
df.rename(columns=column_names, inplace=True)
6676

67-
# Convert date column to a datetime object
68-
df["date"] = pd.to_datetime(df["date"], format="%Y/%m/%d")
6977
# Fill empty cells with zeros
70-
df.fillna(0, inplace=True)
71-
# Convert float to int
72-
df["Debit"] = df["Debit"].astype(int)
73-
df["Credit"] = df["Credit"].astype(int)
78+
df.replace(
79+
to_replace="",
80+
value={"Debit": "0", "Credit": "0", "Balance": "0"},
81+
inplace=True,
82+
)
83+
# Remove thousands marker
84+
df.replace(
85+
to_replace=",",
86+
value={"Debit": "", "Credit": "", "Balance": ""},
87+
inplace=True,
88+
regex=True,
89+
)
90+
91+
df["Debit"] = df["Debit"].apply(Decimal)
92+
df["Credit"] = df["Credit"].apply(Decimal)
93+
df["Balance"] = df["Balance"].apply(Decimal)
94+
7495
df["amount"] = df["Credit"] - df["Debit"]
7596

7697
# Reverse row order because the oldest transaction is on the bottom

‎src/beancount_multitool/SumishinNetBank.py

+18-12
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
from decimal import Decimal
22
from pathlib import Path
3-
import uuid
4-
import sys
53

64
import pandas as pd
75

@@ -44,14 +42,13 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
4442
pd.DataFrame
4543
A dataframe after pre-processing.
4644
"""
47-
df = pd.read_csv(file_name,
48-
encoding="shift_jis",
49-
converters={
50-
"日付": pd.to_datetime,
51-
"出金金額(円)": str,
52-
"入金金額(円)": str,
53-
"残高(円)": str,
54-
})
45+
converters = {
46+
"日付": pd.to_datetime,
47+
"出金金額(円)": str,
48+
"入金金額(円)": str,
49+
"残高(円)": str,
50+
}
51+
df = pd.read_csv(file_name, encoding="shift_jis", converters=converters)
5552
print(f"Found {len(df.index)} data transactions")
5653
# Rename column names to English.
5754
# "日付","内容","出金金額(円)","入金金額(円)","残高(円)","メモ"
@@ -67,8 +64,17 @@ def read_transaction(self, file_name: str) -> pd.DataFrame:
6764
}
6865
df.rename(columns=columns, inplace=True)
6966

70-
df.replace(to_replace="", value={"Debit": "0", "Credit": "0", "Balance": "0"}, inplace=True)
71-
df.replace(to_replace=",", value={"Debit": "", "Credit": "", "Balance": ""}, inplace=True, regex=True)
67+
df.replace(
68+
to_replace="",
69+
value={"Debit": "0", "Credit": "0", "Balance": "0"},
70+
inplace=True,
71+
)
72+
df.replace(
73+
to_replace=",",
74+
value={"Debit": "", "Credit": "", "Balance": ""},
75+
inplace=True,
76+
regex=True,
77+
)
7278
df["Debit"] = df["Debit"].apply(Decimal)
7379
df["Credit"] = df["Credit"].apply(Decimal)
7480
df["Balance"] = df["Balance"].apply(Decimal)

‎src/beancount_multitool/__version__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.5.0"
1+
__version__ = "0.6.0"

‎src/beancount_multitool/as_transaction.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,9 @@
55

66

77
def make_hashtags(func):
8-
"""Add # prefix if missing from any tags.
9-
"""
10-
def add_hash(*args, **kwargs):
8+
"""Add # prefix if missing from any tags."""
119

10+
def add_hash(*args, **kwargs):
1211
hashtags = []
1312
for x in kwargs["tags"]:
1413
if len(x):
@@ -20,14 +19,14 @@ def add_hash(*args, **kwargs):
2019

2120
result = func(*args, **kwargs)
2221
return result
22+
2323
return add_hash
2424

2525

2626
def reconcile(func):
27-
"""Add an UUID field if #reconcile tag exists
28-
"""
29-
def add_uuid(*args, **kwargs):
27+
"""Add an UUID field if #reconcile tag exists"""
3028

29+
def add_uuid(*args, **kwargs):
3130
# Add UUID for manual transactions reconcilation between accounts
3231
if "#reconcile" in kwargs["tags"]:
3332
if kwargs["amount"] < 0: # a credit
@@ -41,6 +40,7 @@ def add_uuid(*args, **kwargs):
4140

4241
result = func(*args, **kwargs)
4342
return result
43+
4444
return add_uuid
4545

4646

‎src/beancount_multitool/cli.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@ def validate_name(ctx, param, value):
1919
raise click.BadParameter(f"Name must be one of: {__INSTITUTIONS__}")
2020

2121

22-
@click.command(
23-
epilog=f"Note: supported financial institutions are {__INSTITUTIONS__}"
24-
)
22+
@click.command(epilog=f"Note: supported financial institutions are {__INSTITUTIONS__}")
2523
@click.argument("name", type=str, callback=validate_name)
2624
@click.argument("config", type=click.Path(exists=True))
2725
@click.argument("data", type=click.Path(exists=True))

‎src/beancount_multitool/tests/test_read_config.py

-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
#import pytest
21
from beancount_multitool.read_config import read_config
32

43

‎tests/conftest.py

+3
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@
33
# Ref: https://stackoverflow.com/a/25188424
44
def pytest_configure(config):
55
import sys
6+
67
sys._called_from_pytest = True
78

9+
810
def pytest_unconfigure(config):
911
import sys
12+
1013
if hasattr(sys, "_called_from_pytest"):
1114
del sys._called_from_pytest

0 commit comments

Comments
 (0)