From 8fcf709adc6a7cbf76ee18b6704d8a32b08e89d1 Mon Sep 17 00:00:00 2001 From: MaysWind Date: Mon, 28 Oct 2024 01:37:57 +0800 Subject: [PATCH] import transaction from ofx 2.x file --- pkg/converters/ofx/ofx_data.go | 90 ++++++++++++++++++--- pkg/converters/ofx/ofx_data_reader.go | 50 ++++++++++++ pkg/converters/ofx/ofx_transaction_table.go | 81 +++++++++++++------ 3 files changed, 184 insertions(+), 37 deletions(-) create mode 100644 pkg/converters/ofx/ofx_data_reader.go diff --git a/pkg/converters/ofx/ofx_data.go b/pkg/converters/ofx/ofx_data.go index 92313400..cf384298 100644 --- a/pkg/converters/ofx/ofx_data.go +++ b/pkg/converters/ofx/ofx_data.go @@ -1,6 +1,10 @@ package ofx -import "encoding/xml" +import ( + "encoding/xml" + + "github.com/mayswind/ezbookkeeping/pkg/models" +) const ofxVersion1 = "100" const ofxVersion2 = "200" @@ -19,6 +23,47 @@ const ( ofxCertificateOfDepositAccount ofxAccountType = "CD" ) +// ofxTransactionType represents transaction type in open financial exchange (ofx) file +type ofxTransactionType string + +// OFX transaction types +const ( + ofxGenericCreditTransaction ofxTransactionType = "CREDIT" + ofxGenericDebitTransaction ofxTransactionType = "DEBIT" + ofxInterestTransaction ofxTransactionType = "INT" + ofxDividendTransaction ofxTransactionType = "DIV" + ofxFIFeeTransaction ofxTransactionType = "FEE" + ofxServiceChargeTransaction ofxTransactionType = "SRVCHG" + ofxDepositTransaction ofxTransactionType = "DEP" + ofxATMTransaction ofxTransactionType = "ATM" + ofxPOSTransaction ofxTransactionType = "POS" + ofxTransferTransaction ofxTransactionType = "XFER" + ofxCheckTransaction ofxTransactionType = "CHECK" + ofxElectronicPaymentTransaction ofxTransactionType = "PAYMENT" + ofxCashWithdrawalTransaction ofxTransactionType = "CASH" + ofxDirectDepositTransaction ofxTransactionType = "DIRECTDEP" + ofxMerchantInitiatedDebitTransaction ofxTransactionType = "DIRECTDEBIT" + ofxRepeatingPaymentTransaction ofxTransactionType = "REPEATPMT" + ofxHoldTransaction ofxTransactionType = "HOLD" + ofxOtherTransaction ofxTransactionType = "OTHER" +) + +var ofxTransactionTypeMapping = map[ofxTransactionType]models.TransactionType{ + ofxGenericCreditTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxGenericDebitTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxDividendTransaction: models.TRANSACTION_TYPE_INCOME, + ofxFIFeeTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxServiceChargeTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxDepositTransaction: models.TRANSACTION_TYPE_INCOME, + ofxTransferTransaction: models.TRANSACTION_TYPE_TRANSFER, + ofxCheckTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxElectronicPaymentTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxCashWithdrawalTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxDirectDepositTransaction: models.TRANSACTION_TYPE_INCOME, + ofxMerchantInitiatedDebitTransaction: models.TRANSACTION_TYPE_EXPENSE, + ofxRepeatingPaymentTransaction: models.TRANSACTION_TYPE_EXPENSE, +} + // ofxFile represents the struct of open financial exchange (ofx) file type ofxFile struct { XMLName xml.Name `xml:"OFX"` @@ -65,9 +110,9 @@ type ofxBankStatementResponse struct { // ofxCreditCardStatementResponse represents the struct of open financial exchange (ofx) credit card statement response type ofxCreditCardStatementResponse struct { - DefaultCurrency string `xml:"CURDEF"` - AccountFrom *ofxCreditCardAccount `xml:"CCACCTFROM"` - TransactionList *ofxBankTransactionList `xml:"BANKTRANLIST"` + DefaultCurrency string `xml:"CURDEF"` + AccountFrom *ofxCreditCardAccount `xml:"CCACCTFROM"` + TransactionList *ofxCreditCardTransactionList `xml:"BANKTRANLIST"` } // ofxBankAccount represents the struct of open financial exchange (ofx) bank account @@ -92,17 +137,36 @@ type ofxBankTransactionList struct { StatementTransactions []*ofxBankStatementTransaction `xml:"STMTTRN"` } +// ofxCreditCardTransactionList represents the struct of open financial exchange (ofx) credit card transaction list +type ofxCreditCardTransactionList struct { + StartDate string `xml:"DTSTART"` + EndDate string `xml:"DTEND"` + StatementTransactions []*ofxCreditCardStatementTransaction `xml:"STMTTRN"` +} + +// ofxBasicStatementTransaction represents the struct of open financial exchange (ofx) basic statement transaction +type ofxBasicStatementTransaction struct { + TransactionId string `xml:"FITID"` + TransactionType ofxTransactionType `xml:"TRNTYPE"` + PostedDate string `xml:"DTPOSTED"` + Amount string `xml:"TRNAMT"` + Name string `xml:"NAME"` + Payee *ofxPayee `xml:"PAYEE"` + Memo string `xml:"MEMO"` + Currency string `xml:"CURRENCY"` + OriginalCurrency string `xml:"ORIGCURRENCY"` +} + // ofxBankStatementTransaction represents the struct of open financial exchange (ofx) bank statement transaction type ofxBankStatementTransaction struct { - TransactionId string `xml:"FITID"` - TransactionType string `xml:"TRNTYPE"` - PostedDate string `xml:"DTPOSTED"` - Amount string `xml:"TRNAMT"` - Name string `xml:"NAME"` - Payee *ofxPayee `xml:"PAYEE"` - Memo string `xml:"MEMO"` - Currency string `xml:"CURRENCY"` - OriginalCurrency string `xml:"ORIGCURRENCY"` + ofxBasicStatementTransaction + AccountTo *ofxCreditCardAccount `xml:"BANKACCTTO"` +} + +// ofxCreditCardStatementTransaction represents the struct of open financial exchange (ofx) credit card statement transaction +type ofxCreditCardStatementTransaction struct { + ofxBasicStatementTransaction + AccountTo *ofxCreditCardAccount `xml:"CCACCTTO"` } // ofxPayee represents the struct of open financial exchange (ofx) payee info diff --git a/pkg/converters/ofx/ofx_data_reader.go b/pkg/converters/ofx/ofx_data_reader.go new file mode 100644 index 00000000..35489c92 --- /dev/null +++ b/pkg/converters/ofx/ofx_data_reader.go @@ -0,0 +1,50 @@ +package ofx + +import ( + "bytes" + "encoding/xml" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +// ofxFileReader defines the structure of open financial exchange (ofx) file reader +type ofxFileReader struct { + xmlDecoder *xml.Decoder +} + +// read returns the imported open financial exchange (ofx) file +func (r *ofxFileReader) read(ctx core.Context) (*ofxFile, error) { + file := &ofxFile{} + + err := r.xmlDecoder.Decode(&file) + + if err != nil { + return nil, err + } + + return file, nil +} + +func createNewOFXFileReader(data []byte) (*ofxFileReader, error) { + if len(data) > 5 && data[0] == 0x3C && data[1] == 0x3F && data[2] == 0x78 && data[3] == 0x6D && data[4] == 0x6C { // ofx 2.x starts with 13 && string(data[0:13]) == "OFXHEADER:100" { // ofx 1.x starts with OFXHEADER:100 + + } else if len(data) > 5 && string(data[0:5]) == "" { // no ofx header + xmlDecoder := xml.NewDecoder(bytes.NewReader(data)) + xmlDecoder.CharsetReader = utils.IdentReader + + return &ofxFileReader{ + xmlDecoder: xmlDecoder, + }, nil + } + + return nil, errs.ErrInvalidOFXFile +} diff --git a/pkg/converters/ofx/ofx_transaction_table.go b/pkg/converters/ofx/ofx_transaction_table.go index 5204ab76..68d4362f 100644 --- a/pkg/converters/ofx/ofx_transaction_table.go +++ b/pkg/converters/ofx/ofx_transaction_table.go @@ -28,7 +28,7 @@ var ofxTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bo // ofxTransactionData defines the structure of open financial exchange (ofx) transaction data type ofxTransactionData struct { - *ofxBankStatementTransaction + ofxBasicStatementTransaction DefaultCurrency string FromAccountId string ToAccountId string @@ -125,7 +125,31 @@ func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = datetime data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = timezone - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = ofxTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] + + amount, err := utils.ParseAmount(ofxTransaction.Amount) + + if err != nil { + return nil, errs.ErrAmountInvalid + } + + if transactionType, exists := ofxTransactionTypeMapping[ofxTransaction.TransactionType]; exists { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = utils.IntToString(int(transactionType)) + + if data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == ofxTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount) + } else { + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + } + } else { + if amount >= 0 { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = ofxTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount) + } else { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = ofxTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + } + } + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = ofxTransaction.FromAccountId if ofxTransaction.Currency != "" { @@ -134,7 +158,10 @@ func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = ofxTransaction.DefaultCurrency } - //data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amountNum1) + if data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == ofxTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] + data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] + } if ofxTransaction.Memo != "" { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = ofxTransaction.Memo @@ -146,7 +173,7 @@ func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = "" } - return nil, nil + return data, nil } func (t *ofxTransactionDataRowIterator) parseTransactionTimeAndTimeZone(ctx core.Context, datetime string) (string, string, error) { @@ -156,34 +183,26 @@ func (t *ofxTransactionDataRowIterator) parseTransactionTimeAndTimeZone(ctx core var err error var year, month, day string - hour := "0" - minute := "0" - second := "0" + hour := "00" + minute := "00" + second := "00" tzOffset := ofxDefaultTimezoneOffset - if len(datetime) >= 8 { + if len(datetime) >= 8 { // YYYYMMDD year = datetime[0:4] month = datetime[4:6] day = datetime[6:8] } - if len(datetime) >= 14 { + if len(datetime) >= 14 { // YYYYMMDDHHMMSS hour = datetime[8:10] minute = datetime[10:12] second = datetime[12:14] } - if len(month) < 2 { - month = "0" + month - } - - if len(day) < 2 { - day = "0" + day - } - squareBracketStartIndex := strings.Index(datetime, "[") - if squareBracketStartIndex > 0 { + if squareBracketStartIndex > 0 { // YYYYMMDDHHMMSS.XXX [gmt offset[:tz name]] timezoneInfo := datetime[squareBracketStartIndex+1 : len(datetime)-1] timezoneItems := strings.Split(timezoneInfo, ":") tzOffset, err = utils.FormatTimezoneOffsetFromHoursOffset(timezoneItems[0]) @@ -217,10 +236,17 @@ func createNewOFXTransactionDataTable(file *ofxFile) (*ofxTransactionDataTable, } for i := 0; i < len(bankTransactions); i++ { + toAccountId := "" + + if bankTransactions[i].AccountTo != nil { + toAccountId = bankTransactions[i].AccountTo.AccountId + } + allData = append(allData, &ofxTransactionData{ - ofxBankStatementTransaction: bankTransactions[i], - DefaultCurrency: statement.DefaultCurrency, - FromAccountId: fromAccountId, + ofxBasicStatementTransaction: bankTransactions[i].ofxBasicStatementTransaction, + DefaultCurrency: statement.DefaultCurrency, + FromAccountId: fromAccountId, + ToAccountId: toAccountId, }) } } @@ -238,10 +264,17 @@ func createNewOFXTransactionDataTable(file *ofxFile) (*ofxTransactionDataTable, } for i := 0; i < len(bankTransactions); i++ { + toAccountId := "" + + if bankTransactions[i].AccountTo != nil { + toAccountId = bankTransactions[i].AccountTo.AccountId + } + allData = append(allData, &ofxTransactionData{ - ofxBankStatementTransaction: bankTransactions[i], - DefaultCurrency: statement.DefaultCurrency, - FromAccountId: fromAccountId, + ofxBasicStatementTransaction: bankTransactions[i].ofxBasicStatementTransaction, + DefaultCurrency: statement.DefaultCurrency, + FromAccountId: fromAccountId, + ToAccountId: toAccountId, }) } }