From 981a1aac4fec24e88ab34dae0df029eaaaf4fc35 Mon Sep 17 00:00:00 2001 From: MaysWind Date: Sun, 20 Oct 2024 01:47:54 +0800 Subject: [PATCH] import transaction from qif file --- pkg/converters/qif/qif_data.go | 124 +++++ pkg/converters/qif/qif_data_reader.go | 466 +++++++++++++++++ pkg/converters/qif/qif_data_reader_test.go | 477 ++++++++++++++++++ .../qif/qif_transaction_data_file_importer.go | 55 ++ ...qif_transaction_data_file_importer_test.go | 437 ++++++++++++++++ .../qif/qif_transaction_data_table.go | 241 +++++++++ pkg/converters/transaction_data_converters.go | 7 + pkg/errs/converter.go | 1 + pkg/utils/converter.go | 3 + pkg/utils/converter_test.go | 15 + pkg/utils/regexps.go | 30 +- pkg/utils/regexps_test.go | 126 +++++ src/consts/file.js | 15 + src/locales/en.json | 4 + src/locales/zh_Hans.json | 4 + 15 files changed, 1999 insertions(+), 6 deletions(-) create mode 100644 pkg/converters/qif/qif_data.go create mode 100644 pkg/converters/qif/qif_data_reader.go create mode 100644 pkg/converters/qif/qif_data_reader_test.go create mode 100644 pkg/converters/qif/qif_transaction_data_file_importer.go create mode 100644 pkg/converters/qif/qif_transaction_data_file_importer_test.go create mode 100644 pkg/converters/qif/qif_transaction_data_table.go diff --git a/pkg/converters/qif/qif_data.go b/pkg/converters/qif/qif_data.go new file mode 100644 index 00000000..ad065358 --- /dev/null +++ b/pkg/converters/qif/qif_data.go @@ -0,0 +1,124 @@ +package qif + +// qifTransactionClearedStatus represents the quicken interchange format (qif) transaction cleared status +type qifTransactionClearedStatus string + +// Quicken interchange format transaction types +const ( + qifClearedStatusUnreconciled qifTransactionClearedStatus = "" + qifClearedStatusCleared qifTransactionClearedStatus = "C" + qifClearedStatusReconciled qifTransactionClearedStatus = "R" +) + +// qifTransactionType represents the quicken interchange format (qif) transaction type +type qifTransactionType string + +// Quicken interchange format transaction types +const ( + qifInvalidTransactionType qifTransactionType = "" + qifCheckTransactionType qifTransactionType = "KC" + qifDepositTransactionType qifTransactionType = "KD" + qifPaymentTransactionType qifTransactionType = "KP" + qifInvestmentTransactionType qifTransactionType = "KI" + qifElectronicPayeeTransactionType qifTransactionType = "KE" +) + +// qifCategoryType represents the quicken interchange format (qif) category type +type qifCategoryType string + +// Quicken interchange format category types +const ( + qifIncomeTransaction qifCategoryType = "I" + qifExpenseTransaction qifCategoryType = "E" +) + +// qifData defines the structure of quicken interchange format (qif) data +type qifData struct { + bankAccountTransactions []*qifTransactionData + cashAccountTransactions []*qifTransactionData + creditCardAccountTransactions []*qifTransactionData + assetAccountTransactions []*qifTransactionData + liabilityAccountTransactions []*qifTransactionData + memorizedTransactions []*qifMemorizedTransactionData + investmentAccountTransactions []*qifInvestmentTransactionData + accounts []*qifAccountData + categories []*qifCategoryData + classes []*qifClassData +} + +// qifTransactionData defines the structure of quicken interchange format (qif) transaction data +type qifTransactionData struct { + date string + amount string + clearedStatus qifTransactionClearedStatus + num string + payee string + memo string + addresses []string + category string + subTransactionCategory []string + subTransactionMemo []string + subTransactionAmount []string + account *qifAccountData +} + +// qifInvestmentTransactionData defines the structure of quicken interchange format (qif) investment transaction data +type qifInvestmentTransactionData struct { + date string + action string + security string + price string + quantity string + amount string + clearedStatus qifTransactionClearedStatus + text string + memo string + commission string + accountForTransfer string + amountTransferred string + account *qifAccountData +} + +// qifMemorizedTransactionData defines the structure of quicken interchange format (qif) memorized transaction data +type qifMemorizedTransactionData struct { + qifTransactionData + transactionType qifTransactionType + amortization qifMemorizedTransactionAmortizationData +} + +// qifMemorizedTransactionAmortizationData defines the structure of quicken interchange format (qif) memorized transaction amortization data +type qifMemorizedTransactionAmortizationData struct { + firstPaymentDate string + totalYearsForLoan string + numberOfPayments string + numberOfPeriodsPerYear string + interestRate string + currentLoanBalance string + originalLoanAmount string +} + +// qifAccountData defines the structure of quicken interchange format (qif) account data +type qifAccountData struct { + name string + accountType string + description string + creditLimit string + statementBalanceDate string + statementBalanceAmount string +} + +// qifCategoryData defines the structure of quicken interchange format (qif) category data +type qifCategoryData struct { + name string + description string + taxRelated bool + categoryType qifCategoryType + budgetAmount string + taxScheduleInformation string +} + +// qifClassData defines the structure of quicken interchange format (qif) class data +type qifClassData struct { + name string + description string +} diff --git a/pkg/converters/qif/qif_data_reader.go b/pkg/converters/qif/qif_data_reader.go new file mode 100644 index 00000000..efc430b7 --- /dev/null +++ b/pkg/converters/qif/qif_data_reader.go @@ -0,0 +1,466 @@ +package qif + +import ( + "bufio" + "bytes" + "strings" + + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +const qifBankTransactionHeader = "!Type:Bank" +const qifCashTransactionHeader = "!Type:Cash" +const qifCreditCardTransactionHeader = "!Type:CCard" +const qifAssetAccountTransactionHeader = "!Type:Oth A" +const qifLiabilityAccountTransactionHeader = "!Type:Oth L" +const qifMemorizedTransactionHeader = "!Type:Memorized" +const qifInvestmentTransactionHeader = "!Type:Invst" +const qifAccountHeader = "!Account" +const qifCategoryHeader = "!Type:Cat" +const qifClassHeader = "!Type:Class" + +const qifEntryStartRune = '!' +const qifEntryEnd = '^' + +// qifDataReader defines the structure of quicken interchange format (qif) data reader +type qifDataReader struct { + allLines []string +} + +// read returns the imported qif data +func (r *qifDataReader) read(ctx core.Context) (*qifData, error) { + if len(r.allLines) < 1 { + return nil, errs.ErrNotFoundTransactionDataInFile + } + + data := &qifData{} + var currentEntryHeader string + var currentEntryData []string + var currentAccount *qifAccountData + + for i := 0; i < len(r.allLines); i++ { + line := r.allLines[i] + + if len(line) < 1 { + continue + } + + if line[0] == qifEntryStartRune { + if len(currentEntryData) > 0 { + log.Errorf(ctx, "[qif_data_reader.read] read new entry header \"%s\" after unclosed entry", line) + return nil, errs.ErrInvalidQIFFile + } + + if line == qifBankTransactionHeader || + line == qifCashTransactionHeader || + line == qifCreditCardTransactionHeader || + line == qifAssetAccountTransactionHeader || + line == qifLiabilityAccountTransactionHeader || + line == qifMemorizedTransactionHeader || + line == qifInvestmentTransactionHeader || + line == qifAccountHeader || + line == qifCategoryHeader || + line == qifClassHeader { + currentEntryHeader = line + } else { + log.Warnf(ctx, "[qif_data_reader.read] read unsupported entry header line \"%s\" and skip this line", line) + continue + } + } else if line[0] == qifEntryEnd { + entryData := currentEntryData + currentEntryData = nil + + if currentEntryHeader == qifBankTransactionHeader || + currentEntryHeader == qifCashTransactionHeader || + currentEntryHeader == qifCreditCardTransactionHeader || + currentEntryHeader == qifAssetAccountTransactionHeader || + currentEntryHeader == qifLiabilityAccountTransactionHeader { + transactionData, err := r.parseTransaction(ctx, entryData) + + if err != nil { + return nil, err + } + + if transactionData == nil { + continue + } + + transactionData.account = currentAccount + + if currentEntryHeader == qifBankTransactionHeader { + data.bankAccountTransactions = append(data.bankAccountTransactions, transactionData) + } else if currentEntryHeader == qifCashTransactionHeader { + data.cashAccountTransactions = append(data.cashAccountTransactions, transactionData) + } else if currentEntryHeader == qifCreditCardTransactionHeader { + data.creditCardAccountTransactions = append(data.creditCardAccountTransactions, transactionData) + } else if currentEntryHeader == qifAssetAccountTransactionHeader { + data.assetAccountTransactions = append(data.assetAccountTransactions, transactionData) + } else if currentEntryHeader == qifLiabilityAccountTransactionHeader { + data.liabilityAccountTransactions = append(data.liabilityAccountTransactions, transactionData) + } + } else if currentEntryHeader == qifMemorizedTransactionHeader { + transactionData, err := r.parseMemorizedTransaction(ctx, entryData) + + if err != nil { + return nil, err + } + + if transactionData == nil { + continue + } + + transactionData.account = currentAccount + data.memorizedTransactions = append(data.memorizedTransactions, transactionData) + } else if currentEntryHeader == qifInvestmentTransactionHeader { + transactionData, err := r.parseInvestmentTransaction(ctx, entryData) + + if err != nil { + return nil, err + } + + if transactionData == nil { + continue + } + + transactionData.account = currentAccount + data.investmentAccountTransactions = append(data.investmentAccountTransactions, transactionData) + } else if currentEntryHeader == qifAccountHeader { + accountData, err := r.parseAccount(ctx, entryData) + + if err != nil { + return nil, err + } + + if accountData == nil { + continue + } + + currentAccount = accountData + data.accounts = append(data.accounts, accountData) + } else if currentEntryHeader == qifCategoryHeader { + categoryData, err := r.parseCategory(ctx, entryData) + + if err != nil { + return nil, err + } + + if categoryData == nil { + continue + } + + data.categories = append(data.categories, categoryData) + } else if currentEntryHeader == qifClassHeader { + classData, err := r.parseClass(ctx, entryData) + + if err != nil { + return nil, err + } + + if classData == nil { + continue + } + + data.classes = append(data.classes, classData) + } else { + log.Warnf(ctx, "[qif_data_reader.read] read unsupported entry header \"%s\" and skip this entry", currentEntryHeader) + continue + } + } else if currentEntryHeader != "" { + currentEntryData = append(currentEntryData, line) + } else { + log.Warnf(ctx, "[qif_data_reader.read] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + return data, nil +} + +func (r *qifDataReader) parseTransaction(ctx core.Context, data []string) (*qifTransactionData, error) { + if len(data) < 1 { + return nil, nil + } + + transactionData := &qifTransactionData{} + + for i := 0; i < len(data); i++ { + line := data[i] + + if len(line) < 1 { + continue + } + + if line[0] == 'D' { + transactionData.date = line[1:] + } else if line[0] == 'T' { + transactionData.amount = line[1:] + } else if line[0] == 'C' { + transactionData.clearedStatus = r.parseClearedStatus(ctx, line[1:]) + } else if line[0] == 'N' { + transactionData.num = line[1:] + } else if line[0] == 'P' { + transactionData.payee = line[1:] + } else if line[0] == 'M' { + transactionData.memo = line[1:] + } else if line[0] == 'A' { + transactionData.addresses = append(transactionData.addresses, line[1:]) + } else if line[0] == 'L' { + transactionData.category = line[1:] + } else if line[0] == 'S' { + transactionData.subTransactionCategory = append(transactionData.subTransactionCategory, line[1:]) + } else if line[0] == 'E' { + transactionData.subTransactionMemo = append(transactionData.subTransactionMemo, line[1:]) + } else if line[0] == '$' { + transactionData.subTransactionAmount = append(transactionData.subTransactionAmount, line[1:]) + } else { + log.Warnf(ctx, "[qif_data_reader.parseTransaction] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + return transactionData, nil +} + +func (r *qifDataReader) parseMemorizedTransaction(ctx core.Context, data []string) (*qifMemorizedTransactionData, error) { + if len(data) < 1 { + return nil, nil + } + + baseTransactionData, err := r.parseTransaction(ctx, data) + + if err != nil { + return nil, err + } + + transactionData := &qifMemorizedTransactionData{ + qifTransactionData: *baseTransactionData, + amortization: qifMemorizedTransactionAmortizationData{}, + } + + for i := 0; i < len(data); i++ { + line := data[i] + + if len(line) < 1 { + continue + } + + if line[0] == 'K' { + if line == string(qifCheckTransactionType) { + transactionData.transactionType = qifCheckTransactionType + } else if line == string(qifDepositTransactionType) { + transactionData.transactionType = qifDepositTransactionType + } else if line == string(qifPaymentTransactionType) { + transactionData.transactionType = qifPaymentTransactionType + } else if line == string(qifInvestmentTransactionType) { + transactionData.transactionType = qifInvestmentTransactionType + } else if line == string(qifElectronicPayeeTransactionType) { + transactionData.transactionType = qifElectronicPayeeTransactionType + } else { + log.Warnf(ctx, "[qif_data_reader.parseMemorizedTransaction] read unsupported transaction type \"%s\" and skip this line", line) + continue + } + } else if line[0] == '1' { + transactionData.amortization.firstPaymentDate = line[1:] + } else if line[0] == '2' { + transactionData.amortization.totalYearsForLoan = line[1:] + } else if line[0] == '3' { + transactionData.amortization.numberOfPayments = line[1:] + } else if line[0] == '4' { + transactionData.amortization.numberOfPeriodsPerYear = line[1:] + } else if line[0] == '5' { + transactionData.amortization.interestRate = line[1:] + } else if line[0] == '6' { + transactionData.amortization.currentLoanBalance = line[1:] + } else if line[0] == '7' { + transactionData.amortization.originalLoanAmount = line[1:] + } else { + log.Warnf(ctx, "[qif_data_reader.parseMemorizedTransaction] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + return transactionData, nil +} + +func (r *qifDataReader) parseInvestmentTransaction(ctx core.Context, data []string) (*qifInvestmentTransactionData, error) { + if len(data) < 1 { + return nil, nil + } + + transactionData := &qifInvestmentTransactionData{} + + for i := 0; i < len(data); i++ { + line := data[i] + + if len(line) < 1 { + continue + } + + if line[0] == 'D' { + transactionData.date = line[1:] + } else if line[0] == 'N' { + transactionData.action = line[1:] + } else if line[0] == 'Y' { + transactionData.security = line[1:] + } else if line[0] == 'I' { + transactionData.price = line[1:] + } else if line[0] == 'Q' { + transactionData.quantity = line[1:] + } else if line[0] == 'T' { + transactionData.amount = line[1:] + } else if line[0] == 'C' { + transactionData.clearedStatus = r.parseClearedStatus(ctx, line[1:]) + } else if line[0] == 'P' { + transactionData.text = line[1:] + } else if line[0] == 'M' { + transactionData.memo = line[1:] + } else if line[0] == 'O' { + transactionData.commission = line[1:] + } else if line[0] == 'L' { + transactionData.accountForTransfer = line[1:] + } else if line[0] == '$' { + transactionData.amountTransferred = line[1:] + } else { + log.Warnf(ctx, "[qif_data_reader.parseInvestmentTransaction] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + return transactionData, nil +} + +func (r *qifDataReader) parseAccount(ctx core.Context, data []string) (*qifAccountData, error) { + if len(data) < 1 { + return nil, nil + } + + accountData := &qifAccountData{} + + for i := 0; i < len(data); i++ { + line := data[i] + + if len(line) < 1 { + continue + } + + if line[0] == 'N' { + accountData.name = line[1:] + } else if line[0] == 'T' { + accountData.accountType = line[1:] + } else if line[0] == 'D' { + accountData.description = line[1:] + } else if line[0] == 'L' { + accountData.creditLimit = line[1:] + } else if line[0] == '/' { + accountData.statementBalanceDate = line[1:] + } else if line[0] == '$' { + accountData.statementBalanceAmount = line[1:] + } else { + log.Warnf(ctx, "[qif_data_reader.parseAccount] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + return accountData, nil +} + +func (r *qifDataReader) parseCategory(ctx core.Context, data []string) (*qifCategoryData, error) { + if len(data) < 1 { + return nil, nil + } + + categoryData := &qifCategoryData{} + + for i := 0; i < len(data); i++ { + line := data[i] + + if len(line) < 1 { + continue + } + + if line[0] == 'N' { + categoryData.name = line[1:] + } else if line[0] == 'D' { + categoryData.description = line[1:] + } else if line[0] == 'T' { + categoryData.taxRelated = true + } else if line[0] == 'I' { + categoryData.categoryType = qifIncomeTransaction + } else if line[0] == 'E' { + categoryData.categoryType = qifExpenseTransaction + } else if line[0] == 'B' { + categoryData.budgetAmount = line[1:] + } else if line[0] == 'R' { + categoryData.taxScheduleInformation = line[1:] + } else { + log.Warnf(ctx, "[qif_data_reader.parseCategory] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + if categoryData.categoryType == "" { + categoryData.categoryType = qifExpenseTransaction + } + + return categoryData, nil +} + +func (r *qifDataReader) parseClass(ctx core.Context, data []string) (*qifClassData, error) { + if len(data) < 1 { + return nil, nil + } + + classData := &qifClassData{} + + for i := 0; i < len(data); i++ { + line := data[i] + + if len(line) < 1 { + continue + } + + if line[0] == 'N' { + classData.name = line[1:] + } else if line[0] == 'D' { + classData.description = line[1:] + } else { + log.Warnf(ctx, "[qif_data_reader.parseClass] read unsupported line \"%s\" and skip this line", line) + continue + } + } + + return classData, nil +} + +func (r *qifDataReader) parseClearedStatus(ctx core.Context, value string) qifTransactionClearedStatus { + if value == "" { + return qifClearedStatusUnreconciled + } else if value == "*" || strings.ToUpper(value) == "C" { + return qifClearedStatusCleared + } else if strings.ToUpper(value) == "R" || strings.ToUpper(value) == "X" { + return qifClearedStatusReconciled + } else { + log.Warnf(ctx, "[qif_data_reader.parseClearedStatus] read unsupported transaction cleared status \"%s\" and skip this value", value) + return qifClearedStatusUnreconciled + } +} + +func createNewQifDataReader(data []byte) *qifDataReader { + fallback := unicode.UTF8.NewDecoder() + reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) + scanner := bufio.NewScanner(reader) + allLines := make([]string, 0) + + for scanner.Scan() { + allLines = append(allLines, scanner.Text()) + } + + return &qifDataReader{ + allLines: allLines, + } +} diff --git a/pkg/converters/qif/qif_data_reader_test.go b/pkg/converters/qif/qif_data_reader_test.go new file mode 100644 index 00000000..22f776ef --- /dev/null +++ b/pkg/converters/qif/qif_data_reader_test.go @@ -0,0 +1,477 @@ +package qif + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" +) + +func TestQifDataReaderParse(t *testing.T) { + reader := &qifDataReader{ + allLines: []string{ + "!Type:Bank", + "D2024/10/9", + "T-123.45", + "^", + "D2024/10/12", + "T+234.56", + "^", + "!Type:Cash", + "D2024/9/1", + "T100.00", + "POpening Balance", + "L[Wallet]", + "^", + "!Type:Memorized", + "KC", + "T-123.45", + "12024/10/13", + "23", + "31", + "42", + "512.34", + "6100.45", + "7234.56", + "^", + "!Type:Invst", + "D2024/10/14", + "NBuy", + "YTest", + "I12.34", + "Q10", + "T-123.4", + "^", + "!Account", + "NTest Account", + "^", + "NWallet", + "^", + "!Type:Cat", + "NTest Category", + "I", + "^", + "!Type:Class", + "NTest Class", + "DFoo Bar", + "^", + }, + } + context := core.NewNullContext() + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 2, len(actualData.bankAccountTransactions)) + assert.Equal(t, "2024/10/9", actualData.bankAccountTransactions[0].date) + assert.Equal(t, "-123.45", actualData.bankAccountTransactions[0].amount) + assert.Equal(t, "2024/10/12", actualData.bankAccountTransactions[1].date) + assert.Equal(t, "+234.56", actualData.bankAccountTransactions[1].amount) + + assert.Equal(t, 1, len(actualData.cashAccountTransactions)) + assert.Equal(t, "2024/9/1", actualData.cashAccountTransactions[0].date) + assert.Equal(t, "100.00", actualData.cashAccountTransactions[0].amount) + assert.Equal(t, "Opening Balance", actualData.cashAccountTransactions[0].payee) + assert.Equal(t, "[Wallet]", actualData.cashAccountTransactions[0].category) + + assert.Equal(t, 1, len(actualData.memorizedTransactions)) + assert.Equal(t, qifCheckTransactionType, actualData.memorizedTransactions[0].transactionType) + assert.Equal(t, "-123.45", actualData.memorizedTransactions[0].amount) + assert.Equal(t, "2024/10/13", actualData.memorizedTransactions[0].amortization.firstPaymentDate) + assert.Equal(t, "3", actualData.memorizedTransactions[0].amortization.totalYearsForLoan) + assert.Equal(t, "1", actualData.memorizedTransactions[0].amortization.numberOfPayments) + assert.Equal(t, "2", actualData.memorizedTransactions[0].amortization.numberOfPeriodsPerYear) + assert.Equal(t, "12.34", actualData.memorizedTransactions[0].amortization.interestRate) + assert.Equal(t, "100.45", actualData.memorizedTransactions[0].amortization.currentLoanBalance) + assert.Equal(t, "234.56", actualData.memorizedTransactions[0].amortization.originalLoanAmount) + + assert.Equal(t, 1, len(actualData.investmentAccountTransactions)) + assert.Equal(t, "2024/10/14", actualData.investmentAccountTransactions[0].date) + assert.Equal(t, "Buy", actualData.investmentAccountTransactions[0].action) + assert.Equal(t, "Test", actualData.investmentAccountTransactions[0].security) + assert.Equal(t, "12.34", actualData.investmentAccountTransactions[0].price) + assert.Equal(t, "10", actualData.investmentAccountTransactions[0].quantity) + assert.Equal(t, "-123.4", actualData.investmentAccountTransactions[0].amount) + + assert.Equal(t, 2, len(actualData.accounts)) + assert.Equal(t, "Test Account", actualData.accounts[0].name) + assert.Equal(t, "Wallet", actualData.accounts[1].name) + + assert.Equal(t, 1, len(actualData.categories)) + assert.Equal(t, "Test Category", actualData.categories[0].name) + assert.Equal(t, qifIncomeTransaction, actualData.categories[0].categoryType) + + assert.Equal(t, 1, len(actualData.classes)) + assert.Equal(t, "Test Class", actualData.classes[0].name) + assert.Equal(t, "Foo Bar", actualData.classes[0].description) +} + +func TestQifDataReaderParse_AccountEntryBeforeTransaction(t *testing.T) { + reader := &qifDataReader{ + allLines: []string{ + "!Account", + "NTest Account", + "^", + "!Type:Bank", + "D2024/10/9", + "T-123.45", + "^", + "D2024/10/12", + "T+234.56", + "^", + "!Account", + "NWallet", + "^", + "!Type:Cash", + "D2024/9/1", + "T100.00", + "POpening Balance", + "L[Wallet]", + "^", + }, + } + context := core.NewNullContext() + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 2, len(actualData.bankAccountTransactions)) + assert.Equal(t, "2024/10/9", actualData.bankAccountTransactions[0].date) + assert.Equal(t, "-123.45", actualData.bankAccountTransactions[0].amount) + assert.Equal(t, "2024/10/12", actualData.bankAccountTransactions[1].date) + assert.Equal(t, "+234.56", actualData.bankAccountTransactions[1].amount) + + assert.Equal(t, 1, len(actualData.cashAccountTransactions)) + assert.Equal(t, "2024/9/1", actualData.cashAccountTransactions[0].date) + assert.Equal(t, "100.00", actualData.cashAccountTransactions[0].amount) + assert.Equal(t, "Opening Balance", actualData.cashAccountTransactions[0].payee) + assert.Equal(t, "[Wallet]", actualData.cashAccountTransactions[0].category) + + assert.Equal(t, 2, len(actualData.accounts)) + assert.Equal(t, "Test Account", actualData.accounts[0].name) + assert.Equal(t, "Wallet", actualData.accounts[1].name) +} + +func TestQifDataReaderParse_EmptyContent(t *testing.T) { + reader := &qifDataReader{ + allLines: []string{}, + } + context := core.NewNullContext() + + _, err := reader.read(context) + assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) +} + +func TestQifDataReaderParse_EmptyEntry(t *testing.T) { + reader := &qifDataReader{ + allLines: []string{ + "!Type:Bank", + "^", + "!Type:Cash", + "^", + "!Type:Memorized", + "^", + "!Type:Invst", + "^", + "!Account", + "^", + "!Type:Cat", + "^", + "!Type:Class", + "^", + }, + } + context := core.NewNullContext() + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 0, len(actualData.bankAccountTransactions)) + assert.Equal(t, 0, len(actualData.cashAccountTransactions)) + assert.Equal(t, 0, len(actualData.memorizedTransactions)) + assert.Equal(t, 0, len(actualData.investmentAccountTransactions)) + assert.Equal(t, 0, len(actualData.accounts)) + assert.Equal(t, 0, len(actualData.categories)) + assert.Equal(t, 0, len(actualData.classes)) +} + +func TestQifDataReaderParse_NewEntryHeaderAfterUnclosedEntry(t *testing.T) { + reader := &qifDataReader{ + allLines: []string{ + "!Type:Bank", + "D2024/10/9", + "T-123.45", + "!Type:Cash", + "D2024/9/1", + "T100.00", + "POpening Balance", + "L[Wallet]", + "^", + }, + } + context := core.NewNullContext() + + _, err := reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidQIFFile.Message) +} + +func TestQifDataReaderParseTransaction_SupportedFields(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualData, err := reader.parseTransaction(context, []string{ + "D2024/10/12", + "T-123.45", + "C", + "N100", + "PFoo", + "MBar", + "AAddress 1", + "AAddress 2", + "AAddress 3", + "LTest Category", + "SPart1 Category", + "EPart1 Memo", + "$-100.00", + "SPart2 Category", + "EPart2 Memo", + "$-23.45", + }) + + assert.Nil(t, err) + assert.Equal(t, "2024/10/12", actualData.date) + assert.Equal(t, "-123.45", actualData.amount) + assert.Equal(t, qifClearedStatusUnreconciled, actualData.clearedStatus) + assert.Equal(t, "100", actualData.num) + assert.Equal(t, "Foo", actualData.payee) + assert.Equal(t, "Bar", actualData.memo) + assert.Equal(t, 3, len(actualData.addresses)) + assert.Equal(t, "Address 1", actualData.addresses[0]) + assert.Equal(t, "Address 2", actualData.addresses[1]) + assert.Equal(t, "Address 3", actualData.addresses[2]) + assert.Equal(t, "Test Category", actualData.category) + assert.Equal(t, 2, len(actualData.subTransactionCategory)) + assert.Equal(t, "Part1 Category", actualData.subTransactionCategory[0]) + assert.Equal(t, "Part2 Category", actualData.subTransactionCategory[1]) + assert.Equal(t, 2, len(actualData.subTransactionMemo)) + assert.Equal(t, "Part1 Memo", actualData.subTransactionMemo[0]) + assert.Equal(t, "Part2 Memo", actualData.subTransactionMemo[1]) + assert.Equal(t, 2, len(actualData.subTransactionAmount)) + assert.Equal(t, "-100.00", actualData.subTransactionAmount[0]) + assert.Equal(t, "-23.45", actualData.subTransactionAmount[1]) +} + +func TestQifDataReaderParseMemorizedTransaction_SupportedFields(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualData, err := reader.parseMemorizedTransaction(context, []string{ + "KC", + "D2024/10/12", + "T-123.45", + "C*", + "N100", + "PFoo", + "MBar", + "12024/10/13", + "23", + "31", + "42", + "512.34", + "6100.45", + "7234.56", + }) + + assert.Nil(t, err) + assert.Equal(t, qifCheckTransactionType, actualData.transactionType) + assert.Equal(t, "2024/10/12", actualData.date) + assert.Equal(t, "-123.45", actualData.amount) + assert.Equal(t, qifClearedStatusCleared, actualData.clearedStatus) + assert.Equal(t, "100", actualData.num) + assert.Equal(t, "Foo", actualData.payee) + assert.Equal(t, "Bar", actualData.memo) + assert.Equal(t, "2024/10/13", actualData.amortization.firstPaymentDate) + assert.Equal(t, "3", actualData.amortization.totalYearsForLoan) + assert.Equal(t, "1", actualData.amortization.numberOfPayments) + assert.Equal(t, "2", actualData.amortization.numberOfPeriodsPerYear) + assert.Equal(t, "12.34", actualData.amortization.interestRate) + assert.Equal(t, "100.45", actualData.amortization.currentLoanBalance) + assert.Equal(t, "234.56", actualData.amortization.originalLoanAmount) + + actualData, err = reader.parseMemorizedTransaction(context, []string{"KD"}) + assert.Nil(t, err) + assert.Equal(t, qifDepositTransactionType, actualData.transactionType) + + actualData, err = reader.parseMemorizedTransaction(context, []string{"KP"}) + assert.Nil(t, err) + assert.Equal(t, qifPaymentTransactionType, actualData.transactionType) + + actualData, err = reader.parseMemorizedTransaction(context, []string{"KI"}) + assert.Nil(t, err) + assert.Equal(t, qifInvestmentTransactionType, actualData.transactionType) + + actualData, err = reader.parseMemorizedTransaction(context, []string{"KE"}) + assert.Nil(t, err) + assert.Equal(t, qifElectronicPayeeTransactionType, actualData.transactionType) +} + +func TestQifDataReaderParseInvestmentTransaction_SupportedFields(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualData, err := reader.parseInvestmentTransaction(context, []string{ + "D2024/10/12", + "NBuy", + "YTest", + "I12.34", + "Q10", + "T-123.4", + "CR", + "PFoo", + "MBar", + "OTest2", + "LAccount Name", + "$100", + }) + + assert.Nil(t, err) + assert.Equal(t, "2024/10/12", actualData.date) + assert.Equal(t, "Buy", actualData.action) + assert.Equal(t, "Test", actualData.security) + assert.Equal(t, "12.34", actualData.price) + assert.Equal(t, "10", actualData.quantity) + assert.Equal(t, "-123.4", actualData.amount) + assert.Equal(t, qifClearedStatusReconciled, actualData.clearedStatus) + assert.Equal(t, "Foo", actualData.text) + assert.Equal(t, "Bar", actualData.memo) + assert.Equal(t, "Test2", actualData.commission) + assert.Equal(t, "Account Name", actualData.accountForTransfer) + assert.Equal(t, "100", actualData.amountTransferred) +} + +func TestQifDataReaderParseAccount_SupportedFields(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualData, err := reader.parseAccount(context, []string{ + "NAccount Name", + "TAccount Type", + "DSome Text", + "L1234.56", + "/2024/10/12", + "$123.45", + }) + + assert.Nil(t, err) + assert.Equal(t, "Account Name", actualData.name) + assert.Equal(t, "Account Type", actualData.accountType) + assert.Equal(t, "Some Text", actualData.description) + assert.Equal(t, "1234.56", actualData.creditLimit) + assert.Equal(t, "2024/10/12", actualData.statementBalanceDate) + assert.Equal(t, "123.45", actualData.statementBalanceAmount) +} + +func TestQifDataReaderParseCategory_SupportedFields(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualData, err := reader.parseCategory(context, []string{ + "NCategory Name:Sub Category Name", + "DSome Text", + "T", + "I", + "B123.45", + "RTest", + }) + + assert.Nil(t, err) + assert.Equal(t, "Category Name:Sub Category Name", actualData.name) + assert.Equal(t, "Some Text", actualData.description) + assert.Equal(t, true, actualData.taxRelated) + assert.Equal(t, qifIncomeTransaction, actualData.categoryType) + assert.Equal(t, "123.45", actualData.budgetAmount) + assert.Equal(t, "Test", actualData.taxScheduleInformation) + + actualData2, err := reader.parseCategory(context, []string{ + "NCategory Name:Sub Category Name", + "DSome Text", + "E", + }) + + assert.Nil(t, err) + assert.Equal(t, "Category Name:Sub Category Name", actualData2.name) + assert.Equal(t, "Some Text", actualData2.description) + assert.Equal(t, false, actualData2.taxRelated) + assert.Equal(t, qifExpenseTransaction, actualData2.categoryType) + + actualData3, err := reader.parseCategory(context, []string{ + "NCategory Name:Sub Category Name", + "DSome Text", + }) + + assert.Nil(t, err) + assert.Equal(t, "Category Name:Sub Category Name", actualData3.name) + assert.Equal(t, "Some Text", actualData3.description) + assert.Equal(t, qifExpenseTransaction, actualData3.categoryType) +} + +func TestQifDataReaderParseClass_SupportedFields(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualData, err := reader.parseClass(context, []string{ + "NClass Name", + "DSome Text", + }) + + assert.Nil(t, err) + assert.Equal(t, "Class Name", actualData.name) + assert.Equal(t, "Some Text", actualData.description) +} + +func TestQifDataReaderParse_UnsupportedFieldsOrValues(t *testing.T) { + reader := &qifDataReader{} + context := core.NewNullContext() + + actualTransactionData, err := reader.parseTransaction(context, []string{ + "ZTest", + "CZ", + "", + }) + assert.Nil(t, err) + assert.Equal(t, qifClearedStatusUnreconciled, actualTransactionData.clearedStatus) + + actualMemorizedTransactionData, err := reader.parseMemorizedTransaction(context, []string{ + "ZTest", + "KZ", + "", + }) + assert.Nil(t, err) + assert.Equal(t, qifInvalidTransactionType, actualMemorizedTransactionData.transactionType) + + _, err = reader.parseInvestmentTransaction(context, []string{ + "ZTest", + "", + }) + assert.Nil(t, err) + + _, err = reader.parseAccount(context, []string{ + "ZTest", + "", + }) + assert.Nil(t, err) + + _, err = reader.parseCategory(context, []string{ + "ZTest", + "", + }) + assert.Nil(t, err) + + _, err = reader.parseClass(context, []string{ + "ZTest", + "", + }) + assert.Nil(t, err) +} diff --git a/pkg/converters/qif/qif_transaction_data_file_importer.go b/pkg/converters/qif/qif_transaction_data_file_importer.go new file mode 100644 index 00000000..b429791d --- /dev/null +++ b/pkg/converters/qif/qif_transaction_data_file_importer.go @@ -0,0 +1,55 @@ +package qif + +import ( + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +var qifTransactionTypeNameMapping = map[models.TransactionType]string{ + models.TRANSACTION_TYPE_MODIFY_BALANCE: utils.IntToString(int(models.TRANSACTION_TYPE_MODIFY_BALANCE)), + models.TRANSACTION_TYPE_INCOME: utils.IntToString(int(models.TRANSACTION_TYPE_INCOME)), + models.TRANSACTION_TYPE_EXPENSE: utils.IntToString(int(models.TRANSACTION_TYPE_EXPENSE)), + models.TRANSACTION_TYPE_TRANSFER: utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)), +} + +// qifTransactionDataImporter defines the structure of quicken interchange format (qif) importer for transaction data +type qifTransactionDataImporter struct { + dateFormatType qifDateFormatType +} + +// Initialize a quicken interchange format (qif) transaction data importer singleton instance +var ( + QifYearMonthDayTransactionDataImporter = &qifTransactionDataImporter{ + dateFormatType: qifYearMonthDayDateFormat, + } + + QifMonthDayYearTransactionDataImporter = &qifTransactionDataImporter{ + dateFormatType: qifMonthDayYearDateFormat, + } + + QifDayMonthYearTransactionDataImporter = &qifTransactionDataImporter{ + dateFormatType: qifDayMonthYearDateFormat, + } +) + +// ParseImportedData returns the imported data by parsing the quicken interchange format (qif) transaction data +func (c *qifTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { + qifDataReader := createNewQifDataReader(data) + qifData, err := qifDataReader.read(ctx) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + transactionDataTable, err := createNewQifTransactionDataTable(c.dateFormatType, qifData) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + dataTableImporter := datatable.CreateNewSimpleImporter(qifTransactionTypeNameMapping) + + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) +} diff --git a/pkg/converters/qif/qif_transaction_data_file_importer_test.go b/pkg/converters/qif/qif_transaction_data_file_importer_test.go new file mode 100644 index 00000000..49143a54 --- /dev/null +++ b/pkg/converters/qif/qif_transaction_data_file_importer_test.go @@ -0,0 +1,437 @@ +package qif + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +func TestQIFTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T123.45\n"+ + "POpening Balance\n"+ + "L[Test Account]\n"+ + "^\n"+ + "D2024-09-02\n"+ + "T0.12\n"+ + "LTest Category\n"+ + "^\n"+ + "D2024-09-03\n"+ + "T-1.00\n"+ + "LTest Category2\n"+ + "^\n"+ + "D2024-09-04\n"+ + "T-0.05\n"+ + "L[Test Account2]\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 3, len(allNewAccounts)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) + assert.Equal(t, 0, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + assert.Equal(t, "", allNewTransactions[1].OriginalSourceAccountName) + assert.Equal(t, "Test Category", allNewTransactions[1].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + assert.Equal(t, "", allNewTransactions[2].OriginalSourceAccountName) + assert.Equal(t, "Test Category2", allNewTransactions[2].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, "", allNewTransactions[3].OriginalSourceAccountName) + assert.Equal(t, "Test Account2", allNewTransactions[3].OriginalDestinationAccountName) + assert.Equal(t, "", allNewTransactions[3].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "", allNewAccounts[1].Name) + assert.Equal(t, "CNY", allNewAccounts[1].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[2].Uid) + assert.Equal(t, "Test Account2", allNewAccounts[2].Name) + assert.Equal(t, "CNY", allNewAccounts[2].Currency) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "", allNewSubTransferCategories[0].Name) +} + +func TestQIFTransactionDataFileParseImportedData_ParseYearMonthDayDateFormatTime(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "^\n"+ + "D2024-9-2\n"+ + "T-123.45\n"+ + "^\n"+ + "D2024/9/3\n"+ + "T-123.45\n"+ + "^\n"+ + "D2024.9.4\n"+ + "T-123.45\n"+ + "^\n"+ + "D2024'9.5\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 5, len(allNewTransactions)) + + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(1725494400), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime)) +} + +func TestQIFTransactionDataFileParseImportedData_ParseMonthDayYearDateFormatTime(t *testing.T) { + converter := QifMonthDayYearTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D09-01-2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D9-2-2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D9/3/2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D9.4.2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D9.5'2024\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 5, len(allNewTransactions)) + + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(1725494400), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime)) +} + +func TestQIFTransactionDataFileParseImportedData_ParseDayYearMonthDateFormatTime(t *testing.T) { + converter := QifDayMonthYearTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D01-09-2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D2-9-2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D3/9/2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D4.9.2024\n"+ + "T-123.45\n"+ + "^\n"+ + "D5'9.2024\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 5, len(allNewTransactions)) + + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(1725494400), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime)) +} + +func TestQIFTransactionDataFileParseImportedData_ParseInvalidTime(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024 09 01\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) +} + +func TestQIFTransactionDataFileParseImportedData_ParseInvalidAmount(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T-123 45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} + +func TestQIFTransactionDataFileParseImportedData_ParseAccountType(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Cash\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "!Type:CCard\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "!Type:Oth A\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "!Type:Oth L\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) +} + +func TestQIFTransactionDataFileParseImportedData_ParseAccount(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Account\n"+ + "NTest Account\n"+ + "^\n"+ + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, 1, len(allNewAccounts)) + + assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) +} + +func TestQIFTransactionDataFileParseImportedData_ParseAmountWithLeadingPlusSing(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T+123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) +} + +func TestQIFTransactionDataFileParseImportedData_ParseSubCategory(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, allNewSubExpenseCategories, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "LTest Category:Sub Category\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + + assert.Equal(t, "Sub Category", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Sub Category", allNewSubExpenseCategories[0].Name) +} + +func TestQIFTransactionDataFileParseImportedData_ParseDescription(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "T-123.45\n"+ + "PTest\n"+ + "Mfoo bar\t#test\n"+ + "^\n"+ + "D2024-09-02\n"+ + "T-234.56\n"+ + "PTest2\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 2, len(allNewTransactions)) + assert.Equal(t, "foo bar\t#test", allNewTransactions[0].Comment) + assert.Equal(t, "Test2", allNewTransactions[1].Comment) +} + +func TestQIFTransactionDataFileParseImportedData_MissingRequiredFields(t *testing.T) { + converter := QifYearMonthDayTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1, + DefaultCurrency: "CNY", + } + + // Missing Time Field + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "T-123.45\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) + + // Missing Amount Field + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "!Type:Bank\n"+ + "D2024-09-01\n"+ + "^\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} diff --git a/pkg/converters/qif/qif_transaction_data_table.go b/pkg/converters/qif/qif_transaction_data_table.go new file mode 100644 index 00000000..8bd3cf52 --- /dev/null +++ b/pkg/converters/qif/qif_transaction_data_table.go @@ -0,0 +1,241 @@ +package qif + +import ( + "fmt" + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +const qifOpeningBalancePayeeText = "Opening Balance" + +var qifTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true, + datatable.TRANSACTION_DATA_TABLE_CATEGORY: true, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true, +} + +// qifDateFormatType represents the quicken interchange format (qif) date format type +type qifDateFormatType byte + +const ( + qifYearMonthDayDateFormat qifDateFormatType = 0 + qifMonthDayYearDateFormat qifDateFormatType = 1 + qifDayMonthYearDateFormat qifDateFormatType = 2 +) + +// qifTransactionDataTable defines the structure of quicken interchange format (qif) transaction data table +type qifTransactionDataTable struct { + dateFormatType qifDateFormatType + allData []*qifTransactionData +} + +// qifTransactionDataRow defines the structure of quicken interchange format (qif) transaction data row +type qifTransactionDataRow struct { + dataTable *qifTransactionDataTable + data *qifTransactionData + finalItems map[datatable.TransactionDataTableColumn]string +} + +// qifTransactionDataRowIterator defines the structure of quicken interchange format (qif) transaction data row iterator +type qifTransactionDataRowIterator struct { + dataTable *qifTransactionDataTable + currentIndex int +} + +// HasColumn returns whether the transaction data table has specified column +func (t *qifTransactionDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool { + _, exists := qifTransactionSupportedColumns[column] + return exists +} + +// TransactionRowCount returns the total count of transaction data row +func (t *qifTransactionDataTable) TransactionRowCount() int { + return len(t.allData) +} + +// TransactionRowIterator returns the iterator of transaction data row +func (t *qifTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator { + return &qifTransactionDataRowIterator{ + dataTable: t, + currentIndex: -1, + } +} + +// IsValid returns whether this row is valid data for importing +func (r *qifTransactionDataRow) IsValid() bool { + return true +} + +// GetData returns the data in the specified column type +func (r *qifTransactionDataRow) GetData(column datatable.TransactionDataTableColumn) string { + _, exists := qifTransactionSupportedColumns[column] + + if exists { + return r.finalItems[column] + } + + return "" +} + +// HasNext returns whether the iterator does not reach the end +func (t *qifTransactionDataRowIterator) HasNext() bool { + return t.currentIndex+1 < len(t.dataTable.allData) +} + +// Next returns the next imported data row +func (t *qifTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { + if t.currentIndex+1 >= len(t.dataTable.allData) { + return nil, nil + } + + t.currentIndex++ + + data := t.dataTable.allData[t.currentIndex] + rowItems, err := t.parseTransaction(ctx, user, data) + + if err != nil { + return nil, err + } + + return &qifTransactionDataRow{ + dataTable: t.dataTable, + data: data, + finalItems: rowItems, + }, nil +} + +func (t *qifTransactionDataRowIterator) parseTransaction(ctx core.Context, user *models.User, qifTransaction *qifTransactionData) (map[datatable.TransactionDataTableColumn]string, error) { + data := make(map[datatable.TransactionDataTableColumn]string, len(qifTransactionSupportedColumns)) + + transactionTime, err := t.parseTransactionTime(ctx, qifTransaction.date) + + if err != nil { + return nil, err + } + + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = transactionTime + + if qifTransaction.amount == "" { + return nil, errs.ErrAmountInvalid + } + + amount, err := utils.ParseAmount(strings.ReplaceAll(qifTransaction.amount, ",", "")) + + if err != nil { + return nil, errs.ErrAmountInvalid + } + + if qifTransaction.account != nil { + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = qifTransaction.account.name + } else { + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = "" + } + + if len(qifTransaction.category) > 0 && qifTransaction.category[0] == '[' && qifTransaction.category[len(qifTransaction.category)-1] == ']' { + if qifTransaction.payee == qifOpeningBalancePayeeText { // balance modification + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = qifTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount) + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = qifTransaction.category[1 : len(qifTransaction.category)-1] + } else { // transfer to [account name] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = qifTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = qifTransaction.category[1 : len(qifTransaction.category)-1] + } + } else { // income/expense + if amount >= 0 { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = qifTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount) + } else { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = qifTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + } + + if strings.Index(qifTransaction.category, ":") > 0 { // category:subcategory + categories := strings.Split(qifTransaction.category, ":") + data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = categories[0] + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categories[len(categories)-1] + } else { + data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = "" + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = qifTransaction.category + } + } + + if qifTransaction.memo != "" { + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = qifTransaction.memo + } else if qifTransaction.payee != "" && qifTransaction.payee != qifOpeningBalancePayeeText { + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = qifTransaction.payee + } + + return data, nil +} + +func (t *qifTransactionDataRowIterator) parseTransactionTime(ctx core.Context, date string) (string, error) { + var year, month, day string + + if (t.dataTable.dateFormatType == qifYearMonthDayDateFormat && utils.IsValidYearMonthDayLongOrShortDateFormat(date)) || + (t.dataTable.dateFormatType == qifMonthDayYearDateFormat && utils.IsValidMonthDayYearLongOrShortDateFormat(date)) || + (t.dataTable.dateFormatType == qifDayMonthYearDateFormat && utils.IsValidDayMonthYearLongOrShortDateFormat(date)) { + date = strings.ReplaceAll(date, ".", "-") + date = strings.ReplaceAll(date, "/", "-") + date = strings.ReplaceAll(date, "'", "-") + items := strings.Split(date, "-") + + if t.dataTable.dateFormatType == qifYearMonthDayDateFormat { + year = items[0] + month = items[1] + day = items[2] + } else if t.dataTable.dateFormatType == qifMonthDayYearDateFormat { + month = items[0] + day = items[1] + year = items[2] + } else if t.dataTable.dateFormatType == qifDayMonthYearDateFormat { + day = items[0] + month = items[1] + year = items[2] + } + } + + if year == "" || month == "" || day == "" { + log.Errorf(ctx, "[qif_transaction_data_table.parseTransactionTime] cannot parse date \"%s\"", date) + return "", errs.ErrTransactionTimeInvalid + } + + if len(month) < 2 { + month = "0" + month + } + + if len(day) < 2 { + day = "0" + day + } + + return fmt.Sprintf("%s-%s-%s 00:00:00", year, month, day), nil +} + +func createNewQifTransactionDataTable(dateFormatType qifDateFormatType, qifData *qifData) (*qifTransactionDataTable, error) { + if qifData == nil { + return nil, errs.ErrNotFoundTransactionDataInFile + } + + allData := make([]*qifTransactionData, 0) + allData = append(allData, qifData.bankAccountTransactions...) + allData = append(allData, qifData.cashAccountTransactions...) + allData = append(allData, qifData.creditCardAccountTransactions...) + allData = append(allData, qifData.assetAccountTransactions...) + allData = append(allData, qifData.liabilityAccountTransactions...) + + return &qifTransactionDataTable{ + dateFormatType: dateFormatType, + allData: allData, + }, nil +} diff --git a/pkg/converters/transaction_data_converters.go b/pkg/converters/transaction_data_converters.go index c4240fd0..db861369 100644 --- a/pkg/converters/transaction_data_converters.go +++ b/pkg/converters/transaction_data_converters.go @@ -6,6 +6,7 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/converters/default" "github.com/mayswind/ezbookkeeping/pkg/converters/feidee" "github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII" + "github.com/mayswind/ezbookkeeping/pkg/converters/qif" "github.com/mayswind/ezbookkeeping/pkg/converters/wechat" "github.com/mayswind/ezbookkeeping/pkg/errs" ) @@ -27,6 +28,12 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter, return _default.DefaultTransactionDataCSVFileConverter, nil } else if fileType == "ezbookkeeping_tsv" { return _default.DefaultTransactionDataTSVFileConverter, nil + } else if fileType == "qif_ymd" { + return qif.QifYearMonthDayTransactionDataImporter, nil + } else if fileType == "qif_mdy" { + return qif.QifMonthDayYearTransactionDataImporter, nil + } else if fileType == "qif_dmy" { + return qif.QifDayMonthYearTransactionDataImporter, nil } else if fileType == "firefly_iii_csv" { return fireflyIII.FireflyIIITransactionDataCsvFileImporter, nil } else if fileType == "feidee_mymoney_csv" { diff --git a/pkg/errs/converter.go b/pkg/errs/converter.go index c681f09f..d31d1dff 100644 --- a/pkg/errs/converter.go +++ b/pkg/errs/converter.go @@ -16,4 +16,5 @@ var ( ErrInvalidCSVFile = NewNormalError(NormalSubcategoryConverter, 9, http.StatusBadRequest, "invalid csv file") ErrRelatedIdCannotBeBlank = NewNormalError(NormalSubcategoryConverter, 10, http.StatusBadRequest, "related id cannot be blank") ErrFoundRecordNotHasRelatedRecord = NewNormalError(NormalSubcategoryConverter, 11, http.StatusBadRequest, "found some transactions without related records") + ErrInvalidQIFFile = NewNormalError(NormalSubcategoryConverter, 12, http.StatusBadRequest, "invalid qif file") ) diff --git a/pkg/utils/converter.go b/pkg/utils/converter.go index b339fe56..2ddc3ef5 100644 --- a/pkg/utils/converter.go +++ b/pkg/utils/converter.go @@ -140,6 +140,9 @@ func ParseAmount(amount string) (int64, error) { if amount[0] == '-' { amount = amount[1:] sign = -1 + } else if amount[0] == '+' { + amount = amount[1:] + sign = 1 } if len(amount) < 1 { diff --git a/pkg/utils/converter_test.go b/pkg/utils/converter_test.go index 441215b6..a40cda75 100644 --- a/pkg/utils/converter_test.go +++ b/pkg/utils/converter_test.go @@ -278,6 +278,21 @@ func TestParseAmount(t *testing.T) { actualValue, err = ParseAmount("-12.34") assert.Nil(t, err) assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(0) + actualValue, err = ParseAmount("+0") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(12) + actualValue, err = ParseAmount("+0.12") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(1234) + actualValue, err = ParseAmount("+12.34") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) } func TestParseAmount_InvalidAmount(t *testing.T) { diff --git a/pkg/utils/regexps.go b/pkg/utils/regexps.go index 6f2ad057..49ec3313 100644 --- a/pkg/utils/regexps.go +++ b/pkg/utils/regexps.go @@ -3,12 +3,15 @@ package utils import "regexp" var ( - usernamePattern = regexp.MustCompile("^(?i)[a-z0-9_-]+$") - emailPattern = regexp.MustCompile("^(?i)(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])$") - hexRGBColorPattern = regexp.MustCompile("^(?i)([0-9a-f]{6}|[0-9a-f]{3})$") - longDateTimePattern = regexp.MustCompile("^([1-9][0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01]) ([0-1][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])$") - longDateTimeWithoutSecondPattern = regexp.MustCompile("^([1-9][0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01]) ([0-1][0-9]|2[0-3]):([0-5][0-9])$") - longDatePattern = regexp.MustCompile("^([1-9][0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01])$") + usernamePattern = regexp.MustCompile("^(?i)[a-z0-9_-]+$") + emailPattern = regexp.MustCompile("^(?i)(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])$") + hexRGBColorPattern = regexp.MustCompile("^(?i)([0-9a-f]{6}|[0-9a-f]{3})$") + longDateTimePattern = regexp.MustCompile("^([1-9][0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01]) ([0-1][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])$") + longDateTimeWithoutSecondPattern = regexp.MustCompile("^([1-9][0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01]) ([0-1][0-9]|2[0-3]):([0-5][0-9])$") + longDatePattern = regexp.MustCompile("^([1-9][0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01])$") + longOrShortYearMonthDayDatePattern = regexp.MustCompile("^([1-9][0-9]{3})[-/.']([1-9]|0[1-9]|1[0-2])[-/.']([1-9]|0[1-9]|1[0-9]|2[0-9]|3[01])$") + longOrShortMonthDayYearDatePattern = regexp.MustCompile("^([1-9]|0[1-9]|1[0-2])[-/.']([1-9]|0[1-9]|1[0-9]|2[0-9]|3[01])[-/.']([1-9][0-9]{3})$") + longOrShortDayMonthYearDatePattern = regexp.MustCompile("^([1-9]|0[1-9]|1[0-9]|2[0-9]|3[01])[-/.']([1-9]|0[1-9]|1[0-2])[-/.']([1-9][0-9]{3})$") ) // IsValidUsername reports whether username is valid @@ -40,3 +43,18 @@ func IsValidLongDateTimeWithoutSecondFormat(datetime string) bool { func IsValidLongDateFormat(date string) bool { return longDatePattern.MatchString(date) } + +// IsValidYearMonthDayLongOrShortDateFormat reports long date is valid format +func IsValidYearMonthDayLongOrShortDateFormat(date string) bool { + return longOrShortYearMonthDayDatePattern.MatchString(date) +} + +// IsValidMonthDayYearLongOrShortDateFormat reports long date is valid format +func IsValidMonthDayYearLongOrShortDateFormat(date string) bool { + return longOrShortMonthDayYearDatePattern.MatchString(date) +} + +// IsValidDayMonthYearLongOrShortDateFormat reports long date is valid format +func IsValidDayMonthYearLongOrShortDateFormat(date string) bool { + return longOrShortDayMonthYearDatePattern.MatchString(date) +} diff --git a/pkg/utils/regexps_test.go b/pkg/utils/regexps_test.go index 3da8d5af..d480cf66 100644 --- a/pkg/utils/regexps_test.go +++ b/pkg/utils/regexps_test.go @@ -229,3 +229,129 @@ func TestIsValidLongDateFormat_InvalidLongDateFormat(t *testing.T) { actualValue = IsValidLongDateFormat(datetime) assert.Equal(t, expectedValue, actualValue) } + +func TestIsValidYearMonthDayLongOrShortDateFormat_ValidFormat(t *testing.T) { + datetime := "2024-09-01" + expectedValue := true + actualValue := IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "2024-09-1" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "2024-9-01" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "2024-9-1" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "9999-12-31" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "2024/09/01" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "2024.09.01" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "2024'09.01" + expectedValue = true + actualValue = IsValidYearMonthDayLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) +} + +func TestIsValidMonthDayYearLongOrShortDateFormat_ValidFormat(t *testing.T) { + datetime := "09-01-2024" + expectedValue := true + actualValue := IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "09-1-2024" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "9-01-2024" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "9-1-2024" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "12-31-9999" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "09/01/2024" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "09.01.2024" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "09/01'2024" + expectedValue = true + actualValue = IsValidMonthDayYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) +} + +func TestIsValidDayMonthYearLongDateFormat_ValidLongDateFormat(t *testing.T) { + datetime := "01-09-2024" + expectedValue := true + actualValue := IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "1-09-2024" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "01-9-2024" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "1-9-2024" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "31-12-9999" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "01/09/2024" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "01.09.2024" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) + + datetime = "01/09'2024" + expectedValue = true + actualValue = IsValidDayMonthYearLongOrShortDateFormat(datetime) + assert.Equal(t, expectedValue, actualValue) +} diff --git a/src/consts/file.js b/src/consts/file.js index 20d8aee7..557e6535 100644 --- a/src/consts/file.js +++ b/src/consts/file.js @@ -28,6 +28,21 @@ const supportedImportFileTypes = [ anchor: 'how-to-get-firefly-iii-data-export-file' } }, + { + type: 'qif_ymd', + name: 'Quicken Interchange Format (QIF) File (Year-month-day format)', + extensions: '.qif' + }, + { + type: 'qif_mdy', + name: 'Quicken Interchange Format (QIF) File (Month-day-year format)', + extensions: '.qif' + }, + { + type: 'qif_dmy', + name: 'Quicken Interchange Format (QIF) File (Day-month-year format)', + extensions: '.qif' + }, { type: 'feidee_mymoney_csv', name: 'Feidee MyMoney (App) Data Export File', diff --git a/src/locales/en.json b/src/locales/en.json index d2c73671..35ad3776 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1126,6 +1126,7 @@ "invalid csv file": "Invalid CSV file", "related id cannot be blank": "Related ID cannot be blank", "found some transactions without related records": "There are some transactions which don't have related records", + "invalid qif file": "Invalid QIF file", "query items cannot be blank": "There are no query items", "query items too much": "There are too many query items", "query items have invalid item": "There is invalid item in query items", @@ -1517,6 +1518,9 @@ "How to export this file?": "How to export this file?", "ezbookkeeping Data Export File (CSV)": "ezbookkeeping Data Export File (CSV)", "ezbookkeeping Data Export File (TSV)": "ezbookkeeping Data Export File (TSV)", + "Quicken Interchange Format (QIF) File (Year-month-day format)": "Quicken Interchange Format (QIF) File (Year-month-day format)", + "Quicken Interchange Format (QIF) File (Month-day-year format)": "Quicken Interchange Format (QIF) File (Month-day-year format)", + "Quicken Interchange Format (QIF) File (Day-month-year format)": "Quicken Interchange Format (QIF) File (Day-month-year format)", "Firefly III Data Export File": "Firefly III Data Export File", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App) Data Export File", "Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web) Data Export File", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index 36561682..33812df5 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -1126,6 +1126,7 @@ "invalid csv file": "无效的 CSV 文件", "related id cannot be blank": "关联Id不能为空", "found some transactions without related records": "有一些交易没有关联记录", + "invalid qif file": "无效的 QIF 文件", "query items cannot be blank": "请求项目不能为空", "query items too much": "请求项目过多", "query items have invalid item": "请求项目中有非法项目", @@ -1517,6 +1518,9 @@ "How to export this file?": "如何导出该文件？", "ezbookkeeping Data Export File (CSV)": "ezbookkeeping 数据导出文件 (CSV)", "ezbookkeeping Data Export File (TSV)": "ezbookkeeping 数据导出文件 (TSV)", + "Quicken Interchange Format (QIF) File (Year-month-day format)": "Quicken Interchange Format (QIF) 文件 (年-月-日 格式)", + "Quicken Interchange Format (QIF) File (Month-day-year format)": "Quicken Interchange Format (QIF) 文件 (月-日-年 格式)", + "Quicken Interchange Format (QIF) File (Day-month-year format)": "Quicken Interchange Format (QIF) 文件 (日-月-年 格式)", "Firefly III Data Export File": "Firefly III 数据导出文件", "Feidee MyMoney (App) Data Export File": "金蝶随手记 (App) 数据导出文件", "Feidee MyMoney (Web) Data Export File": "金蝶随手记 (Web版) 数据导出文件",