diff --git a/pkg/converters/alipay/alipay_transaction_csv_data_table.go b/pkg/converters/alipay/alipay_transaction_csv_data_table.go index 468b0895..0bb93a58 100644 --- a/pkg/converters/alipay/alipay_transaction_csv_data_table.go +++ b/pkg/converters/alipay/alipay_transaction_csv_data_table.go @@ -2,10 +2,10 @@ package alipay import ( "encoding/csv" - "fmt" "io" "strings" + csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -53,31 +53,20 @@ type alipayTransactionColumnNames struct { // alipayTransactionDataTable defines the structure of alipay transaction plain text data table type alipayTransactionDataTable struct { - allOriginalLines [][]string - originalHeaderLineColumnNames []string - originalTimeColumnIndex int - originalCategoryColumnIndex int - originalTargetNameColumnIndex int - originalProductNameColumnIndex int - originalAmountColumnIndex int - originalTypeColumnIndex int - originalRelatedAccountColumnIndex int - originalStatusColumnIndex int - originalDescriptionColumnIndex int + innerDataTable datatable.CommonDataTable + columns alipayTransactionColumnNames } // alipayTransactionDataRow defines the structure of alipay transaction plain text data row type alipayTransactionDataRow struct { - dataTable *alipayTransactionDataTable - isValid bool - originalItems []string - finalItems map[datatable.TransactionDataTableColumn]string + isValid bool + finalItems map[datatable.TransactionDataTableColumn]string } // alipayTransactionDataRowIterator defines the structure of alipay transaction plain text data row iterator type alipayTransactionDataRowIterator struct { - dataTable *alipayTransactionDataTable - currentIndex int + dataTable *alipayTransactionDataTable + innerIterator datatable.CommonDataRowIterator } // HasColumn returns whether the transaction data table has specified column @@ -88,18 +77,14 @@ func (t *alipayTransactionDataTable) HasColumn(column datatable.TransactionDataT // TransactionRowCount returns the total count of transaction data row func (t *alipayTransactionDataTable) TransactionRowCount() int { - if len(t.allOriginalLines) < 1 { - return 0 - } - - return len(t.allOriginalLines) - 1 + return t.innerDataTable.DataRowCount() } // TransactionRowIterator returns the iterator of transaction data row func (t *alipayTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator { return &alipayTransactionDataRowIterator{ - dataTable: t, - currentIndex: 0, + dataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), } } @@ -121,94 +106,85 @@ func (r *alipayTransactionDataRow) GetData(column datatable.TransactionDataTable // HasNext returns whether the iterator does not reach the end func (t *alipayTransactionDataRowIterator) HasNext() bool { - return t.currentIndex+1 < len(t.dataTable.allOriginalLines) + return t.innerIterator.HasNext() } // Next returns the next imported data row func (t *alipayTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { - if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) { + importedRow := t.innerIterator.Next() + + if importedRow == nil { return nil, nil } - t.currentIndex++ + finalItems, isValid, err := t.dataTable.parseTransactionData(ctx, user, importedRow, t.innerIterator.CurrentRowId()) - rowItems := t.dataTable.allOriginalLines[t.currentIndex] - isValid := true - - if t.dataTable.originalTypeColumnIndex >= 0 && - rowItems[t.dataTable.originalTypeColumnIndex] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && - rowItems[t.dataTable.originalTypeColumnIndex] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && - rowItems[t.dataTable.originalTypeColumnIndex] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { - log.Warnf(ctx, "[alipay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because type is \"%s\"", t.currentIndex, rowItems[t.dataTable.originalTypeColumnIndex]) - isValid = false - } - - if t.dataTable.originalStatusColumnIndex >= 0 && - rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusSuccessName && - rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusPaymentSuccessName && - rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusRepaymentSuccessName && - rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusClosedName && - rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusRefundSuccessName && - rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusTaxRefundSuccessName { - log.Warnf(ctx, "[alipay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because status is \"%s\"", t.currentIndex, rowItems[t.dataTable.originalStatusColumnIndex]) - isValid = false + if err != nil { + return nil, err } - var finalItems map[datatable.TransactionDataTableColumn]string - var errMsg string + return &alipayTransactionDataRow{ + isValid: isValid, + finalItems: finalItems, + }, nil +} - if isValid { - finalItems, errMsg = t.dataTable.parseTransactionData(ctx, user, rowItems) +func (t *alipayTransactionDataTable) hasOriginalColumn(columnName string) bool { + return columnName != "" && t.innerDataTable.HasColumn(columnName) +} - if finalItems == nil { - log.Warnf(ctx, "[alipay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because %s", t.currentIndex, errMsg) - isValid = false - } +func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, dataRow datatable.CommonDataRow, rowId string) (map[datatable.TransactionDataTableColumn]string, bool, error) { + if dataRow.GetData(t.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && + dataRow.GetData(t.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && + dataRow.GetData(t.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + log.Warnf(ctx, "[alipay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because type is \"%s\"", rowId, dataRow.GetData(t.columns.typeColumnName)) + return nil, false, nil } - return &alipayTransactionDataRow{ - dataTable: t.dataTable, - isValid: isValid, - originalItems: rowItems, - finalItems: finalItems, - }, nil -} + if dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusSuccessName && + dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusPaymentSuccessName && + dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusRepaymentSuccessName && + dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusClosedName && + dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusRefundSuccessName && + dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusTaxRefundSuccessName { + log.Warnf(ctx, "[alipay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because status is \"%s\"", rowId, dataRow.GetData(t.columns.statusColumnName)) + return nil, false, nil + } -func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.TransactionDataTableColumn]string, string) { data := make(map[datatable.TransactionDataTableColumn]string, len(alipayTransactionSupportedColumns)) - if t.originalTimeColumnIndex >= 0 && t.originalTimeColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex] + if t.hasOriginalColumn(t.columns.timeColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = dataRow.GetData(t.columns.timeColumnName) } - if t.originalCategoryColumnIndex >= 0 && t.originalCategoryColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex] + if t.hasOriginalColumn(t.columns.categoryColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(t.columns.categoryColumnName) } else { data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" } - if t.originalAmountColumnIndex >= 0 && t.originalAmountColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex] + if t.hasOriginalColumn(t.columns.amountColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = dataRow.GetData(t.columns.amountColumnName) } - if t.originalDescriptionColumnIndex >= 0 && t.originalDescriptionColumnIndex < len(items) && items[t.originalDescriptionColumnIndex] != "" { - data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex] - } else if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) && items[t.originalProductNameColumnIndex] != "" { - data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex] + if t.hasOriginalColumn(t.columns.descriptionColumnName) && dataRow.GetData(t.columns.descriptionColumnName) != "" { + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(t.columns.descriptionColumnName) + } else if t.hasOriginalColumn(t.columns.productNameColumnName) && dataRow.GetData(t.columns.productNameColumnName) != "" { + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(t.columns.productNameColumnName) } else { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = "" } relatedAccountName := "" - if t.originalRelatedAccountColumnIndex >= 0 && t.originalRelatedAccountColumnIndex < len(items) { - relatedAccountName = items[t.originalRelatedAccountColumnIndex] + if t.hasOriginalColumn(t.columns.relatedAccountColumnName) { + relatedAccountName = dataRow.GetData(t.columns.relatedAccountColumnName) } statusName := "" - if t.originalStatusColumnIndex >= 0 && t.originalStatusColumnIndex < len(items) { - statusName = items[t.originalStatusColumnIndex] + if t.hasOriginalColumn(t.columns.statusColumnName) { + statusName = dataRow.GetData(t.columns.statusColumnName) } locale := user.Language @@ -219,12 +195,13 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user localeTextItems := locales.GetLocaleTextItems(locale) - if t.originalTypeColumnIndex >= 0 && t.originalTypeColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex] + if t.hasOriginalColumn(t.columns.typeColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(t.columns.typeColumnName) - if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { + if dataRow.GetData(t.columns.typeColumnName) == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { if statusName == alipayTransactionDataStatusClosedName { - return nil, fmt.Sprintf("income transaction is closed") + log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because income transaction is closed", rowId) + return nil, false, nil } if statusName == alipayTransactionDataStatusSuccessName { @@ -234,20 +211,21 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = "" data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" } - } else if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + } else if dataRow.GetData(t.columns.typeColumnName) == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { if statusName == alipayTransactionDataStatusClosedName { - return nil, fmt.Sprintf("non-income/expense transaction is closed") + log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because non-income/expense transaction is closed", rowId) + return nil, false, nil } targetName := "" productName := "" - if t.originalTargetNameColumnIndex >= 0 && t.originalTargetNameColumnIndex < len(items) { - targetName = items[t.originalTargetNameColumnIndex] + if t.hasOriginalColumn(t.columns.targetNameColumnName) { + targetName = dataRow.GetData(t.columns.targetNameColumnName) } - if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) { - productName = items[t.originalProductNameColumnIndex] + if t.hasOriginalColumn(t.columns.productNameColumnName) { + productName = dataRow.GetData(t.columns.productNameColumnName) } if statusName == alipayTransactionDataStatusRefundSuccessName { @@ -271,7 +249,8 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName } else { - return nil, fmt.Sprintf("product name (\"%s\") is unknown", productName) + log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because product name (\"%s\") is unknown", rowId, productName) + return nil, false, nil } } } else { @@ -291,79 +270,33 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user } } - return data, "" + return data, true, nil } func createNewAlipayTransactionDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune, originalColumnNames alipayTransactionColumnNames) (*alipayTransactionDataTable, error) { - allOriginalLines, err := parseAllLinesFromAlipayTransactionPlainText(ctx, reader, fileHeaderLine, dataHeaderStartContent, dataBottomEndLineRune) + dataTable, err := createNewAlipayImportedDataTable(ctx, reader, fileHeaderLine, dataHeaderStartContent, dataBottomEndLineRune) if err != nil { return nil, err } - if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayTransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1") - return nil, errs.ErrNotFoundTransactionDataInFile - } - - originalHeaderItems := allOriginalLines[0] - originalHeaderItemMap := make(map[string]int) - - for i := 0; i < len(originalHeaderItems); i++ { - originalHeaderItemMap[originalHeaderItems[i]] = i - } + commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable) - timeColumnIdx, timeColumnExists := originalHeaderItemMap[originalColumnNames.timeColumnName] - categoryColumnIdx, categoryColumnExists := originalHeaderItemMap[originalColumnNames.categoryColumnName] - targetNameColumnIdx, targetNameColumnExists := originalHeaderItemMap[originalColumnNames.targetNameColumnName] - productNameColumnIdx, productNameColumnExists := originalHeaderItemMap[originalColumnNames.productNameColumnName] - amountColumnIdx, amountColumnExists := originalHeaderItemMap[originalColumnNames.amountColumnName] - typeColumnIdx, typeColumnExists := originalHeaderItemMap[originalColumnNames.typeColumnName] - relatedAccountColumnIdx, relatedAccountColumnExists := originalHeaderItemMap[originalColumnNames.relatedAccountColumnName] - statusColumnIdx, statusColumnExists := originalHeaderItemMap[originalColumnNames.statusColumnName] - descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap[originalColumnNames.descriptionColumnName] - - if !timeColumnExists || !amountColumnExists || !typeColumnExists || !statusColumnExists { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayTransactionPlainTextDataTable] cannot parse alipay csv data, because missing essential columns in header row") + if !commonDataTable.HasColumn(originalColumnNames.timeColumnName) || + !commonDataTable.HasColumn(originalColumnNames.amountColumnName) || + !commonDataTable.HasColumn(originalColumnNames.typeColumnName) || + !commonDataTable.HasColumn(originalColumnNames.statusColumnName) { + log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayTransactionDataTable] cannot parse alipay csv data, because missing essential columns in header row") return nil, errs.ErrMissingRequiredFieldInHeaderRow } - if originalColumnNames.categoryColumnName == "" || !categoryColumnExists { - categoryColumnIdx = -1 - } - - if originalColumnNames.targetNameColumnName == "" || !targetNameColumnExists { - targetNameColumnIdx = -1 - } - - if originalColumnNames.productNameColumnName == "" || !productNameColumnExists { - productNameColumnIdx = -1 - } - - if originalColumnNames.relatedAccountColumnName == "" || !relatedAccountColumnExists { - relatedAccountColumnIdx = -1 - } - - if originalColumnNames.descriptionColumnName == "" || !descriptionColumnExists { - descriptionColumnIdx = -1 - } - return &alipayTransactionDataTable{ - allOriginalLines: allOriginalLines, - originalHeaderLineColumnNames: originalHeaderItems, - originalTimeColumnIndex: timeColumnIdx, - originalCategoryColumnIndex: categoryColumnIdx, - originalTargetNameColumnIndex: targetNameColumnIdx, - originalProductNameColumnIndex: productNameColumnIdx, - originalAmountColumnIndex: amountColumnIdx, - originalTypeColumnIndex: typeColumnIdx, - originalRelatedAccountColumnIndex: relatedAccountColumnIdx, - originalStatusColumnIndex: statusColumnIdx, - originalDescriptionColumnIndex: descriptionColumnIdx, + innerDataTable: commonDataTable, + columns: originalColumnNames, }, nil } -func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) ([][]string, error) { +func createNewAlipayImportedDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) (datatable.ImportedDataTable, error) { csvReader := csv.NewReader(reader) csvReader.FieldsPerRecord = -1 @@ -379,7 +312,7 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea } if err != nil { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.parseAllLinesFromAlipayTransactionPlainText] cannot parse alipay csv data, because %s", err.Error()) + log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse alipay csv data, because %s", err.Error()) return nil, errs.ErrInvalidCSVFile } @@ -390,7 +323,7 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea hasFileHeader = true continue } else { - log.Warnf(ctx, "[alipay_transaction_csv_data_table.parseAllLinesFromAlipayTransactionPlainText] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) + log.Warnf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) continue } } @@ -418,7 +351,7 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea } if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.parseAllLinesFromAlipayTransactionPlainText] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) + log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } @@ -430,5 +363,12 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea return nil, errs.ErrInvalidFileHeader } - return allOriginalLines, nil + if len(allOriginalLines) < 2 { + log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse import data, because data table row count is less 1") + return nil, errs.ErrNotFoundTransactionDataInFile + } + + dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines) + + return dataTable, nil } diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go index cf19a934..03ecb004 100644 --- a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go @@ -364,6 +364,56 @@ func TestAlipayCsvFileImporterParseImportedData_ParseCategory(t *testing.T) { assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) } +func TestAlipayCsvFileImporterParseImportedData_ParseRelatedAccount(t *testing.T) { + converter := AlipayAppTransactionDataCsvFileImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + data1, err := simplifiedchinese.GB18030.NewEncoder().String("------------------------------------------------------------------------------------\n" + + "导出信息：\n" + + "姓名：xxx\n" + + "支付宝账户：xxx@xxx.xxx\n" + + "起始时间：[2024-01-01 00:00:00] 终止时间：[2024-09-01 23:59:59]\n" + + "导出交易类型：[全部]\n" + + "------------------------支付宝（中国）网络技术有限公司 电子客户回单------------------------\n" + + "交易时间,商品说明,收/支,金额,收/付款方式,交易状态,\n" + + "2024-09-01 03:45:07,余额宝-单次转入,不计收支,0.01,Test Account,交易成功,\n" + + "2024-09-01 05:07:29,信用卡还款,不计收支,0.02,Test Account2,交易成功,\n") + assert.Nil(t, err) + + allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + + assert.Equal(t, 2, len(allNewTransactions)) + assert.Equal(t, 3, len(allNewAccounts)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, int64(1), allNewTransactions[0].Amount) + assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[0].OriginalDestinationAccountName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, int64(2), allNewTransactions[1].Amount) + assert.Equal(t, "Test Account2", allNewTransactions[1].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[1].OriginalDestinationAccountName) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "", allNewAccounts[1].Name) + assert.Equal(t, "CNY", allNewAccounts[1].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[2].Uid) + assert.Equal(t, "Test Account2", allNewAccounts[2].Name) + assert.Equal(t, "CNY", allNewAccounts[2].Currency) +} + func TestAlipayCsvFileImporterParseImportedData_ParseDescription(t *testing.T) { converter := AlipayWebTransactionDataCsvFileImporter context := core.NewNullContext() @@ -479,3 +529,23 @@ func TestAlipayCsvFileImporterParseImportedData_MissingRequiredColumn(t *testing _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data4), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) } + +func TestAlipayCsvFileImporterParseImportedData_NoTransactionData(t *testing.T) { + converter := AlipayWebTransactionDataCsvFileImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1, + DefaultCurrency: "CNY", + } + + // Missing Time Column + data1, err := simplifiedchinese.GB18030.NewEncoder().String("支付宝交易记录明细查询\n" + + "账号:[xxx@xxx.xxx]\n" + + "起始日期:[2024-01-01 00:00:00] 终止日期:[2024-09-01 23:59:59]\n" + + "---------------------------------交易记录明细列表------------------------------------\n" + + "交易创建时间 ,金额（元）,收/支 ,交易状态 ,\n" + + "------------------------------------------------------------------------------------\n") + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) +} diff --git a/pkg/converters/csv/csv_file_imported_data_table.go b/pkg/converters/csv/csv_file_imported_data_table.go index c0d50d30..4816f644 100644 --- a/pkg/converters/csv/csv_file_imported_data_table.go +++ b/pkg/converters/csv/csv_file_imported_data_table.go @@ -2,6 +2,7 @@ package csv import ( "encoding/csv" + "fmt" "io" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" @@ -72,6 +73,11 @@ func (t *CsvFileImportedDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allLines) } +// CurrentRowId returns current index +func (t *CsvFileImportedDataRowIterator) CurrentRowId() string { + return fmt.Sprintf("line#%d", t.currentIndex) +} + // Next returns the next imported data row func (t *CsvFileImportedDataRowIterator) Next() datatable.ImportedDataRow { if t.currentIndex+1 >= len(t.dataTable.allLines) { @@ -88,11 +94,18 @@ func (t *CsvFileImportedDataRowIterator) Next() datatable.ImportedDataRow { } } -// CreateNewCsvDataTable returns comma separated values data table by io readers -func CreateNewCsvDataTable(ctx core.Context, reader io.Reader) (*CsvFileImportedDataTable, error) { +// CreateNewCsvImportedDataTable returns comma separated values data table by io readers +func CreateNewCsvImportedDataTable(ctx core.Context, reader io.Reader) (*CsvFileImportedDataTable, error) { return createNewCsvFileDataTable(ctx, reader, ',') } +// CreateNewCustomCsvImportedDataTable returns character separated values data table by io readers +func CreateNewCustomCsvImportedDataTable(allLines [][]string) *CsvFileImportedDataTable { + return &CsvFileImportedDataTable{ + allLines: allLines, + } +} + func createNewCsvFileDataTable(ctx core.Context, reader io.Reader, separator rune) (*CsvFileImportedDataTable, error) { csvReader := csv.NewReader(reader) csvReader.Comma = separator diff --git a/pkg/converters/datatable/common_data_table.go b/pkg/converters/datatable/common_data_table.go new file mode 100644 index 00000000..14bf9ef4 --- /dev/null +++ b/pkg/converters/datatable/common_data_table.go @@ -0,0 +1,40 @@ +package datatable + +// CommonDataTable defines the structure of common data table +type CommonDataTable interface { + // HeaderColumnCount returns the total count of column in header row + HeaderColumnCount() int + + // HasColumn returns whether the common data table has specified column name + HasColumn(columnName string) bool + + // DataRowCount returns the total count of common data row + DataRowCount() int + + // DataRowIterator returns the iterator of common data row + DataRowIterator() CommonDataRowIterator +} + +// CommonDataRow defines the structure of common data row +type CommonDataRow interface { + // ColumnCount returns the total count of column in this data row + ColumnCount() int + + // HasData returns whether the common data row has specified column data + HasData(columnName string) bool + + // GetData returns the data in the specified column name + GetData(columnName string) string +} + +// CommonDataRowIterator defines the structure of common data row iterator +type CommonDataRowIterator interface { + // HasNext returns whether the iterator does not reach the end + HasNext() bool + + // CurrentRowId returns current row id + CurrentRowId() string + + // Next returns the next common data row + Next() CommonDataRow +} diff --git a/pkg/converters/datatable/imported_common_data_table.go b/pkg/converters/datatable/imported_common_data_table.go new file mode 100644 index 00000000..2a827103 --- /dev/null +++ b/pkg/converters/datatable/imported_common_data_table.go @@ -0,0 +1,107 @@ +package datatable + +// ImportedCommonDataTable defines the structure of imported common data table +type ImportedCommonDataTable struct { + innerDataTable ImportedDataTable + dataColumnIndexes map[string]int +} + +// ImportedCommonDataRow defines the structure of imported common data row +type ImportedCommonDataRow struct { + rowData map[string]string +} + +// ImportedCommonDataRowIterator defines the structure of imported common data row iterator +type ImportedCommonDataRowIterator struct { + commonDataTable *ImportedCommonDataTable + innerIterator ImportedDataRowIterator +} + +// HeaderColumnCount returns the total count of column in header row +func (t *ImportedCommonDataTable) HeaderColumnCount() int { + return len(t.innerDataTable.HeaderColumnNames()) +} + +// HasColumn returns whether the data table has specified column name +func (t *ImportedCommonDataTable) HasColumn(columnName string) bool { + index, exists := t.dataColumnIndexes[columnName] + return exists && index >= 0 +} + +// DataRowCount returns the total count of common data row +func (t *ImportedCommonDataTable) DataRowCount() int { + return t.innerDataTable.DataRowCount() +} + +// DataRowIterator returns the iterator of common data row +func (t *ImportedCommonDataTable) DataRowIterator() CommonDataRowIterator { + return &ImportedCommonDataRowIterator{ + commonDataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), + } +} + +// HasData returns whether the common data row has specified column data +func (r *ImportedCommonDataRow) HasData(columnName string) bool { + _, exists := r.rowData[columnName] + return exists +} + +// ColumnCount returns the total count of column in this data row +func (r *ImportedCommonDataRow) ColumnCount() int { + return len(r.rowData) +} + +// GetData returns the data in the specified column name +func (r *ImportedCommonDataRow) GetData(columnName string) string { + return r.rowData[columnName] +} + +// HasNext returns whether the iterator does not reach the end +func (t *ImportedCommonDataRowIterator) HasNext() bool { + return t.innerIterator.HasNext() +} + +// CurrentRowId returns current row id +func (t *ImportedCommonDataRowIterator) CurrentRowId() string { + return t.innerIterator.CurrentRowId() +} + +// Next returns the next common data row +func (t *ImportedCommonDataRowIterator) Next() CommonDataRow { + importedRow := t.innerIterator.Next() + + if importedRow == nil { + return nil + } + + rowData := make(map[string]string, len(t.commonDataTable.dataColumnIndexes)) + + for column, columnIndex := range t.commonDataTable.dataColumnIndexes { + if columnIndex < 0 || columnIndex >= importedRow.ColumnCount() { + continue + } + + value := importedRow.GetData(columnIndex) + rowData[column] = value + } + + return &ImportedCommonDataRow{ + rowData: rowData, + } +} + +// CreateNewImportedCommonDataTable returns common data table from imported data table +func CreateNewImportedCommonDataTable(dataTable ImportedDataTable) *ImportedCommonDataTable { + headerLineItems := dataTable.HeaderColumnNames() + dataColumnIndexes := make(map[string]int, len(headerLineItems)) + + for i := 0; i < len(headerLineItems); i++ { + dataColumnIndexes[headerLineItems[i]] = i + } + + return &ImportedCommonDataTable{ + innerDataTable: dataTable, + dataColumnIndexes: dataColumnIndexes, + } +} diff --git a/pkg/converters/datatable/imported_data_table.go b/pkg/converters/datatable/imported_data_table.go index 80a39ee6..9e6992b7 100644 --- a/pkg/converters/datatable/imported_data_table.go +++ b/pkg/converters/datatable/imported_data_table.go @@ -26,6 +26,9 @@ type ImportedDataRowIterator interface { // HasNext returns whether the iterator does not reach the end HasNext() bool + // CurrentRowId returns current row id + CurrentRowId() string + // Next returns the next imported data row Next() ImportedDataRow } diff --git a/pkg/converters/datatable/imported_transaction_data_table.go b/pkg/converters/datatable/imported_transaction_data_table.go index 76b1cf44..4b393c5a 100644 --- a/pkg/converters/datatable/imported_transaction_data_table.go +++ b/pkg/converters/datatable/imported_transaction_data_table.go @@ -143,13 +143,13 @@ func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models }, nil } -// CreateImportedTransactionDataTable returns transaction data table from imported data table -func CreateImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable { - return CreateImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil) +// CreateNewImportedTransactionDataTable returns transaction data table from imported data table +func CreateNewImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable { + return CreateNewImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil) } -// CreateImportedTransactionDataTableWithRowParser returns transaction data table from imported data table -func CreateImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable { +// CreateNewImportedTransactionDataTableWithRowParser returns transaction data table from imported data table +func CreateNewImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable { headerLineItems := dataTable.HeaderColumnNames() headerItemMap := make(map[string]int, len(headerLineItems)) diff --git a/pkg/converters/default/default_transaction_data_plain_text_converter.go b/pkg/converters/default/default_transaction_data_plain_text_converter.go index a8be68eb..2d97a966 100644 --- a/pkg/converters/default/default_transaction_data_plain_text_converter.go +++ b/pkg/converters/default/default_transaction_data_plain_text_converter.go @@ -93,7 +93,7 @@ func (c *defaultTransactionDataPlainTextConverter) ParseImportedData(ctx core.Co return nil, nil, nil, nil, nil, nil, err } - transactionDataTable := datatable.CreateImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping) + transactionDataTable := datatable.CreateNewImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping) dataTableImporter := datatable.CreateNewImporter( ezbookkeepingTransactionTypeNameMapping, diff --git a/pkg/converters/default/default_transaction_plain_text_data_table.go b/pkg/converters/default/default_transaction_plain_text_data_table.go index 0195c6e6..c0d88d8c 100644 --- a/pkg/converters/default/default_transaction_plain_text_data_table.go +++ b/pkg/converters/default/default_transaction_plain_text_data_table.go @@ -78,6 +78,11 @@ func (t *defaultPlainTextDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allLines) } +// CurrentRowId returns current index +func (t *defaultPlainTextDataRowIterator) CurrentRowId() string { + return fmt.Sprintf("line#%d", t.currentIndex) +} + // Next returns the next imported data row func (t *defaultPlainTextDataRowIterator) Next() datatable.ImportedDataRow { if t.currentIndex+1 >= len(t.dataTable.allLines) { diff --git a/pkg/converters/excel/excel_file_imported_data_table.go b/pkg/converters/excel/excel_file_imported_data_table.go index 75bcc0f9..30efc597 100644 --- a/pkg/converters/excel/excel_file_imported_data_table.go +++ b/pkg/converters/excel/excel_file_imported_data_table.go @@ -2,6 +2,7 @@ package excel import ( "bytes" + "fmt" "github.com/shakinm/xlsReader/xls" @@ -115,6 +116,11 @@ func (t *ExcelFileDataRowIterator) HasNext() bool { return false } +// CurrentRowId returns current index +func (t *ExcelFileDataRowIterator) CurrentRowId() string { + return fmt.Sprintf("table#%d-row#%d", t.currentTableIndex, t.currentRowIndexInTable) +} + // Next returns the next imported data row func (t *ExcelFileDataRowIterator) Next() datatable.ImportedDataRow { allSheets := t.dataTable.workbook.GetSheets() diff --git a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go index 88b2e762..d5a83f68 100644 --- a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go @@ -5,6 +5,7 @@ import ( "io" "strings" + csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -12,14 +13,35 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) -const feideeMymoneyTransactionDataCsvFileHeader = "随手记导出文件(headers:v5;" -const feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + feideeMymoneyTransactionDataCsvFileHeader - -const feideeMymoneyCsvFileTransactionTypeModifyBalanceText = "余额变更" -const feideeMymoneyCsvFileTransactionTypeIncomeText = "收入" -const feideeMymoneyCsvFileTransactionTypeExpenseText = "支出" -const feideeMymoneyCsvFileTransactionTypeTransferInText = "转入" -const feideeMymoneyCsvFileTransactionTypeTransferOutText = "转出" +const feideeMymoneyAppTransactionDataCsvFileHeader = "随手记导出文件(headers:v5;" +const feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + feideeMymoneyAppTransactionDataCsvFileHeader + +const feideeMymoneyAppTransactionTimeColumnName = "日期" +const feideeMymoneyAppTransactionTypeColumnName = "交易类型" +const feideeMymoneyAppTransactionCategoryColumnName = "类别" +const feideeMymoneyAppTransactionSubCategoryColumnName = "子类别" +const feideeMymoneyAppTransactionAccountNameColumnName = "账户" +const feideeMymoneyAppTransactionAccountCurrencyColumnName = "账户币种" +const feideeMymoneyAppTransactionAmountColumnName = "金额" +const feideeMymoneyAppTransactionDescriptionColumnName = "备注" +const feideeMymoneyAppTransactionRelatedIdColumnName = "关联Id" + +const feideeMymoneyAppTransactionTypeModifyBalanceText = "余额变更" +const feideeMymoneyAppTransactionTypeIncomeText = "收入" +const feideeMymoneyAppTransactionTypeExpenseText = "支出" +const feideeMymoneyAppTransactionTypeTransferInText = "转入" +const feideeMymoneyAppTransactionTypeTransferOutText = "转出" + +var feideeMymoneyAppDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: feideeMymoneyAppTransactionTimeColumnName, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: feideeMymoneyAppTransactionTypeColumnName, + datatable.TRANSACTION_DATA_TABLE_CATEGORY: feideeMymoneyAppTransactionCategoryColumnName, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: feideeMymoneyAppTransactionSubCategoryColumnName, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: feideeMymoneyAppTransactionAccountNameColumnName, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: feideeMymoneyAppTransactionAccountCurrencyColumnName, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: feideeMymoneyAppTransactionAmountColumnName, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: feideeMymoneyAppTransactionDescriptionColumnName, +} // feideeMymoneyAppTransactionDataCsvFileImporter defines the structure of feidee mymoney app csv importer for transaction data type feideeMymoneyAppTransactionDataCsvFileImporter struct{} @@ -32,125 +54,162 @@ var ( // ParseImportedData returns the imported data by parsing the feidee mymoney app transaction csv data func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { content := string(data) + dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, content) - if strings.Index(content, feideeMymoneyTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom) != 0 { - return nil, nil, nil, nil, nil, nil, errs.ErrInvalidFileHeader + if err != nil { + return nil, nil, nil, nil, nil, nil, err } - allLines, err := c.parseAllLinesFromCsvData(ctx, content) + commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable) + + if !commonDataTable.HasColumn(feideeMymoneyAppTransactionTimeColumnName) || + !commonDataTable.HasColumn(feideeMymoneyAppTransactionTypeColumnName) || + !commonDataTable.HasColumn(feideeMymoneyAppTransactionSubCategoryColumnName) || + !commonDataTable.HasColumn(feideeMymoneyAppTransactionAccountNameColumnName) || + !commonDataTable.HasColumn(feideeMymoneyAppTransactionAmountColumnName) || + !commonDataTable.HasColumn(feideeMymoneyAppTransactionRelatedIdColumnName) { + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data, because missing essential columns in header row") + return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + } + + transactionDataTable, err := c.createNewFeideeMymoneyAppTransactionDataTable(ctx, commonDataTable) if err != nil { return nil, nil, nil, nil, nil, nil, err } - if len(allLines) < 2 { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile + dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping) + + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) +} + +func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, content string) (datatable.ImportedDataTable, error) { + if strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) != 0 { + return nil, errs.ErrInvalidFileHeader } - headerLineItems := allLines[0] - headerItemMap := make(map[string]int) + csvReader := csv.NewReader(strings.NewReader(content)) + csvReader.FieldsPerRecord = -1 + + allOriginalLines := make([][]string, 0) + hasFileHeader := false + + for { + items, err := csvReader.Read() + + if err == io.EOF { + break + } + + if err != nil { + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse feidee mymoney csv data, because %s", err.Error()) + return nil, errs.ErrInvalidCSVFile + } + + if !hasFileHeader { + if len(items) <= 0 { + continue + } else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) == 0 { + hasFileHeader = true + continue + } else { + log.Warnf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) + } + } - for i := 0; i < len(headerLineItems); i++ { - headerItemMap[headerLineItems[i]] = i + allOriginalLines = append(allOriginalLines, items) } - timeColumnIdx, timeColumnExists := headerItemMap["日期"] - typeColumnIdx, typeColumnExists := headerItemMap["交易类型"] - categoryColumnIdx, categoryColumnExists := headerItemMap["类别"] - subCategoryColumnIdx, subCategoryColumnExists := headerItemMap["子类别"] - accountColumnIdx, accountColumnExists := headerItemMap["账户"] - accountCurrencyColumnIdx, accountCurrencyColumnExists := headerItemMap["账户币种"] - amountColumnIdx, amountColumnExists := headerItemMap["金额"] - descriptionColumnIdx, descriptionColumnExists := headerItemMap["备注"] - relatedIdColumnIdx, relatedIdColumnExists := headerItemMap["关联Id"] - - if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists || - !accountColumnExists || !amountColumnExists || !relatedIdColumnExists { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + if !hasFileHeader { + return nil, errs.ErrInvalidFileHeader } + if len(allOriginalLines) < 2 { + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse import data, because data table row count is less 1") + return nil, errs.ErrNotFoundTransactionDataInFile + } + + dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines) + + return dataTable, nil +} + +func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppTransactionDataTable(ctx core.Context, commonDataTable datatable.CommonDataTable) (datatable.TransactionDataTable, error) { newColumns := make([]datatable.TransactionDataTableColumn, 0, 11) newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME) - if categoryColumnExists { + if commonDataTable.HasColumn(feideeMymoneyAppTransactionCategoryColumnName) { newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_CATEGORY) } newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY) newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME) - if accountCurrencyColumnExists { + if commonDataTable.HasColumn(feideeMymoneyAppTransactionAccountCurrencyColumnName) { newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) } newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_AMOUNT) newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME) - if accountCurrencyColumnExists { + if commonDataTable.HasColumn(feideeMymoneyAppTransactionAccountCurrencyColumnName) { newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) } newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT) - if descriptionColumnExists { + if commonDataTable.HasColumn(feideeMymoneyAppTransactionDescriptionColumnName) { newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_DESCRIPTION) } transactionRowParser := createFeideeMymoneyTransactionDataRowParser() - dataTable := datatable.CreateNewWritableTransactionDataTableWithRowParser(newColumns, transactionRowParser) + transactionDataTable := datatable.CreateNewWritableTransactionDataTableWithRowParser(newColumns, transactionRowParser) transferTransactionsMap := make(map[string]map[datatable.TransactionDataTableColumn]string, 0) - for i := 1; i < len(allLines); i++ { - items := allLines[i] + commonDataTableIterator := commonDataTable.DataRowIterator() + + for commonDataTableIterator.HasNext() { + dataRow := commonDataTableIterator.Next() + rowId := commonDataTableIterator.CurrentRowId() + + if dataRow.ColumnCount() < commonDataTable.HeaderColumnCount() { + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse row \"%s\", because may missing some columns (column count %d in data row is less than header column count %d)", rowId, dataRow.ColumnCount(), commonDataTable.HeaderColumnCount()) + return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + } + + data := make(map[datatable.TransactionDataTableColumn]string, 11) + relatedId := "" - if len(items) < len(headerLineItems) { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", i, user.Uid, len(items), len(headerLineItems)) - return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + for columnType, columnName := range feideeMymoneyAppDataColumnNameMapping { + if dataRow.HasData(columnName) { + data[columnType] = dataRow.GetData(columnName) + } } - data, relatedId := c.parseTransactionData(items, - timeColumnIdx, - timeColumnExists, - typeColumnIdx, - typeColumnExists, - categoryColumnIdx, - categoryColumnExists, - subCategoryColumnIdx, - subCategoryColumnExists, - accountColumnIdx, - accountColumnExists, - accountCurrencyColumnIdx, - accountCurrencyColumnExists, - amountColumnIdx, - amountColumnExists, - descriptionColumnIdx, - descriptionColumnExists, - relatedIdColumnIdx, - relatedIdColumnExists, - ) + if dataRow.HasData(feideeMymoneyAppTransactionRelatedIdColumnName) { + relatedId = dataRow.GetData(feideeMymoneyAppTransactionRelatedIdColumnName) + } transactionType := data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] - if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText || transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText || transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText { - if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText { + if transactionType == feideeMymoneyAppTransactionTypeModifyBalanceText || transactionType == feideeMymoneyAppTransactionTypeIncomeText || transactionType == feideeMymoneyAppTransactionTypeExpenseText { + if transactionType == feideeMymoneyAppTransactionTypeModifyBalanceText { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] - } else if transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText { + } else if transactionType == feideeMymoneyAppTransactionTypeIncomeText { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] - } else if transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText { + } else if transactionType == feideeMymoneyAppTransactionTypeExpenseText { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] } data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = "" data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = "" - dataTable.Add(data) - } else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText || transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText { + transactionDataTable.Add(data) + } else if transactionType == feideeMymoneyAppTransactionTypeTransferInText || transactionType == feideeMymoneyAppTransactionTypeTransferOutText { if relatedId == "" { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] transfer transaction has blank related id in row \"index:%d\" for user \"uid:%d\"", i, user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrRelatedIdCannotBeBlank + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] transfer transaction has blank related id in row \"%s\"", rowId) + return nil, errs.ErrRelatedIdCannotBeBlank } relatedData, exists := transferTransactionsMap[relatedId] @@ -160,140 +219,39 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c continue } - if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferOutText { + if transactionType == feideeMymoneyAppTransactionTypeTransferInText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyAppTransactionTypeTransferOutText { relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] - dataTable.Add(relatedData) + transactionDataTable.Add(relatedData) delete(transferTransactionsMap, relatedId) - } else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferInText { + } else if transactionType == feideeMymoneyAppTransactionTypeTransferOutText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyAppTransactionTypeTransferInText { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] - dataTable.Add(data) + transactionDataTable.Add(data) delete(transferTransactionsMap, relatedId) } else { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] transfer transaction type \"%s\" is not expected in row \"index:%d\" for user \"uid:%d\"", transactionType, i, user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTypeInvalid + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] transfer transaction type \"%s\" is not expected in row \"%s\"", transactionType, rowId) + return nil, errs.ErrTransactionTypeInvalid } } else { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse transaction type \"%s\" in row \"index:%d\" for user \"uid:%d\"", transactionType, i, user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTypeInvalid + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse transaction type \"%s\" in row \"%s\"", transactionType, rowId) + return nil, errs.ErrTransactionTypeInvalid } } if len(transferTransactionsMap) > 0 { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] there are %d transactions (related id is %s) which don't have related records", len(transferTransactionsMap), c.getRelatedIds(transferTransactionsMap)) - return nil, nil, nil, nil, nil, nil, errs.ErrFoundRecordNotHasRelatedRecord - } - - dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping) - - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) -} - -func (c *feideeMymoneyAppTransactionDataCsvFileImporter) parseAllLinesFromCsvData(ctx core.Context, content string) ([][]string, error) { - csvReader := csv.NewReader(strings.NewReader(content)) - csvReader.FieldsPerRecord = -1 - - allLines := make([][]string, 0) - hasFileHeader := false - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.parseAllLinesFromCsvData] cannot parse feidee mymoney csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - if !hasFileHeader { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], feideeMymoneyTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom) == 0 { - hasFileHeader = true - continue - } else { - log.Warnf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.parseAllLinesFromCsvData] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) - } - } - - allLines = append(allLines, items) - } - - return allLines, nil -} - -func (c *feideeMymoneyAppTransactionDataCsvFileImporter) parseTransactionData( - items []string, - timeColumnIdx int, - timeColumnExists bool, - typeColumnIdx int, - typeColumnExists bool, - categoryColumnIdx int, - categoryColumnExists bool, - subCategoryColumnIdx int, - subCategoryColumnExists bool, - accountColumnIdx int, - accountColumnExists bool, - accountCurrencyColumnIdx int, - accountCurrencyColumnExists bool, - amountColumnIdx int, - amountColumnExists bool, - descriptionColumnIdx int, - descriptionColumnExists bool, - relatedIdColumnIdx int, - relatedIdColumnExists bool, -) (map[datatable.TransactionDataTableColumn]string, string) { - data := make(map[datatable.TransactionDataTableColumn]string, 11) - relatedId := "" - - if timeColumnExists && timeColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx] - } - - if typeColumnExists && typeColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx] - } - - if categoryColumnExists && categoryColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = items[categoryColumnIdx] - } - - if subCategoryColumnExists && subCategoryColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[subCategoryColumnIdx] - } - - if accountColumnExists && accountColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = items[accountColumnIdx] - } - - if accountCurrencyColumnExists && accountCurrencyColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = items[accountCurrencyColumnIdx] - } - - if amountColumnExists && amountColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[amountColumnIdx] - } - - if descriptionColumnExists && descriptionColumnIdx < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx] - } - - if relatedIdColumnExists && relatedIdColumnIdx < len(items) { - relatedId = items[relatedIdColumnIdx] + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] there are %d transactions (related id is %s) which don't have related records", len(transferTransactionsMap), c.getFeideeMymoneyAppRelatedTransactionIds(transferTransactionsMap)) + return nil, errs.ErrFoundRecordNotHasRelatedRecord } - return data, relatedId + return transactionDataTable, nil } -func (c *feideeMymoneyAppTransactionDataCsvFileImporter) getRelatedIds(transferTransactionsMap map[string]map[datatable.TransactionDataTableColumn]string) string { +func (c *feideeMymoneyAppTransactionDataCsvFileImporter) getFeideeMymoneyAppRelatedTransactionIds(transferTransactionsMap map[string]map[datatable.TransactionDataTableColumn]string) string { builder := strings.Builder{} for relatedId := range transferTransactionsMap { diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go deleted file mode 100644 index d52bc924..00000000 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go +++ /dev/null @@ -1,24 +0,0 @@ -package feidee - -import ( - "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/models" -) - -var feideeMymoneyDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ - datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "日期", - datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "交易类型", - datatable.TRANSACTION_DATA_TABLE_CATEGORY: "分类", - datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "子分类", - datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "账户1", - datatable.TRANSACTION_DATA_TABLE_AMOUNT: "金额", - datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2", - datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "备注", -} - -var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{ - models.TRANSACTION_TYPE_MODIFY_BALANCE: "余额变更", - models.TRANSACTION_TYPE_INCOME: "收入", - models.TRANSACTION_TYPE_EXPENSE: "支出", - models.TRANSACTION_TYPE_TRANSFER: "转账", -} diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go index d20dacd9..d704d6ff 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go @@ -7,6 +7,13 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/utils" ) +var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{ + models.TRANSACTION_TYPE_MODIFY_BALANCE: "余额变更", + models.TRANSACTION_TYPE_INCOME: "收入", + models.TRANSACTION_TYPE_EXPENSE: "支出", + models.TRANSACTION_TYPE_TRANSFER: "转账", +} + // feideeMymoneyTransactionDataRowParser defines the structure of feidee mymoney transaction data row parser type feideeMymoneyTransactionDataRowParser struct { } diff --git a/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go b/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go index 6b48c89f..5ec6542a 100644 --- a/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go @@ -7,6 +7,17 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) +var feideeMymoneyWebDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "日期", + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "交易类型", + datatable.TRANSACTION_DATA_TABLE_CATEGORY: "分类", + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "子分类", + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "账户1", + datatable.TRANSACTION_DATA_TABLE_AMOUNT: "金额", + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2", + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "备注", +} + // feideeMymoneyWebTransactionDataXlsFileImporter defines the structure of feidee mymoney (web) xls importer for transaction data type feideeMymoneyWebTransactionDataXlsFileImporter struct { datatable.DataTableTransactionDataImporter @@ -26,7 +37,7 @@ func (c *feideeMymoneyWebTransactionDataXlsFileImporter) ParseImportedData(ctx c } transactionRowParser := createFeideeMymoneyTransactionDataRowParser() - transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyDataColumnNameMapping, transactionRowParser) + transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyWebDataColumnNameMapping, transactionRowParser) dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping) return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go index 1483e99e..20dfac0e 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go @@ -41,14 +41,14 @@ var ( // ParseImportedData returns the imported data by parsing the firefly III transaction csv data func (c *fireflyIIITransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { reader := bytes.NewReader(data) - dataTable, err := csv.CreateNewCsvDataTable(ctx, reader) + dataTable, err := csv.CreateNewCsvImportedDataTable(ctx, reader) if err != nil { return nil, nil, nil, nil, nil, nil, err } transactionRowParser := createFireflyIIITransactionDataRowParser() - transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser) + transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser) dataTableImporter := datatable.CreateNewImporter(fireflyIIITransactionTypeNameMapping, "", ",") return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) diff --git a/pkg/converters/wechat/wechat_pay_transaction_csv_data_table.go b/pkg/converters/wechat/wechat_pay_transaction_csv_data_table.go index b997aed9..ec299534 100644 --- a/pkg/converters/wechat/wechat_pay_transaction_csv_data_table.go +++ b/pkg/converters/wechat/wechat_pay_transaction_csv_data_table.go @@ -2,10 +2,10 @@ package wechat import ( "encoding/csv" - "fmt" "io" "strings" + csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -19,6 +19,15 @@ const wechatPayTransactionDataCsvFileHeader = "微信支付账单明细" const wechatPayTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + wechatPayTransactionDataCsvFileHeader const wechatPayTransactionDataHeaderStartContentBeginning = "----------------------微信支付账单明细列表--------------------" +const wechatPayTransactionTimeColumnName = "交易时间" +const wechatPayTransactionCategoryColumnName = "交易类型" +const wechatPayTransactionProductNameColumnName = "商品" +const wechatPayTransactionTypeColumnName = "收/支" +const wechatPayTransactionAmountColumnName = "金额(元)" +const wechatPayTransactionRelatedAccountColumnName = "支付方式" +const wechatPayTransactionStatusColumnName = "当前状态" +const wechatPayTransactionDescriptionColumnName = "备注" + const wechatPayTransactionDataCategoryTransferToWeChatWallet = "零钱充值" const wechatPayTransactionDataCategoryTransferFromWeChatWallet = "零钱提现" @@ -34,33 +43,21 @@ var wechatPayTransactionSupportedColumns = map[datatable.TransactionDataTableCol datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true, } -// wechatPayTransactionDataTable defines the structure of wechatPay transaction plain text data table +// wechatPayTransactionDataTable defines the structure of wechat pay transaction plain text data table type wechatPayTransactionDataTable struct { - allOriginalLines [][]string - originalHeaderLineColumnNames []string - originalTimeColumnIndex int - originalCategoryColumnIndex int - originalTargetNameColumnIndex int - originalProductNameColumnIndex int - originalTypeColumnIndex int - originalAmountColumnIndex int - originalRelatedAccountColumnIndex int - originalStatusColumnIndex int - originalDescriptionColumnIndex int + innerDataTable datatable.CommonDataTable } -// wechatPayTransactionDataRow defines the structure of wechatPay transaction plain text data row +// wechatPayTransactionDataRow defines the structure of wechat pay transaction plain text data row type wechatPayTransactionDataRow struct { - dataTable *wechatPayTransactionDataTable - isValid bool - originalItems []string - finalItems map[datatable.TransactionDataTableColumn]string + isValid bool + finalItems map[datatable.TransactionDataTableColumn]string } -// wechatPayTransactionDataRowIterator defines the structure of wechatPay transaction plain text data row iterator +// wechatPayTransactionDataRowIterator defines the structure of wechat pay transaction plain text data row iterator type wechatPayTransactionDataRowIterator struct { - dataTable *wechatPayTransactionDataTable - currentIndex int + dataTable *wechatPayTransactionDataTable + innerIterator datatable.CommonDataRowIterator } // HasColumn returns whether the transaction data table has specified column @@ -71,18 +68,14 @@ func (t *wechatPayTransactionDataTable) HasColumn(column datatable.TransactionDa // TransactionRowCount returns the total count of transaction data row func (t *wechatPayTransactionDataTable) TransactionRowCount() int { - if len(t.allOriginalLines) < 1 { - return 0 - } - - return len(t.allOriginalLines) - 1 + return t.innerDataTable.DataRowCount() } // TransactionRowIterator returns the iterator of transaction data row func (t *wechatPayTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator { return &wechatPayTransactionDataRowIterator{ - dataTable: t, - currentIndex: 0, + dataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), } } @@ -104,89 +97,80 @@ func (r *wechatPayTransactionDataRow) GetData(column datatable.TransactionDataTa // HasNext returns whether the iterator does not reach the end func (t *wechatPayTransactionDataRowIterator) HasNext() bool { - return t.currentIndex+1 < len(t.dataTable.allOriginalLines) + return t.innerIterator.HasNext() } // Next returns the next imported data row func (t *wechatPayTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { - if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) { - return nil, nil - } - - t.currentIndex++ + importedRow := t.innerIterator.Next() - rowItems := t.dataTable.allOriginalLines[t.currentIndex] - isValid := true - - if t.dataTable.originalTypeColumnIndex >= 0 && - rowItems[t.dataTable.originalTypeColumnIndex] != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && - rowItems[t.dataTable.originalTypeColumnIndex] != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && - rowItems[t.dataTable.originalTypeColumnIndex] != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { - log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because type is \"%s\"", t.currentIndex, rowItems[t.dataTable.originalTypeColumnIndex]) - isValid = false + if importedRow == nil { + return nil, nil } - var finalItems map[datatable.TransactionDataTableColumn]string - var errMsg string + finalItems, isValid, err := t.dataTable.parseTransactionData(ctx, user, importedRow, t.innerIterator.CurrentRowId()) - if isValid { - finalItems, errMsg = t.dataTable.parseTransactionData(ctx, user, rowItems) - - if finalItems == nil { - log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because %s", t.currentIndex, errMsg) - isValid = false - } + if err != nil { + return nil, err } return &wechatPayTransactionDataRow{ - dataTable: t.dataTable, - isValid: isValid, - originalItems: rowItems, - finalItems: finalItems, + isValid: isValid, + finalItems: finalItems, }, nil } -func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.TransactionDataTableColumn]string, string) { +func (t *wechatPayTransactionDataTable) hasOriginalColumn(columnName string) bool { + return columnName != "" && t.innerDataTable.HasColumn(columnName) +} + +func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, dataRow datatable.CommonDataRow, rowId string) (map[datatable.TransactionDataTableColumn]string, bool, error) { + if dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && + dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && + dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because type is \"%s\"", rowId, dataRow.GetData(wechatPayTransactionTypeColumnName)) + return nil, false, nil + } + data := make(map[datatable.TransactionDataTableColumn]string, len(wechatPayTransactionSupportedColumns)) - if t.originalTimeColumnIndex >= 0 && t.originalTimeColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex] + if t.hasOriginalColumn(wechatPayTransactionTimeColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = dataRow.GetData(wechatPayTransactionTimeColumnName) } - if t.originalCategoryColumnIndex >= 0 && t.originalCategoryColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex] - } else { - data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" + if t.hasOriginalColumn(wechatPayTransactionCategoryColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(wechatPayTransactionCategoryColumnName) } - if t.originalAmountColumnIndex >= 0 && t.originalAmountColumnIndex < len(items) { - amount, success := utils.ParseFirstConsecutiveNumber(items[t.originalAmountColumnIndex]) + if t.hasOriginalColumn(wechatPayTransactionAmountColumnName) { + amount, success := utils.ParseFirstConsecutiveNumber(dataRow.GetData(wechatPayTransactionAmountColumnName)) - if success { - data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = amount - } else { - data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex] + if !success { + log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] cannot parse amount \"%s\" of transaction in row \"%s\"", dataRow.GetData(wechatPayTransactionAmountColumnName), rowId) + return nil, false, errs.ErrAmountInvalid } + + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = amount } - if t.originalDescriptionColumnIndex >= 0 && t.originalDescriptionColumnIndex < len(items) && items[t.originalDescriptionColumnIndex] != "" && items[t.originalDescriptionColumnIndex] != "/" { - data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex] - } else if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) && items[t.originalProductNameColumnIndex] != "" && items[t.originalProductNameColumnIndex] != "/" { - data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex] + if t.hasOriginalColumn(wechatPayTransactionDescriptionColumnName) && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "" && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "/" { + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(wechatPayTransactionDescriptionColumnName) + } else if t.hasOriginalColumn(wechatPayTransactionProductNameColumnName) && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "" && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "/" { + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(wechatPayTransactionProductNameColumnName) } else { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = "" } relatedAccountName := "" - if t.originalRelatedAccountColumnIndex >= 0 && t.originalRelatedAccountColumnIndex < len(items) { - relatedAccountName = items[t.originalRelatedAccountColumnIndex] + if t.hasOriginalColumn(wechatPayTransactionRelatedAccountColumnName) { + relatedAccountName = dataRow.GetData(wechatPayTransactionRelatedAccountColumnName) } statusName := "" - if t.originalStatusColumnIndex >= 0 && t.originalStatusColumnIndex < len(items) { - statusName = items[t.originalStatusColumnIndex] + if t.hasOriginalColumn(wechatPayTransactionStatusColumnName) { + statusName = dataRow.GetData(wechatPayTransactionStatusColumnName) } locale := user.Language @@ -197,17 +181,17 @@ func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, u localeTextItems := locales.GetLocaleTextItems(locale) - if t.originalTypeColumnIndex >= 0 && t.originalTypeColumnIndex < len(items) { - data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex] + if t.hasOriginalColumn(wechatPayTransactionTypeColumnName) { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(wechatPayTransactionTypeColumnName) - if items[t.originalTypeColumnIndex] == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { + if dataRow.GetData(wechatPayTransactionTypeColumnName) == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { if relatedAccountName == "" || relatedAccountName == "/" { data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.WeChatWallet data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" } else { data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName } - } else if items[t.originalTypeColumnIndex] == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + } else if dataRow.GetData(wechatPayTransactionTypeColumnName) == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { if data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == wechatPayTransactionDataCategoryTransferToWeChatWallet { data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.WeChatWallet @@ -215,7 +199,8 @@ func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, u data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.WeChatWallet data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedAccountName } else { - return nil, fmt.Sprintf("unkown transfer transaction category") + log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because unkown transfer transaction category \"%s\"", rowId, data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY]) + return nil, false, nil } } else { data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName @@ -234,75 +219,33 @@ func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, u } } - return data, "" + return data, true, nil } func createNewWeChatPayTransactionDataTable(ctx core.Context, reader io.Reader) (*wechatPayTransactionDataTable, error) { - allOriginalLines, err := parseAllLinesFromWechatPayTransactionPlainText(ctx, reader) + dataTable, err := createNewWeChatPayImportedDataTable(ctx, reader) if err != nil { return nil, err } - if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewwechatPayTransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1") - return nil, errs.ErrNotFoundTransactionDataInFile - } - - originalHeaderItems := allOriginalLines[0] - originalHeaderItemMap := make(map[string]int) + commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable) - for i := 0; i < len(originalHeaderItems); i++ { - originalHeaderItemMap[originalHeaderItems[i]] = i - } - - timeColumnIdx, timeColumnExists := originalHeaderItemMap["交易时间"] - categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["交易类型"] - targetNameColumnIdx, targetNameColumnExists := originalHeaderItemMap["交易对方"] - productNameColumnIdx, productNameColumnExists := originalHeaderItemMap["商品"] - typeColumnIdx, typeColumnExists := originalHeaderItemMap["收/支"] - amountColumnIdx, amountColumnExists := originalHeaderItemMap["金额(元)"] - relatedAccountColumnIdx, relatedAccountColumnExists := originalHeaderItemMap["支付方式"] - statusColumnIdx, statusColumnExists := originalHeaderItemMap["当前状态"] - descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["备注"] - - if !timeColumnExists || !categoryColumnExists || !typeColumnExists || !amountColumnExists || !statusColumnExists { - log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewwechatPayTransactionPlainTextDataTable] cannot parse wechat pay csv data, because missing essential columns in header row") + if !commonDataTable.HasColumn(wechatPayTransactionTimeColumnName) || + !commonDataTable.HasColumn(wechatPayTransactionCategoryColumnName) || + !commonDataTable.HasColumn(wechatPayTransactionTypeColumnName) || + !commonDataTable.HasColumn(wechatPayTransactionAmountColumnName) || + !commonDataTable.HasColumn(wechatPayTransactionStatusColumnName) { + log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayTransactionDataTable] cannot parse wechat pay csv data, because missing essential columns in header row") return nil, errs.ErrMissingRequiredFieldInHeaderRow } - if !targetNameColumnExists { - targetNameColumnIdx = -1 - } - - if !productNameColumnExists { - productNameColumnIdx = -1 - } - - if !relatedAccountColumnExists { - relatedAccountColumnIdx = -1 - } - - if !descriptionColumnExists { - descriptionColumnIdx = -1 - } - return &wechatPayTransactionDataTable{ - allOriginalLines: allOriginalLines, - originalHeaderLineColumnNames: originalHeaderItems, - originalTimeColumnIndex: timeColumnIdx, - originalCategoryColumnIndex: categoryColumnIdx, - originalTargetNameColumnIndex: targetNameColumnIdx, - originalProductNameColumnIndex: productNameColumnIdx, - originalAmountColumnIndex: amountColumnIdx, - originalTypeColumnIndex: typeColumnIdx, - originalRelatedAccountColumnIndex: relatedAccountColumnIdx, - originalStatusColumnIndex: statusColumnIdx, - originalDescriptionColumnIndex: descriptionColumnIdx, + innerDataTable: commonDataTable, }, nil } -func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) { +func createNewWeChatPayImportedDataTable(ctx core.Context, reader io.Reader) (datatable.ImportedDataTable, error) { csvReader := csv.NewReader(reader) csvReader.FieldsPerRecord = -1 @@ -318,7 +261,7 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io. } if err != nil { - log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.parseAllLinesFromWechatPayTransactionPlainText] cannot parse wechat pay csv data, because %s", err.Error()) + log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] cannot parse wechat pay csv data, because %s", err.Error()) return nil, errs.ErrInvalidCSVFile } @@ -329,7 +272,7 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io. hasFileHeader = true continue } else { - log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseAllLinesFromWechatPayTransactionPlainText] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) + log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) continue } } @@ -355,7 +298,7 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io. } if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { - log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.parseAllLinesFromWechatPayTransactionPlainText] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) + log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } @@ -367,5 +310,12 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io. return nil, errs.ErrInvalidFileHeader } - return allOriginalLines, nil + if len(allOriginalLines) < 2 { + log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] cannot parse import data, because data table row count is less 1") + return nil, errs.ErrNotFoundTransactionDataInFile + } + + dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines) + + return dataTable, nil }