import transaction from firefly iii

pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go

@@ -0,0 +1,40 @@
+package fireflyIII
+
+import (
+	"bytes"
+
+	"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
+	"github.com/mayswind/ezbookkeeping/pkg/core"
+	"github.com/mayswind/ezbookkeeping/pkg/models"
+)
+
+// fireflyiiiTransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data
+type fireflyIIITransactionDataCsvImporter struct{}
+
+// Initialize a firefly III transaction data csv file importer singleton instance
+var (
+	FireflyIIITransactionDataCsvImporter = &fireflyIIITransactionDataCsvImporter{}
+)
+
+// ParseImportedData returns the imported data by parsing the firefly iii transaction csv data
+func (c *fireflyIIITransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
+	reader := bytes.NewReader(data)
+
+	dataTable, err := createNewFireflyIIITransactionPlainTextDataTable(
+		ctx,
+		reader,
+	)
+
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, err
+	}
+
+	dataTableImporter := datatable.CreateNewImporter(
+		dataTable.GetDataColumnMapping(),
+		fireflyIIITransactionTypeNameMapping,
+		"",
+		",",
+	)
+
+	return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
+}

pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go

@@ -0,0 +1 @@
+package fireflyIII

pkg/converters/fireflyIII/fireflyiii_transaction_data_plain_text_data_table.go

@@ -0,0 +1,311 @@
+package fireflyIII
+
+import (
+	"encoding/csv"
+	"io"
+	"time"
+
+	"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
+	"github.com/mayswind/ezbookkeeping/pkg/core"
+	"github.com/mayswind/ezbookkeeping/pkg/errs"
+	"github.com/mayswind/ezbookkeeping/pkg/log"
+	"github.com/mayswind/ezbookkeeping/pkg/models"
+	"github.com/mayswind/ezbookkeeping/pkg/utils"
+)
+
+var fireflyIIITransactionSupportedColumns = []datatable.DataTableColumn{
+	datatable.DATA_TABLE_TRANSACTION_TIME,
+	datatable.DATA_TABLE_TRANSACTION_TYPE,
+	datatable.DATA_TABLE_SUB_CATEGORY,
+	datatable.DATA_TABLE_ACCOUNT_NAME,
+	datatable.DATA_TABLE_ACCOUNT_CURRENCY,
+	datatable.DATA_TABLE_AMOUNT,
+	datatable.DATA_TABLE_RELATED_ACCOUNT_NAME,
+	datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY,
+	datatable.DATA_TABLE_RELATED_AMOUNT,
+	datatable.DATA_TABLE_TAGS,
+	datatable.DATA_TABLE_DESCRIPTION,
+}
+
+var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
+	models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance",
+	models.TRANSACTION_TYPE_INCOME:         "Deposit",
+	models.TRANSACTION_TYPE_EXPENSE:        "Withdrawal",
+	models.TRANSACTION_TYPE_TRANSFER:       "Transfer",
+}
+
+// fireflyIIITransactionPlainTextDataTable defines the structure of firefly III transaction plain text data table
+type fireflyIIITransactionPlainTextDataTable struct {
+	allOriginalLines              [][]string
+	originalHeaderLineColumnNames []string
+	originalColumnIndex           map[datatable.DataTableColumn]int
+}
+
+// fireflyIIITransactionPlainTextDataRow defines the structure of firefly III transaction plain text data row
+type fireflyIIITransactionPlainTextDataRow struct {
+	dataTable     *fireflyIIITransactionPlainTextDataTable
+	originalItems []string
+	finalItems    map[datatable.DataTableColumn]string
+}
+
+// fireflyIIITransactionPlainTextDataRowIterator defines the structure of firefly III transaction plain text data row iterator
+type fireflyIIITransactionPlainTextDataRowIterator struct {
+	dataTable    *fireflyIIITransactionPlainTextDataTable
+	currentIndex int
+}
+
+// DataRowCount returns the total count of data row
+func (t *fireflyIIITransactionPlainTextDataTable) DataRowCount() int {
+	if len(t.allOriginalLines) < 1 {
+		return 0
+	}
+
+	return len(t.allOriginalLines) - 1
+}
+
+// GetDataColumnMapping returns data column map for data importer
+func (t *fireflyIIITransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string {
+	dataColumnMapping := make(map[datatable.DataTableColumn]string, len(fireflyIIITransactionSupportedColumns))
+
+	for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
+		column := fireflyIIITransactionSupportedColumns[i]
+
+		if t.originalColumnIndex[column] < 0 {
+			continue
+		}
+
+		dataColumnMapping[column] = utils.IntToString(int(column))
+	}
+
+	return dataColumnMapping
+}
+
+// HeaderLineColumnNames returns the header column name list
+func (t *fireflyIIITransactionPlainTextDataTable) HeaderLineColumnNames() []string {
+	columnIndexes := make([]string, len(fireflyIIITransactionSupportedColumns))
+
+	for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
+		column := fireflyIIITransactionSupportedColumns[i]
+
+		if t.originalColumnIndex[column] < 0 {
+			continue
+		}
+
+		columnIndexes[i] = utils.IntToString(int(column))
+	}
+
+	return columnIndexes
+}
+
+// DataRowIterator returns the iterator of data row
+func (t *fireflyIIITransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
+	return &fireflyIIITransactionPlainTextDataRowIterator{
+		dataTable:    t,
+		currentIndex: 0,
+	}
+}
+
+// IsValid returns whether this row contains valid data for importing
+func (r *fireflyIIITransactionPlainTextDataRow) IsValid() bool {
+	return true
+}
+
+// ColumnCount returns the total count of column in this data row
+func (r *fireflyIIITransactionPlainTextDataRow) ColumnCount() int {
+	return len(r.finalItems)
+}
+
+// GetData returns the data in the specified column index
+func (r *fireflyIIITransactionPlainTextDataRow) GetData(columnIndex int) string {
+	if columnIndex >= len(fireflyIIITransactionSupportedColumns) {
+		return ""
+	}
+
+	dataColumn := fireflyIIITransactionSupportedColumns[columnIndex]
+
+	return r.finalItems[dataColumn]
+}
+
+// GetTime returns the time in the specified column index
+func (r *fireflyIIITransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
+	return utils.ParseFromLongDateTimeWithTimezone(r.GetData(columnIndex))
+}
+
+// GetTimezoneOffset returns the time zone offset in the specified column index
+func (r *fireflyIIITransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
+	return nil, errs.ErrNotSupported
+}
+
+// HasNext returns whether the iterator does not reach the end
+func (t *fireflyIIITransactionPlainTextDataRowIterator) HasNext() bool {
+	return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
+}
+
+// Next returns the next imported data row
+func (t *fireflyIIITransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
+	if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
+		return nil
+	}
+
+	t.currentIndex++
+
+	rowItems := t.dataTable.allOriginalLines[t.currentIndex]
+	finalItems := t.dataTable.parseTransactionData(rowItems)
+
+	return &fireflyIIITransactionPlainTextDataRow{
+		dataTable:     t.dataTable,
+		originalItems: rowItems,
+		finalItems:    finalItems,
+	}
+}
+
+func (t *fireflyIIITransactionPlainTextDataTable) parseTransactionData(items []string) map[datatable.DataTableColumn]string {
+	data := make(map[datatable.DataTableColumn]string, 12)
+
+	data[datatable.DATA_TABLE_SUB_CATEGORY] = ""
+
+	for column, index := range t.originalColumnIndex {
+		data[column] = items[index]
+	}
+
+	// trim trailing zero in decimal
+	if data[datatable.DATA_TABLE_AMOUNT] != "" {
+		data[datatable.DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_AMOUNT])
+		amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT])
+
+		if err == nil {
+			data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
+		}
+	}
+
+	if data[datatable.DATA_TABLE_RELATED_AMOUNT] != "" {
+		data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_RELATED_AMOUNT])
+		amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_RELATED_AMOUNT])
+
+		if err == nil {
+			data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount)
+		}
+	} else {
+		data[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT]
+	}
+
+	// the related account currency field is foreign currency in firefly iii actually
+	if data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
+		data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY]
+	}
+
+	// the destination account of modify balance transaction in firefly iii is the asset account
+	if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
+		data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
+	}
+
+	// the destination account of income transaction in firefly iii is the asset account
+	if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
+		data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
+	}
+
+	return data
+}
+
+func createNewFireflyIIITransactionPlainTextDataTable(ctx core.Context, reader io.Reader) (*fireflyIIITransactionPlainTextDataTable, error) {
+	allOriginalLines, err := parseAllLinesFromFireflyIIITransactionPlainText(ctx, reader)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if len(allOriginalLines) < 2 {
+		log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1")
+		return nil, errs.ErrNotFoundTransactionDataInFile
+	}
+
+	originalHeaderItems := allOriginalLines[0]
+	originalHeaderItemMap := make(map[string]int)
+
+	for i := 0; i < len(originalHeaderItems); i++ {
+		originalHeaderItemMap[originalHeaderItems[i]] = i
+	}
+
+	typeColumnIdx, typeColumnExists := originalHeaderItemMap["type"]
+	amountColumnIdx, amountColumnExists := originalHeaderItemMap["amount"]
+	foreignAmountColumnIdx, foreignAmountColumnExists := originalHeaderItemMap["foreign_amount"]
+	currencyColumnIdx, currencyColumnExists := originalHeaderItemMap["currency_code"]
+	foreignCurrencyColumnIdx, foreignCurrencyColumnExists := originalHeaderItemMap["foreign_currency_code"]
+	descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["description"]
+	dateColumnIdx, dateColumnExists := originalHeaderItemMap["date"]
+	sourceNameColumnIdx, sourceNameColumnExists := originalHeaderItemMap["source_name"]
+	destinationNameColumnIdx, destinationNameColumnExists := originalHeaderItemMap["destination_name"]
+	categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["category"]
+	tagsColumnIdx, tagsColumnExists := originalHeaderItemMap["tags"]
+
+	if !typeColumnExists || !amountColumnExists || !dateColumnExists || !sourceNameColumnExists || !destinationNameColumnExists {
+		log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse firefly III csv data, because missing essential columns in header row")
+		return nil, errs.ErrMissingRequiredFieldInHeaderRow
+	}
+
+	if !foreignAmountColumnExists {
+		foreignAmountColumnIdx = -1
+	}
+
+	if !currencyColumnExists {
+		currencyColumnIdx = -1
+	}
+
+	if !foreignCurrencyColumnExists {
+		foreignCurrencyColumnIdx = -1
+	}
+
+	if !descriptionColumnExists {
+		descriptionColumnIdx = -1
+	}
+
+	if !categoryColumnExists {
+		categoryColumnIdx = -1
+	}
+
+	if !tagsColumnExists {
+		tagsColumnIdx = -1
+	}
+
+	return &fireflyIIITransactionPlainTextDataTable{
+		allOriginalLines:              allOriginalLines,
+		originalHeaderLineColumnNames: originalHeaderItems,
+		originalColumnIndex: map[datatable.DataTableColumn]int{
+			datatable.DATA_TABLE_TRANSACTION_TIME:         dateColumnIdx,
+			datatable.DATA_TABLE_TRANSACTION_TYPE:         typeColumnIdx,
+			datatable.DATA_TABLE_SUB_CATEGORY:             categoryColumnIdx,
+			datatable.DATA_TABLE_ACCOUNT_NAME:             sourceNameColumnIdx,
+			datatable.DATA_TABLE_ACCOUNT_CURRENCY:         currencyColumnIdx,
+			datatable.DATA_TABLE_AMOUNT:                   amountColumnIdx,
+			datatable.DATA_TABLE_RELATED_ACCOUNT_NAME:     destinationNameColumnIdx,
+			datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: foreignCurrencyColumnIdx,
+			datatable.DATA_TABLE_RELATED_AMOUNT:           foreignAmountColumnIdx,
+			datatable.DATA_TABLE_TAGS:                     tagsColumnIdx,
+			datatable.DATA_TABLE_DESCRIPTION:              descriptionColumnIdx,
+		},
+	}, nil
+}
+
+func parseAllLinesFromFireflyIIITransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) {
+	csvReader := csv.NewReader(reader)
+	csvReader.FieldsPerRecord = -1
+
+	allOriginalLines := make([][]string, 0)
+
+	for {
+		items, err := csvReader.Read()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.parseAllLinesFromFireflyIIITransactionPlainText] cannot parse firefly III csv data, because %s", err.Error())
+			return nil, errs.ErrInvalidCSVFile
+		}
+
+		allOriginalLines = append(allOriginalLines, items)
+	}
+
+	return allOriginalLines, nil
+}

pkg/converters/transaction_data_converters.go

@@ -5,6 +5,7 @@ import (
 	"github.com/mayswind/ezbookkeeping/pkg/converters/base"
 	"github.com/mayswind/ezbookkeeping/pkg/converters/default"
 	"github.com/mayswind/ezbookkeeping/pkg/converters/feidee"
+	"github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII"
 	"github.com/mayswind/ezbookkeeping/pkg/errs"
 )
 
@@ -25,6 +26,8 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter,
 		return _default.EzBookKeepingTransactionDataCSVFileConverter, nil
 	} else if fileType == "ezbookkeeping_tsv" {
 		return _default.EzBookKeepingTransactionDataTSVFileConverter, nil
+	} else if fileType == "firefly_iii_csv" {
+		return fireflyIII.FireflyIIITransactionDataCsvImporter, nil
 	} else if fileType == "feidee_mymoney_csv" {
 		return feidee.FeideeMymoneyTransactionDataCsvImporter, nil
 	} else if fileType == "feidee_mymoney_xls" {

pkg/utils/datetimes.go

@@ -10,6 +10,7 @@ import (
 
 const (
 	longDateTimeFormat              = "2006-01-02 15:04:05"
+	longDateTimeWithTimezoneFormat  = "2006-01-02T15:04:05Z07:00"
 	longDateTimeWithoutSecondFormat = "2006-01-02 15:04"
 	shortDateTimeFormat             = "2006-1-2 15:4:5"
 	yearMonthDateTimeFormat         = "2006-01"
@@ -135,6 +136,11 @@ func ParseFromLongDateTime(t string, utcOffset int16) (time.Time, error) {
 	return time.ParseInLocation(longDateTimeFormat, t, timezone)
 }
 
+// ParseFromLongDateTimeWithTimezone parses a formatted string in long date time format
+func ParseFromLongDateTimeWithTimezone(t string) (time.Time, error) {
+	return time.Parse(longDateTimeWithTimezoneFormat, t)
+}
+
 // ParseFromLongDateTimeWithoutSecond parses a formatted string in long date time format (no second)
 func ParseFromLongDateTimeWithoutSecond(t string, utcOffset int16) (time.Time, error) {
 	timezone := time.FixedZone("Timezone", int(utcOffset)*60)