fix: remember known chunks and do not send them twice.

go.mod

@@ -3,6 +3,7 @@ module proxmoxbackupgo
 go 1.19
 
 require (
+	github.com/cornelk/hashmap v1.0.8
 	github.com/dchest/siphash v1.2.3
 	github.com/gen2brain/beeep v0.0.0-20230907135156-1a38885a97fc
 	github.com/getlantern/systray v1.2.2

go.sum

@@ -1,5 +1,7 @@
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
 github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/cornelk/hashmap v1.0.8 h1:nv0AWgw02n+iDcawr5It4CjQIAcdMMKRrs10HOJYlrc=
+github.com/cornelk/hashmap v1.0.8/go.mod h1:RfZb7JO3RviW/rT6emczVuC/oxpdz4UsSB2LJSclR1k=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=

main.go

@@ -10,7 +10,9 @@ import (
 	"hash"
 	"os"
 	"runtime"
+	"sync/atomic"
 
+	"github.com/cornelk/hashmap"
 	"github.com/gen2brain/beeep"
 	"github.com/getlantern/systray"
 	"github.com/tawesoft/golib/v2/dialog"
@@ -46,9 +48,9 @@ func (c *ChunkState) Init() {
 }
 
 func main() {
-	newchunk := 0
-	reusechunk := 0
-	known_chunks_digest := make(map[string]bool)
+	var newchunk atomic.Uint64
+	var reusechunk atomic.Uint64
+	knownChunks := hashmap.New[string, bool]()
 
 	// Define command-line flags
 	baseURLFlag := flag.String("baseurl", "", "Base URL for the proxmox backup server, example: https://192.168.1.10:8007")
@@ -101,10 +103,12 @@ func main() {
 
 	backupdir := *backupSourceDirFlag
 
-	backupdir = createVSSSnapshot(backupdir)
-
 	fmt.Printf("Starting backup of %s\n", backupdir)
 
+	backupdir = createVSSSnapshot(backupdir)
+	//Remove VSS snapshot on windows, on linux for now NOP
+	defer VSSCleanup()
+
 	client.Connect(false)
 
 	A := &PXARArchive{}
@@ -137,11 +141,11 @@ func main() {
 			e.digest = previous_didx[i*40+8 : i*40+40]
 			shahash := hex.EncodeToString(e.digest)
 			fmt.Printf("Previous: %s\n", shahash)
-			known_chunks_digest[shahash] = true
+			knownChunks.Set(shahash, true)
 		}
 	}
 
-	fmt.Printf("Known chunks: %d!\n", len(known_chunks_digest))
+	fmt.Printf("Known chunks: %d!\n", knownChunks.Len())
 	f := &os.File{}
 	if *pxarOut != "" {
 		f, _ = os.Create(*pxarOut)
@@ -171,13 +175,14 @@ func main() {
 				bindigest := h.Sum(nil)
 				shahash := hex.EncodeToString(bindigest)
 
-				if !known_chunks_digest[shahash] {
+				if _, ok := knownChunks.GetOrInsert(shahash, true); ok {
 					fmt.Printf("New chunk[%s] %d bytes\n", shahash, len(PXAR_CHK.current_chunk))
-					newchunk++
+					newchunk.Add(1)
+
 					client.UploadCompressedChunk(PXAR_CHK.wrid, shahash, PXAR_CHK.current_chunk)
 				} else {
 					fmt.Printf("Reuse chunk[%s] %d bytes\n", shahash, len(PXAR_CHK.current_chunk))
-					reusechunk++
+					reusechunk.Add(1)
 				}
 
 				binary.Write(PXAR_CHK.chunkdigests, binary.LittleEndian, (PXAR_CHK.pos + uint64(len(PXAR_CHK.current_chunk))))
@@ -212,7 +217,7 @@ func main() {
 				h.Write(PCAT1_CHK.current_chunk)
 				shahash := hex.EncodeToString(h.Sum(nil))
 
-				fmt.Printf("Catalog: New chunk[%s] %d bytes\n", shahash, len(PCAT1_CHK.current_chunk))
+				fmt.Printf("Catalog: New chunk[%s] %d bytes, pos %d\n", shahash, len(PCAT1_CHK.current_chunk), chunkpos)
 
 				client.UploadCompressedChunk(PCAT1_CHK.wrid, shahash, PCAT1_CHK.current_chunk)
 				binary.Write(PCAT1_CHK.chunkdigests, binary.LittleEndian, (PCAT1_CHK.pos + uint64(len(PCAT1_CHK.current_chunk))))
@@ -245,13 +250,13 @@ func main() {
 		binary.Write(PXAR_CHK.chunkdigests, binary.LittleEndian, (PXAR_CHK.pos + uint64(len(PXAR_CHK.current_chunk))))
 		PXAR_CHK.chunkdigests.Write(h.Sum(nil))
 
-		if !known_chunks_digest[shahash] {
+		if _, ok := knownChunks.GetOrInsert(shahash, true); ok {
 			fmt.Printf("New chunk[%s] %d bytes\n", shahash, len(PXAR_CHK.current_chunk))
 			client.UploadCompressedChunk(PXAR_CHK.wrid, shahash, PXAR_CHK.current_chunk)
-			newchunk++
+			newchunk.Add(1)
 		} else {
 			fmt.Printf("Reuse chunk[%s] %d bytes\n", shahash, len(PXAR_CHK.current_chunk))
-			reusechunk++
+			reusechunk.Add(1)
 		}
 		PXAR_CHK.assignments_offset = append(PXAR_CHK.assignments_offset, PXAR_CHK.pos)
 		PXAR_CHK.assignments = append(PXAR_CHK.assignments, shahash)
@@ -267,7 +272,7 @@ func main() {
 		binary.Write(PCAT1_CHK.chunkdigests, binary.LittleEndian, (PCAT1_CHK.pos + uint64(len(PCAT1_CHK.current_chunk))))
 		PCAT1_CHK.chunkdigests.Write(h.Sum(nil))
 
-		fmt.Printf("New chunk[%s] %d bytes\n", shahash, len(PCAT1_CHK.current_chunk))
+		fmt.Printf("Catalog: New chunk[%s] %d bytes\n", shahash, len(PCAT1_CHK.current_chunk))
 		PCAT1_CHK.assignments_offset = append(PCAT1_CHK.assignments_offset, PCAT1_CHK.pos)
 		PCAT1_CHK.assignments = append(PCAT1_CHK.assignments, shahash)
 		PCAT1_CHK.pos += uint64(len(PCAT1_CHK.current_chunk))
@@ -299,13 +304,10 @@ func main() {
 	client.UploadManifest()
 	client.Finish()
 
-	fmt.Printf("New %d , Reused %d\n", newchunk, reusechunk)
-
-	//Remove VSS snapshot on windows, on linux for now NOP
-	VSSCleanup()
+	fmt.Printf("New %d , Reused %d\n", newchunk.Load(), reusechunk.Load())
 	if runtime.GOOS == "windows" {
 		systray.Quit()
-		beeep.Notify("Proxmox Backup Go", fmt.Sprintf("Backup complete\nChunks New %d , Reused %d\n", newchunk, reusechunk), "")
+		beeep.Notify("Proxmox Backup Go", fmt.Sprintf("Backup complete\nChunks New %d , Reused %d\n", newchunk.Load(), reusechunk.Load()), "")
 	}
 
 }

pbsapi.go

@@ -377,7 +377,7 @@ func (pbs *PBSClient) Connect(reader bool) {
 					conn.Write([]byte("Upgrade: proxmox-backup-reader-protocol-v1\r\n"))
 				}
 				conn.Write([]byte("Connection: Upgrade\r\n\r\n"))
-				fmt.Println("Reading response to upgrade...\n")
+				fmt.Printf("Reading response to upgrade...\n")
 				buf := make([]byte, 0)
 				for !strings.HasSuffix(string(buf), "\r\n\r\n") && !strings.HasSuffix(string(buf), "\n\n") {
 					//fmt.Println(buf)

pxar.go

@@ -284,7 +284,6 @@ func (a *PXARArchive) WriteDir(path string, dirname string, toplevel bool) Catal
 	a.Flush()
 
 	goodbyteitems := make([]GoodByeItem, 0)
-
 	catalog_files := make([]CatalogFile, 0)
 	catalog_dirs := make([]CatalogDir, 0)
 
@@ -301,8 +300,8 @@ func (a *PXARArchive) WriteDir(path string, dirname string, toplevel bool) Catal
 			})
 		} else {
 			F := a.WriteFile(filepath.Join(path, file.Name()), file.Name())
-			catalog_files = append(catalog_files, F)
 
+			catalog_files = append(catalog_files, F)
 			goodbyteitems = append(goodbyteitems, GoodByeItem{
 				offset: startpos,
 				hash:   siphash.Hash(0x83ac3f1cfbb450db, 0xaa4f1b6879369fbd, []byte(file.Name())),