Df dev remove bolt

cmd/gleaner/main.go

@@ -11,7 +11,6 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/viper"
-	bolt "go.etcd.io/bbolt"
 
 	"github.com/gleanerio/gleaner/internal/common"
 	"github.com/gleanerio/gleaner/internal/objects"
@@ -179,13 +178,6 @@ func main() {
 	//	os.Exit(1)
 	//}
 
-	// setup the KV store to hold a record of indexed resources
-	db, err := bolt.Open("gleaner.db", 0600, nil)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer db.Close()
-
 	// Defer a function to be called on successful ending.  Note, if gleaner crashes, this will NOT
 	// get called, do consideration must be taken in such a cases.  Some errors in such cases should
 	// be sent to stdout to be captured by docker, k8s, Airflow etc in case they are being used.
@@ -196,7 +188,7 @@ func main() {
 	}()
 
 	//cli(mc, v1, db)
-	pkg.Cli(mc, v1, db) // move to a common call in batch.go
+	pkg.Cli(mc, v1) // move to a common call in batch.go
 }
 
 func cleanUp() {

cmd/glweb/main.go

@@ -12,7 +12,6 @@ import (
 	"github.com/minio/minio-go/v7"
 	"github.com/rs/xid"
 	"github.com/spf13/viper"
-	bolt "go.etcd.io/bbolt"
 
 	"github.com/gleanerio/gleaner/internal/common"
 	"github.com/gleanerio/gleaner/internal/millers"
@@ -136,13 +135,7 @@ func cli(mc *minio.Client, v1 *viper.Viper) {
 	mcfg := v1.GetStringMapString("gleaner")
 
 	if mcfg["summon"] == "true" {
-		// setup the KV store to hold a record of indexed resources
-		db, err := bolt.Open("gleaner.db", 0600, nil)
-		if err != nil {
-			log.Fatal(err)
-		}
-		defer db.Close()
-		summoner.Summoner(mc, v1, db)
+		summoner.Summoner(mc, v1)
 	}
 
 	if mcfg["mill"] == "true" {

cmd/husker/main.go

@@ -10,7 +10,6 @@ import (
 	"github.com/gleanerio/gleaner/internal/summoner/acquire"
 
 	"github.com/spf13/viper"
-	bolt "go.etcd.io/bbolt"
 )
 
 var viperVal string
@@ -58,17 +57,12 @@ func main() {
 	//	buf    bytes.Buffer
 	//	logger = log.New(&buf, "logger: ", log.Lshortfile)
 	//)
-	// setup the KV store to hold a record of indexed resources
-	db, err := bolt.Open("gleaner.db", 0600, nil)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer db.Close()
+
 	rlogginer, _ := common.LogIssues(v1, k)
 
 	runStats := common.NewRunStats()
 	repostats := runStats.Add(k)
-	err = acquire.PageRenderAndUpload(v1, mc, 45*time.Second, url, k, db, rlogginer, repostats)
+	err = acquire.PageRenderAndUpload(v1, mc, 45*time.Second, url, k, rlogginer, repostats)
 	if err != nil {
 		panic(fmt.Errorf("error when reading config: %v", err))
 	}

go.mod

@@ -14,9 +14,10 @@ require (
 	github.com/gocarina/gocsv v0.0.0-20211020200912-82fc2684cc48
 	github.com/gorilla/mux v1.8.0
 	github.com/gosuri/uiprogress v0.0.1
+	github.com/k4s/webrowser v0.0.0-20160107091637-934d526d0f27
 	github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042
 	github.com/mafredri/cdp v0.32.0
-	github.com/minio/minio-go/v7 v7.0.51
+	github.com/minio/minio-go/v7 v7.0.52
 	github.com/ohler55/ojg v1.14.5
 	github.com/piprate/json-gold v0.5.0
 	github.com/rs/xid v1.4.0
@@ -27,7 +28,6 @@ require (
 	github.com/valyala/fasttemplate v1.2.2
 	github.com/xitongsys/parquet-go v1.6.0
 	github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371
-	go.etcd.io/bbolt v1.3.6
 	golang.org/x/crypto v0.6.0
 )
 
@@ -57,6 +57,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k4s/phantomgo v0.0.0-20161104020322-11963773aa04 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
@@ -98,11 +99,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-// just using bolt github.com/boltdb/bolt would be ok... but it complains if we mix.
-//replace  github.com/boltdb/bolt v1.3.1 => "go.etcd.io/bbolt" v1.3.6
-
 require (
-	github.com/boltdb/bolt v1.3.1
 	github.com/gleanerio/nabu v0.0.0-20230316180932-b68225cabb13
 	github.com/orandin/lumberjackrus v1.0.1
 	github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4

go.sum

@@ -124,7 +124,6 @@ github.com/blevesearch/zap/v12 v12.0.14/go.mod h1:rOnuZOiMKPQj18AEKEHJxuI14236tT
 github.com/blevesearch/zap/v13 v13.0.6/go.mod h1:L89gsjdRKGyGrRN6nCpIScCvvkyxvmeDCwZRcjjPCrw=
 github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY=
 github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -387,6 +386,10 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
+github.com/k4s/phantomgo v0.0.0-20161104020322-11963773aa04 h1:tA/Xc0VnJtHIdxAML0WraKG+ErOYVgJ6oDcuxOloZOM=
+github.com/k4s/phantomgo v0.0.0-20161104020322-11963773aa04/go.mod h1:YWxksSger0gUVO0tKEY/mVkyBTPoKAf4KX/S8Vt7ndc=
+github.com/k4s/webrowser v0.0.0-20160107091637-934d526d0f27 h1:YBHxM4fmxQghvs3Ty/rQIPnY+tdCFheIOMj/h0Zw0A8=
+github.com/k4s/webrowser v0.0.0-20160107091637-934d526d0f27/go.mod h1:kd1f/k6xHQrfwfszgeiZklsPzBNJJj/el6cjp86YowQ=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/godepgraph v0.0.0-20190626013829-57a7e4a651a9/go.mod h1:Gb5YEgxqiSSVrXKWQxDcKoCM94NO5QAwOwTaVmIUAMI=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -451,8 +454,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
 github.com/minio/minio-go/v7 v7.0.15/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
-github.com/minio/minio-go/v7 v7.0.51 h1:eSewrwc23TqUDEH8aw8Bwp4f+JDdozRrPWcKR7DZhmY=
-github.com/minio/minio-go/v7 v7.0.51/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
+github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
+github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
@@ -645,8 +648,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -851,7 +852,6 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/check/check.go

@@ -111,7 +111,6 @@ func Setup(mc *minio.Client, v1 *viper.Viper) error {
 
 /*
 Check to see we can connect to s3 instance, and that buckets exist
-Might also be used to flight check bolt database, and if containers are up
 */
 func PreflightChecks(mc *minio.Client, v1 *viper.Viper) error {
 	// Validate Minio access

internal/summoner/acquire/acquire.go

@@ -1,7 +1,6 @@
 package acquire
 
 import (
-	"fmt"
 	"github.com/gleanerio/gleaner/internal/common"
 	"net/http"
 	"net/url"
@@ -17,14 +16,13 @@ import (
 	"github.com/minio/minio-go/v7"
 	"github.com/schollz/progressbar/v3"
 	"github.com/spf13/viper"
-	bolt "go.etcd.io/bbolt"
 )
 
 const EarthCubeAgent = "EarthCube_DataBot/1.0"
 const JSONContentType = "application/ld+json"
 
 // ResRetrieve is a function to pull down the data graphs at resources
-func ResRetrieve(v1 *viper.Viper, mc *minio.Client, m map[string][]string, db *bolt.DB, runStats *common.RunStats) {
+func ResRetrieve(v1 *viper.Viper, mc *minio.Client, m map[string][]string, runStats *common.RunStats) {
 	wg := sync.WaitGroup{}
 
 	// Why do I pass the wg pointer?   Just make a new one
@@ -47,7 +45,7 @@ func ResRetrieve(v1 *viper.Viper, mc *minio.Client, m map[string][]string, db *b
 		}
 		wg.Add(1)
 		//go getDomain(v1, mc, urls, domain, &wg, db)
-		go getDomain(v1, mc, urls, domain, &wg, db, repologger, r)
+		go getDomain(v1, mc, urls, domain, &wg, repologger, r)
 	}
 
 	wg.Wait()
@@ -92,17 +90,7 @@ func getConfig(v1 *viper.Viper, sourceName string) (string, int, int64, error) {
 }
 
 func getDomain(v1 *viper.Viper, mc *minio.Client, urls []string, sourceName string,
-	wg *sync.WaitGroup, db *bolt.DB, repologger *log.Logger, repoStats *common.RepoStats) {
-	//func getDomain(v1 *viper.Viper, mc *minio.Client, urls []string, sourceName string, wg *sync.WaitGroup, db *bolt.DB) {
-
-	// make the bucket (if it doesn't exist)
-	db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucket([]byte(sourceName))
-		if err != nil {
-			return fmt.Errorf("create bucket: %s", err)
-		}
-		return nil
-	})
+	wg *sync.WaitGroup, repologger *log.Logger, repoStats *common.RepoStats) {
 
 	bucketName, tc, delay, err := getConfig(v1, sourceName)
 	if err != nil {
@@ -178,17 +166,12 @@ func getDomain(v1 *viper.Viper, mc *minio.Client, urls []string, sourceName stri
 				// TODO is her where I then try headless, and scope the following for into an else?
 				log.WithFields(log.Fields{"url": urlloc, "contentType": "Direct access failed, trying headless']"}).Info("Direct access failed, trying headless for ", urlloc)
 				repologger.WithFields(log.Fields{"url": urlloc, "contentType": "Direct access failed, trying headless']"}).Error() // this needs to go into the issues file
-				err := PageRenderAndUpload(v1, mc, 60*time.Second, urlloc, sourceName, db, repologger, repoStats)                  // TODO make delay configurable
+				err := PageRenderAndUpload(v1, mc, 60*time.Second, urlloc, sourceName, repologger, repoStats)                      // TODO make delay configurable
 
 				if err != nil {
 					log.WithFields(log.Fields{"url": urlloc, "issue": "converting json ld"}).Error("PageRenderAndUpload ", urlloc, "::", err)
 					repologger.WithFields(log.Fields{"url": urlloc, "issue": "converting json ld"}).Error(err)
 				}
-				db.Update(func(tx *bolt.Tx) error {
-					b := tx.Bucket([]byte(sourceName))
-					err := b.Put([]byte(urlloc), []byte(fmt.Sprintf("NILL: %s", urlloc))) // no JOSN-LD found at this URL
-					return err
-				})
 				if err != nil {
 					log.Error("DB Update", urlloc, "::", err)
 				}
@@ -199,7 +182,7 @@ func getDomain(v1 *viper.Viper, mc *minio.Client, urls []string, sourceName stri
 				repoStats.Inc(common.Summoned)
 			}
 
-			UploadWrapper(v1, mc, bucketName, sourceName, urlloc, db, repologger, repoStats, jsonlds)
+			UploadWrapper(v1, mc, bucketName, sourceName, urlloc, repologger, repoStats, jsonlds)
 
 			bar.Add(1)                                          // bar.Incr()
 			log.Trace("#", i, "thread for", urlloc)             // print an message containing the index (won't keep order)
@@ -251,7 +234,7 @@ func FindJSONInResponse(v1 *viper.Viper, urlloc string, repologger *log.Logger,
 	return jsonlds, nil
 }
 
-func UploadWrapper(v1 *viper.Viper, mc *minio.Client, bucketName string, sourceName string, urlloc string, db *bolt.DB, repologger *log.Logger, repoStats *common.RepoStats, jsonlds []string) {
+func UploadWrapper(v1 *viper.Viper, mc *minio.Client, bucketName string, sourceName string, urlloc string, repologger *log.Logger, repoStats *common.RepoStats, jsonlds []string) {
 	for i, jsonld := range jsonlds {
 		if jsonld != "" { // traps out the root domain...   should do this different
 			logFields := log.Fields{"url": urlloc, "issue": "Uploading"}
@@ -269,28 +252,12 @@ func UploadWrapper(v1 *viper.Viper, mc *minio.Client, bucketName string, sourceN
 				log.WithFields(logFields).Info("Successfully put ", sha, " in summoned bucket for ", urlloc)
 				repoStats.Inc(common.Stored)
 			}
-			// TODO  Is here where to add an entry to the KV store
-			db.Update(func(tx *bolt.Tx) error {
-				b := tx.Bucket([]byte(sourceName))
-				err := b.Put([]byte(urlloc), []byte(sha))
-				if err != nil {
-					log.Error("Error writing to bolt ", err)
-				}
-				return nil
-			})
+
 		} else {
 			logFields := log.Fields{"url": urlloc, "issue": "Empty JSON-LD document found "}
 			log.WithFields(logFields).Info("Empty JSON-LD document found. Continuing.")
 			repologger.WithFields(logFields).Error("Empty JSON-LD document found. Continuing.")
-			// TODO  Is here where to add an entry to the KV store
-			db.Update(func(tx *bolt.Tx) error {
-				b := tx.Bucket([]byte(sourceName))
-				err := b.Put([]byte(urlloc), []byte(fmt.Sprintf("NULL: %s", urlloc))) // no JSON-LD found at this URL
-				if err != nil {
-					log.Error("Error writing to bolt ", err)
-				}
-				return nil
-			})
+
 		}
 	}
 }

internal/summoner/acquire/api.go

@@ -7,7 +7,6 @@ import (
 	"github.com/minio/minio-go/v7"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	bolt "go.etcd.io/bbolt"
 	"net/http"
 	"sync"
 	"time"
@@ -33,7 +32,7 @@ func RetrieveAPIEndpoints(v1 *viper.Viper) ([]configTypes.Sources, error) {
 
 // given a paged API url template, iterate through the pages until we get
 // all the results we want.
-func RetrieveAPIData(apiSources []configTypes.Sources, mc *minio.Client, db *bolt.DB, runStats *common.RunStats, v1 *viper.Viper) {
+func RetrieveAPIData(apiSources []configTypes.Sources, mc *minio.Client, runStats *common.RunStats, v1 *viper.Viper) {
 	wg := sync.WaitGroup{}
 
 	for _, source := range apiSources {
@@ -50,21 +49,13 @@ func RetrieveAPIData(apiSources []configTypes.Sources, mc *minio.Client, db *bol
 			repologger.Info("Queuing API calls for ", source.Name)
 		}
 		wg.Add(1)
-		go getAPISource(v1, mc, source, &wg, db, repologger, r)
+		go getAPISource(v1, mc, source, &wg, repologger, r)
 	}
 
 	wg.Wait()
 }
 
-func getAPISource(v1 *viper.Viper, mc *minio.Client, source configTypes.Sources, wg *sync.WaitGroup, db *bolt.DB, repologger *log.Logger, repoStats *common.RepoStats) {
-	// make the bucket (if it doesn't exist)
-	db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucket([]byte(source.Name))
-		if err != nil {
-			return fmt.Errorf("create bucket: %s", err)
-		}
-		return nil
-	})
+func getAPISource(v1 *viper.Viper, mc *minio.Client, source configTypes.Sources, wg *sync.WaitGroup, repologger *log.Logger, repoStats *common.RepoStats) {
 
 	bucketName, tc, delay, err := getConfig(v1, source.Name)
 	if err != nil {
@@ -140,19 +131,13 @@ func getAPISource(v1 *viper.Viper, mc *minio.Client, source configTypes.Sources,
 				log.WithFields(log.Fields{"url": urlloc, "contentType": "No JSON-LD found']"}).Info("No JSON-LD found at ", urlloc)
 				repologger.WithFields(log.Fields{"url": urlloc, "contentType": "No JSON-LD found']"}).Error() // this needs to go into the issues file
 
-				db.Update(func(tx *bolt.Tx) error {
-					b := tx.Bucket([]byte(sourceName))
-					err := b.Put([]byte(urlloc), []byte(fmt.Sprintf("NILL: %s", urlloc))) // no JSON-LD found at this URL
-					return err
-				})
-
 			} else {
 				log.WithFields(log.Fields{"url": urlloc, "issue": "Indexed"}).Trace("Indexed ", urlloc)
 				repologger.WithFields(log.Fields{"url": urlloc, "issue": "Indexed"}).Trace()
 				repoStats.Inc(common.Summoned)
 			}
 
-			UploadWrapper(v1, mc, bucketName, sourceName, urlloc, db, repologger, repoStats, jsonlds)
+			UploadWrapper(v1, mc, bucketName, sourceName, urlloc, repologger, repoStats, jsonlds)
 
 			log.Trace("#", i, "thread for", urlloc)             // print an message containing the index (won't keep order)
 			time.Sleep(time.Duration(delay) * time.Millisecond) // sleep a bit if directed to by the provider