fix: pull git issue

crawlab-team/crawlab#1224
crawlab-team · Jun 4, 2023 · ed9c369 · ed9c369
1 parent 25b7a85
commit ed9c369
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 3 deletions.
diff --git a/controllers/spider.go b/controllers/spider.go
@@ -1089,6 +1089,11 @@ func (ctx *spiderContext) _gitPull(gitClient *vcs.GitClient, remote, branch stri
 		return trace.TraceError(err)
 	}
 
+	// reset
+	if err := gitClient.Reset(); err != nil {
+		return trace.TraceError(err)
+	}
+
 	return nil
 }
 

diff --git a/go.mod b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484
 	github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73
 	github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c
-	github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230603042545-1d886de7e8c0
+	github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230604071631-1bf1ab20e7be
 	github.com/crawlab-team/go-trace v0.1.1
 	github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69
 	github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e

diff --git a/go.sum b/go.sum
@@ -156,6 +156,8 @@ github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230603041819-f743a203c7d5 h1:CcY2
 github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230603041819-f743a203c7d5/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I=
 github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230603042545-1d886de7e8c0 h1:0q8z1+Xgk27oX3lSryKi4wVpSYO1HDGum5ye0ebRrsM=
 github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230603042545-1d886de7e8c0/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I=
+github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230604071631-1bf1ab20e7be h1:yqakkCNjkKRJc7/Ds1L6z6th5B8p4qOKp7TqGoydT0w=
+github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230604071631-1bf1ab20e7be/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I=
 github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8=
 github.com/crawlab-team/go-trace v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8=
 github.com/crawlab-team/go-trace v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug=

diff --git a/models/models/spider.go b/models/models/spider.go
@@ -23,7 +23,6 @@ type Spider struct {
 	Mode         string               `json:"mode" bson:"mode"`                     // default Task.Mode
 	NodeIds      []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
 	Stat         *SpiderStat          `json:"stat,omitempty" bson:"-"`
-	GitId        primitive.ObjectID   `json:"git_id" bson:"git_id"`
 
 	// execution
 	Cmd      string `json:"cmd" bson:"cmd"`     // execute command

diff --git a/spider/admin/service.go b/spider/admin/service.go
@@ -208,7 +208,26 @@ func (svc *Service) syncGit() {
 		svc.syncLock = false
 	}()
 
-	gits, err := svc.modelSvc.GetGitList(bson.M{"auto_pull": true}, nil)
+	// spiders
+	spiders, err := svc.modelSvc.GetSpiderList(nil, nil)
+	if err != nil {
+		trace.PrintError(err)
+		return
+	}
+
+	// spider ids
+	var spiderIds []primitive.ObjectID
+	for _, s := range spiders {
+		spiderIds = append(spiderIds, s.Id)
+	}
+
+	// gits
+	gits, err := svc.modelSvc.GetGitList(bson.M{
+		"_id": bson.M{
+			"$in": spiderIds,
+		},
+		"auto_pull": true,
+	}, nil)
 	if err != nil {
 		trace.PrintError(err)
 		return
@@ -228,6 +247,8 @@ func (svc *Service) syncGit() {
 }
 
 func (svc *Service) syncGitOne(g models.Git) {
+	log.Infof("[SpiderAdminService] sync git %s", g.Id)
+
 	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
 	defer cancel()
 
@@ -261,6 +282,10 @@ func (svc *Service) syncGitOne(g models.Git) {
 			trace.PrintError(err)
 			return
 		}
+		if err := gitClient.Reset(); err != nil {
+			trace.PrintError(err)
+			return
+		}
 		if err := fsSvc.GetFsService().SyncToFs(); err != nil {
 			trace.PrintError(err)
 			return