Merge pull request #242 from wo10378931/develop

Develop
crawlab-team · Oct 7, 2019 · 9a973ab · 9a973ab
2 parents b9f316c + 4a40d38
commit 9a973ab
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/backend/routes/spider.go b/backend/routes/spider.go
@@ -135,7 +135,7 @@ func PutSpider(c *gin.Context) {
 	// 以防tmp目录不存在
 	tmpPath := viper.GetString("other.tmppath")
 	if !utils.Exists(tmpPath) {
-		if err := os.Mkdir(tmpPath, os.ModePerm); err != nil {
+		if err := os.MkdirAll(tmpPath, os.ModePerm); err != nil {
 			log.Error("mkdir other.tmppath dir error:" + err.Error())
 			debug.PrintStack()
 			HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error"))

diff --git a/backend/services/spider.go b/backend/services/spider.go
@@ -16,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime/debug"
+	"strings"
 )
 
 type SpiderFileData struct {
@@ -144,7 +145,11 @@ func PublishSpider(spider model.Spider) {
 	}
 	// md5值不一样，则下载
 	md5Str := utils.ReadFileOneLine(md5)
+	// 去掉空格以及换行符
+	md5Str = strings.Replace(md5Str, " ", "", -1)
+	md5Str = strings.Replace(md5Str, "\n", "", -1)
 	if gfFile.Md5 != md5Str {
+		log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str)
 		spiderSync.RemoveSpiderFile()
 		spiderSync.Download()
 		spiderSync.CreateMd5File(gfFile.Md5)