更新蜜源数据采集

app/task/init.go

@@ -88,6 +88,7 @@ func initTasks() {
 	jobs[taskMd.DataCollectCronHaodankuRealNews10] = taskDataCollectCronHaodankuRealNews10 //
 
 	jobs[taskMd.DataCollectCronMiyuanRealNews] = taskDataCollectCronMiyuanRealNews     //
+	jobs[taskMd.DataCollectCronMiyuanRealNews1] = taskDataCollectCronMiyuanRealNews1   //
 	jobs[taskMd.DataCollectCronMiyuanRealNews10] = taskDataCollectCronMiyuanRealNews10 //
 
 }

app/task/md/cron_key.go

@@ -5,5 +5,6 @@ const (
 	DataCollectCronHaodankuRealNews10 = "data_collect_cron_haodanku_real_news10"
 
 	DataCollectCronMiyuanRealNews   = "data_collect_cron_miyuan_real_news"
+	DataCollectCronMiyuanRealNews1  = "data_collect_cron_miyuan_real_news1"
 	DataCollectCronMiyuanRealNews10 = "data_collect_cron_miyuan_real_news10"
 )

app/task/svc/svc_data_collect_cron_miyuan_real_news.go

@@ -15,7 +15,62 @@ import (
 func DataCollectCronMiyuanRealNews(types int) {
 	key := md.ZhiosRealNewsData
 	//es.CreateIndexIfNotExists(key, md.ZhiosRealNewsDataField)
-	ids := []string{"MC", "TM", "JD", "PDD", "CD", "ZD", "MS", "QYH", "TM", "BJ", "HJ"}
+	ids := []string{"CD", "MS", "TM", "JD", "PDD"}
+	//ids = []string{"CD"}
+	num := 10
+	if types == 0 {
+		ids = []string{""}
+		num = 3
+	}
+	size := 10
+	now := time.Now()
+	fmt.Println("============", time.Since(now))
+	for _, v := range ids {
+		for i := 1; i <= num; i++ {
+			time.Sleep(time.Microsecond * 10) // 等待10毫秒
+			newsData, err := gzmiyuan.GetGzmiRealData(utils.IntToStr(size), utils.IntToStr(i), v)
+			utils.FilePutContents("my", utils.SerializeStr(newsData))
+			if err != nil {
+				utils.FilePutContents("my", err.Error())
+			}
+			if newsData == nil || err != nil {
+				continue
+			}
+			platformMap := map[string]string{"TM": "10010", "BJ": "10009", "HJ": "10008", "MC": "10001", "TB": "10003", "JD": "10004", "PDD": "10005", "CD": "10006", "ZD": "10002", "MS": "10007", "QYH": "10000"}
+			for _, item := range newsData {
+				var tmp = md2.RealNewsData{
+					Id:         item.Id,
+					CateId:     utils.SerializeStr([]string{platformMap[item.OpenType]}),
+					Images:     utils.SerializeStr(strings.Split(item.XianbaoImageUrl, ",")),
+					Platform:   "my",
+					StartTime:  utils.TimeStdParseUnix(item.CreateTime),
+					OldContent: item.XianbaoContent,
+				}
+				tmp.Key = php2go.Md5("hdk_real_news_my_" + utils.IntToStr(item.Id))
+				var uniqueId = tmp.Key
+				doc, _ := es.FirstDoc(key, uniqueId)
+				if doc == nil {
+					goodsList := ToContent(item.XianbaoContent)
+
+					tmp.Content = utils.SerializeStr(goodsList)
+					fmt.Println(tmp)
+					createDocRet, err := es.CreateDoc(key, uniqueId, tmp)
+					if err != nil {
+						return
+					}
+					fmt.Printf("CreateDoc ==> %+v \n\n", createDocRet)
+				}
+
+			}
+		}
+
+	}
+	fmt.Println("==========================end", time.Since(now))
+}
+func DataCollectCronMiyuanRealNews1(types int) {
+	key := md.ZhiosRealNewsData
+	//es.CreateIndexIfNotExists(key, md.ZhiosRealNewsDataField)
+	ids := []string{"ZD", "MC", "QYH", "TM", "BJ", "HJ"}
 	//ids = []string{"CD"}
 	num := 10
 	if types == 0 {

app/task/task_data_collect_cron_miyuan_real_news1.go

@@ -0,0 +1,21 @@
+package task
+
+import (
+	"applet/app/task/svc"
+	"math/rand"
+	"time"
+)
+
+func taskDataCollectCronMiyuanRealNews1() {
+	for {
+		if len(ch) > workerNum {
+			time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000)))
+		} else {
+			goto START
+		}
+	}
+START:
+	ch <- 1
+	svc.DataCollectCronMiyuanRealNews1(1)
+	<-ch
+}