diff --git a/app/task/init.go b/app/task/init.go index df674b7..072f36d 100644 --- a/app/task/init.go +++ b/app/task/init.go @@ -87,4 +87,7 @@ func initTasks() { jobs[taskMd.DataCollectCronHaodankuRealNews] = taskDataCollectCronHaodankuRealNews // jobs[taskMd.DataCollectCronHaodankuRealNews10] = taskDataCollectCronHaodankuRealNews10 // + jobs[taskMd.DataCollectCronMiyuanRealNews] = taskDataCollectCronMiyuanRealNews // + jobs[taskMd.DataCollectCronMiyuanRealNews10] = taskDataCollectCronMiyuanRealNews10 // + } diff --git a/app/task/md/cron_key.go b/app/task/md/cron_key.go index d780827..8c010f7 100644 --- a/app/task/md/cron_key.go +++ b/app/task/md/cron_key.go @@ -3,4 +3,7 @@ package md const ( DataCollectCronHaodankuRealNews = "data_collect_cron_haodanku_real_news" DataCollectCronHaodankuRealNews10 = "data_collect_cron_haodanku_real_news10" + + DataCollectCronMiyuanRealNews = "data_collect_cron_miyuan_real_news" + DataCollectCronMiyuanRealNews10 = "data_collect_cron_miyuan_real_news10" ) diff --git a/app/task/svc/svc_data_collect_cron_miyuan_real_news.go b/app/task/svc/svc_data_collect_cron_miyuan_real_news.go new file mode 100644 index 0000000..7cb4a91 --- /dev/null +++ b/app/task/svc/svc_data_collect_cron_miyuan_real_news.go @@ -0,0 +1,124 @@ +package svc + +import ( + "applet/app/es/md" + md2 "applet/app/task/md" + "applet/app/utils" + "code.fnuoos.com/go_rely_warehouse/zyos_go_es.git/es" + "code.fnuoos.com/go_rely_warehouse/zyos_go_third_party_api.git/gzmiyuan" + "fmt" + "github.com/syyongx/php2go" + "strings" + "time" +) + +func DataCollectCronMiyuanRealNews(types int) { + key := md.ZhiosRealNewsData + //es.CreateIndexIfNotExists(key, md.ZhiosRealNewsDataField) + ids := []string{"MC", "TM", "JD", "PDD", "CD", "ZD", "MS", "QYH"} + ids = []string{"JD"} + num := 10 + if types == 0 { + ids = []string{""} + num = 3 + } + size := 10 + now := time.Now() + fmt.Println("============", time.Since(now)) + for _, v := range ids { + for i := 1; i <= num; i++ { + newsData, err := gzmiyuan.GetGzmiRealData(utils.IntToStr(size), utils.IntToStr(i), v) + if newsData == nil || err != nil { + continue + } + platformMap := map[string]string{"MC": "10001", "TM": "10003", "JD": "10004", "PDD": "10005", "CD": "10006", "ZD": "10002", "MS": "10007", "QYH": "10000"} + for _, item := range newsData { + var tmp = md2.RealNewsData{ + Id: item.Id, + CateId: utils.SerializeStr([]string{platformMap[item.OpenType]}), + Images: utils.SerializeStr(strings.Split(item.XianbaoImageUrl, ",")), + Platform: "my", + StartTime: utils.TimeStdParseUnix(item.CreateTime), + OldContent: item.XianbaoContent, + } + tmp.Key = php2go.Md5("hdk_real_news_my_" + utils.IntToStr(item.Id)) + var uniqueId = tmp.Key + doc, _ := es.FirstDoc(key, uniqueId) + if doc == nil { + goodsList := ToContent(item.XianbaoContent) + + tmp.Content = utils.SerializeStr(goodsList) + fmt.Println(tmp) + createDocRet, err := es.CreateDoc(key, uniqueId, tmp) + if err != nil { + return + } + fmt.Printf("CreateDoc ==> %+v \n\n", createDocRet) + } + + } + } + + } + fmt.Println("==========================end", time.Since(now)) +} +func To(str, str1, str2 string) string { + tmp := strings.Split(str, str1) + if len(tmp) > 0 { + tmp1 := strings.Split(tmp[1], str2) + if len(tmp1) > 0 { + return tmp1[0] + } + } + return "" +} +func ToContent(str string) []map[string]string { + platformTypeList := map[string]string{ + "JD": "jd", + "PDD": "pdd", + "TB": "taobao", + "TM": "tmall", + } + str = strings.ReplaceAll(str, "
", "\n") + strs := strings.Split(str, "
") + for _, v1 := range tmps { + if strings.Contains(v1, "href=") { + url := To(v1, "href=\"", "\"") + gid := To(v1, "item_id=\"", "\"") + platformType := To(v1, "open_type=\"", "\"") + goodsListTmp = map[string]string{ + "is_click": "1", + "content": "去购买>\n", + "gid": gid, + "url": "", + "platform_type": platformTypeList[platformType], + } + if platformType == "JD" && gid == "" { + goodsListTmp = map[string]string{ + "is_click": "0", + "content": url, + "gid": "", + "url": "", + "platform_type": platformTypeList[platformType], + } + } + } + } + } + goodsList = append(goodsList, goodsListTmp) + + } + return goodsList +} diff --git a/app/task/task_data_collect_cron_miyuan_real_news.go b/app/task/task_data_collect_cron_miyuan_real_news.go new file mode 100644 index 0000000..ad9a6bc --- /dev/null +++ b/app/task/task_data_collect_cron_miyuan_real_news.go @@ -0,0 +1,21 @@ +package task + +import ( + "applet/app/task/svc" + "math/rand" + "time" +) + +func taskDataCollectCronMiyuanRealNews() { + for { + if len(ch) > workerNum { + time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000))) + } else { + goto START + } + } +START: + ch <- 1 + svc.DataCollectCronMiyuanRealNews(1) + <-ch +} diff --git a/app/task/task_data_collect_cron_miyuan_real_news10.go b/app/task/task_data_collect_cron_miyuan_real_news10.go new file mode 100644 index 0000000..2af1015 --- /dev/null +++ b/app/task/task_data_collect_cron_miyuan_real_news10.go @@ -0,0 +1,21 @@ +package task + +import ( + "applet/app/task/svc" + "math/rand" + "time" +) + +func taskDataCollectCronMiyuanRealNews10() { + for { + if len(ch) > workerNum { + time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000))) + } else { + goto START + } + } +START: + ch <- 1 + svc.DataCollectCronMiyuanRealNews(0) + <-ch +} diff --git a/go.mod b/go.mod index 3fb222f..001e2c7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( code.fnuoos.com/go_rely_warehouse/zyos_go_es.git v1.0.1-0.20230707081910-52e70aa52998 - code.fnuoos.com/go_rely_warehouse/zyos_go_third_party_api.git v1.1.21-0.20231221064048-3937c0824e59 + code.fnuoos.com/go_rely_warehouse/zyos_go_third_party_api.git v1.1.21-0.20240103061829-86ef8b1a29c1 github.com/360EntSecGroup-Skylar/excelize v1.4.1 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/boombuler/barcode v1.0.1