From 2c96fe85166ecca078f33c28ab4b14a90e28f60a Mon Sep 17 00:00:00 2001
From: huangjiajun <582604932@qq.com>
Date: Wed, 3 Jan 2024 14:56:54 +0800
Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E8=9C=9C=E6=BA=90=E6=95=B0?=
=?UTF-8?q?=E6=8D=AE=E9=87=87=E9=9B=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
app/task/init.go | 3 +
app/task/md/cron_key.go | 3 +
.../svc_data_collect_cron_miyuan_real_news.go | 124 ++++++++++++++++++
...task_data_collect_cron_miyuan_real_news.go | 21 +++
...sk_data_collect_cron_miyuan_real_news10.go | 21 +++
go.mod | 2 +-
6 files changed, 173 insertions(+), 1 deletion(-)
create mode 100644 app/task/svc/svc_data_collect_cron_miyuan_real_news.go
create mode 100644 app/task/task_data_collect_cron_miyuan_real_news.go
create mode 100644 app/task/task_data_collect_cron_miyuan_real_news10.go
diff --git a/app/task/init.go b/app/task/init.go
index df674b7..072f36d 100644
--- a/app/task/init.go
+++ b/app/task/init.go
@@ -87,4 +87,7 @@ func initTasks() {
jobs[taskMd.DataCollectCronHaodankuRealNews] = taskDataCollectCronHaodankuRealNews //
jobs[taskMd.DataCollectCronHaodankuRealNews10] = taskDataCollectCronHaodankuRealNews10 //
+ jobs[taskMd.DataCollectCronMiyuanRealNews] = taskDataCollectCronMiyuanRealNews //
+ jobs[taskMd.DataCollectCronMiyuanRealNews10] = taskDataCollectCronMiyuanRealNews10 //
+
}
diff --git a/app/task/md/cron_key.go b/app/task/md/cron_key.go
index d780827..8c010f7 100644
--- a/app/task/md/cron_key.go
+++ b/app/task/md/cron_key.go
@@ -3,4 +3,7 @@ package md
const (
DataCollectCronHaodankuRealNews = "data_collect_cron_haodanku_real_news"
DataCollectCronHaodankuRealNews10 = "data_collect_cron_haodanku_real_news10"
+
+ DataCollectCronMiyuanRealNews = "data_collect_cron_miyuan_real_news"
+ DataCollectCronMiyuanRealNews10 = "data_collect_cron_miyuan_real_news10"
)
diff --git a/app/task/svc/svc_data_collect_cron_miyuan_real_news.go b/app/task/svc/svc_data_collect_cron_miyuan_real_news.go
new file mode 100644
index 0000000..7cb4a91
--- /dev/null
+++ b/app/task/svc/svc_data_collect_cron_miyuan_real_news.go
@@ -0,0 +1,124 @@
+package svc
+
+import (
+ "applet/app/es/md"
+ md2 "applet/app/task/md"
+ "applet/app/utils"
+ "code.fnuoos.com/go_rely_warehouse/zyos_go_es.git/es"
+ "code.fnuoos.com/go_rely_warehouse/zyos_go_third_party_api.git/gzmiyuan"
+ "fmt"
+ "github.com/syyongx/php2go"
+ "strings"
+ "time"
+)
+
+func DataCollectCronMiyuanRealNews(types int) {
+ key := md.ZhiosRealNewsData
+ //es.CreateIndexIfNotExists(key, md.ZhiosRealNewsDataField)
+ ids := []string{"MC", "TM", "JD", "PDD", "CD", "ZD", "MS", "QYH"}
+ ids = []string{"JD"}
+ num := 10
+ if types == 0 {
+ ids = []string{""}
+ num = 3
+ }
+ size := 10
+ now := time.Now()
+ fmt.Println("============", time.Since(now))
+ for _, v := range ids {
+ for i := 1; i <= num; i++ {
+ newsData, err := gzmiyuan.GetGzmiRealData(utils.IntToStr(size), utils.IntToStr(i), v)
+ if newsData == nil || err != nil {
+ continue
+ }
+ platformMap := map[string]string{"MC": "10001", "TM": "10003", "JD": "10004", "PDD": "10005", "CD": "10006", "ZD": "10002", "MS": "10007", "QYH": "10000"}
+ for _, item := range newsData {
+ var tmp = md2.RealNewsData{
+ Id: item.Id,
+ CateId: utils.SerializeStr([]string{platformMap[item.OpenType]}),
+ Images: utils.SerializeStr(strings.Split(item.XianbaoImageUrl, ",")),
+ Platform: "my",
+ StartTime: utils.TimeStdParseUnix(item.CreateTime),
+ OldContent: item.XianbaoContent,
+ }
+ tmp.Key = php2go.Md5("hdk_real_news_my_" + utils.IntToStr(item.Id))
+ var uniqueId = tmp.Key
+ doc, _ := es.FirstDoc(key, uniqueId)
+ if doc == nil {
+ goodsList := ToContent(item.XianbaoContent)
+
+ tmp.Content = utils.SerializeStr(goodsList)
+ fmt.Println(tmp)
+ createDocRet, err := es.CreateDoc(key, uniqueId, tmp)
+ if err != nil {
+ return
+ }
+ fmt.Printf("CreateDoc ==> %+v \n\n", createDocRet)
+ }
+
+ }
+ }
+
+ }
+ fmt.Println("==========================end", time.Since(now))
+}
+func To(str, str1, str2 string) string {
+ tmp := strings.Split(str, str1)
+ if len(tmp) > 0 {
+ tmp1 := strings.Split(tmp[1], str2)
+ if len(tmp1) > 0 {
+ return tmp1[0]
+ }
+ }
+ return ""
+}
+func ToContent(str string) []map[string]string {
+ platformTypeList := map[string]string{
+ "JD": "jd",
+ "PDD": "pdd",
+ "TB": "taobao",
+ "TM": "tmall",
+ }
+ str = strings.ReplaceAll(str, "
", "\n")
+ strs := strings.Split(str, "")
+ for _, v1 := range tmps {
+ if strings.Contains(v1, "href=") {
+ url := To(v1, "href=\"", "\"")
+ gid := To(v1, "item_id=\"", "\"")
+ platformType := To(v1, "open_type=\"", "\"")
+ goodsListTmp = map[string]string{
+ "is_click": "1",
+ "content": "去购买>\n",
+ "gid": gid,
+ "url": "",
+ "platform_type": platformTypeList[platformType],
+ }
+ if platformType == "JD" && gid == "" {
+ goodsListTmp = map[string]string{
+ "is_click": "0",
+ "content": url,
+ "gid": "",
+ "url": "",
+ "platform_type": platformTypeList[platformType],
+ }
+ }
+ }
+ }
+ }
+ goodsList = append(goodsList, goodsListTmp)
+
+ }
+ return goodsList
+}
diff --git a/app/task/task_data_collect_cron_miyuan_real_news.go b/app/task/task_data_collect_cron_miyuan_real_news.go
new file mode 100644
index 0000000..ad9a6bc
--- /dev/null
+++ b/app/task/task_data_collect_cron_miyuan_real_news.go
@@ -0,0 +1,21 @@
+package task
+
+import (
+ "applet/app/task/svc"
+ "math/rand"
+ "time"
+)
+
+func taskDataCollectCronMiyuanRealNews() {
+ for {
+ if len(ch) > workerNum {
+ time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000)))
+ } else {
+ goto START
+ }
+ }
+START:
+ ch <- 1
+ svc.DataCollectCronMiyuanRealNews(1)
+ <-ch
+}
diff --git a/app/task/task_data_collect_cron_miyuan_real_news10.go b/app/task/task_data_collect_cron_miyuan_real_news10.go
new file mode 100644
index 0000000..2af1015
--- /dev/null
+++ b/app/task/task_data_collect_cron_miyuan_real_news10.go
@@ -0,0 +1,21 @@
+package task
+
+import (
+ "applet/app/task/svc"
+ "math/rand"
+ "time"
+)
+
+func taskDataCollectCronMiyuanRealNews10() {
+ for {
+ if len(ch) > workerNum {
+ time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000)))
+ } else {
+ goto START
+ }
+ }
+START:
+ ch <- 1
+ svc.DataCollectCronMiyuanRealNews(0)
+ <-ch
+}
diff --git a/go.mod b/go.mod
index 3fb222f..001e2c7 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.15
require (
code.fnuoos.com/go_rely_warehouse/zyos_go_es.git v1.0.1-0.20230707081910-52e70aa52998
- code.fnuoos.com/go_rely_warehouse/zyos_go_third_party_api.git v1.1.21-0.20231221064048-3937c0824e59
+ code.fnuoos.com/go_rely_warehouse/zyos_go_third_party_api.git v1.1.21-0.20240103061829-86ef8b1a29c1
github.com/360EntSecGroup-Skylar/excelize v1.4.1
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5
github.com/boombuler/barcode v1.0.1