Merge branch 'gtm' of wangfan/adweb3-server into master

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/common/util/CommonUtil.java

@@ -1,11 +1,16 @@
 package org.jeecg.modules.adweb.common.util;
 
+import static java.util.function.Predicate.*;
+
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
 import com.google.common.net.InternetDomainName;
 
 import lombok.extern.slf4j.Slf4j;
 
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.List;
 
 /**
  * 常用工具类
@@ -31,4 +36,19 @@ public class CommonUtil {
             return InternetDomainName.from(host).topPrivateDomain().toString();
         }
     }
+
+    /**
+     * 对一个字符串进行split and trim，过滤空串，返回{@link List}
+     *
+     * @param str
+     * @param separator
+     * @return
+     */
+    public static List<String> splitAndTrim(String str, final String separator) {
+        return Splitter.on(separator)
+                .splitToStream(str)
+                .map(String::trim)
+                .filter(not(Strings::isNullOrEmpty))
+                .toList();
+    }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/controller/GTMController.java

@@ -0,0 +1,50 @@
+package org.jeecg.modules.adweb.dmp.controller;
+
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
+import org.jeecg.modules.adweb.dmp.service.google.GTMAdminService;
+import org.jeecg.modules.adweb.site.entity.AdwebSite;
+import org.jeecg.modules.adweb.site.service.IAdwebSiteService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Objects;
+
+/**
+ * GTM容器和标签管理
+ *
+ * @author wfansh
+ */
+@Tag(name = "Google Tag Manager")
+@RestController
+@RequestMapping("/gtm")
+@Slf4j
+public class GTMController {
+
+    @Autowired private IAdwebSiteService adwebSiteService;
+
+    @Autowired private GTMAdminService gtmAdminService;
+
+    @RequestMapping(value = "/add", method = RequestMethod.POST)
+    @ResponseBody
+    public Result<Pair<String, String>> addContainer(String siteId) {
+        AdwebSite adwebSite = adwebSiteService.getById(siteId);
+        if (Objects.isNull(adwebSite)) {
+            return Result.error("站点未找到" + siteId);
+        }
+
+        // 如果GoogleGTM表中已存在，不再创建，返回现有记录
+        GoogleGTM googleGTM =
+                gtmAdminService.createContainer(
+                        adwebSite.getCode(), adwebSite.getDomain(), adwebSite.getName());
+        return Result.ok(gtmAdminService.getSnippets(googleGTM.getGtmTagId()));
+    }
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/analytics/GAAccountDTO.java

@@ -1,5 +1,7 @@
 package org.jeecg.modules.adweb.dmp.dto.google.analytics;
 
+import com.google.common.base.Joiner;
+
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -24,6 +26,8 @@ public class GAAccountDTO extends ResourceDTO {
     private long updateTime;
 
     public static String toResourceName(String id) {
-        return StringUtils.isNumeric(id) ? "accounts" + RESOURCE_NAME_SPLITTER + id : id;
+        return StringUtils.isNumeric(id)
+                ? Joiner.on(RESOURCE_NAME_SPLITTER).join("accounts", id)
+                : id;
     }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/analytics/GAPropertyDTO.java

@@ -1,5 +1,7 @@
 package org.jeecg.modules.adweb.dmp.dto.google.analytics;
 
+import com.google.common.base.Joiner;
+
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -42,6 +44,8 @@ public class GAPropertyDTO extends ResourceDTO {
     private List<GADataStreamDTO> dataStreams;
 
     public static String toResourceName(String id) {
-        return StringUtils.isNumeric(id) ? "properties" + RESOURCE_NAME_SPLITTER + id : id;
+        return StringUtils.isNumeric(id)
+                ? Joiner.on(RESOURCE_NAME_SPLITTER).join("properties", id)
+                : id;
     }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/gtm/GTMAccountDTO.java

@@ -1,5 +1,7 @@
 package org.jeecg.modules.adweb.dmp.dto.google.gtm;
 
+import com.google.common.base.Joiner;
+
 import lombok.*;
 import lombok.experimental.SuperBuilder;
 
@@ -21,6 +23,8 @@ public class GTMAccountDTO extends ResourceDTO {
     private boolean supportMultipleContainers;
 
     public static String toResourceName(String id) {
-        return StringUtils.isNumeric(id) ? "accounts" + RESOURCE_NAME_SPLITTER + id : id;
+        return StringUtils.isNumeric(id)
+                ? Joiner.on(RESOURCE_NAME_SPLITTER).join("accounts", id)
+                : id;
     }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/gtm/GTMContainerDTO.java

@@ -1,5 +1,7 @@
 package org.jeecg.modules.adweb.dmp.dto.google.gtm;
 
+import com.google.common.base.Joiner;
+
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -41,7 +43,10 @@ public class GTMContainerDTO extends ResourceDTO {
 
     private List<GTMTagDTO> tags;
 
-    public static String toResourceName(String id) {
-        return StringUtils.isNumeric(id) ? "accounts" + RESOURCE_NAME_SPLITTER + id : id;
+    public static String toResourceName(String accountId, String containerId) {
+        return StringUtils.isNumeric(accountId) && StringUtils.isNumeric(containerId)
+                ? Joiner.on(RESOURCE_NAME_SPLITTER)
+                        .join("accounts", accountId, "containers", containerId)
+                : containerId;
     }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/google/GAReportService.java

@@ -3,6 +3,7 @@ package org.jeecg.modules.adweb.dmp.service.google;
 import static org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportConstant.*;
 
 import com.baomidou.mybatisplus.annotation.TableName;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.google.common.annotations.VisibleForTesting;
 
@@ -16,6 +17,7 @@ import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.FastJsonUtil;
 import org.jeecg.modules.adweb.common.mapper.CommonMapper;
 import org.jeecg.modules.adweb.common.util.DateUtil;
+import org.jeecg.modules.adweb.common.util.ListUtil;
 import org.jeecg.modules.adweb.common.util.RestTemplateUtil;
 import org.jeecg.modules.adweb.dmp.dto.OpenAPIRequest;
 import org.jeecg.modules.adweb.dmp.dto.OpenAPIResponse;
@@ -86,19 +88,21 @@ public class GAReportService {
         this.restTemplate = RestTemplateUtil.getRestTemplate(60, 60, dataBridgeApiToken);
     }
 
-    /** 拉取并同步Google Analytics报表 */
-    public void syncGAReport() {
-        List<GoogleGTM> googleGTMS =
-                googleGTMService.list().stream()
-                        // 判断网站状态
-                        .filter(
-                                googleGTM ->
-                                        adwebSiteService
-                                                .getAllActiveSiteCodes()
-                                                .contains(googleGTM.getSiteCode()))
-                        .toList();
-
-        for (GoogleGTM googleGTM : googleGTMS) {
+    /**
+     * 拉取并同步Google Analytics报表
+     *
+     * @param siteCodes 待执行的网站codes，为空时执行所有网站
+     */
+    public void syncGAReport(List<String> siteCodes) {
+        if (ListUtil.isEmpty(siteCodes)) {
+            siteCodes = adwebSiteService.getAllActiveSiteCodes();
+        }
+
+        List<GoogleGTM> googleGTMs =
+                googleGTMService.list(
+                        new LambdaQueryWrapper<GoogleGTM>().in(GoogleGTM::getSiteCode, siteCodes));
+
+        for (GoogleGTM googleGTM : googleGTMs) {
             // 每个帐号同步更新三张报表
             try {
                 this.syncGACountryReport(googleGTM);
@@ -335,7 +339,9 @@ public class GAReportService {
      * <p>2. 表中最大时间减一天 - 如10月10号凌晨2点执行，最大时间可能是10号；同时9号数据GA侧有可能更新
      */
     private Date getReportStartDate(String tableName, String siteCode) {
-        Date maxDate = commonMapper.getMaxDate(tableName, "date", "site_code = " + siteCode);
+        Date maxDate =
+                commonMapper.getMaxDate(
+                        tableName, "date", String.format("site_code = '%s'", siteCode));
         if (Objects.isNull(maxDate)) {
             // 1. 一年前
             return DateUtil.addDays(new Date(), -365);

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/google/GTMAdminService.java

@@ -18,6 +18,7 @@ import org.jeecg.modules.adweb.dmp.dto.google.analytics.CreatePropertyRequestDTO
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAAccountDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAPropertyDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.gtm.CreateContainerRequestDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.gtm.GTMAccountDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.gtm.GTMContainerDTO;
 import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
 import org.jeecg.modules.adweb.dmp.service.IGoogleGTMService;
@@ -43,8 +44,9 @@ import java.util.List;
 public class GTMAdminService {
 
     private static final String GA_CREATE_PROPERTY_API_PATH = "/api/google/ga/properties/create";
+    private static final String GA_DELETE_PROPERTY_API_PATH = "/api/google/ga/properties/delete";
     private static final String GTM_CREATE_CONTAINER_API_PATH = "/api/google/gtm/containers/create";
-    private static final String GTM_GET_CONTAINER_API_PATH = "/api/google/gtm/containers/get";
+    private static final String GTM_DELETE_CONTAINER_API_PATH = "/api/google/gtm/containers/delete";
 
     @Value("${data-bridge.api.host}")
     private String dataBridgeApiHost;
@@ -93,7 +95,6 @@ public class GTMAdminService {
         List<GoogleGTM> googleGTMs =
                 googleGTMService.list(
                         new LambdaQueryWrapper<GoogleGTM>().eq(GoogleGTM::getSiteCode, siteCode));
-
         if (ListUtil.notEmpty(googleGTMs)) {
             log.info("站点 {} 对应的GoogleGTM已存在，ID = {}", siteCode, googleGTMs.get(0).getId());
             return googleGTMs.get(0);
@@ -109,7 +110,6 @@ public class GTMAdminService {
                         .displayName(siteName)
                         .url(siteUrl)
                         .build());
-
         GAPropertyDTO gaProperty =
                 RestTemplateUtil.postForObject(
                                 restTemplate,
@@ -126,11 +126,10 @@ public class GTMAdminService {
         createGTMContainerRequest.setRequestTime(System.currentTimeMillis());
         createGTMContainerRequest.setData(
                 CreateContainerRequestDTO.builder()
-                        .accountResourceName(GTMContainerDTO.toResourceName(gtmAccountId))
+                        .accountResourceName(GTMAccountDTO.toResourceName(gtmAccountId))
                         .displayName(siteName)
                         .googleTagId(gaProperty.getDataStreams().get(0).getStreamMeasurementId())
                         .build());
-
         GTMContainerDTO gtmContainer =
                 RestTemplateUtil.postForObject(
                                 restTemplate,
@@ -176,5 +175,50 @@ public class GTMAdminService {
      * <li>2. 删除{@link GoogleGTM}表
      */
     @VisibleForTesting
-    void deleteContainer(String siteCode) {}
+    boolean deleteContainer(String siteCode) {
+        List<GoogleGTM> googleGTMs =
+                googleGTMService.list(
+                        new LambdaQueryWrapper<GoogleGTM>().eq(GoogleGTM::getSiteCode, siteCode));
+        if (ListUtil.isEmpty(googleGTMs)) {
+            log.info("站点 {} 对应的GoogleGTM不存在", siteCode);
+            return false;
+        }
+
+        GoogleGTM googleGTM = googleGTMs.get(0);
+
+        // 1. 删除GA property - 通过data-bridge API
+        OpenAPIRequest<String> deleteGAPropertyRequest = new OpenAPIRequest<>();
+        deleteGAPropertyRequest.setRequestServer(this.getClass().getSimpleName());
+        deleteGAPropertyRequest.setRequestTime(System.currentTimeMillis());
+        deleteGAPropertyRequest.setData(GAPropertyDTO.toResourceName(googleGTM.getGaPropertyId()));
+        String gaPropertyResourceName =
+                RestTemplateUtil.postForObject(
+                                restTemplate,
+                                dataBridgeApiHost + GA_DELETE_PROPERTY_API_PATH,
+                                deleteGAPropertyRequest,
+                                new ParameterizedTypeReference<OpenAPIResponse<String>>() {})
+                        .getData();
+        log.info("为站点 {} 删除GA property: {}", siteCode, gaPropertyResourceName);
+
+        // 2. 删除GTM container - 通过data-bridge API
+        OpenAPIRequest<String> deleteGTMContainerRequest = new OpenAPIRequest<>();
+        deleteGTMContainerRequest.setRequestServer(this.getClass().getSimpleName());
+        deleteGTMContainerRequest.setRequestTime(System.currentTimeMillis());
+        deleteGTMContainerRequest.setData(
+                GTMContainerDTO.toResourceName(
+                        googleGTM.getGtmAccountId(), googleGTM.getGtmContainerId()));
+        String gtmContainerResourceName =
+                RestTemplateUtil.postForObject(
+                                restTemplate,
+                                dataBridgeApiHost + GTM_DELETE_CONTAINER_API_PATH,
+                                deleteGTMContainerRequest,
+                                new ParameterizedTypeReference<OpenAPIResponse<String>>() {})
+                        .getData();
+        log.info("为站点 {} 删除GTM container: {}", siteCode, gtmContainerResourceName);
+
+        // 3. 更新数据库
+        googleGTMService.removeById(googleGTM.getId());
+
+        return true;
+    }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/enquiry/controller/EnquiryController.java

@@ -1,7 +0,0 @@
-package org.jeecg.modules.adweb.enquiry.controller;
-
-/**
- * @author wfansh
- */
-public class EnquiryController {
-}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/seo/service/dataforseo/DataForSEOService.java

@@ -27,7 +27,6 @@ import org.jeecg.modules.adweb.seo.entity.SeoKeywordsSerp;
 import org.jeecg.modules.adweb.seo.mapper.SeoKeywordsMapper;
 import org.jeecg.modules.adweb.seo.service.ISeoKeywordsSerpService;
 import org.jeecg.modules.adweb.seo.service.ISeoKeywordsService;
-import org.jeecg.modules.adweb.site.entity.AdwebSite;
 import org.jeecg.modules.adweb.site.service.IAdwebSiteService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -95,20 +94,13 @@ public class DataForSEOService {
     /**
      * 全局查询更新DataForSEO keywords Serp数据 - 启动Serp任务并保存到Redis
      *
+     * @param siteCodes 待执行的网站codes，为空时执行所有网站
      * @param keywordType 1 - 指定词; 2 - 长尾词
      * @param limit 最大查询条数
      */
     public void runKeywordsSerpTasks(List<String> siteCodes, int keywordType, int limit) {
         if (ListUtil.isEmpty(siteCodes)) {
-            siteCodes =
-                    adwebSiteService
-                            .list(
-                                    new LambdaQueryWrapper<AdwebSite>()
-                                            .eq(AdwebSite::getStatus, 1)
-                                            .eq(AdwebSite::getRunStatus, 1))
-                            .stream()
-                            .map(AdwebSite::getCode)
-                            .toList();
+            siteCodes = adwebSiteService.getAllActiveSiteCodes();
         }
 
         List<SeoKeywords> seoKeywords =

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/xxl/DataForSEOJob.java

@@ -6,15 +6,11 @@ import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 
 import org.jeecg.modules.adweb.common.constant.AdwebConstant;
+import org.jeecg.modules.adweb.common.util.CommonUtil;
 import org.jeecg.modules.adweb.seo.service.dataforseo.DataForSEOService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
 /**
  * DataForSEO Serp查询及同步任务，{@link DataForSEOService}
  *
@@ -36,7 +32,7 @@ public class DataForSEOJob {
     public ReturnT<String> runLongTailKeywordsSerpTasksHandler(String siteCodes) {
         log.info("执行长尾词Serp查询..., site codes = {}", siteCodes);
         dataForSEOService.runKeywordsSerpTasks(
-                this.parseSiteCodes(siteCodes),
+                CommonUtil.splitAndTrim(siteCodes, ","),
                 AdwebConstant.KEYWORD_TYPE_LONG_TAIL,
                 Integer.MAX_VALUE);
         log.info("执行长尾词Serp查询结束");
@@ -54,7 +50,7 @@ public class DataForSEOJob {
     public ReturnT<String> runAppointKeywordsSerpTasksHandler(String siteCodes) {
         log.info("执行指定词Serp查询..., site codes = {}", siteCodes);
         dataForSEOService.runKeywordsSerpTasks(
-                this.parseSiteCodes(siteCodes),
+                CommonUtil.splitAndTrim(siteCodes, ","),
                 AdwebConstant.KEYWORD_TYPE_APPOINT,
                 Integer.MAX_VALUE);
         log.info("执行指定词Serp查询结束");
@@ -75,10 +71,4 @@ public class DataForSEOJob {
 
         return ReturnT.SUCCESS;
     }
-
-    private List<String> parseSiteCodes(String param) {
-        return Objects.nonNull(param)
-                ? Arrays.stream(param.split(",")).map(siteCode -> siteCode.trim()).toList()
-                : Collections.EMPTY_LIST;
-    }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/xxl/GAReportJob.java

@@ -5,6 +5,7 @@ import com.xxl.job.core.handler.annotation.XxlJob;
 
 import lombok.extern.slf4j.Slf4j;
 
+import org.jeecg.modules.adweb.common.util.CommonUtil;
 import org.jeecg.modules.adweb.dmp.service.google.GAReportService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
@@ -23,13 +24,13 @@ public class GAReportJob {
     /**
      * 查询GA数据 - 报表以为天为单位，且覆盖旧数据，建议每天执行一至三次
      *
-     * @param param
+     * @param siteCodes
      * @return
      */
     @XxlJob("syncGAReportHandler")
-    public ReturnT<String> syncGAReportHandler(String param) {
+    public ReturnT<String> syncGAReportHandler(String siteCodes) {
         log.info("同步GA报表数据...");
-        gaReportService.syncGAReport();
+        gaReportService.syncGAReport(CommonUtil.splitAndTrim(siteCodes, ","));
         log.info("同步GA报表数据结束");
 
         return ReturnT.SUCCESS;

jeecg-module-system/jeecg-system-start/src/test/java/org/jeecg/modules/adweb/dmp/service/google/GoogleServiceTest.java

@@ -1,9 +1,12 @@
 package org.jeecg.modules.adweb.dmp.service.google;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.jeecg.common.util.FastJsonUtil;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.GAReportRequestDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportType;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.data.CountryChartData;
+import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -32,4 +35,24 @@ public class GoogleServiceTest {
 
         System.out.println(FastJsonUtil.toJSONString(countryReport));
     }
+
+    @Test
+    @Disabled("GTM + GA帐户资源限制")
+    public void testCreateAndDeleteGTMContainer() {
+        GoogleGTM googleGTM =
+                gtmAdminService.createContainer(
+                        "230206anit40",
+                        "https://www.sourcingstone.com",
+                        "泉州索兴石业有限公司#" + this.getClass().getSimpleName());
+        System.out.println(FastJsonUtil.toJSONString(googleGTM));
+
+        gtmAdminService.deleteContainer(googleGTM.getSiteCode());
+    }
+
+    @Test
+    public void testGetGTMSnippets() {
+        Pair<String, String> snippets = gtmAdminService.getSnippets("GTM-T87NNQ4F");
+        System.out.println(snippets.getLeft());
+        System.out.println(snippets.getRight());
+    }
 }

jeecg-module-system/jeecg-system-start/src/test/java/org/jeecg/modules/adweb/seo/service/DataForSEOTest.java

@@ -17,14 +17,14 @@ public class DataForSEOTest {
     @Autowired private DataForSEOService dataForSEOService;
 
     @Test
-    @Disabled
+    @Disabled("DataForSEO创建Serp任务收费")
     public void testRunKeywordsSerpTasks() {
         dataForSEOService.runKeywordsSerpTasks(
                 Collections.EMPTY_LIST, AdwebConstant.KEYWORD_TYPE_APPOINT, 10);
     }
 
     @Test
-    @Disabled
+    @Disabled("DataForSEO读取Serp结果免费 - 但用例影响生产数据")
     public void testSyncKeywordsSerpResults() {
         dataForSEOService.syncKeywordsSerpResults();
     }