Merge branch 'gtm' of wangfan/adweb3-server into master

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/analytics/CreatePropertyRequestDTO.java

@@ -1,11 +1,13 @@
 package org.jeecg.modules.adweb.dmp.dto.google.analytics;
 
+import lombok.Builder;
 import lombok.Data;
 
 /**
  * @author wfansh
  */
 @Data
+@Builder
 public class CreatePropertyRequestDTO {
 
     private String accountResourceName;

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/gtm/CreateContainerRequestDTO.java

@@ -1,11 +1,13 @@
 package org.jeecg.modules.adweb.dmp.dto.google.gtm;
 
+import lombok.Builder;
 import lombok.Data;
 
 /**
  * @author wfansh
  */
 @Data
+@Builder
 public class CreateContainerRequestDTO {
 
     private String accountResourceName;

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/google/GTMAdminService.java

@@ -1,15 +1,36 @@
 package org.jeecg.modules.adweb.dmp.service.google;
 
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.Resources;
+
 import jakarta.annotation.PostConstruct;
 
+import lombok.extern.slf4j.Slf4j;
+
 import org.apache.commons.lang3.tuple.Pair;
+import org.jeecg.common.util.FastJsonUtil;
+import org.jeecg.modules.adweb.common.util.ListUtil;
 import org.jeecg.modules.adweb.common.util.RestTemplateUtil;
+import org.jeecg.modules.adweb.dmp.dto.OpenAPIRequest;
+import org.jeecg.modules.adweb.dmp.dto.OpenAPIResponse;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.CreatePropertyRequestDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAAccountDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAPropertyDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.gtm.CreateContainerRequestDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.gtm.GTMContainerDTO;
+import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
+import org.jeecg.modules.adweb.dmp.service.IGoogleGTMService;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
 /**
  * Google Tag Manager + Google Analytics帐号及标签管理
  *
@@ -17,12 +38,13 @@ import org.springframework.web.client.RestTemplate;
  *
  * @author wfansh
  */
+@Slf4j
 @Service
 public class GTMAdminService {
 
-    private static final String GA_PROPERTY_CREATE_API_PATH = "/api/google/ga/properties/create";
-    private static final String GTM_CONTAINER_CREATE_API_PATH = "/api/google/gtm/containers/create";
-    private static final String GTM_CONTAINER_GET_API_PATH = "/api/google/gtm/containers/create";
+    private static final String GA_CREATE_PROPERTY_API_PATH = "/api/google/ga/properties/create";
+    private static final String GTM_CREATE_CONTAINER_API_PATH = "/api/google/gtm/containers/create";
+    private static final String GTM_GET_CONTAINER_API_PATH = "/api/google/gtm/containers/get";
 
     @Value("${data-bridge.api.host}")
     private String dataBridgeApiHost;
@@ -30,17 +52,129 @@ public class GTMAdminService {
     @Value("${data-bridge.api.token}")
     private String dataBridgeApiToken;
 
+    @Value("${data-bridge.gtm.account-id}")
+    private String gtmAccountId;
+
+    @Value("${data-bridge.ga.account-id}")
+    private String gaAccountId;
+
+    @Autowired private IGoogleGTMService googleGTMService;
+
     private RestTemplate restTemplate;
 
+    private String headSnippetTemplate;
+
+    private String bodySnippetTemplate;
+
     @PostConstruct
-    private void init() {
+    private void init() throws IOException {
         this.restTemplate = RestTemplateUtil.getRestTemplate(60, 60, dataBridgeApiToken);
+
+        headSnippetTemplate =
+                Resources.toString(
+                        this.getClass()
+                                .getClassLoader()
+                                .getResource("google/gtm/head-snippet.tmpl"),
+                        StandardCharsets.UTF_8);
+        bodySnippetTemplate =
+                Resources.toString(
+                        this.getClass()
+                                .getClassLoader()
+                                .getResource("google/gtm/body-snippet.tmpl"),
+                        StandardCharsets.UTF_8);
     }
 
-    /** 创建Google Analytics + Google Tag Manager标签 */
-    public Pair<GAPropertyDTO, GTMContainerDTO> createTags(
-            String siteCode, String siteUrl, String siteName) {
+    /**
+     *
+     * <li>1. 创建GTM container + GA property
+     * <li>2. 更新{@link GoogleGTM}表
+     */
+    public GoogleGTM createContainer(String siteCode, String siteUrl, String siteName) {
+        List<GoogleGTM> googleGTMs =
+                googleGTMService.list(
+                        new LambdaQueryWrapper<GoogleGTM>().eq(GoogleGTM::getSiteCode, siteCode));
 
-        // TODO
+        if (ListUtil.notEmpty(googleGTMs)) {
+            log.info("站点 {} 对应的GoogleGTM已存在，ID = {}", siteCode, googleGTMs.get(0).getId());
+            return googleGTMs.get(0);
+        }
+
+        // 1. 创建GA property - 通过data-bridge API
+        OpenAPIRequest<CreatePropertyRequestDTO> createGAPropertyRequest = new OpenAPIRequest<>();
+        createGAPropertyRequest.setRequestServer(this.getClass().getSimpleName());
+        createGAPropertyRequest.setRequestTime(System.currentTimeMillis());
+        createGAPropertyRequest.setData(
+                CreatePropertyRequestDTO.builder()
+                        .accountResourceName(GAAccountDTO.toResourceName(gaAccountId))
+                        .displayName(siteName)
+                        .url(siteUrl)
+                        .build());
+
+        GAPropertyDTO gaProperty =
+                RestTemplateUtil.postForObject(
+                                restTemplate,
+                                dataBridgeApiHost + GA_CREATE_PROPERTY_API_PATH,
+                                createGAPropertyRequest,
+                                new ParameterizedTypeReference<OpenAPIResponse<GAPropertyDTO>>() {})
+                        .getData();
+        log.info("为站点 {} 创建GA property: {}", siteCode, FastJsonUtil.toJSONString(gaProperty));
+
+        // 2. 创建GTM container - 通过data-bridge API
+        OpenAPIRequest<CreateContainerRequestDTO> createGTMContainerRequest =
+                new OpenAPIRequest<>();
+        createGTMContainerRequest.setRequestServer(this.getClass().getSimpleName());
+        createGTMContainerRequest.setRequestTime(System.currentTimeMillis());
+        createGTMContainerRequest.setData(
+                CreateContainerRequestDTO.builder()
+                        .accountResourceName(GTMContainerDTO.toResourceName(gtmAccountId))
+                        .displayName(siteName)
+                        .googleTagId(gaProperty.getDataStreams().get(0).getStreamMeasurementId())
+                        .build());
+
+        GTMContainerDTO gtmContainer =
+                RestTemplateUtil.postForObject(
+                                restTemplate,
+                                dataBridgeApiHost + GTM_CREATE_CONTAINER_API_PATH,
+                                createGTMContainerRequest,
+                                new ParameterizedTypeReference<
+                                        OpenAPIResponse<GTMContainerDTO>>() {})
+                        .getData();
+        log.info("为站点 {} 创建GTM container: {}", siteCode, FastJsonUtil.toJSONString(gtmContainer));
+
+        // 3. 更新数据库
+        GoogleGTM googleGTM = new GoogleGTM();
+        googleGTM.setSiteCode(siteCode);
+        googleGTM.setUid(null); // TODO
+        googleGTM.setGtmAccountId(gtmAccountId);
+        googleGTM.setGtmContainerId(gtmContainer.getId());
+        googleGTM.setGtmTagId(gtmContainer.getPublicId());
+        googleGTM.setGaAccountId(gaAccountId);
+        googleGTM.setGaPropertyId(gaProperty.getId());
+        googleGTM.setGaVersion("V4"); // GA4 https://support.google.com/analytics/answer/10089681
+        googleGTM.setGaSiteUrl(siteUrl);
+        googleGTM.setGaTagId(gaProperty.getDataStreams().get(0).getStreamMeasurementId());
+        googleGTMService.save(googleGTM);
+
+        return googleGTM;
+    }
+
+    /**
+     * 返回GTM container的head + body snippet代码
+     *
+     * @param gtmTagId
+     * @return
+     */
+    public Pair<String, String> getSnippets(String gtmTagId) {
+        return Pair.of(
+                String.format(headSnippetTemplate, gtmTagId),
+                String.format(bodySnippetTemplate, gtmTagId));
     }
+
+    /**
+     * 测试专用 - 删除相关资源
+     * <li>1. 删除GTM container + GA property
+     * <li>2. 删除{@link GoogleGTM}表
+     */
+    @VisibleForTesting
+    void deleteContainer(String siteCode) {}
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/seo/service/dataforseo/DataForSEOService.java

@@ -1,7 +1,6 @@
 package org.jeecg.modules.adweb.seo.service.dataforseo;
 
 import cn.hutool.core.collection.CollectionUtil;
-import cn.hutool.json.JSONUtil;
 
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
@@ -18,6 +17,7 @@ import jakarta.annotation.PostConstruct;
 import lombok.extern.slf4j.Slf4j;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jeecg.common.util.FastJsonUtil;
 import org.jeecg.modules.adweb.common.util.AdwebRedisUtil;
 import org.jeecg.modules.adweb.common.util.CommonUtil;
 import org.jeecg.modules.adweb.common.util.DateUtil;
@@ -167,7 +167,7 @@ public class DataForSEOService {
                     serpApi.googleOrganicTaskPost(serpTaskRequestInfoList);
             log.info(
                     "创建DataForSEO Serp任务，response = {}",
-                    JSONUtil.toJsonStr(serpTaskPostResponseInfo));
+                    FastJsonUtil.toJSONString(serpTaskPostResponseInfo));
             if (serpTaskPostResponseInfo.getStatusCode() != SERP_STATUS_CODE_SUCCESS) {
                 throw new ApiException(serpTaskPostResponseInfo.getStatusMessage());
             }
@@ -223,7 +223,7 @@ public class DataForSEOService {
             String taskId = redisUtil.getString(serpTaskRedisKey);
             SerpGoogleOrganicTaskGetRegularTaskInfo serpTask =
                     serpApi.googleOrganicTaskGetRegular(taskId).getTasks().get(0);
-            log.info("获取DataForSEO Serp任务，response = {}", JSONUtil.toJsonStr(serpTask));
+            log.info("获取DataForSEO Serp任务，response = {}", FastJsonUtil.toJSONString(serpTask));
             if (serpTask.getStatusCode() != SERP_STATUS_CODE_SUCCESS) {
                 log.info(
                         "DataForSEO Serp任务 {} 状态为 {} {}",

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/xxl/DataForSEOJob.java

@@ -26,6 +26,12 @@ public class DataForSEOJob {
 
     @Autowired private DataForSEOService dataForSEOService;
 
+    /**
+     * 查询长尾关键词，Serp更新频率较低，建议三天左右执行一次
+     *
+     * @param siteCodes
+     * @return
+     */
     @XxlJob("runLongTailKeywordsSerpTasksHandler")
     public ReturnT<String> runLongTailKeywordsSerpTasksHandler(String siteCodes) {
         log.info("执行长尾词Serp查询..., site codes = {}", siteCodes);
@@ -38,6 +44,12 @@ public class DataForSEOJob {
         return ReturnT.SUCCESS;
     }
 
+    /**
+     * 查询指定关键词，Serp更新频率较低，建议三天左右执行一次
+     *
+     * @param siteCodes
+     * @return
+     */
     @XxlJob("runAppointKeywordsSerpTasksHandler")
     public ReturnT<String> runAppointKeywordsSerpTasksHandler(String siteCodes) {
         log.info("执行指定词Serp查询..., site codes = {}", siteCodes);
@@ -50,6 +62,11 @@ public class DataForSEOJob {
         return ReturnT.SUCCESS;
     }
 
+    /**
+     * 同步Serp查询结果，基于Redis中的剩余任务 - 负载较低，建议一天执行多次
+     *
+     * @return
+     */
     @XxlJob("syncKeywordsSerpResultsHandler")
     public ReturnT<String> syncKeywordsSerpResultsHandler(String param) {
         log.info("同步关键词Serp查询结果...");

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/xxl/GAReportJob.java

@@ -20,6 +20,12 @@ public class GAReportJob {
 
     @Autowired private GAReportService gaReportService;
 
+    /**
+     * 查询GA数据 - 报表以为天为单位，且覆盖旧数据，建议每天执行一至三次
+     *
+     * @param param
+     * @return
+     */
     @XxlJob("syncGAReportHandler")
     public ReturnT<String> syncGAReportHandler(String param) {
         log.info("同步GA报表数据...");

jeecg-module-system/jeecg-system-biz/src/main/resources/google/gtm/body-snippet.tmpl

@@ -0,0 +1,4 @@
+<!-- Google Tag Manager (noscript) -->
+<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=%s"
+height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
+<!-- End Google Tag Manager (noscript) -->

jeecg-module-system/jeecg-system-biz/src/main/resources/google/gtm/head-snippet.tmpl

@@ -0,0 +1,7 @@
+<!-- Google Tag Manager -->
+<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+})(window,document,'script','dataLayer','%s');</script>
+<!-- End Google Tag Manager -->

jeecg-module-system/jeecg-system-start/src/main/resources/application-dev.yml

@@ -355,6 +355,10 @@ data-bridge:
   api:
     host: http://data-bridge.v3.adwebcloud.com:9002
     token: lgoXX9APqgPLGMPECiNoxaPx
+  gtm:
+    account-id: 6000226571
+  ga:
+    account-id: 191734056
 
 ##GEOIP MMDB 静态数据库文件
 geoip:

jeecg-module-system/jeecg-system-start/src/main/resources/application-prod.yml

@@ -343,6 +343,10 @@ data-bridge:
   api:
     host: http://data-bridge.v3.adwebcloud.com:9002
     token: lgoXX9APqgPLGMPECiNoxaPx
+  gtm:
+    account-id: 6000226571
+  ga:
+    account-id: 191734056
 
 ##GEOIP MMDB 静态数据库文件
 geoip:

jeecg-module-system/jeecg-system-start/src/main/resources/application-test.yml

@@ -340,6 +340,10 @@ data-bridge:
   api:
     host: http://data-bridge.v3.adwebcloud.com:9002
     token: lgoXX9APqgPLGMPECiNoxaPx
+  gtm:
+    account-id: 6000226571
+  ga:
+    account-id: 191734056
 
 ### dataforseo
 dataforseo:

jeecg-module-system/jeecg-system-start/src/test/java/org/jeecg/modules/adweb/dmp/service/google/GoogleServiceTest.java

@@ -17,6 +17,8 @@ public class GoogleServiceTest {
 
     @Autowired GAReportService gaReportService;
 
+    @Autowired GTMAdminService gtmAdminService;
+
     @Test
     public void testGAReport() {
         GAReportRequestDTO gaReportRequest = new GAReportRequestDTO();