Merge branch 'dmp' of wangfan/adweb3-server into master

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/common/enums/CountryCode.java

@@ -272,10 +272,18 @@ public enum CountryCode {
             Stream.of(CountryCode.values())
                     .collect(Collectors.toMap(CountryCode::getCode, Function.identity()));
 
+    private static final Map<String, CountryCode> NAME_MAP =
+            Stream.of(CountryCode.values())
+                    .collect(Collectors.toMap(CountryCode::getName, Function.identity()));
+
     public static CountryCode valueOfCode(String code) {
         return CODE_MAP.getOrDefault(code, UNK);
     }
 
+    public static CountryCode valueOfName(String name) {
+        return NAME_MAP.getOrDefault(name, UNK);
+    }
+
     private final String code;
     private final String name;
     private final String cnName;

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/controller/DMPDataController.java

@@ -13,6 +13,7 @@ import org.jeecg.modules.adweb.common.util.DateUtil;
 import org.jeecg.modules.adweb.dmp.service.*;
 import org.jeecg.modules.adweb.dmp.vo.report.*;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.format.annotation.DateTimeFormat;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -35,12 +36,16 @@ import java.util.stream.Collectors;
 @Slf4j
 public class DMPDataController {
 
+    @Value("${v3.dmp.realtimeReport}")
+    private boolean realtimeReport;
+
     @Autowired private IGADailyReportService gaDailyReportService;
     @Autowired private IGACountryReportService gaCountryReportService;
     @Autowired private IGASourceMediumReportService gaSourceMediumReportService;
     @Autowired private IGAPagePathReportService gaPagePathReportService;
     @Autowired private IGADeviceReportService gaDeviceReportService;
     @Autowired private IEnquiryReportService enquiryReportService;
+    @Autowired private RealtimeReportService realtimeReportService;
 
     /** 首页：网站流量按时间段分析统计 - 今天，昨天，本周，上周等 */
     @GetMapping("/site-periodic/stats")
@@ -69,9 +74,11 @@ public class DMPDataController {
             end = dateRange.getRight();
         }
 
-        // 2. 查询GA Daily Report和Enquiries
+        // 2. TODO: 实时或DB离线查询GA Daily Report和Enquiries
         List<DailyStatsVO> dailyStatsVOs =
-                gaDailyReportService.getDailyStatsForDateRange(siteCode, start, end);
+                realtimeReport
+                        ? realtimeReportService.getDailyStats(siteCode, start, end)
+                        : gaDailyReportService.getDailyStats(siteCode, start, end);
 
         // 3. 生成SiteOverviewStatsVO并返回
         return Result.ok(SiteOverviewStatsVO.fromDailyStats(dailyStatsVOs));
@@ -90,8 +97,11 @@ public class DMPDataController {
             end = dateRange.getRight();
         }
 
-        // 2. 查询并返回
-        return Result.ok(gaCountryReportService.getCountryStats(siteCode, start, end));
+        // 2. TODO: 实时或DB离线查询
+        return Result.ok(
+                realtimeReport
+                        ? realtimeReportService.getCountryStats(siteCode, start, end)
+                        : gaCountryReportService.getCountryStats(siteCode, start, end));
     }
 
     @GetMapping("/source-medium/stats")
@@ -107,8 +117,11 @@ public class DMPDataController {
             end = dateRange.getRight();
         }
 
-        // 2. 查询并返回
-        return Result.ok(gaSourceMediumReportService.getSourceMediumStats(siteCode, start, end));
+        // 2. TODO: 实时或DB离线查询
+        return Result.ok(
+                realtimeReport
+                        ? realtimeReportService.getSourceMediumStats(siteCode, start, end)
+                        : gaSourceMediumReportService.getSourceMediumStats(siteCode, start, end));
     }
 
     @GetMapping("/page-path/stats")
@@ -125,10 +138,12 @@ public class DMPDataController {
             end = dateRange.getRight();
         }
 
-        // 2. 查询并返回
+        // 2. TODO: 实时或DB离线查询
+        limit = limit >= 0 ? limit : 10;
         return Result.ok(
-                gaPagePathReportService.getPagePathStats(
-                        siteCode, start, end, limit >= 0 ? limit : 10));
+                realtimeReport
+                        ? realtimeReportService.getPagePathStats(siteCode, start, end, limit)
+                        : gaPagePathReportService.getPagePathStats(siteCode, start, end, limit));
     }
 
     @GetMapping("/device/stats")
@@ -144,8 +159,11 @@ public class DMPDataController {
             end = dateRange.getRight();
         }
 
-        // 2. 查询并返回
-        return Result.ok(gaDeviceReportService.getDeviceStats(siteCode, start, end));
+        // 2. TODO: 实时或DB离线查询
+        return Result.ok(
+                realtimeReport
+                        ? realtimeReportService.getDeviceStats(siteCode, start, end)
+                        : gaDeviceReportService.getDeviceStats(siteCode, start, end));
     }
 
     @GetMapping("/enquiry-country/stats")

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/analytics/report/GAReportRequestDTO.java

@@ -32,4 +32,6 @@ public class GAReportRequestDTO {
 
     /** Use default values in {@link ReportType} instead. */
     @Deprecated private Boolean orderByDesc;
+
+    private int limit;
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/IGADailyReportService.java

@@ -22,7 +22,7 @@ public interface IGADailyReportService extends IService<GADailyReport> {
      *
      * <p>填充每日询盘数据
      */
-    List<DailyStatsVO> getDailyStatsForDateRange(String siteCode, Date start, Date end);
+    List<DailyStatsVO> getDailyStats(String siteCode, Date start, Date end);
 
     /**
      * 查询分时间段的{@link PeriodicStatsVO} - 今天，昨天，本周，上周，本月，上月，全部

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/RealtimeReportService.java

@@ -0,0 +1,488 @@
+package org.jeecg.modules.adweb.dmp.service;
+
+import static org.jeecg.modules.adweb.dmp.vo.report.SiteOverviewStatsVO.DailyStatsVO;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
+import org.jeecg.modules.adweb.common.enums.CountryCode;
+import org.jeecg.modules.adweb.common.util.AdwebRedisUtil;
+import org.jeecg.modules.adweb.common.util.DateUtil;
+import org.jeecg.modules.adweb.common.util.NumberUtil;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAPropertyDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.GAReportRequestDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.OrderByType;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportConstant;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportType;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.data.CustomReportData;
+import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
+import org.jeecg.modules.adweb.dmp.service.google.GAReportService;
+import org.jeecg.modules.adweb.dmp.vo.report.*;
+import org.jeecg.modules.adweb.enquiry.mapper.AdwebEnquiryMapper;
+import org.jeecg.modules.adweb.site.entity.AdwebSite;
+import org.jeecg.modules.adweb.site.service.IAdwebSiteService;
+import org.jeecg.modules.redis.CacheConfig;
+import org.jeecg.modules.redis.TTLCacheManager;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * 从DataBridge实时拉取GA报表数据，非DB离线查询 - http://data-bridge.v3.adwebcloud.com:9002/swagger-ui/index.html
+ *
+ * @author wfansh
+ */
+@Service
+@Slf4j
+public class RealtimeReportService {
+
+    @Autowired private IAdwebSiteService adwebSiteService;
+    @Autowired private IGoogleGTMService googleGTMService;
+    @Autowired private AdwebEnquiryMapper adwebEnquiryMapper;
+    @Autowired private GAReportService gaReportService;
+    @Autowired private AdwebRedisUtil redisUtil;
+
+    /**
+     * 拉取Google Analytics - Daily报表
+     *
+     * <p>如数据库中某天数据缺失，需填充空数据
+     *
+     * <p>填充每日询盘数据
+     */
+    @Cacheable(
+            cacheManager = CacheConfig.TTL_CACHE_MANAGER,
+            cacheNames =
+                    "dmp:realtime:getDailyStats"
+                            + TTLCacheManager.TTL_SPLITTER
+                            + 60 * 10) // Redis TTL为10分钟
+    public List<SiteOverviewStatsVO.DailyStatsVO> getDailyStats(
+            String siteCode, Date start, Date end) {
+        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+            return Collections.EMPTY_LIST;
+        }
+
+        // 0. 起止日期为空时，设置默认区间 - start = 网站发布日期 & end = 今天
+        if (start == null || end == null) {
+            AdwebSite adwebSite = adwebSiteService.getSiteByCode(siteCode);
+            start =
+                    DateUtil.getTodayZeroTime(
+                            Optional.ofNullable(adwebSite.getIssueTime())
+                                    .orElse(adwebSite.getCtime()));
+            end = DateUtil.getTodayZeroTime(new Date());
+        }
+
+        // 1. 构建GA报表请求参数
+        GAReportRequestDTO gaReportRequest =
+                this.buildGAReportRequest(
+                        googleGTM.getGaPropertyId(),
+                        start,
+                        end,
+                        List.of(
+                                ReportConstant.METRIC_TOTAL_USERS,
+                                ReportConstant.METRIC_SESSIONS,
+                                ReportConstant.METRIC_BOUNCE_RATE,
+                                ReportConstant.METRIC_AVG_SESSION_DURATION,
+                                ReportConstant.METRIC_SCREEN_PAGE_VIEWS,
+                                ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION),
+                        List.of(ReportConstant.DIMENSION_DATE),
+                        OrderByType.DIMENSIONS,
+                        ReportConstant.DIMENSION_DATE,
+                        false);
+
+        // 2. 请求API接口
+        List<CustomReportData> reportDataList =
+                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+
+        // 3. 转化为VOs
+        Map<String, DailyStatsVO> dailyStatsVOs = Maps.newHashMap();
+        for (CustomReportData reportData : reportDataList) {
+            DailyStatsVO dailyStatsVO = new DailyStatsVO();
+            dailyStatsVO.setDate(reportData.get(ReportConstant.DIMENSION_DATE));
+            dailyStatsVO.setTotalUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+            dailyStatsVO.setSessions(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SESSIONS)));
+            dailyStatsVO.setBounceRate(
+                    Double.parseDouble(reportData.get(ReportConstant.METRIC_BOUNCE_RATE)));
+            dailyStatsVO.setAvgSessionDuration(
+                    Double.parseDouble(reportData.get(ReportConstant.METRIC_AVG_SESSION_DURATION)));
+            dailyStatsVO.setPageViews(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS)));
+            dailyStatsVO.setPageViewsPerSession(
+                    Double.parseDouble(
+                            reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION)));
+
+            dailyStatsVOs.put(dailyStatsVO.getDate(), dailyStatsVO);
+        }
+
+        // 4. 缺失日期补充
+        Set<String> absentDays =
+                Sets.difference(
+                        DateUtil.getAllDaysBetween(start, end).stream()
+                                .map(date -> DateUtil.formatDate(date, DateUtil.DATE_FORMAT))
+                                .collect(Collectors.toSet()),
+                        dailyStatsVOs.keySet());
+        absentDays.forEach(
+                date -> {
+                    DailyStatsVO dailyStatsVO = new DailyStatsVO();
+                    dailyStatsVO.setDate(date); // 其它数值字段设置为默认值
+                    dailyStatsVOs.put(date, dailyStatsVO);
+                });
+
+        // 5. 询盘数据补充
+        List<ImmutableTriple<String, Long, Long>> enquiryDailyCounts =
+                adwebEnquiryMapper.getEnquiryDailyCounts(siteCode, start, end);
+        for (ImmutableTriple<String, Long, Long> enquiryDailyCount : enquiryDailyCounts) {
+            if (!dailyStatsVOs.containsKey(enquiryDailyCount.getLeft())) {
+                continue;
+            }
+
+            DailyStatsVO dailyStatsVO = dailyStatsVOs.get(enquiryDailyCount.getLeft());
+            dailyStatsVO.setEnquires(enquiryDailyCount.getMiddle().intValue());
+            dailyStatsVO.setUnreadEnquires(enquiryDailyCount.getRight().intValue());
+        }
+
+        // 6. 根据日期排序并返回
+        return dailyStatsVOs.values().stream()
+                .sorted(Comparator.comparing(DailyStatsVO::getDate)) // 排日期升序排序
+                .collect(Collectors.toList()); // 使用toList()方法返回UnmodifiableList，导致Redis类型解析异常
+    }
+
+    /** 拉取Google Analytics - Country报表 */
+    @Cacheable(
+            cacheManager = CacheConfig.TTL_CACHE_MANAGER,
+            cacheNames =
+                    "dmp:realtime:getCountryStats"
+                            + TTLCacheManager.TTL_SPLITTER
+                            + 60 * 10) // Redis TTL为10分钟
+    public List<CountryStatsVO> getCountryStats(String siteCode, Date start, Date end) {
+        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+            return Collections.EMPTY_LIST;
+        }
+
+        // 1. 构建GA报表请求参数
+        GAReportRequestDTO gaReportRequest =
+                this.buildGAReportRequest(
+                        googleGTM.getGaPropertyId(),
+                        start,
+                        end,
+                        List.of(ReportConstant.METRIC_TOTAL_USERS),
+                        List.of(ReportConstant.DIMENSION_COUNTRY),
+                        OrderByType.METRICS,
+                        ReportConstant.METRIC_TOTAL_USERS,
+                        true);
+
+        // 2. 请求API接口
+        List<CustomReportData> reportDataList =
+                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+
+        // 3.1 转化为VOs
+        List<CountryStatsVO> countryStatsVOs = Lists.newArrayList();
+        for (CustomReportData reportData : reportDataList) {
+            CountryStatsVO countryStatsVO = new CountryStatsVO();
+            countryStatsVO.setCountry(reportData.get(ReportConstant.DIMENSION_COUNTRY));
+
+            CountryCode countryCode = CountryCode.valueOfName(countryStatsVO.getCountry());
+            countryStatsVO.setCountryName(countryCode.getCnName());
+            countryStatsVO.setCountryCode(countryCode.getCode());
+
+            countryStatsVO.setTotalUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+
+            countryStatsVOs.add(countryStatsVO);
+        }
+
+        // 3.2 时间区间内所有国家totalUsers总数
+        int totalUsersSum = countryStatsVOs.stream().mapToInt(CountryStatsVO::getTotalUsers).sum();
+
+        // 3.3 VO数据计算填充
+        for (CountryStatsVO countryStatsVO : countryStatsVOs) {
+            countryStatsVO.setTotalUsersProportion(
+                    NumberUtil.formatPercentage(
+                            NumberUtil.safeDivide(countryStatsVO.getTotalUsers(), totalUsersSum),
+                            2));
+        }
+
+        return countryStatsVOs;
+    }
+
+    /** 拉取Google Analytics - Session Source Medium报表 */
+    @Cacheable(
+            cacheManager = CacheConfig.TTL_CACHE_MANAGER,
+            cacheNames =
+                    "dmp:realtime:getSourceMediumStats"
+                            + TTLCacheManager.TTL_SPLITTER
+                            + 60 * 10) // Redis TTL为10分钟
+    public List<SourceMediumStatsVO> getSourceMediumStats(String siteCode, Date start, Date end) {
+        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+            return Collections.EMPTY_LIST;
+        }
+
+        // 1. 构建GA报表请求参数
+        GAReportRequestDTO gaReportRequest =
+                this.buildGAReportRequest(
+                        googleGTM.getGaPropertyId(),
+                        start,
+                        end,
+                        List.of(
+                                ReportConstant.METRIC_TOTAL_USERS,
+                                ReportConstant.METRIC_NEW_USERS,
+                                ReportConstant.METRIC_SESSIONS,
+                                ReportConstant.METRIC_SCREEN_PAGE_VIEWS,
+                                ReportConstant.METRIC_AVG_SESSION_DURATION,
+                                ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION),
+                        List.of(ReportConstant.DIMENSION_SESSION_SOURCE_MEDIUM),
+                        OrderByType.METRICS,
+                        ReportConstant.METRIC_TOTAL_USERS,
+                        true);
+
+        // 2. 请求API接口
+        List<CustomReportData> reportDataList =
+                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+
+        // 3.1 转化为VOs
+        List<SourceMediumStatsVO> sourceMediumStatsVOs = Lists.newArrayList();
+        for (CustomReportData reportData : reportDataList) {
+            SourceMediumStatsVO sourceMediumStatsVO = new SourceMediumStatsVO();
+            sourceMediumStatsVO.setType(
+                    reportData.get(ReportConstant.DIMENSION_SESSION_SOURCE_MEDIUM));
+            sourceMediumStatsVO.setTotalUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+            sourceMediumStatsVO.setNewUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_NEW_USERS)));
+            sourceMediumStatsVO.setSessions(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SESSIONS)));
+            sourceMediumStatsVO.setPageViews(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS)));
+            sourceMediumStatsVO.setAvgSessionDuration(
+                    NumberUtil.formatDecimal(
+                                    Double.parseDouble(
+                                            reportData.get(
+                                                    ReportConstant.METRIC_AVG_SESSION_DURATION)),
+                                    2)
+                            .doubleValue());
+            sourceMediumStatsVO.setPageViewsPerSession(
+                    NumberUtil.formatDecimal(
+                                    Double.parseDouble(
+                                            reportData.get(
+                                                    ReportConstant
+                                                            .METRIC_SCREEN_PAGE_VIEWS_PER_SESSION)),
+                                    2)
+                            .doubleValue());
+
+            sourceMediumStatsVOs.add(sourceMediumStatsVO);
+        }
+
+        // 3.2 时间区间内所有媒介totalUsers总数
+        int totalUsersSum =
+                sourceMediumStatsVOs.stream().mapToInt(SourceMediumStatsVO::getTotalUsers).sum();
+
+        // 3.3 VO数据计算填充
+        for (SourceMediumStatsVO sourceMediumStatsVO : sourceMediumStatsVOs) {
+            int totalUsers = sourceMediumStatsVO.getTotalUsers();
+            sourceMediumStatsVO.setTotalUsersProportion(
+                    NumberUtil.formatPercentage(
+                            NumberUtil.safeDivide(totalUsers, totalUsersSum), 2));
+
+            sourceMediumStatsVO.setNewUsersRatio(
+                    NumberUtil.formatPercentage(
+                            NumberUtil.safeDivide(sourceMediumStatsVO.getNewUsers(), totalUsers),
+                            2));
+        }
+
+        return sourceMediumStatsVOs;
+    }
+
+    /** 拉取Google Analytics - Page Path报表 */
+    @Cacheable(
+            cacheManager = CacheConfig.TTL_CACHE_MANAGER,
+            cacheNames =
+                    "dmp:realtime:getPagePathStats"
+                            + TTLCacheManager.TTL_SPLITTER
+                            + 60 * 10) // Redis TTL为10分钟
+    public List<PagePathStatsVO> getPagePathStats(
+            String siteCode, Date start, Date end, int limit) {
+        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+            return Collections.EMPTY_LIST;
+        }
+
+        // 1. 构建GA报表请求参数
+        GAReportRequestDTO gaReportRequest =
+                this.buildGAReportRequest(
+                        googleGTM.getGaPropertyId(),
+                        start,
+                        end,
+                        List.of(
+                                ReportConstant.METRIC_SCREEN_PAGE_VIEWS,
+                                ReportConstant.METRIC_USER_ENGAGEMENT_DURATION),
+                        List.of(ReportConstant.DIMENSION_PAGE_PATH),
+                        OrderByType.METRICS,
+                        ReportConstant.METRIC_SCREEN_PAGE_VIEWS,
+                        true);
+
+        // 2. 请求API接口
+        List<CustomReportData> reportDataList =
+                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+
+        // 3.1 转化为VOs
+        List<PagePathStatsVO> pagePathStatsVOs = Lists.newArrayList();
+        for (CustomReportData reportData : reportDataList) {
+            PagePathStatsVO pagePathStatsVO = new PagePathStatsVO();
+            pagePathStatsVO.setPagePath(reportData.get(ReportConstant.DIMENSION_PAGE_PATH));
+            pagePathStatsVO.setPageViews(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS)));
+            pagePathStatsVO.setAvgTimeOnPage(
+                    NumberUtil.formatDecimal(
+                                    Double.parseDouble(
+                                            reportData.get(
+                                                    ReportConstant
+                                                            .METRIC_USER_ENGAGEMENT_DURATION)),
+                                    2)
+                            .doubleValue());
+
+            pagePathStatsVOs.add(pagePathStatsVO);
+        }
+
+        // 3.2. 网站URL及格式化
+        AdwebSite adwebSite = adwebSiteService.getSiteByCode(siteCode);
+        String siteUrl = StringUtils.removeEnd(Strings.nullToEmpty(adwebSite.getDomain()), "/");
+
+        // 3.3. 时间区间内PV总数
+        int totalPVs = pagePathStatsVOs.stream().mapToInt(PagePathStatsVO::getPageViews).sum();
+
+        // 3.4 VO数据计算填充
+        for (PagePathStatsVO pagePathStatsVO : pagePathStatsVOs) {
+            pagePathStatsVO.setPagePath(siteUrl + pagePathStatsVO.getPagePath());
+            pagePathStatsVO.setPvProportion(
+                    NumberUtil.formatPercentage(
+                            NumberUtil.safeDivide(pagePathStatsVO.getPageViews(), totalPVs), 2));
+        }
+
+        return pagePathStatsVOs.stream()
+                .limit(limit)
+                .collect(Collectors.toList()); // 使用toList()方法返回UnmodifiableList，导致Redis类型解析异常
+    }
+
+    /** 拉取Google Analytics - Device报表 */
+    @Cacheable(
+            cacheManager = CacheConfig.TTL_CACHE_MANAGER,
+            cacheNames =
+                    "dmp:realtime:getDeviceStats"
+                            + TTLCacheManager.TTL_SPLITTER
+                            + 60 * 10) // Redis TTL为10分钟
+    public List<DeviceStatsVO> getDeviceStats(String siteCode, Date start, Date end) {
+        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+            return Collections.EMPTY_LIST;
+        }
+
+        // 1. 构建GA报表请求参数
+        GAReportRequestDTO gaReportRequest =
+                this.buildGAReportRequest(
+                        googleGTM.getGaPropertyId(),
+                        start,
+                        end,
+                        List.of(
+                                ReportConstant.METRIC_TOTAL_USERS,
+                                ReportConstant.METRIC_SESSIONS,
+                                ReportConstant.METRIC_SCREEN_PAGE_VIEWS),
+                        List.of(ReportConstant.DIMENSION_PLATFORM_DEVICE_CATEGORY),
+                        OrderByType.METRICS,
+                        ReportConstant.METRIC_TOTAL_USERS,
+                        true);
+
+        // 2. 请求API接口
+        List<CustomReportData> reportDataList =
+                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+
+        // 3. 转化为VOs
+        List<DeviceStatsVO> deviceStatsVOs = Lists.newArrayList();
+        for (CustomReportData reportData : reportDataList) {
+            DeviceStatsVO deviceStatsVO = new DeviceStatsVO();
+            deviceStatsVO.setDevice(
+                    reportData.get(ReportConstant.DIMENSION_PLATFORM_DEVICE_CATEGORY));
+            deviceStatsVO.setTotalUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+            deviceStatsVO.setSessions(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SESSIONS)));
+            deviceStatsVO.setPageViews(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS)));
+
+            deviceStatsVOs.add(deviceStatsVO);
+        }
+
+        return deviceStatsVOs;
+    }
+
+    /**
+     * 获取{@link GoogleGTM}帐号信息 - 放入Redis缓存，避免同一页面多个HTTP请求频繁DB查询
+     *
+     * <p>使用AdwebRedisUtil操作 - @Cacheable注解无法嵌套使用
+     */
+    private synchronized GoogleGTM getGTMAccount(String siteCode) {
+        final String redisKey = String.format("%s:account:%s", "googlegtm", siteCode);
+
+        GoogleGTM googleGTM = (GoogleGTM) redisUtil.get(redisKey);
+        if (Objects.nonNull(googleGTM)) {
+            return googleGTM;
+        }
+
+        // DB查询
+        googleGTM =
+                googleGTMService.getOne(
+                        new LambdaQueryWrapper<GoogleGTM>().eq(GoogleGTM::getSiteCode, siteCode),
+                        false);
+        googleGTM = Objects.nonNull(googleGTM) ? googleGTM : new GoogleGTM(); // Redis存储对象不允许为空
+        redisUtil.set(redisKey, googleGTM, 30); // Redis TTL为30秒
+
+        return googleGTM;
+    }
+
+    private GAReportRequestDTO buildGAReportRequest(
+            String propertyId,
+            Date start,
+            Date end,
+            List<String> metrics,
+            List<String> dimensions,
+            OrderByType orderByType,
+            String orderBy,
+            boolean orderByDesc) {
+        GAReportRequestDTO gaReportRequest = new GAReportRequestDTO();
+        gaReportRequest.setPropertyResourceName(GAPropertyDTO.toResourceName(propertyId));
+        gaReportRequest.setReportType(ReportType.ADWEB_CUSTOM_REPORT);
+        gaReportRequest.setStartDate(
+                DateUtil.formatDate(
+                        Optional.ofNullable(start)
+                                // 如果start为空，设置为无限早时间
+                                .orElse(DateUtil.parseDate("2016-01-01", DateUtil.DATE_FORMAT)),
+                        DateUtil.DATE_FORMAT));
+        gaReportRequest.setEndDate(
+                DateUtil.formatDate(
+                        Optional.ofNullable(end).orElse(new Date()), DateUtil.DATE_FORMAT));
+        gaReportRequest.setMetrics(metrics);
+        gaReportRequest.setDimensions(dimensions);
+        gaReportRequest.setOrderByType(orderByType);
+        gaReportRequest.setOrderBy(orderBy);
+        gaReportRequest.setOrderByDesc(orderByDesc);
+        return gaReportRequest;
+    }
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/google/GAReportService.java

@@ -3,7 +3,6 @@ package org.jeecg.modules.adweb.dmp.service.google;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.google.common.annotations.VisibleForTesting;
 
 import jakarta.annotation.PostConstruct;
 
@@ -439,8 +438,7 @@ public class GAReportService {
      * @return
      * @param <T>
      */
-    @VisibleForTesting
-    <T extends GAReportDataDTO> List<T> runGAReport(
+    public <T extends GAReportDataDTO> List<T> runGAReport(
             GAReportRequestDTO gaReportRequest, Class<T> reportDataClass) {
         log.info("runGAReport: report request = {}", FastJsonUtil.toJSONString(gaReportRequest));
 

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/impl/GADailyReportServiceImpl.java

@@ -53,10 +53,8 @@ public class GADailyReportServiceImpl extends ServiceImpl<GADailyReportMapper, G
     @Cacheable(
             cacheManager = CacheConfig.TTL_CACHE_MANAGER,
             cacheNames =
-                    "dmp:getDailyStatsWithinPeriod"
-                            + TTLCacheManager.TTL_SPLITTER
-                            + 60 * 10) // Redis TTL为10分钟
-    public List<DailyStatsVO> getDailyStatsForDateRange(String siteCode, Date start, Date end) {
+                    "dmp:getDailyStats" + TTLCacheManager.TTL_SPLITTER + 60 * 10) // Redis TTL为10分钟
+    public List<DailyStatsVO> getDailyStats(String siteCode, Date start, Date end) {
         Map<String, DailyStatsVO> dailyStatsVOs = Maps.newHashMap();
 
         // 0. 起止日期为空时，设置默认区间 - start = 网站发布日期 & end = 今天
@@ -88,8 +86,6 @@ public class GADailyReportServiceImpl extends ServiceImpl<GADailyReportMapper, G
             dailyStatsVO.setAvgSessionDuration(gaDailyReport.getAvgSessionDuration());
             dailyStatsVO.setPageViews(gaDailyReport.getPageViews());
             dailyStatsVO.setPageViewsPerSession(gaDailyReport.getPageViewsPerSession());
-            // TODO: 待后续步骤设置
-            //  dailyStatsVO.setEnquires(0);
 
             dailyStatsVOs.put(date, dailyStatsVO);
         }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/impl/GASourceMediumReportServiceImpl.java

@@ -42,7 +42,7 @@ public class GASourceMediumReportServiceImpl
             return Collections.EMPTY_LIST;
         }
 
-        // 1. 时间区间内所有国家totalUsers总数
+        // 1. 时间区间内所有媒介totalUsers总数
         int totalUsersSum =
                 sourceMediumStatsVOs.stream().mapToInt(SourceMediumStatsVO::getTotalUsers).sum();
 

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/marketing/googleads/service/GoogleAdsReportService.java

@@ -311,7 +311,7 @@ public class GoogleAdsReportService {
                 DateUtil.formatDate(
                         Optional.ofNullable(start)
                                 // 如果start为空，设置为无限早时间
-                                .orElse(DateUtil.parseDate("2010-01-01", DateUtil.DATE_FORMAT)),
+                                .orElse(DateUtil.parseDate("2016-01-01", DateUtil.DATE_FORMAT)),
                         DateUtil.DATE_FORMAT));
         reportRequest.setEndDate(
                 DateUtil.formatDate(

jeecg-module-system/jeecg-system-start/src/main/resources/application-dev.yml

@@ -2,7 +2,7 @@ server:
   port: 8080
   undertow:
     # 平替 tomcat server.tomcat.max-swallow-siz， undertow该值默认为-1
-    #    max-http-post-size: 10MB
+    # max-http-post-size: 10MB
     threads:
       io: 16 # 4核CPU标准配置
       worker: 256
@@ -68,11 +68,11 @@ spring:
   quartz:
     job-store-type: jdbc
     initialize-schema: embedded
-    #定时任务启动开关，true-开  false-关
+    # 定时任务启动开关，true-开  false-关
     auto-startup: false
-    #延迟1秒启动定时任务
+    # 延迟1秒启动定时任务
     startup-delay: 1s
-    #启动时更新己存在的Job
+    # 启动时更新己存在的Job
     overwrite-existing-jobs: true
     properties:
       org:
@@ -92,7 +92,7 @@ spring:
             threadCount: 10
             threadPriority: 5
             threadsInheritContextClassLoaderOfInitializingThread: true
-  #json 时间戳统一转换
+  # json 时间戳统一转换
   jackson:
     date-format: yyyy-MM-dd HH:mm:ss
     time-zone: GMT+8
@@ -100,7 +100,7 @@ spring:
     open-in-view: false
   aop:
     proxy-target-class: true
-  #配置freemarker
+  # 配置freemarker
   freemarker:
     # 设置模板后缀名
     suffix: .ftl
@@ -118,7 +118,7 @@ spring:
   # 设置静态文件路径，js,css等
   mvc:
     static-path-pattern: /**
-    #Spring Boot 2.6+后映射匹配的默认策略已从AntPathMatcher更改为PathPatternParser,需要手动指定为ant-path-matcher
+    # Spring Boot 2.6+后映射匹配的默认策略已从AntPathMatcher更改为PathPatternParser,需要手动指定为ant-path-matcher
     pathmatch:
       matching-strategy: ant_path_matcher
   resource:
@@ -172,12 +172,12 @@ spring:
           password: Advich@2024,.
           driver-class-name: com.mysql.cj.jdbc.Driver
           # 多数据源配置
-          #multi-datasource1:
-          #url: jdbc:mysql://localhost:3306/jeecg-boot2?useUnicode=true&characterEncoding=utf8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
-          #username: root
-          #password: root
-          #driver-class-name: com.mysql.cj.jdbc.Driver
-  #redis 配置
+          # multi-datasource1:
+          #   url: jdbc:mysql://localhost:3306/jeecg-boot2?useUnicode=true&characterEncoding=utf8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+          #   username: root
+          #   password: root
+          #   driver-class-name: com.mysql.cj.jdbc.Driver
+  # redis 配置
   data:
     redis:
       database: 0
@@ -193,23 +193,25 @@ spring:
     publisher-confirm-type: correlated
     publisher-returns: true
     virtual-host: /adweb3-dev
-#mybatis plus 设置
+
+# mybatis plus 设置
 mybatis-plus:
   mapper-locations: classpath*:org/jeecg/**/xml/*Mapper.xml
   global-config:
     # 关闭MP3.0自带的banner
     banner: false
     db-config:
-      #主键类型  0:"数据库ID自增",1:"该类型为未设置主键类型", 2:"用户输入ID",3:"全局唯一ID (数字类型唯一ID)", 4:"全局唯一ID UUID",5:"字符串全局唯一ID (idWorker 的字符串表示)";
+      # 主键类型  0:"数据库ID自增",1:"该类型为未设置主键类型", 2:"用户输入ID",3:"全局唯一ID (数字类型唯一ID)", 4:"全局唯一ID UUID",5:"字符串全局唯一ID (idWorker 的字符串表示)";
       id-type: ASSIGN_ID
       # 默认数据库表下划线命名
       table-underline: true
   configuration:
     # 这个配置会将执行的sql打印出来，在开发或测试的时候可以用
-    #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
+    # log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
     # 返回类型为Map,显示null对应的字段
     call-setters-on-nulls: true
-#jeecg专用配置
+
+# jeecg专用配置
 minidao:
   base-package: org.jeecg.modules.jmreport.*,org.jeecg.modules.drag.*
 jeecg:
@@ -221,7 +223,7 @@ jeecg:
     lowCodeMode: dev
   # 签名密钥串(前后端要一致，正式发布请自行修改)
   signatureSecret: dd05f1c54d63749eda95f9fa6d49v442a
-  #签名拦截接口
+  # 签名拦截接口
   signUrls: /sys/dict/getDictItems/*,/sys/dict/loadDict/*,/sys/dict/loadDictOrderByValue/*,/sys/dict/loadDictItem/*,/sys/dict/loadTreeData,/sys/api/queryTableDictItemsByCode,/sys/api/queryFilterTableDictInfo,/sys/api/queryTableDictByKeys,/sys/api/translateDictFromTable,/sys/api/translateDictFromTableByKeys,/sys/sendChangePwdSms,/sys/user/sendChangePhoneSms,/sys/sms,/desform/api/sendVerifyCode
   # 本地：local、Minio：minio、阿里云：alioss
   uploadType: local
@@ -230,11 +232,11 @@ jeecg:
     pc: http://localhost:3100
     app: http://localhost:8051
   path:
-    #文件上传根目录 设置
+    # 文件上传根目录 设置
     upload: D:/ProjectsCode/adweb3-server/upload
-    #webapp文件路径
+    # webapp文件路径
     webapp: D:/ProjectsCode/adweb3-server/upload
-  #阿里云oss存储和大鱼短信秘钥配置
+  # 阿里云oss存储和大鱼短信秘钥配置
   oss:
     accessKey: ??
     secretKey: ??
@@ -262,9 +264,9 @@ jeecg:
     bucketName: sync-adwebcloud-bucket
     dataPrefix: https://data.adwebcloud.com/
     originDataPrefix: https://advich-wordpress-static-resources.s3.us-west-2.amazonaws.com/
-  #大屏报表参数设置
+  # 大屏报表参数设置
   jmreport:
-    #多租户模式，默认值为空(created:按照创建人隔离、tenant:按照租户隔离) (v1.6.2+ 新增)
+    # 多租户模式，默认值为空(created:按照创建人隔离、tenant:按照租户隔离) (v1.6.2+ 新增)
     saasMode:
     # 平台上线安全配置(v1.6.2+ 新增)
     firewall:
@@ -272,7 +274,7 @@ jeecg:
       dataSourceSafe: false
       # 低代码开发模式（dev:开发模式，prod:发布模式—关闭在线报表设计功能，分配角色admin、lowdeveloper可以放开限制）
       lowCodeMode: dev
-  #xxl-job配置 - jeecg-boot-starter3-job
+  # xxl-job配置 - jeecg-boot-starter3-job
   xxljob:
     enabled: false
     adminAddresses: http://localhost:8090/xxl-job-admin/
@@ -287,7 +289,7 @@ jeecg:
     port: 9999
     logPath: logs/jeecg/job/jobhandler/
     logRetentionDays: 30
-  #分布式锁配置
+  # 分布式锁配置
   redisson:
     address: 127.0.0.1:6379
     password:
@@ -304,29 +306,33 @@ jeecg:
     # 超时时间单位:s。默认 60s
     timeout: 60
 
-  # 本地代理地址
-#    proxy:
-#      host: "http://127.0.0.1"
-#      port: "7890"
-#cas单点登录
+    # 本地代理地址
+    # proxy:
+    #   host: "http://127.0.0.1"
+    #   port: "7890"
+
+# cas单点登录
 cas:
   prefixUrl: http://cas.example.org:8443/cas
-#Mybatis输出sql日志
+
+# Mybatis输出sql日志
 logging:
   level:
     org.flywaydb: debug
     org.jeecg.modules.system.mapper: debug
-#swagger
+
+# swagger
 knife4j:
-  #开启增强配置
+  # 开启增强配置
   enable: true
-  #开启生产环境屏蔽
+  # 开启生产环境屏蔽
   production: false
   basic:
     enable: false
     username: jeecg
     password: jeecg1314
-#第三方登录
+
+# 第三方登录
 justauth:
   enabled: true
   type:
@@ -371,13 +377,13 @@ data-bridge:
   ga:
     account-id: 191734056
 
-##GEOIP MMDB 静态数据库文件
+### GEOIP MMDB 静态数据库文件
 geoip:
   static:
     city:
       mmdb: D:\Advich\GeoLite2-City.mmdb
 
-# 机器人预警url
+### 机器人预警url
 robot:
   enquiry-url: https://open.feishu.cn/open-apis/bot/v2/hook/042cf8d0-0072-435c-bf5c-656b9368a5ac  #询盘拉取失败通知
   market-plan-missing-url: https://open.feishu.cn/open-apis/bot/v2/hook/4a7110ff-9121-49fc-b0b6-0d809b548ad4  #缺少营销方案导致的拉取站点失败
@@ -386,19 +392,19 @@ robot:
   localize-product-fail-url: https://open.feishu.cn/open-apis/bot/v2/hook/46e780fc-6ed9-45ec-b39f-1e332a236386 #产品图片本地化失败通知
   gtm-url: https://open.feishu.cn/open-apis/bot/v2/hook/e29a515d-7331-428e-b890-5b5549ffcab5 #跟踪gtm通知
 
-#垃圾询盘判断规则
+### 垃圾询盘判断规则
 judge_waste_enquiry:
   email:
     tenMinNum: 3 # 10分钟内最多发送3次
     oneDayNum: 5 # 一天最多发送5封邮件
-    notBlackListNum: 3 #不在黑名单中的邮箱发送次数
-    notBlackListDate: 7 #不在邮箱黑名单中的垃圾询盘计数的天数
+    notBlackListNum: 3 # 不在黑名单中的邮箱发送次数
+    notBlackListDate: 7 # 不在邮箱黑名单中的垃圾询盘计数的天数
   ip:
     tenMinNum: 3 # 10分钟内最多发送次数 >
     oneDayNum: 5 # 一天内最多发送次数 >
-    notBlackListNum: 3 #不在Ip黑名单中垃圾询盘计数次数 >
-    notBlackListDate: 7 #不在Ip黑名单中垃圾询盘计数的天数
-    delOrdIpDate: 90 #删除时效性低的Ip时间
+    notBlackListNum: 3 # 不在Ip黑名单中垃圾询盘计数次数 >
+    notBlackListDate: 7 # 不在Ip黑名单中垃圾询盘计数的天数
+    delOrdIpDate: 90 # 删除时效性低的Ip时间
 
 ### dataforseo
 dataforseo:
@@ -420,20 +426,23 @@ tradesparq:
 
 ### 询盘列表配置
 enquiry:
-  disable-admin-read: false #询盘的消息是否阅读后更改阅读状态
-  demoFlag: false #询盘的示例关键消息是否脱敏处理
+  disable-admin-read: false # 询盘的消息是否阅读后更改阅读状态
+  demoFlag: false # 询盘的示例关键消息是否脱敏处理
 
-# 亚马逊翻译api
+### 亚马逊翻译api
 aws:
   translate:
     accessKey: AKIAS37NJDKDETZ7PPEN
     secretKey: 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
 
-#v3项目路径
+### v3系统配置
 v3:
   projectPath: D:/Advich/pem
+  dmp:
+    # DMP模块实时报表开关。true-从DataBridge实时读取 false-从DB离线读取
+    realtimeReport: true
 
-#建站链接测试环境配置
+### 建站链接测试环境配置
 AdwebSiteConnect:
   host: 35.87.155.71
   port: 22
@@ -443,4 +452,4 @@ AdwebSiteConnect:
   tempCname: devci2.adwebcloud.com
 
 serverIp:
-  test: 35.87.155.71
+  test: 35.87.155.71

jeecg-module-system/jeecg-system-start/src/main/resources/application-prod.yml

@@ -2,7 +2,7 @@ server:
   port: 8080
   undertow:
     # 平替 tomcat server.tomcat.max-swallow-siz， undertow该值默认为-1
-    #    max-http-post-size: 10MB
+    # max-http-post-size: 10MB
     threads:
       io: 16 # 4核CPU标准配置
       worker: 256
@@ -68,11 +68,11 @@ spring:
   quartz:
     job-store-type: jdbc
     initialize-schema: embedded
-    #定时任务开关，true-开  false-关
+    # 定时任务启动开关，true-开  false-关
     auto-startup: false
-    #延迟1秒启动定时任务
+    # 延迟1秒启动定时任务
     startup-delay: 1s
-    #启动时更新己存在的Job
+    # 启动时更新己存在的Job
     overwrite-existing-jobs: true
     properties:
       org:
@@ -92,7 +92,7 @@ spring:
             threadCount: 10
             threadPriority: 5
             threadsInheritContextClassLoaderOfInitializingThread: true
-  #json 时间戳统一转换
+  # json 时间戳统一转换
   jackson:
     date-format: yyyy-MM-dd HH:mm:ss
     time-zone: GMT+8
@@ -100,7 +100,7 @@ spring:
     open-in-view: false
   aop:
     proxy-target-class: true
-  #配置freemarker
+  # 配置freemarker
   freemarker:
     # 设置模板后缀名
     suffix: .ftl
@@ -118,7 +118,7 @@ spring:
   # 设置静态文件路径，js,css等
   mvc:
     static-path-pattern: /**
-    #Spring Boot 2.6+后映射匹配的默认策略已从AntPathMatcher更改为PathPatternParser,需要手动指定为ant-path-matcher
+    # Spring Boot 2.6+后映射匹配的默认策略已从AntPathMatcher更改为PathPatternParser,需要手动指定为ant-path-matcher
     pathmatch:
       matching-strategy: ant_path_matcher
   resource:
@@ -167,18 +167,17 @@ spring:
           slow-sql-millis: 5000
       datasource:
         master:
-          # TODO: 待切换生产RDS
           url: jdbc:mysql://adweb3-prod.c0stwhsmuvxv.rds.cn-northwest-1.amazonaws.com.cn:3306/adweb_v3?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
           username: root
           password: xZjenmZAvKJCBDCTrWIb
           driver-class-name: com.mysql.cj.jdbc.Driver
           # 多数据源配置
-          #multi-datasource1:
-          #url: jdbc:mysql://localhost:3306/jeecg-boot2?useUnicode=true&characterEncoding=utf8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
-          #username: root
-          #password: root
-          #driver-class-name: com.mysql.cj.jdbc.Driver
-  #redis 配置
+          # multi-datasource1:
+          #   url: jdbc:mysql://localhost:3306/jeecg-boot2?useUnicode=true&characterEncoding=utf8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+          #   username: root
+          #   password: root
+          #   driver-class-name: com.mysql.cj.jdbc.Driver
+  # redis 配置
   data:
     redis:
       database: 0
@@ -194,23 +193,25 @@ spring:
     publisher-confirm-type: correlated
     publisher-returns: true
     virtual-host: /adweb3
-#mybatis plus 设置
+
+# mybatis plus 设置
 mybatis-plus:
   mapper-locations: classpath*:org/jeecg/**/xml/*Mapper.xml
   global-config:
     # 关闭MP3.0自带的banner
     banner: false
     db-config:
-      #主键类型  0:"数据库ID自增",1:"该类型为未设置主键类型", 2:"用户输入ID",3:"全局唯一ID (数字类型唯一ID)", 4:"全局唯一ID UUID",5:"字符串全局唯一ID (idWorker 的字符串表示)";
+      # 主键类型  0:"数据库ID自增",1:"该类型为未设置主键类型", 2:"用户输入ID",3:"全局唯一ID (数字类型唯一ID)", 4:"全局唯一ID UUID",5:"字符串全局唯一ID (idWorker 的字符串表示)";
       id-type: ASSIGN_ID
       # 默认数据库表下划线命名
       table-underline: true
   configuration:
     # 这个配置会将执行的sql打印出来，在开发或测试的时候可以用
-    #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
+    # log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
     # 返回类型为Map,显示null对应的字段
     call-setters-on-nulls: true
-#jeecg专用配置
+
+# jeecg专用配置
 minidao:
   base-package: org.jeecg.modules.jmreport.*,org.jeecg.modules.drag.*
 jeecg:
@@ -222,27 +223,27 @@ jeecg:
     lowCodeMode: prod
   # 签名密钥串(前后端要一致，正式发布请自行修改)
   signatureSecret: dd05f1c54d63749eda95f9fa6d49v442a
-  #签名拦截接口
+  # 签名拦截接口
   signUrls: /sys/dict/getDictItems/*,/sys/dict/loadDict/*,/sys/dict/loadDictOrderByValue/*,/sys/dict/loadDictItem/*,/sys/dict/loadTreeData,/sys/api/queryTableDictItemsByCode,/sys/api/queryFilterTableDictInfo,/sys/api/queryTableDictByKeys,/sys/api/translateDictFromTable,/sys/api/translateDictFromTableByKeys,/sys/sendChangePwdSms,/sys/user/sendChangePhoneSms,/sys/sms,/desform/api/sendVerifyCode
-  # local\minio\alioss
+  # 本地：local、Minio：minio、阿里云：alioss
   uploadType: local
   # 前端访问地址
   domainUrl:
     pc: http://localhost:3100
     app: http://localhost:8051
   path:
-    #文件上传根目录 设置
+    # 文件上传根目录 设置
     upload: /opt/adweb3/upload
-    #webapp文件路径
+    # webapp文件路径
     webapp: /opt/adweb3/webapp
-  #阿里云oss存储和大鱼短信秘钥配置
+  # 阿里云oss存储和大鱼短信秘钥配置
   oss:
     accessKey: ??
     secretKey: ??
     endpoint: oss-cn-beijing.aliyuncs.com
     bucketName: jeecgdev
     staticDomain: https://static.jeecg.com
-  # ElasticSearch 设置
+  # ElasticSearch 6设置
   elasticsearch:
     cluster-name: jeecg-ES
     cluster-nodes: 127.0.0.1:9200
@@ -264,9 +265,9 @@ jeecg:
     bucketName: sync-adwebcloud-bucket
     dataPrefix: https://data.adwebcloud.com/
     originDataPrefix: https://advich-wordpress-static-resources.s3.us-west-2.amazonaws.com/
-  #大屏报表参数设置
+  # 大屏报表参数设置
   jmreport:
-    #多租户模式，默认值为空(created:按照创建人隔离、tenant:按照租户隔离) (v1.6.2+ 新增)
+    # 多租户模式，默认值为空(created:按照创建人隔离、tenant:按照租户隔离) (v1.6.2+ 新增)
     saasMode:
     # 平台上线安全配置(v1.6.2+ 新增)
     firewall:
@@ -274,7 +275,7 @@ jeecg:
       dataSourceSafe: true
       # 低代码开发模式（dev:开发模式，prod:发布模式—关闭在线报表设计功能，分配角色admin、lowdeveloper可以放开限制）
       lowCodeMode: prod
-  #xxl-job配置 - jeecg-boot-starter3-job
+  # xxl-job配置 - jeecg-boot-starter3-job
   xxljob:
     enabled: true
     adminAddresses: http://xxl-job.v3.adwebcloud.com/xxl-job-admin/
@@ -289,7 +290,7 @@ jeecg:
     port: 9999
     logPath: logs/jeecg/job/jobhandler/
     logRetentionDays: 30
-  #分布式锁配置
+  # 分布式锁配置
   redisson:
     address: 127.0.0.1:6379
     password:
@@ -306,25 +307,33 @@ jeecg:
     # 超时时间单位:s。默认 60s
     timeout: 60
 
-#cas单点登录
+    # 本地代理地址
+    # proxy:
+    #   host: "http://127.0.0.1"
+    #   port: "7890"
+
+# cas单点登录
 cas:
   prefixUrl: http://cas.example.org:8443/cas
-#Mybatis输出sql日志
+
+# Mybatis输出sql日志
 logging:
   level:
     org.flywaydb: debug
     org.jeecg.modules.system.mapper: info
-#swagger
+
+# swagger
 knife4j:
-  #开启增强配置
+  # 开启增强配置
   enable: true
-  #开启生产环境屏蔽
+  # 开启生产环境屏蔽
   production: true
   basic:
     enable: true
     username: jeecg
     password: jeecg1314
-#第三方登录
+
+# 第三方登录
 justauth:
   enabled: true
   type:
@@ -370,13 +379,13 @@ data-bridge:
   ga:
     account-id: 191734056
 
-##GEOIP MMDB 静态数据库文件
+### GEOIP MMDB 静态数据库文件
 geoip:
   static:
     city:
       mmdb: /opt/adweb3/sharing/GeoLite2-City.mmdb
 
-# 机器人预警url
+### 机器人预警url
 robot:
   enquiry-url: https://open.feishu.cn/open-apis/bot/v2/hook/042cf8d0-0072-435c-bf5c-656b9368a5ac  #询盘拉取失败通知
   market-plan-missing-url: https://open.feishu.cn/open-apis/bot/v2/hook/4a7110ff-9121-49fc-b0b6-0d809b548ad4  #缺少营销方案导致的拉取站点失败
@@ -385,19 +394,19 @@ robot:
   localize-product-fail-url: https://open.feishu.cn/open-apis/bot/v2/hook/46e780fc-6ed9-45ec-b39f-1e332a236386
   gtm-url: https://open.feishu.cn/open-apis/bot/v2/hook/e29a515d-7331-428e-b890-5b5549ffcab5
 
-#垃圾询盘判断规则
+### 垃圾询盘判断规则
 judge_waste_enquiry:
   email:
     tenMinNum: 3 # 10分钟内最多发送3次
     oneDayNum: 5 # 一天最多发送5封邮件
-    notBlackListNum: 3 #不在黑名单中的邮箱发送次数
-    notBlackListDate: 7 #不在邮箱黑名单中的垃圾询盘计数的天数
+    notBlackListNum: 3 # 不在黑名单中的邮箱发送次数
+    notBlackListDate: 7 # 不在邮箱黑名单中的垃圾询盘计数的天数
   ip:
     tenMinNum: 3 # 10分钟内最多发送次数 >
     oneDayNum: 5 # 一天内最多发送次数 >
-    notBlackListNum: 3 #不在Ip黑名单中垃圾询盘计数次数 >
-    notBlackListDate: 7 #不在Ip黑名单中垃圾询盘计数的天数
-    delOrdIpDate: 90 #删除时效性低的Ip时间
+    notBlackListNum: 3 # 不在Ip黑名单中垃圾询盘计数次数 >
+    notBlackListDate: 7 # 不在Ip黑名单中垃圾询盘计数的天数
+    delOrdIpDate: 90 # 删除时效性低的Ip时间
 
 ### dataforseo
 dataforseo:
@@ -419,22 +428,25 @@ tradesparq:
 
 ### 询盘列表配置
 enquiry:
-  disable-admin-read: true
-  demoFlag: false
+  disable-admin-read: true # 询盘的消息是否阅读后更改阅读状态
+  demoFlag: false # 询盘的示例关键消息是否脱敏处理
   token: zQ3jJqtIexedIw6tZGk6p
   pullCount: 5
 
-# 亚马逊翻译api
+### 亚马逊翻译api
 aws:
   translate:
     accessKey: AKIAS37NJDKDETZ7PPEN
     secretKey: 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
 
-#v2项目路径
+### v3系统配置
 v3:
   projectPath: /opt/adweb3/pem
+  dmp:
+    # DMP模块实时报表开关。true-从DataBridge实时读取 false-从DB离线读取
+    realtimeReport: false
 
-#建站链接测试环境配置
+### 建站链接测试环境配置
 AdwebSiteConnect:
   host: 35.87.155.71
   port: 22

jeecg-module-system/jeecg-system-start/src/main/resources/application-test.yml

@@ -2,7 +2,7 @@ server:
   port: 8081
   undertow:
     # 平替 tomcat server.tomcat.max-swallow-siz， undertow该值默认为-1
-    #    max-http-post-size: 10MB
+    # max-http-post-size: 10MB
     threads:
       io: 16 # 4核CPU标准配置
       worker: 256
@@ -67,11 +67,11 @@ spring:
   quartz:
     job-store-type: jdbc
     initialize-schema: embedded
-    #定时任务启动开关，true-开  false-关
+    # 定时任务启动开关，true-开  false-关
     auto-startup: false
-    #延迟1秒启动定时任务
+    # 延迟1秒启动定时任务
     startup-delay: 1s
-    #启动时更新己存在的Job
+    # 启动时更新己存在的Job
     overwrite-existing-jobs: true
     properties:
       org:
@@ -91,7 +91,7 @@ spring:
             threadCount: 10
             threadPriority: 5
             threadsInheritContextClassLoaderOfInitializingThread: true
-  #json 时间戳统一转换
+  # json 时间戳统一转换
   jackson:
     date-format: yyyy-MM-dd HH:mm:ss
     time-zone: GMT+8
@@ -99,7 +99,7 @@ spring:
     open-in-view: false
   aop:
     proxy-target-class: true
-  #配置freemarker
+  # 配置freemarker
   freemarker:
     # 设置模板后缀名
     suffix: .ftl
@@ -117,7 +117,7 @@ spring:
   # 设置静态文件路径，js,css等
   mvc:
     static-path-pattern: /**
-    #Spring Boot 2.6+后映射匹配的默认策略已从AntPathMatcher更改为PathPatternParser,需要手动指定为ant-path-matcher
+    # Spring Boot 2.6+后映射匹配的默认策略已从AntPathMatcher更改为PathPatternParser,需要手动指定为ant-path-matcher
     pathmatch:
       matching-strategy: ant_path_matcher
   resource:
@@ -171,12 +171,12 @@ spring:
           password: Advich@2024,.
           driver-class-name: com.mysql.cj.jdbc.Driver
           # 多数据源配置
-          #multi-datasource1:
-          #url: jdbc:mysql://localhost:3306/jeecg-boot2?useUnicode=true&characterEncoding=utf8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
-          #username: root
-          #password: root
-          #driver-class-name: com.mysql.cj.jdbc.Driver
-  #redis 配置
+          # multi-datasource1:
+          #   url: jdbc:mysql://localhost:3306/jeecg-boot2?useUnicode=true&characterEncoding=utf8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+          #   username: root
+          #   password: root
+          #   driver-class-name: com.mysql.cj.jdbc.Driver
+  # redis 配置
   data:
     redis:
       database: 1
@@ -192,23 +192,25 @@ spring:
     publisher-confirm-type: correlated
     publisher-returns: true
     virtual-host: /adweb3-test
-#mybatis plus 设置
+
+# mybatis plus 设置
 mybatis-plus:
   mapper-locations: classpath*:org/jeecg/**/xml/*Mapper.xml
   global-config:
     # 关闭MP3.0自带的banner
     banner: false
     db-config:
-      #主键类型  0:"数据库ID自增",1:"该类型为未设置主键类型", 2:"用户输入ID",3:"全局唯一ID (数字类型唯一ID)", 4:"全局唯一ID UUID",5:"字符串全局唯一ID (idWorker 的字符串表示)";
+      # 主键类型  0:"数据库ID自增",1:"该类型为未设置主键类型", 2:"用户输入ID",3:"全局唯一ID (数字类型唯一ID)", 4:"全局唯一ID UUID",5:"字符串全局唯一ID (idWorker 的字符串表示)";
       id-type: ASSIGN_ID
       # 默认数据库表下划线命名
       table-underline: true
   configuration:
     # 这个配置会将执行的sql打印出来，在开发或测试的时候可以用
-    #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
+    # log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
     # 返回类型为Map,显示null对应的字段
     call-setters-on-nulls: true
-#jeecg专用配置
+
+# jeecg专用配置
 minidao:
   base-package: org.jeecg.modules.jmreport.*,org.jeecg.modules.drag.*
 jeecg:
@@ -220,7 +222,7 @@ jeecg:
     lowCodeMode: dev
   # 签名密钥串(前后端要一致，正式发布请自行修改)
   signatureSecret: dd05f1c54d63749eda95f9fa6d49v442a
-  #签名拦截接口
+  # 签名拦截接口
   signUrls: /sys/dict/getDictItems/*,/sys/dict/loadDict/*,/sys/dict/loadDictOrderByValue/*,/sys/dict/loadDictItem/*,/sys/dict/loadTreeData,/sys/api/queryTableDictItemsByCode,/sys/api/queryFilterTableDictInfo,/sys/api/queryTableDictByKeys,/sys/api/translateDictFromTable,/sys/api/translateDictFromTableByKeys,/sys/sendChangePwdSms,/sys/user/sendChangePhoneSms,/sys/sms,/desform/api/sendVerifyCode
   # 本地：local、Minio：minio、阿里云：alioss
   uploadType: local
@@ -229,11 +231,11 @@ jeecg:
     pc: http://localhost:3100
     app: http://localhost:8051
   path:
-    #文件上传根目录 设置
+    # 文件上传根目录 设置
     upload: /opt/adweb3_test/upload
-    #webapp文件路径
+    # webapp文件路径
     webapp: /opt/adweb3_test/webapp
-  #阿里云oss存储和大鱼短信秘钥配置
+  # 阿里云oss存储和大鱼短信秘钥配置
   oss:
     accessKey: ??
     secretKey: ??
@@ -261,9 +263,9 @@ jeecg:
     bucketName: sync-adwebcloud-bucket
     dataPrefix: https://data.adwebcloud.com/
     originDataPrefix: https://advich-wordpress-static-resources.s3.us-west-2.amazonaws.com/
-  #大屏报表参数设置
+  # 大屏报表参数设置
   jmreport:
-    #多租户模式，默认值为空(created:按照创建人隔离、tenant:按照租户隔离) (v1.6.2+ 新增)
+    # 多租户模式，默认值为空(created:按照创建人隔离、tenant:按照租户隔离) (v1.6.2+ 新增)
     saasMode:
     # 平台上线安全配置(v1.6.2+ 新增)
     firewall:
@@ -271,7 +273,7 @@ jeecg:
       dataSourceSafe: false
       # 低代码开发模式（dev:开发模式，prod:发布模式—关闭在线报表设计功能，分配角色admin、lowdeveloper可以放开限制）
       lowCodeMode: dev
-  #xxl-job配置 - jeecg-boot-starter3-job
+  # xxl-job配置 - jeecg-boot-starter3-job
   xxljob:
     enabled: false
     adminAddresses: http://localhost:8090/xxl-job-admin/
@@ -286,7 +288,7 @@ jeecg:
     port: 9999
     logPath: logs/jeecg/job/jobhandler/
     logRetentionDays: 30
-  #分布式锁配置
+  # 分布式锁配置
   redisson:
     address: 127.0.0.1:6379
     password:
@@ -303,29 +305,33 @@ jeecg:
     # 超时时间单位:s。默认 60s
     timeout: 60
 
-  # 本地代理地址
-#    proxy:
-#      host: "http://127.0.0.1"
-#      port: "7890"
-#cas单点登录
+    # 本地代理地址
+    # proxy:
+    #   host: "http://127.0.0.1"
+    #   port: "7890"
+
+# cas单点登录
 cas:
   prefixUrl: http://cas.example.org:8443/cas
-#Mybatis输出sql日志
+
+# Mybatis输出sql日志
 logging:
   level:
     org.flywaydb: debug
     org.jeecg.modules.system.mapper: debug
-#swagger
+
+# swagger
 knife4j:
-  #开启增强配置
+  # 开启增强配置
   enable: true
-  #开启生产环境屏蔽
+  # 开启生产环境屏蔽
   production: false
   basic:
     enable: false
     username: jeecg
     password: jeecg1314
-#第三方登录
+
+# 第三方登录
 justauth:
   enabled: true
   type:
@@ -370,13 +376,13 @@ data-bridge:
   ga:
     account-id: 191734056
 
-##GEOIP MMDB 静态数据库文件
+### GEOIP MMDB 静态数据库文件
 geoip:
   static:
     city:
       mmdb: /opt/adweb3/sharing/GeoLite2-City.mmdb
 
-# 机器人预警url
+### 机器人预警url
 robot:
   enquiry-url: https://open.feishu.cn/open-apis/bot/v2/hook/042cf8d0-0072-435c-bf5c-656b9368a5ac  #询盘拉取失败通知
   market-plan-missing-url: https://open.feishu.cn/open-apis/bot/v2/hook/4a7110ff-9121-49fc-b0b6-0d809b548ad4  #缺少营销方案导致的拉取站点失败
@@ -385,19 +391,19 @@ robot:
   localize-product-fail-url: https://open.feishu.cn/open-apis/bot/v2/hook/46e780fc-6ed9-45ec-b39f-1e332a236386 #产品图片本地化失败通知
   gtm-url: https://open.feishu.cn/open-apis/bot/v2/hook/e29a515d-7331-428e-b890-5b5549ffcab5 #跟踪gtm通知
 
-#垃圾询盘判断规则
+### 垃圾询盘判断规则
 judge_waste_enquiry:
   email:
     tenMinNum: 3 # 10分钟内最多发送3次
     oneDayNum: 5 # 一天最多发送5封邮件
-    notBlackListNum: 3 #不在黑名单中的邮箱发送次数
-    notBlackListDate: 7 #不在邮箱黑名单中的垃圾询盘计数的天数
+    notBlackListNum: 3 # 不在黑名单中的邮箱发送次数
+    notBlackListDate: 7 # 不在邮箱黑名单中的垃圾询盘计数的天数
   ip:
     tenMinNum: 3 # 10分钟内最多发送次数 >
     oneDayNum: 5 # 一天内最多发送次数 >
-    notBlackListNum: 3 #不在Ip黑名单中垃圾询盘计数次数 >
-    notBlackListDate: 7 #不在Ip黑名单中垃圾询盘计数的天数
-    delOrdIpDate: 90 #删除时效性低的Ip时间
+    notBlackListNum: 3 # 不在Ip黑名单中垃圾询盘计数次数 >
+    notBlackListDate: 7 # 不在Ip黑名单中垃圾询盘计数的天数
+    delOrdIpDate: 90 # 删除时效性低的Ip时间
 
 ### dataforseo
 dataforseo:
@@ -419,22 +425,25 @@ tradesparq:
 
 ### 询盘列表配置
 enquiry:
-  disable-admin-read: true
-  demoFlag: false
+  disable-admin-read: true # 询盘的消息是否阅读后更改阅读状态
+  demoFlag: false # 询盘的示例关键消息是否脱敏处理
   token: zQ3jJqtIexedIw6tZGk6p
   pullCount: 5
 
-# 亚马逊翻译api
+### 亚马逊翻译api
 aws:
   translate:
     accessKey: AKIAS37NJDKDETZ7PPEN
     secretKey: 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
 
-#v3项目路径
+### v3系统配置
 v3:
   projectPath: /opt/adweb3/pem
+  dmp:
+    # DMP模块实时报表开关。true-从DataBridge实时读取 false-从DB离线读取
+    realtimeReport: true
 
-#建站链接测试环境配置
+### 建站链接测试环境配置
 AdwebSiteConnect:
   host: 35.87.155.71
   port: 22