API query instead of DB

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/controller/DMPDataController.java

@@ -41,6 +41,7 @@ public class DMPDataController {
     @Autowired private IGAPagePathReportService gaPagePathReportService;
     @Autowired private IGADeviceReportService gaDeviceReportService;
     @Autowired private IEnquiryReportService enquiryReportService;
+    @Autowired private DMPReportService dmpReportService;
 
     /** 首页：网站流量按时间段分析统计 - 今天，昨天，本周，上周等 */
     @GetMapping("/site-periodic/stats")
@@ -145,7 +146,8 @@ public class DMPDataController {
         }
 
         // 2. 查询并返回
-        return Result.ok(gaDeviceReportService.getDeviceStats(siteCode, start, end));
+        // return Result.ok(gaDeviceReportService.getDeviceStats(siteCode, start, end));
+        return Result.ok(dmpReportService.getDeviceStats(siteCode, start, end));
     }
 
     @GetMapping("/enquiry-country/stats")

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/dto/google/analytics/report/GAReportRequestDTO.java

@@ -32,4 +32,6 @@ public class GAReportRequestDTO {
 
     /** Use default values in {@link ReportType} instead. */
     @Deprecated private Boolean orderByDesc;
+
+    private int limit;
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/DMPReportService.java

@@ -0,0 +1,219 @@
+package org.jeecg.modules.adweb.dmp.service;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.jeecg.modules.adweb.common.util.AdwebRedisUtil;
+import org.jeecg.modules.adweb.common.util.DateUtil;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAPropertyDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.GAReportRequestDTO;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.OrderByType;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportConstant;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportType;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.data.CustomReportData;
+import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
+import org.jeecg.modules.adweb.dmp.service.google.GAReportService;
+import org.jeecg.modules.adweb.dmp.vo.report.DeviceStatsVO;
+import org.jeecg.modules.redis.CacheConfig;
+import org.jeecg.modules.redis.TTLCacheManager;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+
+/**
+ * 从DataBridge直接生成GA报表数据，跳过DB查询 - http://data-bridge.v3.adwebcloud.com:9002/swagger-ui/index.html
+ *
+ * @author wfansh
+ */
+@Service
+@Slf4j
+public class DMPReportService {
+
+    @Autowired private GAReportService gaReportService;
+
+    @Autowired private IGoogleGTMService googleGTMService;
+
+    @Autowired private AdwebRedisUtil redisUtil;
+
+    //
+    //    public List<SourceMediumStatsVO> getSourceMediumStats(String siteCode, Date start, Date
+    // end) {
+    //        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+    //        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+    //            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+    //            return Collections.EMPTY_LIST;
+    //        }
+    //
+    //        // 2. 构建GA报表请求参数
+    //        // 使用ReportType.ADWEB_CUSTOM_REPORT，不是ADWEB_SESSION_SOURCE_MEDIUM_VIEW -
+    // 需要date作为dimension
+    //        GAReportRequestDTO gaReportRequest = new GAReportRequestDTO();
+    //        gaReportRequest.setPropertyResourceName(
+    //                GAPropertyDTO.toResourceName(googleGTM.getGaPropertyId()));
+    //        gaReportRequest.setReportType(ReportType.ADWEB_CUSTOM_REPORT);
+    //        gaReportRequest.setStartDate(
+    //                DateUtil.formatDate(
+    //                        Optional.ofNullable(start)
+    //                                // 如果start为空，设置为无限早时间
+    //                                .orElse(DateUtil.parseDate("2016-01-01",
+    // DateUtil.DATE_FORMAT)),
+    //                        DateUtil.DATE_FORMAT));
+    //        gaReportRequest.setEndDate(
+    //                DateUtil.formatDate(
+    //                        Optional.ofNullable(end).orElse(new Date()), DateUtil.DATE_FORMAT));
+    //        gaReportRequest.setMetrics(
+    //                List.of(
+    //                        ReportConstant.METRIC_TOTAL_USERS,
+    //                        ReportConstant.METRIC_NEW_USERS,
+    //                        ReportConstant.METRIC_SESSIONS,
+    //                        ReportConstant.METRIC_BOUNCE_RATE,
+    //                        ReportConstant.METRIC_AVG_SESSION_DURATION,
+    //                        ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION));
+    //
+    // gaReportRequest.setDimensions(List.of(ReportConstant.DIMENSION_SESSION_SOURCE_MEDIUM));
+    //        List<CustomReportData> reportDataList =
+    //                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+    //
+    //        // 2. 转化为DB entity
+    //        List<SourceMediumStatsVO> sourceMediumReport = Lists.newArrayList();
+    //        for (CustomReportData reportData : reportDataList) {
+    //            SourceMediumStatsVO sourceMediumStatsVO = new SourceMediumStatsVO();
+    //
+    //            sourceMediumStatsVO.setType(
+    //                    reportData.get(ReportConstant.DIMENSION_SESSION_SOURCE_MEDIUM));
+    //
+    //            sourceMediumStatsVO.setTotalUsers(
+    //                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+    //            sourceMediumStatsVO.setNewUsers(
+    //                    Integer.parseInt(reportData.get(ReportConstant.METRIC_NEW_USERS)));
+    //            sourceMediumStatsVO.setSessions(
+    //                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SESSIONS)));
+    //
+    //            sourceMediumStatsVO.setAvgSessionDuration(
+    //
+    // Double.parseDouble(reportData.get(ReportConstant.METRIC_AVG_SESSION_DURATION)));
+    //            sourceMediumStatsVO.setPageViewsPerSession(
+    //                    Double.parseDouble(
+    //
+    // reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION)));
+    //
+    //            sourceMediumReport.add(sourceMediumStatsVO);
+    //        }
+    //
+    //        // 1. 时间区间内所有国家totalUsers总数
+    //        int totalUsersSum =
+    //
+    // sourceMediumReport.stream().mapToInt(SourceMediumStatsVO::getTotalUsers).sum();
+    //
+    //        // 2. VO数据填充
+    //        for (SourceMediumStatsVO sourceMediumStatsVO : sourceMediumReport) {
+    //            int totalUsers = sourceMediumStatsVO.getTotalUsers();
+    //            sourceMediumStatsVO.setTotalUsersProportion(
+    //                    NumberUtil.formatPercentage(
+    //                            NumberUtil.safeDivide(totalUsers, totalUsersSum), 2));
+    //
+    //            sourceMediumStatsVO.setNewUsersRatio(
+    //                    NumberUtil.formatPercentage(
+    //                            NumberUtil.safeDivide(sourceMediumStatsVO.getNewUsers(),
+    // totalUsers),
+    //                            2));
+    //        }
+    //
+    //        return sourceMediumReport;
+    //    }
+
+    /** 拉取Google Analytics - Device报表 */
+    @Cacheable(
+            cacheManager = CacheConfig.TTL_CACHE_MANAGER,
+            cacheNames =
+                    "dmp:getDeviceStats" + TTLCacheManager.TTL_SPLITTER + 60 * 10) // Redis TTL为10分钟
+    public List<DeviceStatsVO> getDeviceStats(String siteCode, Date start, Date end) {
+        GoogleGTM googleGTM = this.getGTMAccount(siteCode);
+        if (StringUtils.isBlank(googleGTM.getGaPropertyId())) {
+            log.info("Google Analytics帐号未配置, siteCode = {}", siteCode);
+            return Collections.EMPTY_LIST;
+        }
+
+        // 1. 构建GA报表请求参数
+        // 使用ReportType.ADWEB_CUSTOM_REPORT
+        Pair<String, String> dateRange = this.getDateRange(start, end);
+
+        GAReportRequestDTO gaReportRequest = new GAReportRequestDTO();
+        gaReportRequest.setPropertyResourceName(
+                GAPropertyDTO.toResourceName(googleGTM.getGaPropertyId()));
+        gaReportRequest.setReportType(ReportType.ADWEB_CUSTOM_REPORT);
+        gaReportRequest.setStartDate(dateRange.getLeft());
+        gaReportRequest.setEndDate(dateRange.getRight());
+        gaReportRequest.setMetrics(
+                List.of(
+                        ReportConstant.METRIC_TOTAL_USERS,
+                        ReportConstant.METRIC_SESSIONS,
+                        ReportConstant.METRIC_SCREEN_PAGE_VIEWS));
+        gaReportRequest.setDimensions(List.of(ReportConstant.DIMENSION_PLATFORM_DEVICE_CATEGORY));
+        gaReportRequest.setOrderByType(OrderByType.METRICS);
+        gaReportRequest.setOrderBy(ReportConstant.METRIC_TOTAL_USERS);
+        gaReportRequest.setOrderByDesc(true);
+
+        // 2. 请求API接口
+        List<CustomReportData> reportDataList =
+                gaReportService.runGAReport(gaReportRequest, CustomReportData.class);
+
+        List<DeviceStatsVO> deviceStatsVOs = Lists.newArrayList();
+        for (CustomReportData reportData : reportDataList) {
+            DeviceStatsVO deviceStatsVO = new DeviceStatsVO();
+            deviceStatsVO.setDevice(
+                    reportData.get(ReportConstant.DIMENSION_PLATFORM_DEVICE_CATEGORY));
+            deviceStatsVO.setTotalUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+            deviceStatsVO.setSessions(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SESSIONS)));
+            deviceStatsVO.setPageViews(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS)));
+
+            deviceStatsVOs.add(deviceStatsVO);
+        }
+
+        return deviceStatsVOs;
+    }
+
+    /**
+     * 获取{@link GoogleGTM}帐号信息 - 放入Redis缓存，避免同一页面多个HTTP请求频繁DB查询
+     *
+     * <p>使用AdwebRedisUtil操作 - @Cacheable注解无法嵌套使用
+     */
+    private synchronized GoogleGTM getGTMAccount(String siteCode) {
+        final String redisKey = String.format("%s:account:%s", "googlegtm", siteCode);
+
+        GoogleGTM googleGTM = (GoogleGTM) redisUtil.get(redisKey);
+        if (Objects.nonNull(googleGTM)) {
+            return googleGTM;
+        }
+
+        // DB查询
+        googleGTM =
+                googleGTMService.getOne(
+                        new LambdaQueryWrapper<GoogleGTM>().eq(GoogleGTM::getSiteCode, siteCode),
+                        false);
+        googleGTM = Objects.nonNull(googleGTM) ? googleGTM : new GoogleGTM(); // Redis存储对象不允许为空
+        redisUtil.set(redisKey, googleGTM, 30); // Redis TTL为30秒
+
+        return googleGTM;
+    }
+
+    private Pair<String, String> getDateRange(Date start, Date end) {
+        return Pair.of(
+                DateUtil.formatDate(
+                        Optional.ofNullable(start)
+                                // 如果start为空，设置为无限早时间
+                                .orElse(DateUtil.parseDate("2016-01-01", DateUtil.DATE_FORMAT)),
+                        DateUtil.DATE_FORMAT),
+                DateUtil.formatDate(
+                        Optional.ofNullable(end).orElse(new Date()), DateUtil.DATE_FORMAT));
+    }
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/marketing/googleads/service/GoogleAdsReportService.java

@@ -311,7 +311,7 @@ public class GoogleAdsReportService {
                 DateUtil.formatDate(
                         Optional.ofNullable(start)
                                 // 如果start为空，设置为无限早时间
-                                .orElse(DateUtil.parseDate("2010-01-01", DateUtil.DATE_FORMAT)),
+                                .orElse(DateUtil.parseDate("2016-01-01", DateUtil.DATE_FORMAT)),
                         DateUtil.DATE_FORMAT));
         reportRequest.setEndDate(
                 DateUtil.formatDate(