Merge branch 'dmp' of wangfan/adweb3-server into master

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/entity/GASourceMediumReport.java

@@ -0,0 +1,82 @@
+package org.jeecg.modules.adweb.dmp.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.fasterxml.jackson.annotation.JsonFormat;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+
+import org.jeecgframework.poi.excel.annotation.Excel;
+import org.springframework.format.annotation.DateTimeFormat;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @Description: dmp_ga_source_medium_report
+ * @Author: jeecg-boot
+ * @Date:   2024-10-11
+ * @Version: V1.0
+ */
+@Data
+@TableName("dmp_ga_source_medium_report")
+@Accessors(chain = true)
+@EqualsAndHashCode(callSuper = false)
+@Schema(description="dmp_ga_source_medium_report")
+public class GASourceMediumReport implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+	/**id*/
+	@TableId(type = IdType.AUTO)
+    @Schema(description = "id")
+    private Long id;
+	/**站点id*/
+	@Excel(name = "站点id", width = 15)
+    @Schema(description = "站点id")
+    private Integer siteId;
+	/**统计时间*/
+	@Excel(name = "统计时间", width = 15, format = "yyyy-MM-dd")
+	@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd")
+    @DateTimeFormat(pattern="yyyy-MM-dd")
+    @Schema(description = "统计时间")
+    private Date date;
+	/**type*/
+	@Excel(name = "type", width = 15)
+    @Schema(description = "type")
+    private String type;
+	/**访问量*/
+	@Excel(name = "访问量", width = 15)
+    @Schema(description = "访问量")
+    private Integer totalUsers;
+	/**新用户数量*/
+	@Excel(name = "新用户数量", width = 15)
+    @Schema(description = "新用户数量")
+    private Integer newUsers;
+	/**会话数量*/
+	@Excel(name = "会话数量", width = 15)
+    @Schema(description = "会话数量")
+    private Integer sessions;
+	/**bounceRate*/
+	@Excel(name = "bounceRate", width = 15)
+    @Schema(description = "bounceRate")
+    private Double bounceRate;
+	/**avgSessionDuration*/
+	@Excel(name = "avgSessionDuration", width = 15)
+    @Schema(description = "avgSessionDuration")
+    private Double avgSessionDuration;
+	/**pageViewsPerSession*/
+	@Excel(name = "pageViewsPerSession", width = 15)
+    @Schema(description = "pageViewsPerSession")
+    private Double pageViewsPerSession;
+	/**ctime*/
+	@Excel(name = "ctime", width = 20, format = "yyyy-MM-dd HH:mm:ss")
+	@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
+    @DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
+    @Schema(description = "ctime")
+    private Date ctime;
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/mapper/GASourceMediumReportMapper.java

@@ -0,0 +1,16 @@
+package org.jeecg.modules.adweb.dmp.mapper;
+
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import org.jeecg.modules.adweb.dmp.entity.GASourceMediumReport;
+
+/**
+ * @Description: dmp_ga_source_medium_report
+ * @Author: jeecg-boot
+ * @Date:   2024-10-11
+ * @Version: V1.0
+ */
+public interface GASourceMediumReportMapper extends BaseMapper<GASourceMediumReport> {
+
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/mapper/xml/GASourceMediumReportMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.jeecg.modules.adweb.dmp.mapper.GASourceMediumReportMapper">
+
+</mapper>

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/IGASourceMediumReportService.java

@@ -0,0 +1,15 @@
+package org.jeecg.modules.adweb.dmp.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+
+import org.jeecg.modules.adweb.dmp.entity.GASourceMediumReport;
+
+/**
+ * @Description: dmp_ga_source_medium_report
+ * @Author: jeecg-boot
+ * @Date:   2024-10-11
+ * @Version: V1.0
+ */
+public interface IGASourceMediumReportService extends IService<GASourceMediumReport> {
+
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/google/GAReportService.java

@@ -8,7 +8,6 @@ import jakarta.annotation.PostConstruct;
 import lombok.extern.slf4j.Slf4j;
 
 import org.apache.commons.compress.utils.Lists;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.reflect.TypeUtils;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.FastJsonUtil;
@@ -17,6 +16,7 @@ import org.jeecg.modules.adweb.common.util.DateUtil;
 import org.jeecg.modules.adweb.common.util.RestTemplateUtil;
 import org.jeecg.modules.adweb.dmp.dto.OpenAPIRequest;
 import org.jeecg.modules.adweb.dmp.dto.OpenAPIResponse;
+import org.jeecg.modules.adweb.dmp.dto.google.analytics.GAPropertyDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.GAReportRequestDTO;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.OrderByType;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportConstant;
@@ -24,8 +24,10 @@ import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.ReportType;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.data.CustomReportData;
 import org.jeecg.modules.adweb.dmp.dto.google.analytics.report.data.GAReportDataDTO;
 import org.jeecg.modules.adweb.dmp.entity.GACountryReport;
+import org.jeecg.modules.adweb.dmp.entity.GASourceMediumReport;
 import org.jeecg.modules.adweb.dmp.entity.GoogleGTM;
 import org.jeecg.modules.adweb.dmp.service.IGACountryReportService;
+import org.jeecg.modules.adweb.dmp.service.IGASourceMediumReportService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.ParameterizedTypeReference;
@@ -55,7 +57,10 @@ public class GAReportService {
 
     @Autowired private IGACountryReportService gaCountryReportService;
 
+    @Autowired private IGASourceMediumReportService gaSourceMediumReportService;
+
     @Autowired private CommonMapper commonMapper;
+
     private RestTemplate restTemplate;
 
     @PostConstruct
@@ -64,22 +69,20 @@ public class GAReportService {
     }
 
     /**
-     * 拉取Google Analytics 国家按日报表
+     * 拉取并同步Google Analytics - Country报表
      *
      * @param googleGTM
      */
-    void queryGACountryDailyReport(GoogleGTM googleGTM) {
-        // 1. 报表时间范围
-        Date startDate =
-                this.getReportStartDate(
-                        "dmp_ga_country_report", "date", "site_id = " + googleGTM.getSiteId());
+    private void syncGACountryReport(GoogleGTM googleGTM) {
+        // 1. 报表时间区间
+        Date startDate = this.getReportStartDate("dmp_ga_country_report", googleGTM.getSiteId());
         Date endDate = new Date();
 
         // 2. 构建GA报表请求参数
-        // 使用ADWEB_CUSTOM_REPORT而不是ADWEB_COUNTRY_CHART，因为需要date作为dimension
+        // 使用ADWEB_CUSTOM_REPORT，不是ADWEB_COUNTRY_CHART - 需要date作为dimension
         GAReportRequestDTO gaReportRequest = new GAReportRequestDTO();
         gaReportRequest.setPropertyResourceName(
-                toGAPropertyResourceName(googleGTM.getGaPropertyId()));
+                GAPropertyDTO.toResourceName(googleGTM.getGaPropertyId()));
         gaReportRequest.setReportType(ReportType.ADWEB_CUSTOM_REPORT);
         gaReportRequest.setStartDate(DateUtils.date2Str(startDate, DateUtils.date_sdf.get()));
         gaReportRequest.setEndDate(DateUtils.date2Str(endDate, DateUtils.date_sdf.get()));
@@ -114,7 +117,79 @@ public class GAReportService {
         queryWrapper.le("date", endDate);
         gaCountryReportService.remove(queryWrapper);
 
-        gaCountryReportService.saveBatch(countryReport);
+        gaCountryReportService.saveBatch(countryReport, countryReport.size());
+    }
+
+    /**
+     * 拉取并同步Google Analytics - Session Source Medium报表
+     *
+     * @param googleGTM
+     */
+    private void syncGASourceMediumReport(GoogleGTM googleGTM) {
+        // 1. 报表时间区间
+        Date startDate =
+                this.getReportStartDate("dmp_ga_source_medium_report", googleGTM.getSiteId());
+        Date endDate = new Date();
+
+        // 2. 构建GA报表请求参数
+        // 使用ADWEB_CUSTOM_REPORT，不是ADWEB_SESSION_SOURCE_MEDIUM_VIEW - 需要date作为dimension
+        GAReportRequestDTO gaReportRequest = new GAReportRequestDTO();
+        gaReportRequest.setPropertyResourceName(
+                GAPropertyDTO.toResourceName(googleGTM.getGaPropertyId()));
+        gaReportRequest.setReportType(ReportType.ADWEB_CUSTOM_REPORT);
+        gaReportRequest.setStartDate(DateUtils.date2Str(startDate, DateUtils.date_sdf.get()));
+        gaReportRequest.setEndDate(DateUtils.date2Str(endDate, DateUtils.date_sdf.get()));
+        gaReportRequest.setMetrics(
+                List.of(
+                        ReportConstant.METRIC_TOTAL_USERS,
+                        ReportConstant.METRIC_NEW_USERS,
+                        ReportConstant.METRIC_SESSIONS,
+                        ReportConstant.METRIC_BOUNCE_RATE,
+                        ReportConstant.METRIC_AVG_SESSION_DURATION,
+                        ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION));
+        gaReportRequest.setDimensions(
+                List.of(
+                        ReportConstant.DIMENSION_DATE,
+                        ReportConstant.DIMENSION_SESSION_SOURCE_MEDIUM));
+        gaReportRequest.setOrderByType(OrderByType.DIMENSIONS);
+        gaReportRequest.setOrderBy(ReportConstant.DIMENSION_DATE);
+        List<CustomReportData> reportDataList =
+                runGAReport(gaReportRequest, CustomReportData.class);
+
+        // 2. 转化为DB entity
+        List<GASourceMediumReport> sourceMediumReport = Lists.newArrayList();
+        for (CustomReportData reportData : reportDataList) {
+            GASourceMediumReport reportRow = new GASourceMediumReport();
+            reportRow.setSiteId(googleGTM.getSiteId());
+            reportRow.setDate(
+                    DateUtils.str2Date(
+                            reportData.get(ReportConstant.DIMENSION_DATE),
+                            DateUtils.date_sdf.get()));
+            reportRow.setType(reportData.get(ReportConstant.DIMENSION_SESSION_SOURCE_MEDIUM));
+            reportRow.setTotalUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_TOTAL_USERS)));
+            reportRow.setNewUsers(
+                    Integer.parseInt(reportData.get(ReportConstant.METRIC_NEW_USERS)));
+            reportRow.setSessions(Integer.parseInt(reportData.get(ReportConstant.METRIC_SESSIONS)));
+            reportRow.setBounceRate(
+                    Double.parseDouble(reportData.get(ReportConstant.METRIC_BOUNCE_RATE)));
+            reportRow.setAvgSessionDuration(
+                    Double.parseDouble(reportData.get(ReportConstant.METRIC_AVG_SESSION_DURATION)));
+            reportRow.setPageViewsPerSession(
+                    Double.parseDouble(
+                            reportData.get(ReportConstant.METRIC_SCREEN_PAGE_VIEWS_PER_SESSION)));
+
+            sourceMediumReport.add(reportRow);
+        }
+
+        // 3. 更新数据库 - 删除旧数据，插入新数据
+        QueryWrapper<GASourceMediumReport> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("site_id", googleGTM.getSiteId());
+        queryWrapper.ge("date", startDate);
+        queryWrapper.le("date", endDate);
+        gaSourceMediumReportService.remove(queryWrapper);
+
+        gaSourceMediumReportService.saveBatch(sourceMediumReport, sourceMediumReport.size());
     }
 
     /**
@@ -159,10 +234,10 @@ public class GAReportService {
      *
      * <p>1. 若表中没有数据，取一年前
      *
-     * <p>2. 表中最大时间减一天 - 如果10月10号凌晨2点执行，最大时间可能是10号，但9号数据GA侧有更新
+     * <p>2. 表中最大时间减一天 - 如10月10号凌晨2点执行，最大时间可能是10号，但9号数据GA侧有更新
      */
-    private Date getReportStartDate(String tableName, String fieldName, String filter) {
-        Date maxDate = commonMapper.getMaxDate(tableName, fieldName, filter);
+    private Date getReportStartDate(String tableName, int siteId) {
+        Date maxDate = commonMapper.getMaxDate(tableName, "date", "site_id = " + siteId);
         if (Objects.isNull(maxDate)) {
             // 1. 一年前
             return DateUtil.plusDays(new Date(), -365);
@@ -170,8 +245,4 @@ public class GAReportService {
         // 2. 最大时间减一天
         return DateUtil.plusDays(maxDate, -1);
     }
-
-    private final String toGAPropertyResourceName(String gaPropertyId) {
-        return StringUtils.isNumeric(gaPropertyId) ? "properties/" + gaPropertyId : gaPropertyId;
-    }
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/adweb/dmp/service/impl/GASourceMediumReportServiceImpl.java

@@ -0,0 +1,20 @@
+package org.jeecg.modules.adweb.dmp.service.impl;
+
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+
+import org.jeecg.modules.adweb.dmp.entity.GASourceMediumReport;
+import org.jeecg.modules.adweb.dmp.mapper.GASourceMediumReportMapper;
+import org.jeecg.modules.adweb.dmp.service.IGASourceMediumReportService;
+import org.springframework.stereotype.Service;
+
+/**
+ * @Description: dmp_ga_source_medium_report
+ * @Author: jeecg-boot
+ * @Date:   2024-10-11
+ * @Version: V1.0
+ */
+@Service
+public class GASourceMediumReportServiceImpl extends ServiceImpl<GASourceMediumReportMapper, GASourceMediumReport> implements IGASourceMediumReportService {
+
+}