|
@@ -1,7 +1,10 @@
|
|
|
package org.jeecg.modules.adweb.seo.service.dataforseo;
|
|
|
|
|
|
+import cn.hutool.core.collection.CollectionUtil;
|
|
|
import cn.hutool.json.JSONUtil;
|
|
|
|
|
|
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|
|
+import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
|
|
|
import com.google.common.collect.Lists;
|
|
|
|
|
|
import io.github.dataforseo.client.ApiClient;
|
|
@@ -14,20 +17,22 @@ import jakarta.annotation.PostConstruct;
|
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
import org.jeecg.common.util.RedisUtil;
|
|
|
+import org.jeecg.modules.adweb.common.util.CommonUtil;
|
|
|
import org.jeecg.modules.adweb.common.util.DateUtil;
|
|
|
import org.jeecg.modules.adweb.common.util.ListUtil;
|
|
|
import org.jeecg.modules.adweb.seo.entity.SeoKeywords;
|
|
|
+import org.jeecg.modules.adweb.seo.entity.SeoKeywordsSerp;
|
|
|
import org.jeecg.modules.adweb.seo.mapper.SeoKeywordsMapper;
|
|
|
import org.jeecg.modules.adweb.seo.service.ISeoKeywordsSerpService;
|
|
|
import org.jeecg.modules.adweb.seo.service.ISeoKeywordsService;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
+import org.springframework.data.redis.core.RedisTemplate;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
-import java.util.Date;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
+import java.util.*;
|
|
|
|
|
|
/**
|
|
|
* DataForSEO Serp查询 - 基于Redis同步
|
|
@@ -39,10 +44,12 @@ import java.util.Map;
|
|
|
@Slf4j
|
|
|
@Service
|
|
|
public class DataForSEOService {
|
|
|
-
|
|
|
private static final int MAX_TASKS_PER_SERP_REQUEST = 100;
|
|
|
- private static final int SERP_REQUEST_CODE_SUCCESS = 20000;
|
|
|
- private static final int SERP_TASK_CODE_SUCCESS = 20100;
|
|
|
+ private static final int SERP_STATUS_CODE_SUCCESS = 20000;
|
|
|
+ private static final int SERP_STATUS_CODE_TASK_CREATED = 20100;
|
|
|
+
|
|
|
+ // Google一页显示搜索结果数量
|
|
|
+ private static final int GOOGLE_SEARCH_PAGE_SIZE = 10;
|
|
|
|
|
|
@Value("${dataforseo.username}")
|
|
|
private String username;
|
|
@@ -61,6 +68,8 @@ public class DataForSEOService {
|
|
|
|
|
|
@Autowired private RedisUtil redisUtil;
|
|
|
|
|
|
+ @Autowired private RedisTemplate<String, Object> redisTemplate;
|
|
|
+
|
|
|
private SerpApi serpApi;
|
|
|
|
|
|
@PostConstruct
|
|
@@ -79,7 +88,7 @@ public class DataForSEOService {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 全局查询更新DataForSEO keywords serp数据
|
|
|
+ * 全局查询更新DataForSEO keywords Serp数据
|
|
|
*
|
|
|
* @param keywordType 1 - 指定词; 2 - 长尾词
|
|
|
* @param limit 最大查询条数
|
|
@@ -96,6 +105,19 @@ public class DataForSEOService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /** 同步DataForSEO Serp结果 - 从Redis中获取正在运行的任务 */
|
|
|
+ public void syncKeywordsSerpResults() {
|
|
|
+ Set<String> serpTaskRedisKeys = redisTemplate.keys(this.getSerpTaskRedisKey("*"));
|
|
|
+
|
|
|
+ if (CollectionUtil.isEmpty(serpTaskRedisKeys)) {
|
|
|
+ log.info("Redis中暂无需要同步的Serp关键词");
|
|
|
+ } else {
|
|
|
+ for (String serpTaskRedisKey : serpTaskRedisKeys) {
|
|
|
+ this.onSerpResult(serpTaskRedisKey);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 向DataForSEO发送Serp请求
|
|
|
*
|
|
@@ -115,7 +137,8 @@ public class DataForSEOService {
|
|
|
SerpTaskRequestInfo serpTaskRequestInfo = new SerpTaskRequestInfo();
|
|
|
serpTaskRequestInfo.setKeyword(seoKeyword.getKeywords());
|
|
|
serpTaskRequestInfo.setSeDomain("google.com");
|
|
|
- serpTaskRequestInfo.setLanguageCode("en");
|
|
|
+ serpTaskRequestInfo.setLanguageCode(
|
|
|
+ StringUtils.defaultIfEmpty(seoKeyword.getLang(), "en"));
|
|
|
serpTaskRequestInfo.setLocationCode(2840); // 美国
|
|
|
serpTaskRequestInfo.setTag(Integer.toString(seoKeyword.getId())); // tag = keywordId
|
|
|
serpTaskRequestInfoList.add(serpTaskRequestInfo);
|
|
@@ -127,7 +150,7 @@ public class DataForSEOService {
|
|
|
log.info(
|
|
|
"创建DataForSEO Serp任务,response = {}",
|
|
|
JSONUtil.toJsonStr(serpTaskPostResponseInfo));
|
|
|
- if (serpTaskPostResponseInfo.getStatusCode() != SERP_REQUEST_CODE_SUCCESS) {
|
|
|
+ if (serpTaskPostResponseInfo.getStatusCode() != SERP_STATUS_CODE_SUCCESS) {
|
|
|
log.error(serpTaskPostResponseInfo.getStatusMessage());
|
|
|
throw new ApiException(serpTaskPostResponseInfo.getStatusMessage());
|
|
|
}
|
|
@@ -135,7 +158,7 @@ public class DataForSEOService {
|
|
|
// 3. 过滤状态为成功的Serp task
|
|
|
List<SerpGoogleOrganicTaskPostTaskInfo> serpingTasks =
|
|
|
serpTaskPostResponseInfo.getTasks().stream()
|
|
|
- .filter(task -> task.getStatusCode() == SERP_TASK_CODE_SUCCESS)
|
|
|
+ .filter(task -> task.getStatusCode() == SERP_STATUS_CODE_TASK_CREATED)
|
|
|
.toList();
|
|
|
List<Integer> serpingKeywordIds = Lists.newArrayList();
|
|
|
|
|
@@ -171,84 +194,113 @@ public class DataForSEOService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- // /** 处理Serp查询结果 */
|
|
|
- // public boolean onSerpResult(String taskId) throws Exception {
|
|
|
- // SerpGoogleOrganicTaskGetAdvancedTaskInfo serpTask =
|
|
|
- // serpApi.googleOrganicTaskGetAdvanced(taskId).getTasks().get(0);
|
|
|
- // Map<String, String> data = (Map<String, String>) serpTask.getData();
|
|
|
- // SerpGoogleOrganicTaskGetAdvancedResultInfo serpResult = serpTask.getResult().get(0);
|
|
|
- //
|
|
|
- // int keywordId = Integer.parseInt(data.get("tag"));
|
|
|
- // String seDomain = serpResult.getSeDomain();
|
|
|
- // String checkUrl = serpResult.getCheckUrl();
|
|
|
- // Date seDatetime =
|
|
|
- // DateUtil.parseDate(serpResult.getDatetime(),
|
|
|
- // DateUtil.ZONED_DATE_TIME_PATTERN);
|
|
|
- //
|
|
|
- // // 1. 查询keyword
|
|
|
- // SeoKeywords seoKeyword = seoKeywordsService.getById(keywordId);
|
|
|
- // if (Objects.isNull(seoKeyword)) {
|
|
|
- // log.info("无法获取关键词 id = {}", keywordId);
|
|
|
- // return false;
|
|
|
- // }
|
|
|
- // String topPrivateDomain = CommonUtil.getTopPrivateDomain(seoKeyword.getDomain()); //
|
|
|
- // 顶级域名
|
|
|
- // OrganicSerpElementItem serpItem =
|
|
|
- // serpResult.getItems().stream()
|
|
|
- // .map(item -> (OrganicSerpElementItem) item)
|
|
|
- // .filter(item -> item.getType().equalsIgnoreCase("organic"))
|
|
|
- // .filter(item -> item.getDomain().contains(topPrivateDomain))
|
|
|
- // .findAny()
|
|
|
- // .orElse(null);
|
|
|
- //
|
|
|
- // // 2. 更新SeoKeywords表
|
|
|
- // UpdateWrapper<SeoKeywords> seoKeywordsUpdateWrapper = new UpdateWrapper<>();
|
|
|
- // seoKeywordsUpdateWrapper.eq("id", keywordId);
|
|
|
- // // TODO: why?
|
|
|
- // seoKeywordsUpdateWrapper.set("last_search_time", seDatetime);
|
|
|
- // seoKeywordsUpdateWrapper.set("last_rank", 0);
|
|
|
- // // 查询结束
|
|
|
- // seoKeywordsUpdateWrapper.set("search_status", 0);
|
|
|
- // if (Objects.nonNull(serpItem)) {
|
|
|
- // seoKeywordsUpdateWrapper.set(
|
|
|
- // "position_url", StringUtils.removeEnd(serpItem.getUrl(), "/"));
|
|
|
- // seoKeywordsUpdateWrapper.set("last_rank", serpItem.getRankGroup());
|
|
|
- // }
|
|
|
- // seoKeywordsService.update(seoKeywordsUpdateWrapper);
|
|
|
- //
|
|
|
- // // 3. 更新SeoSerp表
|
|
|
- // // 3.1 补充与上次更新时间之间的数据
|
|
|
- // seoKeywordsSerpService.fillKeywordsSerpHistory(keywordId, seDatetime);
|
|
|
- // // 3.2 更新Serp表
|
|
|
- // SeoKeywordsSerp seoKeywordsSerp =
|
|
|
- // seoKeywordsSerpService
|
|
|
- // .list(
|
|
|
- // new LambdaQueryWrapper<SeoKeywordsSerp>()
|
|
|
- // .eq(SeoKeywordsSerp::getKeywordsId, keywordId)
|
|
|
- // .eq(
|
|
|
- // SeoKeywordsSerp::getSeDate,
|
|
|
- // DateUtil.formatDate(
|
|
|
- // seDatetime, DateUtil.DATE_PATTERN)))
|
|
|
- // .stream()
|
|
|
- // .findFirst()
|
|
|
- // .orElse(new SeoKeywordsSerp());
|
|
|
- //
|
|
|
- // seoKeywordsSerp.setKeywordsId(keywordsId);
|
|
|
- // seoKeywordsSerp.setSearchUrl(checkUrl);
|
|
|
- // seoKeywordsSerp.setSeDomain(seDomain);
|
|
|
- // seoKeywordsSerp.setLanguageCode(seoKeywords.getLang());
|
|
|
- // seoKeywordsSerp.setType("organic_results");
|
|
|
- // seoKeywordsSerp.setRankAbsolute(rank);
|
|
|
- // seoKeywordsSerp.setPageNumber(rank / PAGE_SIZE + 1);
|
|
|
- // seoKeywordsSerp.setRankType(rank);
|
|
|
- // seoKeywordsSerp.setSeDate(DateUtil.formatDate(seDatetime, DateUtil.DATE_PATTERN));
|
|
|
- // seoKeywordsSerp.setSeDatetime(seDatetime);
|
|
|
- //
|
|
|
- // seoKeywordsSerpService.saveOrUpdate(seoKeywordsSerp);
|
|
|
- // return true;
|
|
|
- // }
|
|
|
+ /**
|
|
|
+ * 处理DataForSEO Serp查询结果
|
|
|
+ *
|
|
|
+ * <p>1. 更新{@link SeoKeywords}表
|
|
|
+ *
|
|
|
+ * <p>2. 更新{@link SeoKeywordsSerp}表
|
|
|
+ *
|
|
|
+ * <p>3. 从Redis删除taskId
|
|
|
+ */
|
|
|
+ private boolean onSerpResult(String serpTaskRedisKey) {
|
|
|
+ try {
|
|
|
+ // 1. 查询Serp task
|
|
|
+ String taskId = redisUtil.get(serpTaskRedisKey).toString();
|
|
|
+ SerpGoogleOrganicTaskGetRegularTaskInfo serpTask =
|
|
|
+ serpApi.googleOrganicTaskGetRegular(taskId).getTasks().get(0);
|
|
|
+ log.info("获取DataForSEO Serp任务,response = {}", JSONUtil.toJsonStr(serpTask));
|
|
|
+ if (serpTask.getStatusCode() != SERP_STATUS_CODE_SUCCESS) {
|
|
|
+ log.error(serpTask.getStatusMessage());
|
|
|
+ throw new ApiException(serpTask.getStatusMessage());
|
|
|
+ }
|
|
|
+ SerpGoogleOrganicTaskGetRegularResultInfo serpResult = serpTask.getResult().get(0);
|
|
|
+
|
|
|
+ // 2. 查询SeoKeywords表,并根据域名过滤Serp result items
|
|
|
+ int keywordId = Integer.parseInt(this.getKeywordIdFromRedisKey(serpTaskRedisKey));
|
|
|
+ SeoKeywords seoKeyword = seoKeywordsService.getById(keywordId);
|
|
|
+ if (Objects.isNull(seoKeyword)) {
|
|
|
+ log.info("无法获取关键词 ID = {}", keywordId);
|
|
|
+
|
|
|
+ redisUtil.del(serpTaskRedisKey);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ String topPrivateDomain =
|
|
|
+ CommonUtil.getTopPrivateDomain(seoKeyword.getDomain()); // 顶级域名
|
|
|
+ OrganicSerpElementItem serpItem =
|
|
|
+ serpResult.getItems().stream()
|
|
|
+ .map(item -> (OrganicSerpElementItem) item)
|
|
|
+ .filter(item -> item.getType().equalsIgnoreCase("organic"))
|
|
|
+ // 根据域名匹配
|
|
|
+ .filter(item -> item.getDomain().contains(topPrivateDomain))
|
|
|
+ .findAny()
|
|
|
+ .orElse(null);
|
|
|
+
|
|
|
+ String seDomain = serpResult.getSeDomain();
|
|
|
+ String checkUrl = serpResult.getCheckUrl();
|
|
|
+ String languageCode = serpResult.getLanguageCode();
|
|
|
+ Date seDatetime =
|
|
|
+ DateUtil.parseDate(serpResult.getDatetime(), DateUtil.ZONED_DATE_TIME_PATTERN);
|
|
|
+ String positionUrl =
|
|
|
+ Objects.nonNull(serpItem) ? StringUtils.remove(serpItem.getUrl(), "/") : null;
|
|
|
+ int rankGroup = Objects.nonNull(serpItem) ? serpItem.getRankGroup() : 0;
|
|
|
+ int rankAbsolute = Objects.nonNull(serpItem) ? serpItem.getRankAbsolute() : 0;
|
|
|
+
|
|
|
+ // 3.更新SeoKeywords表
|
|
|
+ UpdateWrapper<SeoKeywords> seoKeywordsUpdateWrapper = new UpdateWrapper<>();
|
|
|
+ seoKeywordsUpdateWrapper.eq("id", keywordId);
|
|
|
+ // TODO: Why
|
|
|
+ seoKeywordsUpdateWrapper.set("last_search_time", seDatetime);
|
|
|
+ seoKeywordsUpdateWrapper.set("position_url", positionUrl);
|
|
|
+ seoKeywordsUpdateWrapper.set("last_rank", rankGroup);
|
|
|
+ // 状态 -> 查询结束
|
|
|
+ seoKeywordsUpdateWrapper.set("search_status", 0);
|
|
|
+ seoKeywordsService.update(seoKeywordsUpdateWrapper);
|
|
|
+
|
|
|
+ // 4. 更新SeoKeywordsSerp表
|
|
|
+ // 4.1 填充与上次更新时间之间的数据
|
|
|
+ seoKeywordsSerpService.fillKeywordsSerpHistory(keywordId, seDatetime);
|
|
|
+ // 4.2 更新Serp表
|
|
|
+ SeoKeywordsSerp keywordSerp =
|
|
|
+ seoKeywordsSerpService
|
|
|
+ // 如果seDatetime当天有数据,则覆盖
|
|
|
+ .list(
|
|
|
+ new LambdaQueryWrapper<SeoKeywordsSerp>()
|
|
|
+ .eq(SeoKeywordsSerp::getKeywordsId, keywordId)
|
|
|
+ .eq(
|
|
|
+ SeoKeywordsSerp::getSeDate,
|
|
|
+ DateUtil.formatDate(
|
|
|
+ seDatetime, DateUtil.DATE_PATTERN)))
|
|
|
+ .stream()
|
|
|
+ .findFirst()
|
|
|
+ .orElse(new SeoKeywordsSerp());
|
|
|
+
|
|
|
+ keywordSerp.setKeywordsId(keywordId);
|
|
|
+ keywordSerp.setSearchUrl(checkUrl);
|
|
|
+ keywordSerp.setSeDomain(seDomain);
|
|
|
+ keywordSerp.setLanguageCode(languageCode);
|
|
|
+ keywordSerp.setType("organic_results");
|
|
|
+ keywordSerp.setPageNumber(rankGroup > 0 ? rankGroup / GOOGLE_SEARCH_PAGE_SIZE + 1 : 0);
|
|
|
+ keywordSerp.setRankGroup(rankGroup);
|
|
|
+ keywordSerp.setRankAbsolute(rankAbsolute);
|
|
|
+ keywordSerp.setSeDate(DateUtil.formatDate(seDatetime, DateUtil.DATE_PATTERN));
|
|
|
+ keywordSerp.setSeDatetime(seDatetime);
|
|
|
+ seoKeywordsSerpService.save(keywordSerp);
|
|
|
+
|
|
|
+ redisUtil.del(serpTaskRedisKey);
|
|
|
+ return true;
|
|
|
+ } catch (ApiException e) {
|
|
|
+ log.error(e.getMessage(), e);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
private String getSerpTaskRedisKey(String keywordId) {
|
|
|
return String.format("serp_task:%s", keywordId);
|
|
|
}
|
|
|
+
|
|
|
+ private String getKeywordIdFromRedisKey(String serpTaskRedisKey) {
|
|
|
+ return serpTaskRedisKey.split(":")[1];
|
|
|
+ }
|
|
|
}
|