|
@@ -1,7 +1,10 @@
|
|
|
package org.jeecg.modules.adweb.seo.service.dataforseo;
|
|
|
|
|
|
+import cn.hutool.core.collection.CollectionUtil;
|
|
|
import cn.hutool.json.JSONUtil;
|
|
|
|
|
|
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|
|
+import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
|
|
|
import com.google.common.collect.Lists;
|
|
|
|
|
|
import io.github.dataforseo.client.ApiClient;
|
|
@@ -14,20 +17,22 @@ import jakarta.annotation.PostConstruct;
|
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
import org.jeecg.common.util.RedisUtil;
|
|
|
+import org.jeecg.modules.adweb.common.util.CommonUtil;
|
|
|
import org.jeecg.modules.adweb.common.util.DateUtil;
|
|
|
import org.jeecg.modules.adweb.common.util.ListUtil;
|
|
|
import org.jeecg.modules.adweb.seo.entity.SeoKeywords;
|
|
|
+import org.jeecg.modules.adweb.seo.entity.SeoKeywordsSerp;
|
|
|
import org.jeecg.modules.adweb.seo.mapper.SeoKeywordsMapper;
|
|
|
import org.jeecg.modules.adweb.seo.service.ISeoKeywordsSerpService;
|
|
|
import org.jeecg.modules.adweb.seo.service.ISeoKeywordsService;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
+import org.springframework.data.redis.core.RedisTemplate;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
-import java.util.Date;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
+import java.util.*;
|
|
|
|
|
|
|
|
|
* DataForSEO Serp查询 - 基于Redis同步
|
|
@@ -39,10 +44,12 @@ import java.util.Map;
|
|
|
@Slf4j
|
|
|
@Service
|
|
|
public class DataForSEOService {
|
|
|
-
|
|
|
private static final int MAX_TASKS_PER_SERP_REQUEST = 100;
|
|
|
- private static final int SERP_REQUEST_CODE_SUCCESS = 20000;
|
|
|
- private static final int SERP_TASK_CODE_SUCCESS = 20100;
|
|
|
+ private static final int SERP_STATUS_CODE_SUCCESS = 20000;
|
|
|
+ private static final int SERP_STATUS_CODE_TASK_CREATED = 20100;
|
|
|
+
|
|
|
+
|
|
|
+ private static final int GOOGLE_SEARCH_PAGE_SIZE = 10;
|
|
|
|
|
|
@Value("${dataforseo.username}")
|
|
|
private String username;
|
|
@@ -61,6 +68,8 @@ public class DataForSEOService {
|
|
|
|
|
|
@Autowired private RedisUtil redisUtil;
|
|
|
|
|
|
+ @Autowired private RedisTemplate<String, Object> redisTemplate;
|
|
|
+
|
|
|
private SerpApi serpApi;
|
|
|
|
|
|
@PostConstruct
|
|
@@ -79,7 +88,7 @@ public class DataForSEOService {
|
|
|
}
|
|
|
|
|
|
|
|
|
- * 全局查询更新DataForSEO keywords serp数据
|
|
|
+ * 全局查询更新DataForSEO keywords Serp数据
|
|
|
*
|
|
|
* @param keywordType 1 - 指定词; 2 - 长尾词
|
|
|
* @param limit 最大查询条数
|
|
@@ -96,6 +105,19 @@ public class DataForSEOService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+ public void syncKeywordsSerpResults() {
|
|
|
+ Set<String> serpTaskRedisKeys = redisTemplate.keys(this.getSerpTaskRedisKey("*"));
|
|
|
+
|
|
|
+ if (CollectionUtil.isEmpty(serpTaskRedisKeys)) {
|
|
|
+ log.info("Redis中暂无需要同步的Serp关键词");
|
|
|
+ } else {
|
|
|
+ for (String serpTaskRedisKey : serpTaskRedisKeys) {
|
|
|
+ this.onSerpResult(serpTaskRedisKey);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
|
|
|
* 向DataForSEO发送Serp请求
|
|
|
*
|
|
@@ -115,7 +137,8 @@ public class DataForSEOService {
|
|
|
SerpTaskRequestInfo serpTaskRequestInfo = new SerpTaskRequestInfo();
|
|
|
serpTaskRequestInfo.setKeyword(seoKeyword.getKeywords());
|
|
|
serpTaskRequestInfo.setSeDomain("google.com");
|
|
|
- serpTaskRequestInfo.setLanguageCode("en");
|
|
|
+ serpTaskRequestInfo.setLanguageCode(
|
|
|
+ StringUtils.defaultIfEmpty(seoKeyword.getLang(), "en"));
|
|
|
serpTaskRequestInfo.setLocationCode(2840);
|
|
|
serpTaskRequestInfo.setTag(Integer.toString(seoKeyword.getId()));
|
|
|
serpTaskRequestInfoList.add(serpTaskRequestInfo);
|
|
@@ -127,7 +150,7 @@ public class DataForSEOService {
|
|
|
log.info(
|
|
|
"创建DataForSEO Serp任务,response = {}",
|
|
|
JSONUtil.toJsonStr(serpTaskPostResponseInfo));
|
|
|
- if (serpTaskPostResponseInfo.getStatusCode() != SERP_REQUEST_CODE_SUCCESS) {
|
|
|
+ if (serpTaskPostResponseInfo.getStatusCode() != SERP_STATUS_CODE_SUCCESS) {
|
|
|
log.error(serpTaskPostResponseInfo.getStatusMessage());
|
|
|
throw new ApiException(serpTaskPostResponseInfo.getStatusMessage());
|
|
|
}
|
|
@@ -135,7 +158,7 @@ public class DataForSEOService {
|
|
|
|
|
|
List<SerpGoogleOrganicTaskPostTaskInfo> serpingTasks =
|
|
|
serpTaskPostResponseInfo.getTasks().stream()
|
|
|
- .filter(task -> task.getStatusCode() == SERP_TASK_CODE_SUCCESS)
|
|
|
+ .filter(task -> task.getStatusCode() == SERP_STATUS_CODE_TASK_CREATED)
|
|
|
.toList();
|
|
|
List<Integer> serpingKeywordIds = Lists.newArrayList();
|
|
|
|
|
@@ -171,84 +194,113 @@ public class DataForSEOService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
+ * 处理DataForSEO Serp查询结果
|
|
|
+ *
|
|
|
+ * <p>1. 更新{@link SeoKeywords}表
|
|
|
+ *
|
|
|
+ * <p>2. 更新{@link SeoKeywordsSerp}表
|
|
|
+ *
|
|
|
+ * <p>3. 从Redis删除taskId
|
|
|
+ */
|
|
|
+ private boolean onSerpResult(String serpTaskRedisKey) {
|
|
|
+ try {
|
|
|
+
|
|
|
+ String taskId = redisUtil.get(serpTaskRedisKey).toString();
|
|
|
+ SerpGoogleOrganicTaskGetRegularTaskInfo serpTask =
|
|
|
+ serpApi.googleOrganicTaskGetRegular(taskId).getTasks().get(0);
|
|
|
+ log.info("获取DataForSEO Serp任务,response = {}", JSONUtil.toJsonStr(serpTask));
|
|
|
+ if (serpTask.getStatusCode() != SERP_STATUS_CODE_SUCCESS) {
|
|
|
+ log.error(serpTask.getStatusMessage());
|
|
|
+ throw new ApiException(serpTask.getStatusMessage());
|
|
|
+ }
|
|
|
+ SerpGoogleOrganicTaskGetRegularResultInfo serpResult = serpTask.getResult().get(0);
|
|
|
+
|
|
|
+
|
|
|
+ int keywordId = Integer.parseInt(this.getKeywordIdFromRedisKey(serpTaskRedisKey));
|
|
|
+ SeoKeywords seoKeyword = seoKeywordsService.getById(keywordId);
|
|
|
+ if (Objects.isNull(seoKeyword)) {
|
|
|
+ log.info("无法获取关键词 ID = {}", keywordId);
|
|
|
+
|
|
|
+ redisUtil.del(serpTaskRedisKey);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ String topPrivateDomain =
|
|
|
+ CommonUtil.getTopPrivateDomain(seoKeyword.getDomain());
|
|
|
+ OrganicSerpElementItem serpItem =
|
|
|
+ serpResult.getItems().stream()
|
|
|
+ .map(item -> (OrganicSerpElementItem) item)
|
|
|
+ .filter(item -> item.getType().equalsIgnoreCase("organic"))
|
|
|
+
|
|
|
+ .filter(item -> item.getDomain().contains(topPrivateDomain))
|
|
|
+ .findAny()
|
|
|
+ .orElse(null);
|
|
|
+
|
|
|
+ String seDomain = serpResult.getSeDomain();
|
|
|
+ String checkUrl = serpResult.getCheckUrl();
|
|
|
+ String languageCode = serpResult.getLanguageCode();
|
|
|
+ Date seDatetime =
|
|
|
+ DateUtil.parseDate(serpResult.getDatetime(), DateUtil.ZONED_DATE_TIME_PATTERN);
|
|
|
+ String positionUrl =
|
|
|
+ Objects.nonNull(serpItem) ? StringUtils.remove(serpItem.getUrl(), "/") : null;
|
|
|
+ int rankGroup = Objects.nonNull(serpItem) ? serpItem.getRankGroup() : 0;
|
|
|
+ int rankAbsolute = Objects.nonNull(serpItem) ? serpItem.getRankAbsolute() : 0;
|
|
|
+
|
|
|
+
|
|
|
+ UpdateWrapper<SeoKeywords> seoKeywordsUpdateWrapper = new UpdateWrapper<>();
|
|
|
+ seoKeywordsUpdateWrapper.eq("id", keywordId);
|
|
|
+
|
|
|
+ seoKeywordsUpdateWrapper.set("last_search_time", seDatetime);
|
|
|
+ seoKeywordsUpdateWrapper.set("position_url", positionUrl);
|
|
|
+ seoKeywordsUpdateWrapper.set("last_rank", rankGroup);
|
|
|
+
|
|
|
+ seoKeywordsUpdateWrapper.set("search_status", 0);
|
|
|
+ seoKeywordsService.update(seoKeywordsUpdateWrapper);
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ seoKeywordsSerpService.fillKeywordsSerpHistory(keywordId, seDatetime);
|
|
|
+
|
|
|
+ SeoKeywordsSerp keywordSerp =
|
|
|
+ seoKeywordsSerpService
|
|
|
+
|
|
|
+ .list(
|
|
|
+ new LambdaQueryWrapper<SeoKeywordsSerp>()
|
|
|
+ .eq(SeoKeywordsSerp::getKeywordsId, keywordId)
|
|
|
+ .eq(
|
|
|
+ SeoKeywordsSerp::getSeDate,
|
|
|
+ DateUtil.formatDate(
|
|
|
+ seDatetime, DateUtil.DATE_PATTERN)))
|
|
|
+ .stream()
|
|
|
+ .findFirst()
|
|
|
+ .orElse(new SeoKeywordsSerp());
|
|
|
+
|
|
|
+ keywordSerp.setKeywordsId(keywordId);
|
|
|
+ keywordSerp.setSearchUrl(checkUrl);
|
|
|
+ keywordSerp.setSeDomain(seDomain);
|
|
|
+ keywordSerp.setLanguageCode(languageCode);
|
|
|
+ keywordSerp.setType("organic_results");
|
|
|
+ keywordSerp.setPageNumber(rankGroup > 0 ? rankGroup / GOOGLE_SEARCH_PAGE_SIZE + 1 : 0);
|
|
|
+ keywordSerp.setRankGroup(rankGroup);
|
|
|
+ keywordSerp.setRankAbsolute(rankAbsolute);
|
|
|
+ keywordSerp.setSeDate(DateUtil.formatDate(seDatetime, DateUtil.DATE_PATTERN));
|
|
|
+ keywordSerp.setSeDatetime(seDatetime);
|
|
|
+ seoKeywordsSerpService.save(keywordSerp);
|
|
|
+
|
|
|
+ redisUtil.del(serpTaskRedisKey);
|
|
|
+ return true;
|
|
|
+ } catch (ApiException e) {
|
|
|
+ log.error(e.getMessage(), e);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
private String getSerpTaskRedisKey(String keywordId) {
|
|
|
return String.format("serp_task:%s", keywordId);
|
|
|
}
|
|
|
+
|
|
|
+ private String getKeywordIdFromRedisKey(String serpTaskRedisKey) {
|
|
|
+ return serpTaskRedisKey.split(":")[1];
|
|
|
+ }
|
|
|
}
|