数据量分页太大分段处理
前言
Github:https://github.com/HealerJean
1、Java
分段
@Slf4j
public class BigDataSubUtils {
// /**
// * 1、获取分段结果
// */
// public static void main(String[] args) {
// int totalCount = 0;
// int intervalLength = 5;
// List<ImmutablePair<Integer, Integer>> result = intervalPairOfTotalCount(totalCount, intervalLength);
// // result.forEach(item -> System.out.println(item.getLeft() + "-" + item.getRight()));
//
// List<String> list = Lists.newArrayList("1","2","3","4","5");
// List<List<String>> lists = intervalPairOfTotalCount(list, 6);
// lists.forEach(item -> {
// item.forEach(item2->{
// System.out.printf(item2);
// });
// System.out.println();
// });
// }
/**
* 获取分段区间,索引从0开始,注意下游
*
* @param totalCount 总数量
* @param pageSize 区间长度
*/
public static List<ImmutablePair<Integer, Integer>> sizeSub(Integer totalCount, Integer pageSize) {
List<ImmutablePair<Integer, Integer>> result = new ArrayList<>();
Integer pageCount;
if (totalCount % pageSize == 0) {
pageCount = totalCount / pageSize;
} else {
pageCount = totalCount / pageSize + 1;
}
Integer pageNow = 1;
while (pageNow <= pageCount) {
int startIndex = (pageNow - 1) * pageSize;
int endIndex = pageNow.compareTo(pageCount) != 0 ? startIndex + pageSize : totalCount;
ImmutablePair<Integer, Integer> pair = ImmutablePair.of(startIndex, endIndex);
pageNow++;
result.add(pair);
}
return result;
}
/**
* 获取分段区间,索引从0开始,注意下游
*
* @param list 数据
* @param pageSize 区间长度
*/
public static <T> List<List<T>> listSub(List<T> list, Integer pageSize) {
Integer totalCount = list.size();
List<List<T>> result = new ArrayList<>();
Integer pageCount;
if (totalCount % pageSize == 0) {
pageCount = totalCount / pageSize;
} else {
pageCount = totalCount / pageSize + 1;
}
Integer pageNow = 1;
while (pageNow <= pageCount) {
int startIndex = (pageNow - 1) * pageSize;
int endIndex = pageNow.compareTo(pageCount) != 0 ? startIndex + pageSize : totalCount;
List<T> sonList = list.subList(startIndex, endIndex);
pageNow++;
result.add(sonList);
}
return result;
}
}
2、RPC
多次查(分页)
注意下游bug导致无线死循环,防止方法,设置方法请求次数或者设置超时时间
public Set<String> queryAllSigningVendorIds(SignUpStatusQueryReq signUpStatusQueryReq) {
Set<String> result = new HashSet<>();
PageQueryRequest<SignUpStatusQueryReq> pageQueryRequest = new PageQueryRequest<>();
pageQueryRequest.setQuery(signUpStatusQueryReq);
pageQueryRequest.setPageIndex(NumberConstant.ONE);
pageQueryRequest.setPageSize(NumberConstant.TEN_THOUSAND);
try {
while (true) {
PageQueryResponse<String> response = signResource.querySigningVendorIdsPage(pageQueryRequest);
if (response != null && Boolean.TRUE.equals(response.getSuccess())) {
if (CollectionUtils.isEmpty(response.getValues())) {
return result;
}
result.addAll(response.getValues());
if (response.getValues().size() < pageQueryRequest.getPageSize()){
return result;
}
pageQueryRequest.setPageIndex(pageQueryRequest.getPageIndex() + NumberConstant.ONE);
continue;
}
log.error("SignResourceRpc#queryAllSigningVendorIds] FAIL, request:{}, response:{}", JSON.toJSONString(signUpStatusQueryReq), JSON.toJSONString(response));
throw new RuntimeException("查询所有在约的商家Id集合--接口数据异常");
}
} catch (Exception e) {
log.error("SignResourceRpc#queryAllSigningVendorIds] ERROR, request:{}, e:{}", JSON.toJSONString(signUpStatusQueryReq), ExceptionUtils.getStackTrace(e));
throw new RuntimeException("查询在约的商家Id集合-接口调用异常");
}
}
3、数据库多次查询(分页)
public List<RouteInsuranceVendorSupplier> selectByExample(RouteInsuranceVendorSupplierQuery query) {
List<RouteInsuranceVendorSupplier> result = new ArrayList<>();
int pageNo = NumberConstant.ONE;
int pageSize = NumberConstant.TEN_THOUSAND;
while (true) {
long start = System.currentTimeMillis();
int offset = (pageNo - NumberConstant.ONE) * pageSize;
query.setOffset(offset);
query.setPageSize(pageSize);
List<RouteInsuranceVendorSupplier> vendorSuppliers = routeInsuranceVendorSupplierMapper.selectByExample(query);
if (CollectionUtils.isEmpty(vendorSuppliers)) {
return result;
}
log.info("[PolicyStatisticsManager#]selectByExample vendorSuppliers size:{}, cost:{}", vendorSuppliers.size(), System.currentTimeMillis() - start);
pageNo = pageNo + 1;
result.addAll(vendorSuppliers);
if (vendorSuppliers.size() < pageSize){
return result;
}
}
}
4、Java
分页
package com.jd.baoxian.merchant.route.web;
import lombok.Data;
import lombok.experimental.Accessors;
import java.util.List;
/**
* @author zhangyujin
* @date 2022/8/26 14:41.
*/
public class JavaPageUtils {
public static <T> PageDto<T> toPageDto(List<T> list, int pageNow, int pageSize) {
if (list == null || list.isEmpty()) {
return new PageDto<>(null, pageNow, pageSize, 0, 0);
}
int totalCount = list.size();
int pageCount;
if (totalCount % pageSize == 0) {
pageCount = totalCount / pageSize;
} else {
pageCount = totalCount / pageSize + 1;
}
if (pageNow > pageCount) {
return new PageDto<>(null, pageNow, pageSize, totalCount, pageCount);
}
int startIndex = (pageNow - 1) * pageSize;
// 结束索引
int endIndex = pageNow == pageCount ? startIndex + pageSize : totalCount;
List<T> pageList = list.subList(startIndex, endIndex);
return new PageDto<>(pageList, pageNow, pageSize, totalCount, pageCount);
}
@Data
@Accessors(chain = true)
public static class PageDto<T> {
private List<T> datas;
/**
* 当前页码数
*/
private Integer pageNow;
/**
* 每页显示的记录数
*/
private Integer pageSize;
/**
* 总记录数
*/
private Integer totalCount;
/**
* 一共多少页
*/
private Integer pageCount;
public PageDto(List<T> datas, Integer pageNow, Integer pageSize, Integer totalCount, Integer pageCount) {
this.datas = datas;
this.pageNow = pageNow;
this.pageSize = pageSize;
this.totalCount = totalCount;
this.pageCount = pageCount;
}
private PageDto(List<T> datas) {
this.datas = datas;
}
private PageDto() {
}
}
}