新增历史表(0-9),用于分担单表压力。

This commit is contained in:
zhouxiunai
2019-05-27 11:21:32 +08:00
parent 5a05570196
commit c786e83784
9 changed files with 429 additions and 25 deletions
@@ -1,11 +1,17 @@
package com.gzzn.omms.msgexchangeapi.dao;
import org.springframework.data.repository.CrudRepository;
import java.util.List;
import com.gzzn.omms.msgexchangeapi.entity.CminmsgHst;
public interface CminmsgHstDao extends CrudRepository<CminmsgHst, Long> {
public interface CminmsgHstDao {
/**
* 保存到指定的表
* @param tableName 数据库表名
* @param lsCminmsgHsts 要插入的记录列表
* @return true 成功 false 失败
*/
public boolean insertToTable(String tableName,List<CminmsgHst> lsCminmsgHsts);
}
@@ -0,0 +1,80 @@
package com.gzzn.omms.msgexchangeapi.dao;
import java.util.List;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.Query;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
import org.springframework.util.CollectionUtils;
import com.gzzn.omms.msgexchangeapi.entity.CminmsgHst;
@Component
public class CminmsgHstDaoImpl implements CminmsgHstDao {
@PersistenceContext
private EntityManager entityManager;
/**
* 保存到指定的表
* @return 成功或失败
*/
@Transactional
public boolean insertToTable(String tableName,List<CminmsgHst> lsCminmsgHsts)
{
if(CollectionUtils.isEmpty(lsCminmsgHsts))
{
throw new RuntimeException("插入数据库内容不能为空");
}
try {
for (CminmsgHst cminmsgHst : lsCminmsgHsts) {
String sqlStr = "insert into " + tableName + "("
+ "CMINMSGS_ID,"
+ "CMINMSGS_CLOB_MSG,"
+ "CMINMSGS_DATE_RECEIVED,"
+ "CMINMSGS_DATE_PROCESSED,"
+ "CMINMSGS_SUBSYSTEM_NAME,"
+ "CMINMSGS_SUBSYSTEM_SEQUENCE,"
+ "CMINMSGS_SUBSYSTEM_DATE_SENT,"
+ "CMINMSGS_SUBSYSTEM_SUBTYPE,"
+ "CMINMSGS_SUBSYSTEM_TYPE,"
+ "CMINMSGS_STATUS) values("
+ ":CMINMSGS_ID,"
+ ":CMINMSGS_CLOB_MSG,"
+ ":CMINMSGS_DATE_RECEIVED,"
+ ":CMINMSGS_DATE_PROCESSED,"
+ ":CMINMSGS_SUBSYSTEM_NAME,"
+ ":CMINMSGS_SUBSYSTEM_SEQUENCE,"
+ ":CMINMSGS_SUBSYSTEM_DATE_SENT,"
+ ":CMINMSGS_SUBSYSTEM_SUBTYPE,"
+ ":CMINMSGS_SUBSYSTEM_TYPE,"
+ ":CMINMSGS_STATUS);";
Query query = entityManager.createNativeQuery(sqlStr);
query.setParameter("CMINMSGS_ID", cminmsgHst.getCminmsgsId());
query.setParameter("CMINMSGS_CLOB_MSG", cminmsgHst.getCminmsgsClobMsg());
query.setParameter("CMINMSGS_DATE_RECEIVED", cminmsgHst.getCminmsgsDateReceived());
query.setParameter("CMINMSGS_DATE_PROCESSED", cminmsgHst.getCminmsgsDateProcessed());
query.setParameter("CMINMSGS_SUBSYSTEM_NAME", cminmsgHst.getCminmsgsSubsystemName());
query.setParameter("CMINMSGS_SUBSYSTEM_SEQUENCE", cminmsgHst.getCminmsgsSubsystemSequence());
query.setParameter("CMINMSGS_SUBSYSTEM_DATE_SENT", cminmsgHst.getCminmsgsSubsystemDateSent());
query.setParameter("CMINMSGS_SUBSYSTEM_SUBTYPE", cminmsgHst.getCminmsgsSubsystemSubtype());
query.setParameter("CMINMSGS_SUBSYSTEM_TYPE", cminmsgHst.getCminmsgsSubsystemType());
query.setParameter("CMINMSGS_STATUS", cminmsgHst.getCminmsgsStatus());
query.executeUpdate();
}//end for
return true;
} catch (Exception e) {
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
return false;
}
} //end function
}
@@ -30,6 +30,8 @@ public class ROUTHandler extends FlopBaseHandler {
BeanUtils.copyProperties(optRouteDaily, routeDaily);
fltr.getROUT().add(routeDaily);
//完整航线,要加上CTU 成都机场
fltr.getERUT().add(routeDaily);
} //添加新数据
return fltr;
@@ -1,8 +1,9 @@
package com.gzzn.omms.msgexchangeapi.scheduled;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,9 +16,9 @@ import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import com.gzzn.omms.msgexchangeapi.dao.CminmsgDao;
import com.gzzn.omms.msgexchangeapi.dao.CminmsgHstDao;
import com.gzzn.omms.msgexchangeapi.entity.Cminmsg;
import com.gzzn.omms.msgexchangeapi.entity.CminmsgHst;
import com.gzzn.omms.msgexchangeapi.service.ICminmsgHstService;
/**
* cminmsgs 消息转历史
@@ -27,9 +28,9 @@ import com.gzzn.omms.msgexchangeapi.entity.CminmsgHst;
@Component
@Async
public class CminmsgsHisScheduled {
@Autowired
CminmsgHstDao cminmsgHstDao;
ICminmsgHstService cminmsgHstService;
@Autowired
CminmsgDao cminmsgDao;
@@ -41,30 +42,36 @@ public class CminmsgsHisScheduled {
{
logger.info("开始cminmsgs消息转历史..");
Date dateNow = new Date();
List<Cminmsg> lsCminmsg = null;
do
{
Date dateNow = new Date();
lsCminmsg = cminmsgDao.findByCminmsgsDateProcessedIsNotNullAndCminmsgsDateReceivedBefore
(
dateNow,new PageRequest(0,100)); //
if(!CollectionUtils.isEmpty(lsCminmsg))
dateNow,new PageRequest(0,300)
);
if(CollectionUtils.isEmpty(lsCminmsg))
{
//插入历史表
List<CminmsgHst> lsCminmsgHst = new ArrayList();
for(Cminmsg cminmsg : lsCminmsg)
{
CminmsgHst cminmsgHst = new CminmsgHst();
BeanUtils.copyProperties(cminmsg, cminmsgHst);
lsCminmsgHst.add(cminmsgHst);
}
cminmsgHstDao.save(lsCminmsgHst);
continue;
}
List<CminmsgHst> successLs = cminmsgHstService.insert(
cminmsgHstService.toCminmsgHstLs(lsCminmsg)
);
//从原始表删除成功列表
if(!CollectionUtils.isEmpty(successLs))
{
List<Cminmsg> waitDeleteLs = successLs.stream()
.map(node->{
Cminmsg cminmsg = new Cminmsg();
BeanUtils.copyProperties(node, cminmsg);
return cminmsg;
})
.collect(Collectors.toList());
//从原始表删除
cminmsgDao.delete(lsCminmsg);
}
cminmsgDao.delete(waitDeleteLs);
}
}
while(!CollectionUtils.isEmpty(lsCminmsg));
@@ -0,0 +1,74 @@
package com.gzzn.omms.msgexchangeapi.service;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.gzzn.omms.msgexchangeapi.dao.CminmsgHstDao;
import com.gzzn.omms.msgexchangeapi.entity.Cminmsg;
import com.gzzn.omms.msgexchangeapi.entity.CminmsgHst;
/**
* 历史消息服务
* @author zhouxiunai
*
*/
@Service
public class CmoutmsgHstServiceImpl implements ICminmsgHstService {
@Autowired
CminmsgHstDao cminmsgHstDao;
@Override
public List<CminmsgHst> insert(List<CminmsgHst> lsCminmsgHst) {
int tableCount = 10;//历史表数目。必须已经创建有有这个数量的历史表0-9
List<CminmsgHst> successLs = new ArrayList<>();
//按id mod 分隔消息.
Map map = new LinkedHashMap<Integer,List<CminmsgHst>>();
for (int i = 0; i < tableCount; i++) {
map.put(new Integer(i), new ArrayList<CminmsgHst>());
}
for(CminmsgHst cminmsgHst : lsCminmsgHst)
{
Integer key = (int) (cminmsgHst.getCminmsgsId() % tableCount);
List<CminmsgHst> ls = (List<CminmsgHst>) map.get(key);
ls.add(cminmsgHst);
}
//插入历史数据
Iterator iter = map.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry entry = (Map.Entry) iter.next();
String key = String.valueOf(entry.getKey());
List<CminmsgHst> ls = (List<CminmsgHst>) entry.getValue();
if(cminmsgHstDao.insertToTable("cminmsgs_hst_" + key, ls)) {
successLs.addAll(ls);
}
}
return successLs;
}
@Override
public List<CminmsgHst> toCminmsgHstLs(List<Cminmsg> lsCminmsg) {
List<CminmsgHst> lsCminmsgHst = new ArrayList<>(lsCminmsg.size());
for(Cminmsg cminmsg : lsCminmsg)
{
CminmsgHst cminmsgHst = new CminmsgHst();
BeanUtils.copyProperties(cminmsg, cminmsgHst);
lsCminmsgHst.add(cminmsgHst);
}
return lsCminmsgHst;
}
}
@@ -0,0 +1,27 @@
package com.gzzn.omms.msgexchangeapi.service;
import java.util.List;
import com.gzzn.omms.msgexchangeapi.entity.Cminmsg;
import com.gzzn.omms.msgexchangeapi.entity.CminmsgHst;
/**
* 历史消息服务接口
* @author zhouxiunai
*
*/
public interface ICminmsgHstService {
/**
* 保存历史消息(性能问题,分布到不同表)
* @return
*/
public List<CminmsgHst> insert(List<CminmsgHst> lsCminmsg);
/**
* 将Cminmsg列表转换成Cminmsg列表
* @param lsCminmsg cmin 消息
* @return 历史cmin消息列表
*/
public List<CminmsgHst> toCminmsgHstLs(List<Cminmsg> lsCminmsg);
}
@@ -0,0 +1,157 @@
CREATE TABLE `CMINMSGS_HST_0` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_1` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_2` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_3` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_4` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_5` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_6` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_7` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_8` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `CMINMSGS_HST_9` (
`CMINMSGS_ID` int(23) NOT NULL,
`CMINMSGS_CLOB_MSG` longtext NOT NULL,
`CMINMSGS_DATE_RECEIVED` datetime DEFAULT NULL,
`CMINMSGS_DATE_PROCESSED` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_NAME` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SEQUENCE` int(6) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_DATE_SENT` datetime DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_SUBTYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_SUBSYSTEM_TYPE` varchar(4) DEFAULT NULL,
`CMINMSGS_STATUS` varchar(20) DEFAULT NULL,
PRIMARY KEY (`CMINMSGS_ID`),
KEY `index_cminmsgs_date_processed` (`CMINMSGS_DATE_PROCESSED`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -0,0 +1,44 @@
package com.gzzn.omms.msgexchangeapi.service;
import java.util.Date;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.PageRequest;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.util.CollectionUtils;
import com.gzzn.omms.msgexchangeapi.dao.CminmsgDao;
import com.gzzn.omms.msgexchangeapi.entity.Cminmsg;
@RunWith(SpringRunner.class)
@SpringBootTest
public class cminmsgHstServiceTest {
@Autowired
ICminmsgHstService cminmsgHstService;
@Autowired
CminmsgDao cminmsgDao;
@Test
public void insertIntoHst()
{
Date dateNow = new Date();
List<Cminmsg> lsCminmsg = null;
lsCminmsg = cminmsgDao.findByCminmsgsDateProcessedIsNotNullAndCminmsgsDateReceivedBefore
(
dateNow,new PageRequest(0,300)
);
if(CollectionUtils.isEmpty(lsCminmsg))
{
return;
}
cminmsgHstService.insert(
cminmsgHstService.toCminmsgHstLs(lsCminmsg)
);
} //end function
}