master
WuXianChaoPin 6 years ago
parent c4b6841ee3
commit 0a94406f82
  1. 2
      db/src/main/java/db/model/bilibili/ScheduledTaskEntity.java
  2. 8
      pom-webjar.xml
  3. 1
      web/pom.xml
  4. 3
      web/src/main/java/web/aop/Advisor.java
  5. 2
      web/src/main/java/web/config/AbstractWebConfig.java
  6. 11
      web/src/main/java/web/controller/AdminController.java
  7. 37
      web/src/main/java/web/controller/BaseController.java
  8. 10
      web/src/main/java/web/controller/DataController.java
  9. 9
      web/src/main/java/web/controller/MenuController.java
  10. 14
      web/src/main/java/web/controller/TableController.java
  11. 5
      web/src/main/java/web/html/OtherResult.java
  12. 12
      web/src/main/java/web/html/Table.java
  13. 2
      web/src/main/java/web/html/data/DataTable.java
  14. 7
      web/src/main/java/web/html/data/OtherDataResult.java
  15. 2
      web/src/main/java/web/service/DataService.java
  16. 4
      web/src/main/webapp/WEB-INF/jsp/head.jsp
  17. 2
      web/src/main/webapp/WEB-INF/jsp/index.jsp
  18. 5
      web/src/main/webapp/WEB-INF/jsp/menu.jsp
  19. 1
      web/src/main/webapp/index.html
  20. 63
      web/src/test/java/HibernateTest.java
  21. 4
      web/src/test/java/MainTest.java
  22. 69
      web/src/test/java/SpringMVCTest.java
  23. 3
      web/src/test/java/db/model/TaskModel.java
  24. 4
      web/src/test/java/thrift/SpringConfig.java
  25. 2
      web/src/test/java/thrift/ThriftServerDemo.java
  26. 20
      web/src/test/java/thrift/ThriftTest.java

@ -1,5 +1,6 @@
package db.model.bilibili;
import db.annotation.Aliyun;
import db.model.AbstractModel;
import javax.persistence.*;
@ -9,6 +10,7 @@ import java.util.Objects;
@Entity
@Table(name = "scheduled_task")
@Aliyun
public class ScheduledTaskEntity extends AbstractModel {
private int id;
private String description;

@ -10,10 +10,10 @@
<url>https://gitee.com/WuXianChaoPin/WebCrawler</url>
<properties>
<jquery.version>RELEASE</jquery.version>
<bootstrap.version>RELEASE</bootstrap.version>
<fontawesome.version>RELEASE</fontawesome.version>
<jquery-tmpl.version>RELEASE</jquery-tmpl.version>
<jquery.version>3.3.1-1</jquery.version>
<bootstrap.version>4.1.1</bootstrap.version>
<fontawesome.version>5.0.13</fontawesome.version>
<jquery-tmpl.version>beta1.0.0-1</jquery-tmpl.version>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

@ -135,6 +135,7 @@
<version>1.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<finalName>Webcrawler</finalName>

@ -4,6 +4,7 @@ import org.aopalliance.aop.Advice;
import org.springframework.aop.support.StaticMethodMatcherPointcutAdvisor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.ClassUtils;
import web.service.BaseService;
import java.lang.reflect.Method;
@ -21,6 +22,6 @@ public class Advisor extends StaticMethodMatcherPointcutAdvisor {
@Override
public boolean matches(Method method, Class<?> aClass) {
return BaseService.class.isAssignableFrom(aClass);
return BaseService.class.isAssignableFrom(ClassUtils.getUserClass(aClass))&&method.getName().equals("save");
}
}

@ -41,6 +41,8 @@ public class AbstractWebConfig extends AbstractAnnotationConfigDispatcherServlet
ServletRegistration.Dynamic servlet=servletContext.addServlet("DruidStatView", StatViewServlet.class);
servlet.addMapping("/druid/*");
}
}

@ -1,5 +1,7 @@
package web.controller;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
@ -11,21 +13,22 @@ import javax.servlet.http.HttpServletRequest;
* Created by reborn on 2017/9/14.
*/
@Controller
public class AdminController extends BiliController{
public class AdminController{
private Logger log=LogManager.getLogger();
@RequestMapping("index")
@RequestMapping("/index")
private String index(HttpServletRequest request,Model model){
log.info("进入首页");
model.addAttribute("elementPreifx",ElementId.getAll());
return "index";
}
@RequestMapping("test")
@RequestMapping("/test")
private String test(){
return "test";
}
@RequestMapping("test2")
@RequestMapping("/test2")
private String test2(){
return "test2";
}

@ -15,12 +15,18 @@ import org.springframework.core.ResolvableType;
import org.springframework.dao.DataAccessException;
import org.springframework.http.MediaType;
import org.springframework.util.ClassUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import web.service.BaseService;
import java.util.List;
/**
* Created by reborn on 2017/9/14.
*/
public abstract class BaseController<T extends AbstractModel,A extends T,E extends BaseService>{
protected static Logger log=LogManager.getLogger();
@ -45,8 +51,6 @@ public abstract class BaseController<T extends AbstractModel,A extends T,E exten
}
}
@ResponseBody
@RequestMapping(produces = MediaType.APPLICATION_JSON_UTF8_VALUE,method = RequestMethod.POST)
public JsonResult save(T model){
JsonResult result=new JsonResult();
result.setAction(DBAction.C);
@ -60,13 +64,12 @@ public abstract class BaseController<T extends AbstractModel,A extends T,E exten
return result;
}
@ResponseBody
@RequestMapping(produces = MediaType.APPLICATION_JSON_UTF8_VALUE,method = RequestMethod.GET)
@RequestMapping(value = "C",produces = MediaType.APPLICATION_JSON_UTF8_VALUE,method = RequestMethod.GET)
public JsonResult get(T model){
JsonResult result=new JsonResult();
JsonResult<T> result=new JsonResult();
result.setAction(DBAction.R);
try {
service.get(model);
result.setData(service.get(model));
result.setType(Type.success);
} catch (DataAccessException e) {
e.printStackTrace();
@ -76,7 +79,20 @@ public abstract class BaseController<T extends AbstractModel,A extends T,E exten
}
@ResponseBody
@RequestMapping(produces = MediaType.APPLICATION_JSON_UTF8_VALUE,method = RequestMethod.PUT)
@RequestMapping(value = "batch_C",produces = MediaType.APPLICATION_JSON_UTF8_VALUE,method = RequestMethod.GET)
public JsonResult get(A queryCommand, @RequestParam(defaultValue = "0") int firstResult,@RequestParam(defaultValue = "0") int maxResults){
JsonResult<List> result=new JsonResult();
result.setAction(DBAction.R);
try {
result.setData(service.find(getDetachedCriteria(queryCommand), firstResult, maxResults));
result.setType(Type.success);
} catch (DataAccessException e) {
e.printStackTrace();
result.setType(Type.fail);
}
return result;
}
public JsonResult update(T model){
JsonResult result=new JsonResult();
result.setAction(DBAction.U);
@ -90,9 +106,6 @@ public abstract class BaseController<T extends AbstractModel,A extends T,E exten
return result;
}
@ResponseBody
@RequestMapping(produces = MediaType.APPLICATION_JSON_UTF8_VALUE,method = RequestMethod.DELETE)
public JsonResult delete(T model){
JsonResult result=new JsonResult();
result.setAction(DBAction.D);
@ -106,6 +119,8 @@ public abstract class BaseController<T extends AbstractModel,A extends T,E exten
return result;
}
public Class<?> getResolvableType() {
return getResolvableType(0);
}

@ -21,20 +21,18 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import web.html.data.DataTable;
import web.html.data.OtherDataResult;
import web.model.DataModelForm;
import web.service.DataService;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@Controller
@RequestMapping("/data")
public class DataController extends TableController<DataModel, DataModelForm, DataService, OtherDataResult, DataTable<DataModel>> {
public class DataController extends TableController<DataModel, DataModelForm, DataService,DataTable<DataModel>> {
private final String regionJson = IOUtils.toString(DataController.class.getResourceAsStream("/region.json"), Charset.forName("UTF-8"));
@ -113,12 +111,6 @@ public class DataController extends TableController<DataModel, DataModelForm, Da
return service.rowCount(command);
}
@Override
protected List<OtherDataResult> otherDatas(DataModelForm command) {
return new ArrayList<OtherDataResult>() {{
add(new OtherDataResult());
}};
}
@Override
protected void setModel(Model model) {

@ -6,21 +6,12 @@ import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Order;
import org.hibernate.criterion.Property;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@Controller
@RequestMapping("/menu")
public class MenuController extends BiliController<MenuModel> {
@RequestMapping
public String find(Model model, MenuModel queryCommand, @RequestParam Integer firstResult, @RequestParam Integer maxResults){
model.addAttribute("menus", service.find(getDetachedCriteria(queryCommand), firstResult, maxResults).toString());
return "menu";
}
@Override
public boolean checkAction(DBAction action) {
return DBAction.R.equals(action);

@ -7,13 +7,12 @@ import org.springframework.dao.DataAccessException;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
import web.html.OtherResult;
import web.html.Table;
import web.service.BaseService;
import java.util.List;
public abstract class TableController<T extends AbstractModel,A extends T,E extends BaseService,D extends OtherResult,B extends Table<D,T>> extends BaseController<T,A,E> {
public abstract class TableController<T extends AbstractModel,A extends T,E extends BaseService,B extends Table<T>> extends BaseController<T,A,E> {
@RequestMapping("table")
public String find(Model model,@ModelAttribute("command") A command,B pageResult) {
@ -26,13 +25,11 @@ public abstract class TableController<T extends AbstractModel,A extends T,E exte
if(rowCount>0) {
List list = find(command, pageResult.getCurrentPage()*pageResult.getPageSize(), pageResult.getPageSize());
List<D> otherDatas= otherDatas(command);
result.setCount(rowCount);
result.setCurrentPage(pageResult.getCurrentPage()+1);
result.setDatas(list);
result.setPageSize(pageResult.getPageSize());
result.setType(Type.success);
result.setOtherData(otherDatas);
result.setPages();
}else{
result.setType(Type.fail);
@ -72,13 +69,4 @@ public abstract class TableController<T extends AbstractModel,A extends T,E exte
return service.rowCount(getDetachedCriteria(command));
}
/**
* 额外数据
* @param command
* @return
*/
protected List<D> otherDatas(A command){
return null;
}
}

@ -1,5 +0,0 @@
package web.html;
public interface OtherResult {
}

@ -5,7 +5,7 @@ import db.model.AbstractModel;
import java.util.List;
public abstract class Table<E extends OtherResult,T extends AbstractModel> extends PageResult<T> implements TableResult{
public abstract class Table<T extends AbstractModel> extends PageResult<T> implements TableResult{
private static String importJspPath;
@ -16,9 +16,6 @@ public abstract class Table<E extends OtherResult,T extends AbstractModel> exten
private static List<String> columns;
private List<E> otherData;
public String getImportJspPath() {
if(importJspPath==null){
importJspPath=importJspPath();
@ -47,13 +44,6 @@ public abstract class Table<E extends OtherResult,T extends AbstractModel> exten
return columns;
}
public List<E> getOtherData() {
return otherData;
}
public void setOtherData(List<E> otherData) {
this.otherData = otherData;
}
}

@ -6,7 +6,7 @@ import web.html.Table;
import java.util.ArrayList;
import java.util.List;
public class DataTable<T extends DataModel> extends Table<OtherDataResult,T> {
public class DataTable<T extends DataModel> extends Table<T> {
@Override
public int getPageSize() {

@ -1,7 +0,0 @@
package web.html.data;
import web.html.OtherResult;
public class OtherDataResult implements OtherResult {
}

@ -173,4 +173,6 @@ public class DataService extends BaseService {
return getHibernateTemplate().execute(session -> session.createNativeQuery(sql.toString(),DataModel.class).setFirstResult(firstResult).setMaxResults(maxResults>getMaxResults()?getMaxResults():maxResults).getResultList());
}
}

@ -19,7 +19,7 @@
<title>WebCrawler</title>
<!-- Bootstrap Core CSS -->
<link href="/webjars/bootstrap/4.1.0/css/bootstrap.min.css" rel="stylesheet">
<link href="/webjars/bootstrap/4.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- Custom Fonts -->
<link href="/webjars/font-awesome/5.0.13/web-fonts-with-css/css/fontawesome-all.min-jsf.css" rel="stylesheet" type="text/css">
@ -29,7 +29,7 @@
<%--<script src="/webjars/bootstrap/4.1.0/js/bootstrap.min.js"></script>--%>
<script src="/webjars/bootstrap/4.1.0/js/bootstrap.bundle.min.js"></script>
<script src="/webjars/bootstrap/4.1.1/js/bootstrap.bundle.min.js"></script>
<script src="/webjars/jquery-tmpl/beta1.0.0/jquery.tmpl.min.js"></script>

@ -85,7 +85,7 @@
</div>
</div>
<c:import url="/menu?firstResult=0&maxResults=0"></c:import>
<c:import url="menu.jsp"></c:import>

@ -115,9 +115,10 @@
}
$(function () {
addMenu(${menus}['data']);
$.get('/menu/batch_C',function(json){
addMenu(json['data']);
});
$("#datatmlp").tmpl()
});
</script>
</head>

@ -4,7 +4,6 @@
<meta charset="UTF-8">
<title>Title</title>
<script>
alert(123)
location.href='/index'
</script>
</head>

@ -0,0 +1,63 @@
import db.model.TaskModel;
import db.model.bilibili.ScheduledTaskEntity;
import db.util.ExportUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.hibernate.criterion.DetachedCriteria;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.orm.hibernate5.HibernateTemplate;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.web.SpringJUnitWebConfig;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import web.config.AppConfig;
import web.config.SpringConfig;
import web.model.DataModelForm;
import web.service.BiliService;
import web.service.DataService;
import java.io.IOException;
import java.util.ArrayList;
//让测试运行于SpringMVC
@RunWith(SpringJUnit4ClassRunner.class)
@SpringJUnitWebConfig
@ContextConfiguration(classes = {AppConfig.class, SpringConfig.class})
public class HibernateTest {
private Logger log=LogManager.getLogger();
@Autowired
private DataService dataService;
@Autowired
private BiliService biliService;
@Test
public void count() {
DataModelForm form=new DataModelForm();
form.setTypeIds(new ArrayList<Integer>(){{add(32);add(33);}});
long c=dataService.rowCount(form);
log.info(c);
}
@Test
public void exportSql() throws IOException {
ExportUtil sql=new ExportUtil(biliService.getHibernateTemplate(),"E:\\Projects\\JAVA\\WebCrawler\\init\\typeid");
sql.exportType();
}
@Test
public void get(){
TaskModel taskModel=new TaskModel();
taskModel.setId(3);
log.info(biliService.get(taskModel));
}
@Test
public void findAll(){
biliService.find(DetachedCriteria.forClass(ScheduledTaskEntity.class)).forEach(a->log.info(a));
}
}

@ -0,0 +1,4 @@
public class MainTest {
public static void main(String[] args) {
}
}

@ -1,17 +1,9 @@
import core.thrift.MyAsyncMethodCallback;
import core.thrift.ThriftClientPool;
import db.model.bilibili.ScheduledTaskEntity;
import db.util.ExportUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.hibernate.criterion.DetachedCriteria;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
import org.springframework.orm.hibernate5.HibernateTemplate;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.web.SpringJUnitWebConfig;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@ -23,14 +15,8 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import web.config.AppConfig;
import web.config.SpringConfig;
import web.model.DataModelForm;
import web.service.BiliService;
import web.service.DataService;
import java.io.IOException;
import java.util.ArrayList;
//让测试运行于Spring环境
//让测试运行于SpringMVC
@RunWith(SpringJUnit4ClassRunner.class)
@SpringJUnitWebConfig
@ContextConfiguration(classes = {AppConfig.class, SpringConfig.class})
@ -42,16 +28,6 @@ public class SpringMVCTest {
@Autowired
WebApplicationContext wac;
@Autowired
private DataService dataService;
@Autowired
private BiliService biliService;
@Autowired
@Qualifier("Hweb.service.BiliService")
private HibernateTemplate template2;
@Before
public void setup() {
mockMvc = MockMvcBuilders.webAppContextSetup(wac).build();
@ -59,52 +35,17 @@ public class SpringMVCTest {
@Test
public void test() throws Exception {
ResultActions resultActions = this.mockMvc.perform(MockMvcRequestBuilders.get("/data/table").param("currentPage", "0").param("pageSize", "10").param("typeIds","33"));
resultActions.andDo(MockMvcResultHandlers.print()).andReturn();
}
@Test
public void test2() throws Exception {
ResultActions resultActions = this.mockMvc.perform(MockMvcRequestBuilders.get("/data/R").param("cid","9987062"));
public void dataTable() throws Exception {
ResultActions resultActions = mockMvc.perform(MockMvcRequestBuilders.get("/data/table").param("currentPage", "0").param("pageSize", "10").param("typeIds","33"));
resultActions.andDo(MockMvcResultHandlers.print()).andReturn();
}
@Test
public void test3() {
// 查询总条数
// log.info("记录数:"+dataService.rowCount(DetachedCriteria.forClass(DataModel.class)));
// log.info(dataService.rowCount(null));
DataModelForm form=new DataModelForm();
form.setTypeIds(new ArrayList<Integer>(){{add(32);add(33);}});
long c=dataService.rowCount(form);
log.info(c);
}
@Test
public void test4() throws IOException {
ExportUtil sql=new ExportUtil(template2,"E:\\Projects\\JAVA\\WebCrawler\\init\\typeid");
sql.exportType();
}
@Test
public void test5(){
dataService.getHibernateTemplate();
biliService.getHibernateTemplate();
}
@Test
public void test6() throws Exception {
ResultActions resultActions = this.mockMvc.perform(MockMvcRequestBuilders.get("/data/count").param("cids","49052"));
public void get() throws Exception {
ResultActions resultActions=mockMvc.perform(MockMvcRequestBuilders.get("/menu/batch_C"));
resultActions.andDo(MockMvcResultHandlers.print()).andReturn();
}
@Test
public void findAll(){
biliService.find(DetachedCriteria.forClass(ScheduledTaskEntity.class)).forEach(a->log.info(a));
}
}

@ -1,11 +1,14 @@
package db.model;
import db.annotation.Aliyun;
import javax.persistence.*;
import java.io.Serializable;
import java.util.Objects;
@Entity
@Table(name = "task")
@Aliyun
public class TaskModel extends AbstractModel {
private int id;
private String api;

@ -1,3 +1,5 @@
package thrift;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
@ -6,6 +8,6 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@Configuration
@PropertySource({"classpath:config.properties"})
@ComponentScan("core.thrift")
public class SpringTestThrift implements WebMvcConfigurer {
public class SpringConfig implements WebMvcConfigurer {
}

@ -1,3 +1,5 @@
package thrift;
import core.thrift.comment.QueryComment;
import org.apache.thrift.TProcessorFactory;
import org.apache.thrift.protocol.TBinaryProtocol;

@ -1,3 +1,5 @@
package thrift;
import core.thrift.MyAsyncMethodCallback;
import core.thrift.ThriftClientPool;
import core.thrift.comment.QueryComment;
@ -16,7 +18,7 @@ import java.util.ArrayList;
//让测试运行于Spring环境
@RunWith(SpringJUnit4ClassRunner.class)
@SpringJUnitWebConfig
@ContextConfiguration(classes = SpringTestThrift.class)
@ContextConfiguration(classes = SpringConfig.class)
@ComponentScan("core.thrift")
public class ThriftTest {
@ -24,24 +26,26 @@ public class ThriftTest {
@Test
public void commentSum() throws Exception {
MyAsyncMethodCallback<Integer> call = new MyAsyncMethodCallback<>();
ThriftClientPool.doExecute(client -> client.commentSum(49052,call),QueryComment.AsyncClient.class);
log.info(call.getResult());
MyAsyncMethodCallback<Integer> call = new MyAsyncMethodCallback<>();
ThriftClientPool.doExecute(client -> client.commentSum(49052, call), QueryComment.AsyncClient.class);
log.info(call.getResult());
}
@Test
public void download() throws Exception {
MyAsyncMethodCallback<String> call = new MyAsyncMethodCallback<>();
ThriftClientPool.doExecute(client ->client.download(new ArrayList<Integer>(){{
add(49052);
}},"test",call),QueryComment.AsyncClient.class);
ThriftClientPool.doExecute(client -> client.download(new ArrayList<Integer>() {{
add(49052);
}}, "test", call), QueryComment.AsyncClient.class);
log.info(call.getResult());
}
@Test
public void work() throws Exception {
MyAsyncMethodCallback<Boolean> call = new MyAsyncMethodCallback<>();
ThriftClientPool.doExecute(client -> client.qiandao(call),TSDM.AsyncClient.class);
ThriftClientPool.doExecute(client -> client.qiandao(call), TSDM.AsyncClient.class);
log.info(call.getResult());
}
Loading…
Cancel
Save