panqihua
/
webcrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

更新了 项目结构

Browse Source
master
LambdaExpression 7 years ago
parent 09ef84211d
commit b24f09407c
31 changed files with 458 additions and 295 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 57
      makeGif/pom.xml
  2. 216
      pom.xml
  3. 11
      src/main/java/org/pqh/aop/MatcherRule.java
  4. 10
      src/main/webapp/WEB-INF/jsp/index.jsp
  5. 48
      src/test/java/JunitTest.java
  6. 95
      webCrawler-achieve/pom.xml
  7. 2
      webCrawler-achieve/src/main/java/org/pqh/achieve/config/AbstractWebConfig.java
  8. 3
      webCrawler-achieve/src/main/java/org/pqh/achieve/config/SpringConfig.java
  9. 2
      webCrawler-achieve/src/main/java/org/pqh/achieve/config/WebConfigImpl.java
  10. 9
      webCrawler-achieve/src/main/java/org/pqh/achieve/config/WebCrawlerConfig.java
  11. 2
      webCrawler-achieve/src/main/java/org/pqh/achieve/controller/AdminController.java
  12. 16
      webCrawler-achieve/src/main/java/org/pqh/achieve/controller/BaseController.java
  13. 4
      webCrawler-achieve/src/main/java/org/pqh/achieve/controller/ConfigController.java
  14. 4
      webCrawler-achieve/src/main/java/org/pqh/achieve/controller/ParamController.java
  15. 6
      webCrawler-achieve/src/main/resources/config.properties
  16. 0
      webCrawler-achieve/src/main/resources/log4j2.xml
  17. 5
      webCrawler-achieve/src/main/webapp/WEB-INF/jsp/index.jsp
  18. 91
      webCrawler-core/pom.xml
  19. 17
      webCrawler-core/src/main/java/org/pqh/core/annotation/EnableWebCrawlerConfiguration.java
  20. 4
      webCrawler-core/src/main/java/org/pqh/core/aop/InsertAdvice.java
  21. 4
      webCrawler-core/src/main/java/org/pqh/core/aop/InsertAdvisor.java
  22. 10
      webCrawler-core/src/main/java/org/pqh/core/aop/LogAspect.java
  23. 12
      webCrawler-core/src/main/java/org/pqh/core/aop/MatcherRule.java
  24. 31
      webCrawler-core/src/main/java/org/pqh/core/config/HibernateConfig.java
  25. 16
      webCrawler-core/src/main/java/org/pqh/core/dao/BaseDao.java
  26. 2
      webCrawler-core/src/main/java/org/pqh/core/model/AbstractModel.java
  27. 2
      webCrawler-core/src/main/java/org/pqh/core/model/Config.java
  28. 2
      webCrawler-core/src/main/java/org/pqh/core/model/Param.java
  29. 23
      webCrawler-core/src/main/java/org/pqh/core/service/BaseService.java
  30. 2
      webCrawler-core/src/main/java/org/pqh/core/util/DBAction.java
  31. 47
      webCrawler-core/src/main/java/org/pqh/core/util/PropertiesUtil.java

57
makeGif/pom.xml
Unescape View File

@ -1,57 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>webCrawler</artifactId>
<groupId>org.pqh</groupId>
<version>1.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>makeGif</artifactId>
<build>
<finalName>makeGif</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass></mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

216
pom.xml
Unescape View File

@ -1,124 +1,131 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.pqh</groupId> <groupId>org.pqh</groupId>
<artifactId>webCrawler</artifactId> <artifactId>webCrawler</artifactId>
<packaging>war</packaging> <packaging>pom</packaging>
<version>1.0</version> <version>1.0-SNAPSHOT</version>
<description>简单的网络爬虫客户端</description>
<modules> <modules>
<module>makeGif</module> <module>webCrawler-core</module>
<module>webCrawler-achieve</module>
</modules> </modules>
<name>webCrawler Maven Webapp</name>
<url>https://gitee.com/WuXianChaoPin/webCrawler</url>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version> <java.version>1.8</java.version>
<spring.version>4.3.10.RELEASE</spring.version>
<mysql.version>6.0.6</mysql.version> <mysql.version>6.0.6</mysql.version>
<spring.version>RELEASE</spring.version> <commons-dbcp2.version>2.1.1</commons-dbcp2.version>
<hibernate-core.version>5.2.10.Final</hibernate-core.version>
<jackson-databind.version>2.9.0</jackson-databind.version>
<aspectjweaver.version>1.8.10</aspectjweaver.version>
<log4j.version>2.8.2</log4j.version>
<commons-io.version>2.5</commons-io.version>
<junit.version>4.12</junit.version>
<servlet-api.version>3.1.0</servlet-api.version>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework</groupId> <dependency>
<artifactId>spring-web</artifactId> <groupId>org.springframework</groupId>
<version>${spring.version}</version> <artifactId>spring-web</artifactId>
</dependency> <version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId> <dependency>
<artifactId>spring-webmvc</artifactId> <groupId>org.springframework</groupId>
<version>${spring.version}</version> <artifactId>spring-webmvc</artifactId>
</dependency> <version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId> <dependency>
<artifactId>spring-orm</artifactId> <groupId>org.springframework</groupId>
<version>${spring.version}</version> <artifactId>spring-orm</artifactId>
</dependency> <version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId> <dependency>
<artifactId>commons-dbcp2</artifactId> <groupId>org.apache.commons</groupId>
<version>LATEST</version> <artifactId>commons-dbcp2</artifactId>
</dependency> <version>${commons-dbcp2.version}</version>
<dependency> </dependency>
<groupId>org.hibernate</groupId> <dependency>
<artifactId>hibernate-core</artifactId> <groupId>org.hibernate</groupId>
<version>RELEASE</version> <artifactId>hibernate-core</artifactId>
</dependency> <version>${hibernate-core.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId> <dependency>
<artifactId>mysql-connector-java</artifactId> <groupId>mysql</groupId>
<version>${mysql.version}</version> <artifactId>mysql-connector-java</artifactId>
</dependency> <version>${mysql.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId> <dependency>
<artifactId>jackson-databind</artifactId> <groupId>com.fasterxml.jackson.core</groupId>
<version>LATEST</version> <artifactId>jackson-databind</artifactId>
</dependency> <version>${jackson-databind.version}</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId> <dependency>
<artifactId>aspectjweaver</artifactId> <groupId>org.aspectj</groupId>
<version>LATEST</version> <artifactId>aspectjweaver</artifactId>
</dependency> <version>${aspectjweaver.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId> <dependency>
<artifactId>log4j-api</artifactId> <groupId>org.apache.logging.log4j</groupId>
<version>LATEST</version> <artifactId>log4j-api</artifactId>
</dependency> <version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId> <dependency>
<artifactId>log4j-core</artifactId> <groupId>org.apache.logging.log4j</groupId>
<version>LATEST</version> <artifactId>log4j-core</artifactId>
</dependency> <version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId> <dependency>
<artifactId>log4j-web</artifactId> <groupId>org.apache.logging.log4j</groupId>
<version>LATEST</version> <artifactId>log4j-web</artifactId>
</dependency> <version>${log4j.version}</version>
</dependency>
<!--<dependency>-->
<!--<groupId>org.fusesource.jansi</groupId>--> <dependency>
<!--<artifactId>jansi</artifactId>--> <groupId>commons-io</groupId>
<!--<version>LATEST</version>--> <artifactId>commons-io</artifactId>
<!--</dependency>--> <version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId> <dependency>
<artifactId>commons-io</artifactId> <groupId>javax.servlet</groupId>
<version>LATEST</version> <artifactId>javax.servlet-api</artifactId>
</dependency> <version>${servlet-api.version}</version>
<scope>provided</scope>
<dependency> </dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId> <dependency>
<version>3.1.0</version> <groupId>junit</groupId>
<scope>provided</scope> <artifactId>junit</artifactId>
</dependency> <version>${junit.version}</version>
<scope>test</scope>
<!--<dependency>--> </dependency>
<!--<groupId>org.pqh</groupId>--> </dependencies>
<!--<artifactId>makeGif</artifactId>--> </dependencyManagement>
<!---->
<!--</dependency>-->
</dependencies>
<build> <build>
<finalName>webCrawler</finalName> <finalName>webCrawler</finalName>
<plugins> <plugins>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
@ -131,7 +138,4 @@
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

11
src/main/java/org/pqh/aop/MatcherRule.java
Unescape View File

@ -1,11 +0,0 @@
package org.pqh.aop;
import org.aspectj.lang.annotation.Pointcut;
/**
* Created by reborn on 2017/8/3.
*/
public class MatcherRule {
@Pointcut("execution(* org.pqh.dao.BaseDao.*(..))")
public void biliDaoRule(){}
}

10
src/main/webapp/WEB-INF/jsp/index.jsp
Unescape View File

@ -1,10 +0,0 @@
<%@ page contentType="text/html;charset=UTF-8" language="java" %>
<!DOCTYPE html>
<html lang="en">
<head>
<title>Title</title>
</head>
<body>
<h1>首页</h1>
</body>
</html>

48
src/test/java/JunitTest.java
Unescape View File

@ -1,48 +0,0 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.config.xml.XmlConfiguration;
import java.net.URI;
import java.net.URISyntaxException;
/**
* Created by reborn on 2017/8/3.
*/
public class JunitTest {
// private static final boolean flag=init();
private static final Logger rootLogger= LogManager.getLogger();
private static final Logger consoleLogger= LogManager.getLogger("log1");
private static final Logger loggerFile= LogManager.getLogger("log2");
public static void main(String[] args) {
// rootLogger.info("rootLogger");
// consoleLogger.info("consoleLogger");
// loggerFile.info("loggerFile");
// Collection<org.apache.logging.log4j.core.Logger> loggers=loggerContext.getLoggers();
// for(org.apache.logging.log4j.core.Logger logger:loggers){
// logger.info(logger.getName());
// }
}
private static boolean init(){
System.out.println(XmlConfiguration.class.getClassLoader());
LoggerContext loggerContext= (LoggerContext) LogManager.getContext(false);
try {
URI uri=new URI("http://42.56.70.185:8080/master/testlog.xml");
loggerContext.setConfigLocation(uri);
} catch (URISyntaxException e) {
e.printStackTrace();
}
return true;
}
}

95
webCrawler-achieve/pom.xml
Unescape View File

@ -0,0 +1,95 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>webCrawler</artifactId>
<groupId>org.pqh</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>achieve</artifactId>
<packaging>war</packaging>
<name>achieve Maven Webapp</name>
<url>https://gitee.com/WuXianChaoPin/webCrawler</url>
<dependencies>
<dependency>
<groupId>org.pqh</groupId>
<artifactId>core</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-core</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-web</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>achieve</finalName>
</build>
</project>

2
src/main/java/org/pqh/config/AbstractWebConfig.java → webCrawler-achieve/src/main/java/org/pqh/achieve/config/AbstractWebConfig.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.config; package org.pqh.achieve.config;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.springframework.web.filter.CharacterEncodingFilter; import org.springframework.web.filter.CharacterEncodingFilter;

3
src/main/java/org/pqh/config/SpringConfig.java → webCrawler-achieve/src/main/java/org/pqh/achieve/config/SpringConfig.java
Unescape View File

@ -1,5 +1,6 @@
package org.pqh.config; package org.pqh.achieve.config;
import org.pqh.core.config.HibernateConfig;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.*; import org.springframework.context.annotation.*;
import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.EnableWebMvc;

2
src/main/java/org/pqh/config/WebConfigImpl.java → webCrawler-achieve/src/main/java/org/pqh/achieve/config/WebConfigImpl.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.config; package org.pqh.achieve.config;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.springframework.web.WebApplicationInitializer; import org.springframework.web.WebApplicationInitializer;

9
webCrawler-achieve/src/main/java/org/pqh/achieve/config/WebCrawlerConfig.java
Unescape View File

@ -0,0 +1,9 @@
package org.pqh.achieve.config;
import org.pqh.core.annotation.EnableWebCrawlerConfiguration;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableWebCrawlerConfiguration
public class WebCrawlerConfig {
}

2
src/main/java/org/pqh/controller/AdminController.java → webCrawler-achieve/src/main/java/org/pqh/achieve/controller/AdminController.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.controller; package org.pqh.achieve.controller;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;

16
src/main/java/org/pqh/controller/BaseController.java → webCrawler-achieve/src/main/java/org/pqh/achieve/controller/BaseController.java
Unescape View File

@ -1,10 +1,10 @@
package org.pqh.controller; package org.pqh.achieve.controller;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.pqh.model.AbstractModel; import org.pqh.core.model.AbstractModel;
import org.pqh.service.BaseService; import org.pqh.core.service.BaseService;
import org.pqh.util.DBAction; import org.pqh.core.util.DBAction;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
@ -18,15 +18,15 @@ import javax.annotation.Resource;
@Controller @Controller
public class BaseController<T extends AbstractModel> { public class BaseController<T extends AbstractModel> {
protected Logger log= LogManager.getLogger(); protected Logger log = LogManager.getLogger();
@Resource @Resource
protected BaseService<T> baseService; protected BaseService<T> baseService;
@ResponseBody @ResponseBody
@RequestMapping(value = "{action}",produces = "text/html;charset=UTF-8") @RequestMapping(value = "{action}", produces = "text/html;charset=UTF-8")
public String curd(@PathVariable DBAction action,T model) { public String curd(@PathVariable DBAction action, T model) {
log.info(model.tableNote()+"进行"+action+"操作请求"); log.info(model.tableNote() + "进行" + action + "操作请求");
return baseService.curd(action, model); return baseService.curd(action, model);
} }

4
src/main/java/org/pqh/controller/ConfigController.java → webCrawler-achieve/src/main/java/org/pqh/achieve/controller/ConfigController.java
Unescape View File

@ -1,6 +1,6 @@
package org.pqh.controller; package org.pqh.achieve.controller;
import org.pqh.model.Config; import org.pqh.core.model.Config;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;

4
src/main/java/org/pqh/controller/ParamController.java → webCrawler-achieve/src/main/java/org/pqh/achieve/controller/ParamController.java
Unescape View File

@ -1,6 +1,6 @@
package org.pqh.controller; package org.pqh.achieve.controller;
import org.pqh.model.Param; import org.pqh.core.model.Param;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;

6
webCrawler-achieve/src/main/resources/config.properties
Unescape View File

@ -0,0 +1,6 @@
#dbcp2.url=jdbc:mysql://localhost:3306/bilibili?serverTimezone=UTC&&useSSL=true
#dbcp2.username=root
#dbcp2.password=123456
dbcp2.url=jdbc:mysql://172.16.0.50:3306/bb?serverTimezone=UTC&&useSSL=true
dbcp2.username=root
dbcp2.password=123456

0
src/main/resources/log4j2.xml → webCrawler-achieve/src/main/resources/log4j2.xml
Unescape View File

5
webCrawler-achieve/src/main/webapp/WEB-INF/jsp/index.jsp
Unescape View File

@ -0,0 +1,5 @@
<html>
<body>
<h2>Hello World!</h2>
</body>
</html>

91
webCrawler-core/pom.xml
Unescape View File

@ -0,0 +1,91 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>webCrawler</artifactId>
<groupId>org.pqh</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>core</artifactId>
<packaging>jar</packaging>
<name>core Maven Webapp</name>
<url>https://gitee.com/WuXianChaoPin/webCrawler</url>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-core</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-web</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>core</finalName>
</build>
</project>

17
webCrawler-core/src/main/java/org/pqh/core/annotation/EnableWebCrawlerConfiguration.java
Unescape View File

@ -0,0 +1,17 @@
package org.pqh.core.annotation;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import java.lang.annotation.*;
/**
* 爬虫注入组件
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE})
@Documented
@Configuration
@ComponentScan("org.pqh.core.*")
public @interface EnableWebCrawlerConfiguration {
}

4
src/main/java/org/pqh/aop/InsertAdvice.java → webCrawler-core/src/main/java/org/pqh/core/aop/InsertAdvice.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.aop; package org.pqh.core.aop;
import org.springframework.aop.MethodBeforeAdvice; import org.springframework.aop.MethodBeforeAdvice;
@ -12,6 +12,6 @@ public class InsertAdvice implements MethodBeforeAdvice {
@Override @Override
public void before(Method method, Object[] objects, Object o) throws Throwable { public void before(Method method, Object[] objects, Object o) throws Throwable {
System.out.println("类："+o.getClass()+",方法："+method.getName()+"被调用，传入参数："+ Arrays.asList(objects)); System.out.println("类：" + o.getClass() + ",方法：" + method.getName() + "被调用，传入参数：" + Arrays.asList(objects));
} }
} }

4
src/main/java/org/pqh/aop/InsertAdvisor.java → webCrawler-core/src/main/java/org/pqh/core/aop/InsertAdvisor.java
Unescape View File

@ -1,6 +1,6 @@
package org.pqh.aop; package org.pqh.core.aop;
import org.pqh.dao.BaseDao; import org.pqh.core.dao.BaseDao;
import org.springframework.aop.support.StaticMethodMatcherPointcutAdvisor; import org.springframework.aop.support.StaticMethodMatcherPointcutAdvisor;
import java.lang.reflect.Method; import java.lang.reflect.Method;

10
src/main/java/org/pqh/aop/LogAspect.java → webCrawler-core/src/main/java/org/pqh/core/aop/LogAspect.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.aop; package org.pqh.core.aop;
import org.aspectj.lang.JoinPoint; import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Aspect;
@ -12,10 +12,10 @@ import org.springframework.stereotype.Component;
@Component @Component
public class LogAspect { public class LogAspect {
@Before(value = "org.pqh.aop.MatcherRule.biliDaoRule()") @Before(value = "org.pqh.core.aop.MatcherRule.biliDaoRule()")
public void curdCallAction(JoinPoint joinPoint){ public void curdCallAction(JoinPoint joinPoint) {
for(Object arg:joinPoint.getArgs()){ for (Object arg : joinPoint.getArgs()) {
System.out.println("参数"+arg); System.out.println("参数" + arg);
} }
} }

12
webCrawler-core/src/main/java/org/pqh/core/aop/MatcherRule.java
Unescape View File

@ -0,0 +1,12 @@
package org.pqh.core.aop;
import org.aspectj.lang.annotation.Pointcut;
/**
* Created by reborn on 2017/8/3.
*/
public class MatcherRule {
@Pointcut("execution(* org.pqh.core.dao.BaseDao.*(..))")
public void biliDaoRule() {
}
}

31
src/main/java/org/pqh/config/HibernateConfig.java → webCrawler-core/src/main/java/org/pqh/core/config/HibernateConfig.java
Unescape View File

@ -1,6 +1,7 @@
package org.pqh.config; package org.pqh.core.config;
import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.BasicDataSource;
import org.pqh.core.util.PropertiesUtil;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.orm.hibernate5.HibernateTemplate; import org.springframework.orm.hibernate5.HibernateTemplate;
@ -20,19 +21,21 @@ public class HibernateConfig {
private LocalSessionFactoryBean sessionFactoryBean; private LocalSessionFactoryBean sessionFactoryBean;
private BasicDataSource dataSource; private BasicDataSource dataSource;
Properties properties = PropertiesUtil.loadPropertiesFromResourceFile("config.properties");
public HibernateConfig() { public HibernateConfig() {
dataSource=new BasicDataSource(); dataSource = new BasicDataSource();
dataSource.setUrl("jdbc:mysql://localhost:3306/bilibili?serverTimezone=UTC&&useSSL=true"); dataSource.setUrl(properties.getProperty("dbcp2.url"));
dataSource.setUsername(properties.getProperty("dbcp2.username"));
dataSource.setPassword(properties.getProperty("dbcp2.password"));
dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
dataSource.setUsername("root");
dataSource.setPassword("123456");
sessionFactoryBean = new LocalSessionFactoryBean(); sessionFactoryBean = new LocalSessionFactoryBean();
sessionFactoryBean.setDataSource(dataSource); sessionFactoryBean.setDataSource(dataSource);
Properties properties=new Properties(); Properties properties = new Properties();
properties.setProperty("hibernate.connection.pool_size","5"); properties.setProperty("hibernate.connection.pool_size", "5");
properties.setProperty("hibernate.jdbc.fetch_size","50"); properties.setProperty("hibernate.jdbc.fetch_size", "50");
properties.setProperty("hibernate.dialect","org.hibernate.dialect.MySQL57Dialect"); properties.setProperty("hibernate.dialect", "org.hibernate.dialect.MySQL57Dialect");
sessionFactoryBean.setHibernateProperties(properties); sessionFactoryBean.setHibernateProperties(properties);
sessionFactoryBean.setPackagesToScan("org.pqh.model"); sessionFactoryBean.setPackagesToScan("org.pqh.model");
try { try {
@ -43,15 +46,15 @@ public class HibernateConfig {
} }
@Bean("transactionManager") @Bean("transactionManager")
public HibernateTransactionManager hibernateTransactionManager(){ public HibernateTransactionManager hibernateTransactionManager() {
HibernateTransactionManager hibernateTransactionManager=new HibernateTransactionManager(); HibernateTransactionManager hibernateTransactionManager = new HibernateTransactionManager();
hibernateTransactionManager.setSessionFactory(this.sessionFactoryBean.getObject()); hibernateTransactionManager.setSessionFactory(this.sessionFactoryBean.getObject());
return hibernateTransactionManager; return hibernateTransactionManager;
} }
@Bean(name="hibernateTemplate") @Bean(name = "hibernateTemplate")
public HibernateTemplate hibernateTemplate(){ public HibernateTemplate hibernateTemplate() {
HibernateTemplate hibernateTemplate=new HibernateTemplate(); HibernateTemplate hibernateTemplate = new HibernateTemplate();
hibernateTemplate.setSessionFactory(sessionFactoryBean.getObject()); hibernateTemplate.setSessionFactory(sessionFactoryBean.getObject());
return hibernateTemplate; return hibernateTemplate;
} }

16
src/main/java/org/pqh/dao/BaseDao.java → webCrawler-core/src/main/java/org/pqh/core/dao/BaseDao.java
Unescape View File

@ -1,6 +1,6 @@
package org.pqh.dao; package org.pqh.core.dao;
import org.pqh.model.AbstractModel; import org.pqh.core.model.AbstractModel;
import org.springframework.dao.DataAccessException; import org.springframework.dao.DataAccessException;
import org.springframework.orm.hibernate5.HibernateTemplate; import org.springframework.orm.hibernate5.HibernateTemplate;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
@ -18,17 +18,17 @@ public class BaseDao {
@Resource @Resource
private HibernateTemplate hibernateTemplate; private HibernateTemplate hibernateTemplate;
public <T extends AbstractModel>T get(Class<T> T, Serializable pk) throws DataAccessException{ public <T extends AbstractModel> T get(Class<T> T, Serializable pk) throws DataAccessException {
return hibernateTemplate.get(T,pk); return hibernateTemplate.get(T, pk);
} }
public <T extends AbstractModel> void saveOrUpdate(T model)throws DataAccessException{ public <T extends AbstractModel> void saveOrUpdate(T model) throws DataAccessException {
hibernateTemplate.saveOrUpdate(model); hibernateTemplate.saveOrUpdate(model);
} }
public <T extends AbstractModel> void delete(Class<T> T,Serializable key) throws DataAccessException { public <T extends AbstractModel> void delete(Class<T> T, Serializable key) throws DataAccessException {
T model= get(T,key); T model = get(T, key);
if(model!=null){ if (model != null) {
hibernateTemplate.delete(model); hibernateTemplate.delete(model);
} }
} }

2
src/main/java/org/pqh/model/AbstractModel.java → webCrawler-core/src/main/java/org/pqh/core/model/AbstractModel.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.model; package org.pqh.core.model;
import java.io.Serializable; import java.io.Serializable;

2
src/main/java/org/pqh/model/Config.java → webCrawler-core/src/main/java/org/pqh/core/model/Config.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.model; package org.pqh.core.model;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;

2
src/main/java/org/pqh/model/Param.java → webCrawler-core/src/main/java/org/pqh/core/model/Param.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.model; package org.pqh.core.model;
import javax.persistence.Basic; import javax.persistence.Basic;
import javax.persistence.Column; import javax.persistence.Column;

23
src/main/java/org/pqh/service/BaseService.java → webCrawler-core/src/main/java/org/pqh/core/service/BaseService.java
Unescape View File

@ -1,11 +1,10 @@
package org.pqh.service; package org.pqh.core.service;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.pqh.dao.BaseDao; import org.pqh.core.dao.BaseDao;
import org.pqh.model.AbstractModel; import org.pqh.core.model.AbstractModel;
import org.pqh.util.DBAction; import org.pqh.core.util.DBAction;
import org.springframework.dao.DataAccessException;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
@ -16,13 +15,13 @@ import javax.annotation.Resource;
@Service @Service
public class BaseService<T extends AbstractModel> { public class BaseService<T extends AbstractModel> {
protected Logger log= LogManager.getLogger(); protected Logger log = LogManager.getLogger();
@Resource @Resource
private BaseDao baseDao; private BaseDao baseDao;
public String curd(DBAction action,T formModel){ public String curd(DBAction action, T formModel) {
Class<T> tClass= (Class<T>) formModel.getClass(); Class<T> tClass = (Class<T>) formModel.getClass();
try { try {
switch (action) { switch (action) {
case Insert: case Insert:
@ -36,11 +35,11 @@ public class BaseService<T extends AbstractModel> {
baseDao.delete(tClass, formModel.primaryKey()); baseDao.delete(tClass, formModel.primaryKey());
break; break;
} }
return action+" success"; return action + " success";
}catch (DataAccessException e){ } catch (Exception e) {
log.error(formModel.tableNote()+"进行"+action+"操作失败，原因是："+e.getMessage()); log.error(formModel.tableNote() + "进行" + action + "操作失败，原因是：" + e.getMessage());
} }
return action+" fail"; return action + " fail";
} }
} }

2
src/main/java/org/pqh/util/DBAction.java → webCrawler-core/src/main/java/org/pqh/core/util/DBAction.java
Unescape View File

@ -1,4 +1,4 @@
package org.pqh.util; package org.pqh.core.util;
/** /**
* Created by reborn on 2017/8/2. * Created by reborn on 2017/8/2.

47
webCrawler-core/src/main/java/org/pqh/core/util/PropertiesUtil.java
Unescape View File

@ -0,0 +1,47 @@
package org.pqh.core.util;
import java.io.BufferedInputStream;
import java.util.Properties;
/**
* Properties工具类
*/
public final class PropertiesUtil {
// protected static final Logger log = LoggerFactory.getLogger(PropertiesUtil.class);
private PropertiesUtil() {
}
/**
* 根据资源文件名，和name获取对应的value，如果没有对应的值，将返回null
*
* @param resourceFile 资源文件名
* @param name 对应的value
* @return 对应的值
*/
public static String getValue(String resourceFile, String name) {
if (resourceFile == null || name == null) {
return null;
}
return loadPropertiesFromResourceFile(resourceFile).getProperty(name);
}
/**
* 根据资源文件路径，读取资源文件，如果资源文件不存在，将返回默认的[new Properties()]
*
* @param resourceFile 资源文件名
* @return 资源文件
*/
public static Properties loadPropertiesFromResourceFile(String resourceFile) {
Properties properties = new Properties();
try (BufferedInputStream bis = new BufferedInputStream(
PropertiesUtil.class.getClassLoader().getResourceAsStream(resourceFile))) {
properties.load(bis);
return properties;
} catch (Exception e) {
// log.error(e.getMessage(), e);
}
return properties;
}
}
Write Preview
Loading…
Cancel
Save