master
10295
7 years ago
parent
9a27171938
commit
d1d7669bf4
@ -1 +1,2 @@ |
||||
#WebCrawler |
||||
|
||||
 |
||||
@ -1,28 +0,0 @@ |
||||
package org.pqh.core.aop; |
||||
|
||||
import org.aspectj.lang.JoinPoint; |
||||
import org.aspectj.lang.annotation.Aspect; |
||||
import org.aspectj.lang.annotation.Before; |
||||
import org.aspectj.lang.annotation.Pointcut; |
||||
import org.pqh.core.util.LogManger; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/8/3. |
||||
*/ |
||||
@Aspect |
||||
@Component |
||||
public class LogAspect implements LogManger { |
||||
|
||||
@Before(value = "biliDaoRule()") |
||||
public void curdCallAction(JoinPoint joinPoint) { |
||||
for (Object arg : joinPoint.getArgs()) { |
||||
log.info("参数" + arg); |
||||
} |
||||
} |
||||
|
||||
@Pointcut("execution(* org.pqh.core.dao.BaseDao.*(..))") |
||||
public void biliDaoRule() { |
||||
} |
||||
|
||||
} |
||||
@ -1,52 +0,0 @@ |
||||
package org.pqh.core.model; |
||||
|
||||
import javax.persistence.Column; |
||||
import javax.persistence.Entity; |
||||
import javax.persistence.Id; |
||||
import javax.persistence.Table; |
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/8/3. |
||||
*/ |
||||
@Entity |
||||
@Table(name="testjson") |
||||
public class Config extends AbstractModel{ |
||||
|
||||
private String jackson; |
||||
|
||||
@Id |
||||
@Column(name = "jackson") |
||||
public String getJackson() { |
||||
return jackson; |
||||
} |
||||
|
||||
public void setJackson(String jackson) { |
||||
this.jackson = jackson; |
||||
} |
||||
|
||||
@Override |
||||
public boolean equals(Object o) { |
||||
if (this == o) return true; |
||||
if (o == null || getClass() != o.getClass()) return false; |
||||
|
||||
Config config = (Config) o; |
||||
|
||||
return jackson.equals(config.jackson); |
||||
} |
||||
|
||||
@Override |
||||
public int hashCode() { |
||||
return jackson.hashCode(); |
||||
} |
||||
|
||||
@Override |
||||
public Serializable primaryKey() { |
||||
return getJackson(); |
||||
} |
||||
|
||||
@Override |
||||
public String tableNote() { |
||||
return "测试表"; |
||||
} |
||||
} |
||||
@ -1,88 +0,0 @@ |
||||
package org.pqh.core.model; |
||||
|
||||
import javax.persistence.Basic; |
||||
import javax.persistence.Column; |
||||
import javax.persistence.Entity; |
||||
import javax.persistence.Id; |
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/7/31. |
||||
*/ |
||||
@Entity |
||||
public class Param extends AbstractModel{ |
||||
private String key; |
||||
private String value; |
||||
private String desc; |
||||
|
||||
@Id |
||||
@Column(name = "`key`") |
||||
public String getKey() { |
||||
return key; |
||||
} |
||||
|
||||
public void setKey(String key) { |
||||
this.key = key; |
||||
} |
||||
|
||||
@Basic |
||||
@Column(name = "value") |
||||
public String getValue() { |
||||
return value; |
||||
} |
||||
|
||||
public void setValue(String value) { |
||||
this.value = value; |
||||
} |
||||
|
||||
@Basic |
||||
@Column(name = "`desc`") |
||||
public String getDesc() { |
||||
return desc; |
||||
} |
||||
|
||||
public void setDesc(String desc) { |
||||
this.desc = desc; |
||||
} |
||||
|
||||
@Override |
||||
public boolean equals(Object o) { |
||||
if (this == o) return true; |
||||
if (o == null || getClass() != o.getClass()) return false; |
||||
|
||||
Param param = (Param) o; |
||||
|
||||
if (key != null ? !key.equals(param.key) : param.key != null) return false; |
||||
if (value != null ? !value.equals(param.value) : param.value != null) return false; |
||||
if (desc != null ? !desc.equals(param.desc) : param.desc != null) return false; |
||||
|
||||
return true; |
||||
} |
||||
|
||||
@Override |
||||
public int hashCode() { |
||||
int result = key != null ? key.hashCode() : 0; |
||||
result = 31 * result + (value != null ? value.hashCode() : 0); |
||||
result = 31 * result + (desc != null ? desc.hashCode() : 0); |
||||
return result; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "Param{" + |
||||
"key='" + key + '\'' + |
||||
", value='" + value + '\'' + |
||||
", desc='" + desc + '\'' + |
||||
'}'; |
||||
} |
||||
|
||||
@Override |
||||
public Serializable primaryKey() { |
||||
return getKey(); |
||||
} |
||||
|
||||
@Override |
||||
public String tableNote() { |
||||
return "系统参数表"; |
||||
} |
||||
} |
||||
@ -1,227 +0,0 @@ |
||||
package org.pqh.gif; |
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode; |
||||
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
import org.apache.commons.io.FileUtils; |
||||
import org.jsoup.Connection; |
||||
import org.jsoup.Jsoup; |
||||
import org.jsoup.nodes.Document; |
||||
import org.jsoup.select.Elements; |
||||
import org.pqh.core.util.LogManger; |
||||
import org.pqh.gif.gifmaker.AnimatedGifEncoder; |
||||
|
||||
import javax.imageio.ImageIO; |
||||
import javax.net.ssl.HttpsURLConnection; |
||||
import java.awt.*; |
||||
import java.awt.image.BufferedImage; |
||||
import java.io.*; |
||||
import java.util.Arrays; |
||||
import java.util.Enumeration; |
||||
import java.util.List; |
||||
import java.util.zip.ZipEntry; |
||||
import java.util.zip.ZipFile; |
||||
|
||||
import static org.pqh.gif.PixivUtil.Image.big; |
||||
import static org.pqh.gif.PixivUtil.Image.small; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/9/24. |
||||
*/ |
||||
public class PixivUtil implements LogManger{ |
||||
private static final String host="http://i4.pixiv.net"; |
||||
private static final String baseUrl="https://www.pixiv.net/member_illust.php?mode=medium&illust_id="; |
||||
private static final int buffer=10240; |
||||
|
||||
private static final String imgPath="tmp/images/"; |
||||
|
||||
/** |
||||
* 合成动图 |
||||
* @param illust_id |
||||
*/ |
||||
public static void downLoadImg(int illust_id,String phpsessid){ |
||||
|
||||
InputStream is; |
||||
FileOutputStream fos; |
||||
BufferedOutputStream bos; |
||||
ZipFile zipFile = null; |
||||
String imageUrl=baseUrl+illust_id; |
||||
File tmpDir = null; |
||||
try { |
||||
log.info("图片链接："+imageUrl); |
||||
Connection.Response response=Jsoup.connect(imageUrl).cookie("PHPSESSID",phpsessid).execute(); |
||||
if(response.statusCode()== HttpsURLConnection.HTTP_OK) { |
||||
Document document=response.parse(); |
||||
Elements imgEle=document.select("img.original-image"); |
||||
if(imgEle.size()==1){ |
||||
String src=imgEle.first().attr("data-src"); |
||||
downLoadImg(src); |
||||
return; |
||||
} |
||||
|
||||
String script = document.select("#wrapper>script").html(); |
||||
int start = script.indexOf(small.keyword); |
||||
int end = script.indexOf(big.keyword); |
||||
if (start > -1 && end > -1) { |
||||
String json = script.substring(start,end).replaceAll(".*=",""); |
||||
ObjectMapper objectMapper = new ObjectMapper(); |
||||
JsonNode node = objectMapper.readTree(json); |
||||
String src = node.get("src").asText().replace(small.size, big.size); |
||||
log.info("获取到图包地址：" + src); |
||||
|
||||
File file=downLoadImg(src); |
||||
zipFile = new ZipFile(file); |
||||
Enumeration entrys = zipFile.entries(); |
||||
|
||||
log.info("开始解压"); |
||||
|
||||
long a = System.currentTimeMillis(); |
||||
tmpDir = new File(file.getAbsolutePath().replace(".zip", "")); |
||||
File jpgDir=new File(tmpDir.getAbsolutePath() + "/jpg"); |
||||
File pngDir=new File(tmpDir.getAbsolutePath()+"/png"); |
||||
jpgDir.mkdirs(); |
||||
pngDir.mkdirs(); |
||||
while (entrys.hasMoreElements()) { |
||||
ZipEntry zipEntry = (ZipEntry) entrys.nextElement(); |
||||
is = zipFile.getInputStream(zipEntry); |
||||
String jpgPath = jpgDir.getAbsolutePath()+"/"+ zipEntry.getName(); |
||||
File imgFile = new File(jpgPath); |
||||
|
||||
log.info("开始解压图片到：" + imgFile.getAbsolutePath()); |
||||
byte buf[] = new byte[buffer]; |
||||
fos = new FileOutputStream(imgFile); |
||||
bos = new BufferedOutputStream(fos, buffer); |
||||
int count; |
||||
while ((count = is.read(buf)) > -1) { |
||||
bos.write(buf, 0, count); |
||||
} |
||||
bos.close(); |
||||
is.close(); |
||||
File pngFile=new File(jpgPath.replaceAll("jpg","png")); |
||||
ImageIO.write(ImageIO.read(imgFile),"png",pngFile); |
||||
} |
||||
|
||||
|
||||
int length = jpgDir.listFiles().length; |
||||
log.info(length + "张图片解压完毕，耗时：" + (System.currentTimeMillis() - a) + "ms"); |
||||
|
||||
if (length > 0) { |
||||
jpgToGif(document.title().split("/")[0].replaceAll("[「,」]", ""),jpgDir.listFiles()); |
||||
} |
||||
|
||||
} else { |
||||
log.error("图包地址获取失败"); |
||||
} |
||||
}else if(response.statusCode()==HttpsURLConnection.HTTP_NOT_FOUND){ |
||||
log.info("图片链接:"+imageUrl+"不存在"); |
||||
}else{ |
||||
log.error("图片链接:"+imageUrl+"访问异常"); |
||||
} |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
}finally { |
||||
try { |
||||
|
||||
if(zipFile != null){ |
||||
zipFile.close(); |
||||
} |
||||
FileUtils.deleteDirectory(tmpDir); |
||||
} catch (IOException e) { |
||||
log.error(e); |
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
public static File downLoadImg(String src) throws IOException { |
||||
Connection connection = Jsoup.connect(src).header("referer", host).ignoreContentType(true); |
||||
connection.request().maxBodySize(1024 * 1024 * 1024); |
||||
Connection.Response response = connection.execute(); |
||||
log.info("图片资源体积：" + response.header("content-length")); |
||||
String zipName = src.substring(src.lastIndexOf("/") + 1); |
||||
File file = new File(imgPath + zipName); |
||||
FileUtils.writeByteArrayToFile(file, response.bodyAsBytes()); |
||||
log.info("图片资源下载到：" + file.getAbsolutePath()); |
||||
return file; |
||||
} |
||||
|
||||
/** |
||||
* jpg合成gif |
||||
* @param gifName gif文件名 |
||||
* @param |
||||
*/ |
||||
public static void jpgToGif(String gifName,File ...jpgFile){ |
||||
AnimatedGifEncoder animatedGifEncoder=new AnimatedGifEncoder(); |
||||
OutputStream outputStream= null; |
||||
InputStream inputStream; |
||||
|
||||
File gifDir=new File(imgPath+"/gif"); |
||||
gifDir.mkdirs(); |
||||
File gifFile=new File(gifDir.getAbsolutePath()+"/"+gifName+".gif"); |
||||
|
||||
log.info("动图将生成到"+gifFile.getAbsolutePath()); |
||||
try { |
||||
FileUtils.forceMkdirParent(gifFile); |
||||
outputStream = new FileOutputStream(gifFile); |
||||
animatedGifEncoder.start(outputStream); |
||||
//数组转集合
|
||||
List<File> fileList= Arrays.asList(jpgFile); |
||||
long a=System.currentTimeMillis(); |
||||
|
||||
for(File f:fileList){ |
||||
//获取当前帧延迟信息
|
||||
int index=fileList.indexOf(f); |
||||
// int delay=node.get(index).get("delay").asInt();
|
||||
inputStream=new FileInputStream(f.getAbsoluteFile()); |
||||
BufferedImage image = ImageIO.read(inputStream); |
||||
//把帧添加进去合成
|
||||
animatedGifEncoder.addFrame(image); |
||||
// animatedGifEncoder.setDelay(1);
|
||||
|
||||
log.info(gifName+"：正在合成第"+((index+1)+"帧")); |
||||
inputStream.close(); |
||||
} |
||||
//添加完所有帧开始合成
|
||||
animatedGifEncoder.finish(); |
||||
long b=System.currentTimeMillis(); |
||||
log.info("合成"+fileList.size()+"帧花费时间"+(b-a)+"ms"); |
||||
|
||||
} catch (FileNotFoundException e) { |
||||
e.printStackTrace(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
}finally { |
||||
try { |
||||
if(outputStream!=null){ |
||||
outputStream.close(); |
||||
} |
||||
|
||||
|
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
} |
||||
|
||||
if (gifFile.exists()) { |
||||
try { |
||||
Desktop.getDesktop().open(gifFile); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
enum Image{ |
||||
small("pixiv.context.ugokuIllustData","600x600"), |
||||
big("pixiv.context.ugokuIllustFullscreenData","1920x1080"); |
||||
|
||||
private String keyword; |
||||
private String size; |
||||
|
||||
Image(String keyword, String size) { |
||||
this.keyword = keyword; |
||||
this.size = size; |
||||
} |
||||
} |
||||
|
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,36 +0,0 @@ |
||||
package org.pqh.config; |
||||
|
||||
import org.pqh.core.util.LogManger; |
||||
import org.springframework.web.WebApplicationInitializer; |
||||
|
||||
import javax.servlet.ServletContext; |
||||
import javax.servlet.ServletException; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/9/18. |
||||
*/ |
||||
public class WebConfigImpl implements WebApplicationInitializer,LogManger{ |
||||
private String encoding ="UTF-8"; |
||||
|
||||
@Override |
||||
public void onStartup(ServletContext servletContext) throws ServletException { |
||||
log.info("onStartup"); |
||||
// //基于注解配置的上下文
|
||||
// AnnotationConfigWebApplicationContext context = new AnnotationConfigWebApplicationContext();
|
||||
// //注册Spring容器配置类
|
||||
// context.register(SpringConfig.class);
|
||||
// ServletRegistration.Dynamic servlet = servletContext.addServlet("dispatcher", new DispatcherServlet(context));
|
||||
// //配置映射路径
|
||||
// servlet.addMapping("/");
|
||||
// //启动顺序
|
||||
// servlet.setLoadOnStartup(1);
|
||||
//
|
||||
//
|
||||
// servletContext.addListener(new ContextLoaderListener(context));
|
||||
//
|
||||
// FilterRegistration.Dynamic characterEncodingFilter=servletContext.addFilter("CharacterEncodingFilter", CharacterEncodingFilter.class);
|
||||
// servletContext.log("CharacterEncodingFilter设置编码："+encoding);
|
||||
// characterEncodingFilter.setInitParameter("encoding",encoding);
|
||||
// characterEncodingFilter.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST),true,"/*");
|
||||
} |
||||
} |
||||
@ -0,0 +1,10 @@ |
||||
package org.pqh.controller; |
||||
|
||||
import org.pqh.core.model.BangumiEntity; |
||||
import org.springframework.stereotype.Controller; |
||||
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
||||
@Controller |
||||
@RequestMapping("/bangumi") |
||||
public class BangumiController extends BaseController<BangumiEntity> { |
||||
} |
||||
@ -1,14 +0,0 @@ |
||||
package org.pqh.controller; |
||||
|
||||
import org.pqh.core.model.Config; |
||||
import org.springframework.stereotype.Controller; |
||||
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/9/14. |
||||
*/ |
||||
@Controller |
||||
@RequestMapping("/config") |
||||
public class ConfigController extends BaseController<Config> { |
||||
|
||||
} |
||||
@ -1,14 +0,0 @@ |
||||
package org.pqh.controller; |
||||
|
||||
import org.pqh.core.model.Param; |
||||
import org.springframework.stereotype.Controller; |
||||
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
||||
/** |
||||
* Created by reborn on 2017/8/3. |
||||
*/ |
||||
@Controller |
||||
@RequestMapping("/param") |
||||
public class ParamController extends BaseController<Param> { |
||||
|
||||
} |
||||
@ -0,0 +1,81 @@ |
||||
package org.pqh.core.model; |
||||
|
||||
import javax.persistence.*; |
||||
import java.io.Serializable; |
||||
import java.util.Objects; |
||||
|
||||
@Entity |
||||
@Table(name = "bangumi", schema = "bilibili") |
||||
public class BangumiEntity extends AbstractModel{ |
||||
private int bangumiId; |
||||
private Integer seasonId; |
||||
private String title; |
||||
private String allowDownload; |
||||
|
||||
@Id |
||||
@Column(name = "bangumi_id") |
||||
public int getBangumiId() { |
||||
return bangumiId; |
||||
} |
||||
|
||||
public void setBangumiId(int bangumiId) { |
||||
this.bangumiId = bangumiId; |
||||
} |
||||
|
||||
@Basic |
||||
@Column(name = "season_id") |
||||
public Integer getSeasonId() { |
||||
return seasonId; |
||||
} |
||||
|
||||
public void setSeasonId(Integer seasonId) { |
||||
this.seasonId = seasonId; |
||||
} |
||||
|
||||
@Basic |
||||
@Column(name = "title") |
||||
public String getTitle() { |
||||
return title; |
||||
} |
||||
|
||||
public void setTitle(String title) { |
||||
this.title = title; |
||||
} |
||||
|
||||
@Basic |
||||
@Column(name = "allow_download") |
||||
public String getAllowDownload() { |
||||
return allowDownload; |
||||
} |
||||
|
||||
public void setAllowDownload(String allowDownload) { |
||||
this.allowDownload = allowDownload; |
||||
} |
||||
|
||||
@Override |
||||
public boolean equals(Object o) { |
||||
if (this == o) return true; |
||||
if (o == null || getClass() != o.getClass()) return false; |
||||
BangumiEntity that = (BangumiEntity) o; |
||||
return bangumiId == that.bangumiId && |
||||
Objects.equals(seasonId, that.seasonId) && |
||||
Objects.equals(title, that.title) && |
||||
Objects.equals(allowDownload, that.allowDownload); |
||||
} |
||||
|
||||
@Override |
||||
public int hashCode() { |
||||
|
||||
return Objects.hash(bangumiId, seasonId, title, allowDownload); |
||||
} |
||||
|
||||
@Override |
||||
public Serializable primaryKey() { |
||||
return getBangumiId(); |
||||
} |
||||
|
||||
@Override |
||||
public String tableNote() { |
||||
return "bangumi"; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue