panqihua
/
shelltool
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

1.代码通过vscode插件格式化

Browse Source 
2.超大图片压缩无效后使用异常图片处理
3.删除无效变量
4.修改备份时间配置，固定单位为天数。只允许配纯数字x，代表备份x天
master
panqihua 3 years ago
parent 43afbe498c
commit 81295b6393
2 changed files with 270 additions and 308 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      pixiv/config.json.example
  2. 310
      pixiv/pixiv.sh

2
pixiv/config.json.example
Unescape View File

@ -12,7 +12,7 @@
"AccessKeySecret":"OpenAPI AccessKey Secret", "AccessKeySecret":"OpenAPI AccessKey Secret",
"db":"文件上传成功记录数据库文件名", "db":"文件上传成功记录数据库文件名",
"imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560", "imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560",
"backup_time":"1 hour", "backup_time":"7",
"error_file":"error.jpeg" "error_file":"error.jpeg"
}, },
"convertio":{ "convertio":{

310
pixiv/pixiv.sh
Unescape View File

@ -1,60 +1,52 @@
#!/bin/bash #!/bin/bash
config_file=$1 config_file=$1
if [ -z $config_file ] if [ -z $config_file ]; then
then
echo 'The cache directory must be specified' echo 'The cache directory must be specified'
exit 1 exit 1
fi fi
if [ ! -f $config_file ] if [ ! -f $config_file ]; then
then
echo 'config file：$config_file not found' echo 'config file：$config_file not found'
exit 1 exit 1
fi fi
if [ ! -f `which jq` ] if [ ! -f $(which jq) ]; then
then
echo 'To run this script, you need to install jq，https://stedolan.github.io/jq/' echo 'To run this script, you need to install jq，https://stedolan.github.io/jq/'
exit 1 exit 1
else else
echo "jq version:`jq --version`" echo "jq version:$(jq --version)"
fi fi
if [ ! -f `which xmlstarlet` ] if [ ! -f $(which xmlstarlet) ]; then
then
echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php' echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php'
exit 1 exit 1
else else
echo "xmlstarlet version:`xmlstarlet --version`" echo "xmlstarlet version:$(xmlstarlet --version)"
fi fi
# ==================== Config ==================== # ==================== Config ====================
Host=`cat $config_file | jq -r .aliyun_oss.Host` Host=$(cat $config_file | jq -r .aliyun_oss.Host)
accelerateHost=`cat $config_file | jq -r .aliyun_oss.accelerateHost` accelerateHost=$(cat $config_file | jq -r .aliyun_oss.accelerateHost)
bucketname=`cat $config_file | jq -r .aliyun_oss.bucketname` bucketname=$(cat $config_file | jq -r .aliyun_oss.bucketname)
AccessKeyId=`cat $config_file | jq -r .aliyun_oss.AccessKeyId` AccessKeyId=$(cat $config_file | jq -r .aliyun_oss.AccessKeyId)
AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret` AccessKeySecret=$(cat $config_file | jq -r .aliyun_oss.AccessKeySecret)
db_file=`cat $config_file | jq -r .aliyun_oss.db` db_file=$(cat $config_file | jq -r .aliyun_oss.db)
db_file_err=$db_file.err db_file_err=$db_file.err
imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam` imageParam=$(cat $config_file | jq -r .aliyun_oss.imageParam)
backup_time=`cat $config_file | jq -r .aliyun_oss.backup_time` backup_time=$(cat $config_file | jq -r .aliyun_oss.backup_time)
error_file=`cat $config_file | jq -r .aliyun_oss.error_file` error_file=$(cat $config_file | jq -r .aliyun_oss.error_file)
convertio_apikey=`cat $config_file | jq -r .convertio.apikey`
CronitorKey=`cat $config_file | jq -r .Cronitor.API_KEY` convertio_apikey=$(cat $config_file | jq -r .convertio.apikey)
CronitorJobName=`cat $config_file | jq -r .Cronitor.JOB_NAME`
CronitorKey=$(cat $config_file | jq -r .Cronitor.API_KEY)
CronitorJobName=$(cat $config_file | jq -r .Cronitor.JOB_NAME)
basePath=`cat $config_file|jq -r .basePath` basePath=$(cat $config_file | jq -r .basePath)
if [ ! -f $basePath ] if [ ! -f $basePath ]; then
then
mkdir -p $basePath mkdir -p $basePath
fi fi
# ================================================ # ================================================
cd $basePath cd $basePath
if [ ! -f $db_file ] if [ ! -f $db_file ]; then
then
touch $db_file touch $db_file
fi fi
@ -62,16 +54,15 @@ function PutObject(){
VERB="PUT" VERB="PUT"
file=$1 file=$1
Content_MD5="" Content_MD5=""
Content_Type=`file -b --mime-type $file` Content_Type=$(file -b --mime-type $file)
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` Date=$(TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT')
CanonicalizedOSSHeaders="x-oss-object-acl:public-read\n" CanonicalizedOSSHeaders="x-oss-object-acl:public-read\n"
CanonicalizedResource="/$bucketname/$file" CanonicalizedResource="/$bucketname/$file"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` Signature=$(echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64)
Authorization="OSS $AccessKeyId:$Signature" Authorization="OSS $AccessKeyId:$Signature"
http_code=`curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"` http_code=$(curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file")
if [ $http_code -eq "200" ] if [ $http_code -eq "200" ]; then
then
rm -f $file rm -f $file
else else
echo $file >>$db_file_err echo $file >>$db_file_err
@ -82,11 +73,11 @@ function GetBucketV2(){
VERB="GET" VERB="GET"
Content_MD5="" Content_MD5=""
Content_Type="" Content_Type=""
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` Date=$(TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT')
CanonicalizedOSSHeaders="" CanonicalizedOSSHeaders=""
CanonicalizedResource="/$bucketname/" CanonicalizedResource="/$bucketname/"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` Signature=$(echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64)
Authorization="OSS $AccessKeyId:$Signature" Authorization="OSS $AccessKeyId:$Signature"
curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2" curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2"
} }
@ -95,77 +86,67 @@ function DeleteMultipleObjects(){
file="DeleteMultipleObjects.temp" file="DeleteMultipleObjects.temp"
GetBucketV2 | xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n >$file GetBucketV2 | xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n >$file
# 获取备份时间之前的日期时间戳 # 获取备份时间之前的日期时间戳
oldTimestamp=`date -d "-$backup_time" +%s` oldTimestamp=$(date -d "-$backup_time day" +%s)
ObjectNode="" ObjectNode=""
while read line while read line; do
do filename=$(echo $line | awk '{print $1}')
filename=`echo $line|awk '{print $1}'` date=$(echo $line | awk '{print $2}')
date=`echo $line|awk '{print $2}'` timestamp=$(date -d "$date" +%s)
timestamp=`date -d "$date" +%s` if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]]; then
if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]]
then
ObjectNode="${ObjectNode}<Object><Key>${filename}</Key></Object>" ObjectNode="${ObjectNode}<Object><Key>${filename}</Key></Object>"
fi fi
done <$file done <$file
rm $file rm $file
if [ -n "$ObjectNode" ] if [ -n "$ObjectNode" ]; then
then echo "存在${backup_time}天以前的图片"
echo "存在${backup_time}以前的图片"
compress_xml='<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>'${ObjectNode}'</Delete>' compress_xml='<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>'${ObjectNode}'</Delete>'
echo $compress_xml | xmlstarlet format echo $compress_xml | xmlstarlet format
VERB="POST" VERB="POST"
Content_MD5=`echo -n $compress_xml |openssl dgst -md5 -binary|base64` Content_MD5=$(echo -n $compress_xml | openssl dgst -md5 -binary | base64)
Content_Type="application/xml" Content_Type="application/xml"
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` Date=$(TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT')
CanonicalizedOSSHeaders="" CanonicalizedOSSHeaders=""
CanonicalizedResource="/$bucketname/?delete" CanonicalizedResource="/$bucketname/?delete"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
echo "sign:$stringToSign" echo "sign:$stringToSign"
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` Signature=$(echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64)
Authorization="OSS $AccessKeyId:$Signature" Authorization="OSS $AccessKeyId:$Signature"
http_code=`curl -X $VERB -v -w "%{http_code}" -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete"` http_code=$(curl -X $VERB -v -w "%{http_code}" -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete")
if [ $http_code -eq "200" ] if [ $http_code -eq "200" ]; then
then echo "${backup_time}天以前的图片删除成功"
echo "${backup_time}以前的图片删除成功"
else else
echo "${backup_time}以前的图片删除失败" echo "${backup_time}天以前的图片删除失败"
fi fi
echo '' >$db_file echo '' >$db_file
GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n|while read line GetBucketV2 | xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n | while read line; do
do if [ $line != "$error_file" ]; then
if [ $line != "$error_file" ]
then
echo $line >>$db_file echo $line >>$db_file
fi fi
done done
else else
echo "没有${backup_time}以前的图片需要删除" echo "没有${backup_time}天以前的图片需要删除"
fi fi
} }
function fileSizeStr() { function fileSizeStr() {
fileSize=$1 fileSize=$1
if [[ $fileSize -gt 1048576 ]] if [[ $fileSize -gt 1048576 ]]; then
then echo "$(echo "scale=2;$fileSize/1048576" | bc)MB"
echo "`echo "scale=2;$fileSize/1048576"|bc`MB"
else else
echo "$((fileSize / 1024))KB" echo "$((fileSize / 1024))KB"
fi fi
} }
function Request() { function Request() {
while true while true; do
do
echo "发送消息命令：$1" echo "发送消息命令：$1"
result=`bash -c "$1"` result=$(bash -c "$1")
echo "请求响应：$result" echo "请求响应：$result"
if [ `echo $result|jq .ok` = true ] if [ $(echo $result | jq .ok) = true ]; then
then
return 1 return 1
elif [ `echo $result|jq .error_code` -eq 429 ] elif [ $(echo $result | jq .error_code) -eq 429 ]; then
then second=$(echo $result | jq .parameters.retry_after)
second=`echo $result|jq .parameters.retry_after`
sleep $second sleep $second
echo "${second}后再发起请求" echo "${second}后再发起请求"
Request $1 Request $1
@ -176,36 +157,31 @@ function Request(){
done done
} }
telegramToken=$(cat $config_file | jq -r .telegramToken)
telegramToken=`cat $config_file | jq -r .telegramToken`
baseApi="https://api.telegram.org/bot$telegramToken" baseApi="https://api.telegram.org/bot$telegramToken"
chat_id=`cat $config_file | jq .chatId` chat_id=$(cat $config_file | jq .chatId)
mode=`cat $config_file | jq -r .mode` mode=$(cat $config_file | jq -r .mode)
content=`cat $config_file | jq -r .content` content=$(cat $config_file | jq -r .content)
rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&format=json" rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&format=json"
today=`date "+%Y-%m-%d"` today=$(date "+%Y-%m-%d")
_today=`date "+%Y%m%d"` _today=$(date "+%Y%m%d")
rank_json=$today.json rank_json=$today.json
rule='sed -e "s/\\&amp;/\\&/g"|sed -e "s/\\&lt;/\\</g"|sed -e "s/\\&gt;/\\>/g"' rule='sed -e "s/\\&amp;/\\&/g"|sed -e "s/\\&lt;/\\</g"|sed -e "s/\\&gt;/\\>/g"'
Request "curl -v -d chat_id=$chat_id -d parse_mode=HTML -d text=\"Pixiv排行榜已更新，开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>\" $baseApi/sendMessage" Request "curl -v -d chat_id=$chat_id -d parse_mode=HTML -d text=\"Pixiv排行榜已更新，开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>\" $baseApi/sendMessage"
if [ ! -f $rank_json ] if [ ! -f $rank_json ]; then
then
echo "get data from $rank_url" echo "get data from $rank_url"
curl -v $rank_url >$rank_json curl -v $rank_url >$rank_json
fi fi
length=`jq '.contents|length' $rank_json` length=$(jq '.contents|length' $rank_json)
fileList='' fileList=''
media='' media=''
fileCount=0 fileCount=0
maxFileCount=10 maxFileCount=10
tarFile=$today.tar.gz
maxFileSize=20971520 maxFileSize=20971520
maxFileSize_M="$((maxFileSize/1024/1024))M"
start_rank='' start_rank=''
end_rank='' end_rank=''
maxLikeCount=0 maxLikeCount=0
@ -215,117 +191,110 @@ maxBookmarkCountPid=0
maxViewCount=0 maxViewCount=0
maxViewCountPid=0 maxViewCountPid=0
maxRatio=10 maxRatio=10
for index in `seq 1 $length` for index in $(seq 1 $length); do
do
index=$((index - 1)) index=$((index - 1))
pid=`jq --argjson index $index '.contents[$index].illust_id' $rank_json` pid=$(jq --argjson index $index '.contents[$index].illust_id' $rank_json)
artworkLink="https://www.pixiv.net/artworks/$pid" artworkLink="https://www.pixiv.net/artworks/$pid"
rank=`jq --argjson index $index '.contents[$index].rank' $rank_json` rank=$(jq --argjson index $index '.contents[$index].rank' $rank_json)
yes_rank=`jq --argjson index $index '.contents[$index].yes_rank' $rank_json` yes_rank=$(jq --argjson index $index '.contents[$index].yes_rank' $rank_json)
if [ $(((index+1) % 10)) == 1 ] if [ $(((index + 1) % 10)) == 1 ]; then
then
start_rank=$rank start_rank=$rank
fi fi
if [ $(((index+1) % 10)) == 0 ] if [ $(((index + 1) % 10)) == 0 ]; then
then
end_rank=$rank end_rank=$rank
fi fi
if [ $yes_rank -eq 0 ] if [ $yes_rank -eq 0 ]; then
then
rank_info="\#排名$rank \#rank$rank \#首次登场" rank_info="\#排名$rank \#rank$rank \#首次登场"
else else
rank_info="\#排名$rank \#rank$rank 之前 \#排名$yes_rank \#rank$yes_rank" rank_info="\#排名$rank \#rank$rank 之前 \#排名$yes_rank \#rank$yes_rank"
fi fi
echo "pid=$pid,artworkLink=$artworkLink,rank_info=$rank_info" echo "pid=$pid,artworkLink=$artworkLink,rank_info=$rank_info"
png_html_file=$pid.html png_html_file=$pid.html
if [ ! -f $png_html_file ] if [ ! -f $png_html_file ]; then
then
echo "get data from $artworkLink" echo "get data from $artworkLink"
curl -v $artworkLink >$png_html_file curl -v $artworkLink >$png_html_file
fi fi
json_file=$pid.json json_file=$pid.json
if [ ! -f $json_file ] if [ ! -f $json_file ]; then
then
egrep -o "content='{\"timestamp.*" $png_html_file | sed -e "s/content='//" | sed -e "s/..$//" >$json_file egrep -o "content='{\"timestamp.*" $png_html_file | sed -e "s/content='//" | sed -e "s/..$//" >$json_file
fi fi
pageCount=`jq --arg pid $pid '.illust[$pid].pageCount' $json_file` pageCount=$(jq --arg pid $pid '.illust[$pid].pageCount' $json_file)
original_url=`jq -r --arg pid $pid '.illust[$pid].urls.original' $json_file` original_url=$(jq -r --arg pid $pid '.illust[$pid].urls.original' $json_file)
small_url=`jq -r --arg pid $pid '.illust[$pid].urls.small' $json_file` small_url=$(jq -r --arg pid $pid '.illust[$pid].urls.small' $json_file)
title=`jq -r --arg pid $pid '.illust[$pid].title' $json_file|sed -e 's/\"/\\\"/g'|sed -e 's/&lt;//g'|sed -e 's/&gt;//g'` title=$(jq -r --arg pid $pid '.illust[$pid].title' $json_file | sed -e 's/\"/\\\"/g' | sed -e 's/&lt;//g' | sed -e 's/&gt;//g')
title=`bash -c "echo '$title'|$rule"` title=$(bash -c "echo '$title'|$rule")
description=`jq -r --arg pid $pid '.illust[$pid].description' $json_file` description=$(jq -r --arg pid $pid '.illust[$pid].description' $json_file)
userName=`jq -r --arg pid $pid '.illust[$pid].userName' $json_file|sed -e 's/\"/\\\"/g'` userName=$(jq -r --arg pid $pid '.illust[$pid].userName' $json_file | sed -e 's/\"/\\\"/g')
userName=`bash -c "echo '$userName'|$rule"` userName=$(bash -c "echo '$userName'|$rule")
userId=`jq -r --arg pid $pid '.illust[$pid].userId' $json_file` userId=$(jq -r --arg pid $pid '.illust[$pid].userId' $json_file)
likeCount=`jq --arg pid $pid '.illust[$pid].likeCount' $json_file` likeCount=$(jq --arg pid $pid '.illust[$pid].likeCount' $json_file)
if [ $likeCount -gt $maxLikeCount ] if [ $likeCount -gt $maxLikeCount ]; then
then
maxLikeCount=$likeCount maxLikeCount=$likeCount
maxLikeCountPid=$pid maxLikeCountPid=$pid
fi fi
bookmarkCount=`jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file` bookmarkCount=$(jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file)
if [ $bookmarkCount -gt $maxBookmarkCount ] if [ $bookmarkCount -gt $maxBookmarkCount ]; then
then
maxBookmarkCount=$likeCount maxBookmarkCount=$likeCount
maxBookmarkCountPid=$pid maxBookmarkCountPid=$pid
fi fi
viewCount=`jq --arg pid $pid '.illust[$pid].viewCount' $json_file` viewCount=$(jq --arg pid $pid '.illust[$pid].viewCount' $json_file)
if [ $viewCount -gt $maxViewCount ] if [ $viewCount -gt $maxViewCount ]; then
then
maxViewCount=$likeCount maxViewCount=$likeCount
maxViewCountPid=$pid maxViewCountPid=$pid
fi fi
tag=`jq -r --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file|sed -e 's/^/\\#/g'|sed ':a;N;s/\n/ /;t a;'|sed 's/"/\\"/g'` tag=$(jq -r --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file | sed -e 's/^/\\#/g' | sed ':a;N;s/\n/ /;t a;' | sed 's/"/\\"/g')
tag=`bash -c "echo '$tag'|$rule"` tag=$(bash -c "echo '$tag'|$rule")
echo -e "pageCount=$pageCount,original_url=$original_url,small_url=$small_url\n\ echo -e "pageCount=$pageCount,original_url=$original_url,small_url=$small_url\n\
title=$title,description=$description,userName=$userName\n\ title=$title,description=$description,userName=$userName\n\
likeCount=$likeCount,bookmarkCount=$bookmarkCount,viewCount=$viewCount\n\ likeCount=$likeCount,bookmarkCount=$bookmarkCount,viewCount=$viewCount\n\
tag=$tag" tag=$tag"
for page in `seq 1 $pageCount` for page in $(seq 1 $pageCount); do
do
page=$((page - 1)) page=$((page - 1))
page_original_url=`echo $original_url | sed -e "s/p0/p$page/"` page_original_url=$(echo $original_url | sed -e "s/p0/p$page/")
page_small_url=`echo $small_url | sed -e"s/p0/p$page/"` page_small_url=$(echo $small_url | sed -e"s/p0/p$page/")
original_file_name=`echo $page_original_url | egrep -o "$pid.*"` original_file_name=$(echo $page_original_url | egrep -o "$pid.*")
webp_file_name=`echo $original_file_name|sed 's/jpg/webp/'|sed 's/png/webp/'` webp_file_name=$(echo $original_file_name | sed 's/jpg/webp/' | sed 's/png/webp/')
if [ "`cat $db_file|grep $original_file_name`" != "$original_file_name" ] if [ "$(cat $db_file | grep $original_file_name)" != "$original_file_name" ]; then
then
echo "download image file name=$original_file_name,url=$page_original_url" echo "download image file name=$original_file_name,url=$page_original_url"
if [ ! -f $original_file_name ] if [ ! -f $original_file_name ]; then
then
curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name
fi fi
original_file_size=`du -b $original_file_name|awk '{print $1}'` original_file_size=$(du -b $original_file_name | awk '{print $1}')
if [ $original_file_size -gt $maxFileSize ] if [ $original_file_size -gt $maxFileSize ]; then
then
echo "${original_file_name}文件体积：${original_file_size}字节超过${maxFileSize}字节限制，需要在线压缩" echo "${original_file_name}文件体积：${original_file_size}字节超过${maxFileSize}字节限制，需要在线压缩"
result=`curl -v -X POST -d "{\"apikey\": \"$convertio_apikey\", \"input\":\"upload\", \"outputformat\":\"webp\"}" http://api.convertio.co/convert` result=$(curl -v -X POST -d "{\"apikey\": \"$convertio_apikey\", \"input\":\"upload\", \"outputformat\":\"webp\"}" http://api.convertio.co/convert)
if [ `echo $result|jq -r .status` = ok ] if [ $(echo $result | jq -r .status) = ok ]; then
then id=$(echo $result | jq -r .data.id)
id=`echo $result|jq -r .data.id` result=$(curl -v -X PUT --upload-file $original_file_name http://api.convertio.co/convert/$id/$original_file_name)
result=`curl -v -X PUT --upload-file $original_file_name http://api.convertio.co/convert/$id/$original_file_name` if [ $(echo $result | jq -r .status) = ok ]; then
if [ `echo $result|jq -r .status` = ok ] while true; do
then wait_second=10
while true sleep $wait_second
do result=$(curl -v -X GET http://api.convertio.co/convert/$id/status)
sleep 10
result=`curl -v -X GET http://api.convertio.co/convert/$id/status`
echo "转换响应：$result" echo "转换响应：$result"
if [[ `echo $result|jq -r .status` = ok && `echo $result|jq -r .data.step` = finish ]] if [[ $(echo $result | jq -r .status) = ok && $(echo $result | jq -r .data.step) = finish ]]; then
then compress_size=$(echo $result | jq -r .data.output.size)
echo "${original_file_name}文件体积：${original_file_size}字节，压缩后的字节：${compress_size}"
if [ $compress_size -gt $maxFileSize ]; then
echo "压缩无效，返回异常图片"
FileSize=$(fileSizeStr $compress_size)
url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_$(echo "画作pid：$pid" | base64),g_north/watermark,text_$(echo 因为图片文件体积：${FileSize}不合法 | base64 | sed 's/+/-/g; s,/,_,g'),color_FF0000,y_50,g_north/watermark,text_5omA5Lul57yp55Wl5Zu-5peg5rOV5LiK5Lyg5YiwdGVsZWdyYW0=,color_FF0000,y_100,g_north/watermark,text_5bi46KeB5LqO6LaF6ZW_5p2h54q25ryr55S7,y_150,g_north/watermark,text_5rOo5oSP77ya4oCc6aKE6KeI5Y6f5Zu-4oCd44CB4oCc5Yqg6YCf5LiL6L295Y6f5Zu-4oCd,color_008800,y_200,g_north/watermark,text_5Lul5LiK5Yqf6IO954Wn5bi45L2_55So,color_008800,y_250,g_north"
echo "文字水印请求url：$url"
curl -v $url -o $webp_file_name
else
echo $result | jq .data.output.url | xargs curl -v -o $webp_file_name echo $result | jq .data.output.url | xargs curl -v -o $webp_file_name
PutObject $webp_file_name fi
original_file_name=$webp_file_name
break break
elif [ `echo $result|jq -r .status` = error ] elif [ $(echo $result | jq -r .status) = error ]; then
then
break break
else else
echo "10s后重新获取转换结果" echo "${wait_second}s后重新获取转换结果"
fi fi
done done
else else
@ -334,45 +303,39 @@ do
else else
echo "Start a New Conversion error!!!" echo "Start a New Conversion error!!!"
fi fi
else
PutObject $original_file_name
fi fi
PutObject $original_file_name
fi fi
if [ ! -f $webp_file_name ] if [ ! -f $webp_file_name ]; then
then
webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam" webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam"
echo "download image file name=$webp_file_name,url=$webp_url" echo "download image file name=$webp_file_name,url=$webp_url"
curl -v $webp_url -o $webp_file_name curl -v $webp_url -o $webp_file_name
image_ratio=`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'|sed '/^$/d'|awk '{split($0,a,"x");f=a[1]>a[2]?a[1]/a[2]:a[2]/a[1];print f}'` image_size=$(file $webp_file_name | awk '{print $9}' | sed -e 's/,//')
if [[ -n $image_ratio && `echo "$image_ratio > $maxRatio"|bc` -eq 1 ]] image_ratio=$(echo $image_size | sed '/^$/d' | awk '{split($0,a,"x");f=a[1]>a[2]?a[1]/a[2]:a[2]/a[1];print f}')
then if [[ -n $image_ratio && $(echo "$image_ratio > $maxRatio" | bc) -eq 1 ]]; then
echo "图片分辨率`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'`比率：$image_ratio>$maxRatio,视为异常，返回异常图片" echo "图片分辨率：${image_size},分辨率比例：${image_ratio}>${maxRatio},视为异常，返回异常图片"
watermark=`echo "画作pid：$pid"|base64` url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_$(echo "画作pid：$pid" | base64),g_north/watermark,text_$(echo 因为图片分辨率：${image_size}不合法 | base64 | sed 's/+/-/g; s,/,_,g'),color_FF0000,y_50,g_north/watermark,text_5omA5Lul57yp55Wl5Zu-5peg5rOV5LiK5Lyg5YiwdGVsZWdyYW0=,color_FF0000,y_100,g_north/watermark,text_5bi46KeB5LqO6LaF6ZW_5p2h54q25ryr55S7,y_150,g_north/watermark,text_5rOo5oSP77ya4oCc6aKE6KeI5Y6f5Zu-4oCd44CB4oCc5Yqg6YCf5LiL6L295Y6f5Zu-4oCd,color_008800,y_200,g_north/watermark,text_5Lul5LiK5Yqf6IO954Wn5bi45L2_55So,color_008800,y_250,g_north"
url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
echo "文字水印请求url：$url" echo "文字水印请求url：$url"
curl -v $url -o $webp_file_name curl -v $url -o $webp_file_name
fi fi
fi fi
if [ $page -eq 0 ] if [ $page -eq 0 ]; then
then
info_file=$original_file_name.info info_file=$original_file_name.info
original_image_info=`curl -v --output $info_file https://$bucketname.$Host/$original_file_name?x-oss-process=image/info` curl -v -o $info_file https://$bucketname.$Host/$original_file_name?x-oss-process=image/info
FileSize=$(fileSizeStr $(cat $info_file | jq -r .FileSize.value)) FileSize=$(fileSizeStr $(cat $info_file | jq -r .FileSize.value))
ImageHeight=$(cat $info_file | jq -r .ImageHeight.value) ImageHeight=$(cat $info_file | jq -r .ImageHeight.value)
ImageWidth=$(cat $info_file | jq -r .ImageWidth.value) ImageWidth=$(cat $info_file | jq -r .ImageWidth.value)
media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\n原图分辨率：${ImageWidth}X${ImageHeight}(宽X高)，文件体积：$FileSize，<a href=\\\"https://$bucketname.$Host/$original_file_name\\\">预览原图</a> <a href=\\\"https://$bucketname.$accelerateHost/$original_file_name\\\">加速下载原图</a>\"}" media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\n原图分辨率：${ImageWidth}X${ImageHeight}(宽X高)，文件体积：$FileSize，<a href=\\\"https://$bucketname.$Host/$original_file_name\\\">预览原图</a> <a href=\\\"https://$bucketname.$accelerateHost/$original_file_name\\\">加速下载原图</a>\"}"
fileList="$fileList -F $webp_file_name=@$webp_file_name" fileList="$fileList -F $webp_file_name=@$webp_file_name"
fileCount=$((fileCount + 1)) fileCount=$((fileCount + 1))
fileSize=`du $webp_file_name | awk '{print $1}'` fileSize=$(du $webp_file_name | awk '{print $1}')
echo "fileCount=$fileCount" echo "fileCount=$fileCount"
fi fi
if [[ $fileCount -eq $maxFileCount ]] if [[ $fileCount -eq $maxFileCount ]]; then
then Request "curl -v -F chat_id=$chat_id $fileList -F media='[$(echo $media | cut -c 2-)]' $baseApi/sendMediaGroup"
Request "curl -v -F chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup" if [ $(echo $?) -eq 1 ]; then
if [ `echo $?` -eq 1 ]
then
sleep $sleepImage sleep $sleepImage
text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息." text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息."
else else
@ -388,9 +351,8 @@ do
done done
done done
next_expected_at=$(curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:'' | jq .next_expected_at)
Request "curl -v -d chat_id=$chat_id -d text=\"以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是$(date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S')，如有问题请联系管理员。 #date$_today #日期$_today \" $baseApi/sendMessage"
next_expected_at=`curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:''|jq .next_expected_at` find -type f -name "*.jpg" -mtime +$backup_time -o -name "*.png" -mtime +$backup_time -o -name "*.webp" -mtime +$backup_time -o -name "*.html" -mtime +$backup_time -o -name "*.info" -mtime +$backup_time -o -name "*.json" -mtime +$backup_time -not -name $(basename $config_file) | xargs rm -f
Request "curl -v -d chat_id=$chat_id -d text=\"以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today #日期$_today \" $baseApi/sendMessage"
find -type f -mtime +7 -name "*.webp" -o -name "*.html" -o -name "*.info" -o -name "*.json" -not -name `basename $config_file`|xargs rm -f
DeleteMultipleObjects DeleteMultipleObjects
Write Preview
Loading…
Cancel
Save