panqihua
/
shelltool
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

1.代码通过vscode插件格式化

Browse Source 
2.超大图片压缩无效后使用异常图片处理
3.删除无效变量
4.修改备份时间配置，固定单位为天数。只允许配纯数字x，代表备份x天
master
panqihua 3 years ago
parent 43afbe498c
commit 81295b6393
2 changed files with 270 additions and 308 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      pixiv/config.json.example
  2. 576
      pixiv/pixiv.sh

2
pixiv/config.json.example
Unescape View File

@ -12,7 +12,7 @@
"AccessKeySecret":"OpenAPI AccessKey Secret", "AccessKeySecret":"OpenAPI AccessKey Secret",
"db":"文件上传成功记录数据库文件名", "db":"文件上传成功记录数据库文件名",
"imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560", "imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560",
"backup_time":"1 hour", "backup_time":"7",
"error_file":"error.jpeg" "error_file":"error.jpeg"
}, },
"convertio":{ "convertio":{

576
pixiv/pixiv.sh
Unescape View File

@ -1,211 +1,187 @@
#!/bin/bash #!/bin/bash
config_file=$1 config_file=$1
if [ -z $config_file ] if [ -z $config_file ]; then
then
echo 'The cache directory must be specified' echo 'The cache directory must be specified'
exit 1 exit 1
fi fi
if [ ! -f $config_file ] if [ ! -f $config_file ]; then
then echo 'config file：$config_file not found'
echo 'config file：$config_file not found' exit 1
exit 1
fi fi
if [ ! -f `which jq` ] if [ ! -f $(which jq) ]; then
then echo 'To run this script, you need to install jq，https://stedolan.github.io/jq/'
echo 'To run this script, you need to install jq，https://stedolan.github.io/jq/' exit 1
exit 1
else else
echo "jq version:`jq --version`" echo "jq version:$(jq --version)"
fi fi
if [ ! -f `which xmlstarlet` ] if [ ! -f $(which xmlstarlet) ]; then
then echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php'
echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php' exit 1
exit 1
else else
echo "xmlstarlet version:`xmlstarlet --version`" echo "xmlstarlet version:$(xmlstarlet --version)"
fi fi
# ==================== Config ==================== # ==================== Config ====================
Host=`cat $config_file | jq -r .aliyun_oss.Host` Host=$(cat $config_file | jq -r .aliyun_oss.Host)
accelerateHost=`cat $config_file | jq -r .aliyun_oss.accelerateHost` accelerateHost=$(cat $config_file | jq -r .aliyun_oss.accelerateHost)
bucketname=`cat $config_file | jq -r .aliyun_oss.bucketname` bucketname=$(cat $config_file | jq -r .aliyun_oss.bucketname)
AccessKeyId=`cat $config_file | jq -r .aliyun_oss.AccessKeyId` AccessKeyId=$(cat $config_file | jq -r .aliyun_oss.AccessKeyId)
AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret` AccessKeySecret=$(cat $config_file | jq -r .aliyun_oss.AccessKeySecret)
db_file=`cat $config_file | jq -r .aliyun_oss.db` db_file=$(cat $config_file | jq -r .aliyun_oss.db)
db_file_err=$db_file.err db_file_err=$db_file.err
imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam` imageParam=$(cat $config_file | jq -r .aliyun_oss.imageParam)
backup_time=`cat $config_file | jq -r .aliyun_oss.backup_time` backup_time=$(cat $config_file | jq -r .aliyun_oss.backup_time)
error_file=`cat $config_file | jq -r .aliyun_oss.error_file` error_file=$(cat $config_file | jq -r .aliyun_oss.error_file)
convertio_apikey=`cat $config_file | jq -r .convertio.apikey`
CronitorKey=`cat $config_file | jq -r .Cronitor.API_KEY` convertio_apikey=$(cat $config_file | jq -r .convertio.apikey)
CronitorJobName=`cat $config_file | jq -r .Cronitor.JOB_NAME`
CronitorKey=$(cat $config_file | jq -r .Cronitor.API_KEY)
CronitorJobName=$(cat $config_file | jq -r .Cronitor.JOB_NAME)
basePath=`cat $config_file|jq -r .basePath` basePath=$(cat $config_file | jq -r .basePath)
if [ ! -f $basePath ] if [ ! -f $basePath ]; then
then mkdir -p $basePath
mkdir -p $basePath
fi fi
# ================================================ # ================================================
cd $basePath cd $basePath
if [ ! -f $db_file ] if [ ! -f $db_file ]; then
then
touch $db_file touch $db_file
fi fi
function PutObject(){ function PutObject() {
VERB="PUT" VERB="PUT"
file=$1 file=$1
Content_MD5="" Content_MD5=""
Content_Type=`file -b --mime-type $file` Content_Type=$(file -b --mime-type $file)
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` Date=$(TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT')
CanonicalizedOSSHeaders="x-oss-object-acl:public-read\n" CanonicalizedOSSHeaders="x-oss-object-acl:public-read\n"
CanonicalizedResource="/$bucketname/$file" CanonicalizedResource="/$bucketname/$file"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` Signature=$(echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64)
Authorization="OSS $AccessKeyId:$Signature" Authorization="OSS $AccessKeyId:$Signature"
http_code=`curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"` http_code=$(curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file")
if [ $http_code -eq "200" ] if [ $http_code -eq "200" ]; then
then rm -f $file
rm -f $file else
else echo $file >>$db_file_err
echo $file>>$db_file_err fi
fi
} }
function GetBucketV2(){ function GetBucketV2() {
VERB="GET" VERB="GET"
Content_MD5="" Content_MD5=""
Content_Type="" Content_Type=""
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` Date=$(TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT')
CanonicalizedOSSHeaders="" CanonicalizedOSSHeaders=""
CanonicalizedResource="/$bucketname/" CanonicalizedResource="/$bucketname/"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` Signature=$(echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64)
Authorization="OSS $AccessKeyId:$Signature" Authorization="OSS $AccessKeyId:$Signature"
curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2" curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2"
} }
function DeleteMultipleObjects(){ function DeleteMultipleObjects() {
file="DeleteMultipleObjects.temp" file="DeleteMultipleObjects.temp"
GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n > $file GetBucketV2 | xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n >$file
# 获取备份时间之前的日期时间戳 # 获取备份时间之前的日期时间戳
oldTimestamp=`date -d "-$backup_time" +%s` oldTimestamp=$(date -d "-$backup_time day" +%s)
ObjectNode="" ObjectNode=""
while read line while read line; do
do filename=$(echo $line | awk '{print $1}')
filename=`echo $line|awk '{print $1}'` date=$(echo $line | awk '{print $2}')
date=`echo $line|awk '{print $2}'` timestamp=$(date -d "$date" +%s)
timestamp=`date -d "$date" +%s` if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]]; then
if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]] ObjectNode="${ObjectNode}<Object><Key>${filename}</Key></Object>"
then fi
ObjectNode="${ObjectNode}<Object><Key>${filename}</Key></Object>" done <$file
fi rm $file
done < $file
rm $file
if [ -n "$ObjectNode" ] if [ -n "$ObjectNode" ]; then
then echo "存在${backup_time}天以前的图片"
echo "存在${backup_time}以前的图片" compress_xml='<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>'${ObjectNode}'</Delete>'
compress_xml='<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>'${ObjectNode}'</Delete>' echo $compress_xml | xmlstarlet format
echo $compress_xml|xmlstarlet format VERB="POST"
VERB="POST" Content_MD5=$(echo -n $compress_xml | openssl dgst -md5 -binary | base64)
Content_MD5=`echo -n $compress_xml |openssl dgst -md5 -binary|base64` Content_Type="application/xml"
Content_Type="application/xml" Date=$(TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT')
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` CanonicalizedOSSHeaders=""
CanonicalizedOSSHeaders="" CanonicalizedResource="/$bucketname/?delete"
CanonicalizedResource="/$bucketname/?delete" stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" echo "sign:$stringToSign"
echo "sign:$stringToSign" Signature=$(echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64)
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` Authorization="OSS $AccessKeyId:$Signature"
Authorization="OSS $AccessKeyId:$Signature" http_code=$(curl -X $VERB -v -w "%{http_code}" -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete")
http_code=`curl -X $VERB -v -w "%{http_code}" -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete"` if [ $http_code -eq "200" ]; then
if [ $http_code -eq "200" ] echo "${backup_time}天以前的图片删除成功"
then
echo "${backup_time}以前的图片删除成功"
else
echo "${backup_time}以前的图片删除失败"
fi
echo ''>$db_file
GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n|while read line
do
if [ $line != "$error_file" ]
then
echo $line >> $db_file
fi
done
else else
echo "没有${backup_time}以前的图片需要删除" echo "${backup_time}天以前的图片删除失败"
fi fi
} echo '' >$db_file
GetBucketV2 | xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n | while read line; do
if [ $line != "$error_file" ]; then
echo $line >>$db_file
fi
done
else
echo "没有${backup_time}天以前的图片需要删除"
fi
}
function fileSizeStr(){ function fileSizeStr() {
fileSize=$1 fileSize=$1
if [[ $fileSize -gt 1048576 ]] if [[ $fileSize -gt 1048576 ]]; then
then echo "$(echo "scale=2;$fileSize/1048576" | bc)MB"
echo "`echo "scale=2;$fileSize/1048576"|bc`MB" else
else echo "$((fileSize / 1024))KB"
echo "$((fileSize/1024))KB" fi
fi
} }
function Request(){ function Request() {
while true while true; do
do
echo "发送消息命令：$1" echo "发送消息命令：$1"
result=`bash -c "$1"` result=$(bash -c "$1")
echo "请求响应：$result" echo "请求响应：$result"
if [ `echo $result|jq .ok` = true ] if [ $(echo $result | jq .ok) = true ]; then
then
return 1 return 1
elif [ `echo $result|jq .error_code` -eq 429 ] elif [ $(echo $result | jq .error_code) -eq 429 ]; then
then second=$(echo $result | jq .parameters.retry_after)
second=`echo $result|jq .parameters.retry_after`
sleep $second sleep $second
echo "${second}后再发起请求" echo "${second}后再发起请求"
Request $1 Request $1
else else
echo "执行命令遇到未知错误！" echo "执行命令遇到未知错误！"
return 0 return 0
fi fi
done done
} }
telegramToken=$(cat $config_file | jq -r .telegramToken)
telegramToken=`cat $config_file | jq -r .telegramToken`
baseApi="https://api.telegram.org/bot$telegramToken" baseApi="https://api.telegram.org/bot$telegramToken"
chat_id=`cat $config_file | jq .chatId` chat_id=$(cat $config_file | jq .chatId)
mode=`cat $config_file | jq -r .mode` mode=$(cat $config_file | jq -r .mode)
content=`cat $config_file | jq -r .content` content=$(cat $config_file | jq -r .content)
rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&format=json" rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&format=json"
today=`date "+%Y-%m-%d"` today=$(date "+%Y-%m-%d")
_today=`date "+%Y%m%d"` _today=$(date "+%Y%m%d")
rank_json=$today.json rank_json=$today.json
rule='sed -e "s/\\&amp;/\\&/g"|sed -e "s/\\&lt;/\\</g"|sed -e "s/\\&gt;/\\>/g"' rule='sed -e "s/\\&amp;/\\&/g"|sed -e "s/\\&lt;/\\</g"|sed -e "s/\\&gt;/\\>/g"'
Request "curl -v -d chat_id=$chat_id -d parse_mode=HTML -d text=\"Pixiv排行榜已更新，开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>\" $baseApi/sendMessage" Request "curl -v -d chat_id=$chat_id -d parse_mode=HTML -d text=\"Pixiv排行榜已更新，开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>\" $baseApi/sendMessage"
if [ ! -f $rank_json ] if [ ! -f $rank_json ]; then
then echo "get data from $rank_url"
echo "get data from $rank_url" curl -v $rank_url >$rank_json
curl -v $rank_url >$rank_json
fi fi
length=`jq '.contents|length' $rank_json` length=$(jq '.contents|length' $rank_json)
fileList='' fileList=''
media='' media=''
fileCount=0 fileCount=0
maxFileCount=10 maxFileCount=10
tarFile=$today.tar.gz
maxFileSize=20971520 maxFileSize=20971520
maxFileSize_M="$((maxFileSize/1024/1024))M"
start_rank='' start_rank=''
end_rank='' end_rank=''
maxLikeCount=0 maxLikeCount=0
@ -215,182 +191,168 @@ maxBookmarkCountPid=0
maxViewCount=0 maxViewCount=0
maxViewCountPid=0 maxViewCountPid=0
maxRatio=10 maxRatio=10
for index in `seq 1 $length` for index in $(seq 1 $length); do
do index=$((index - 1))
index=$((index-1)) pid=$(jq --argjson index $index '.contents[$index].illust_id' $rank_json)
pid=`jq --argjson index $index '.contents[$index].illust_id' $rank_json` artworkLink="https://www.pixiv.net/artworks/$pid"
artworkLink="https://www.pixiv.net/artworks/$pid" rank=$(jq --argjson index $index '.contents[$index].rank' $rank_json)
rank=`jq --argjson index $index '.contents[$index].rank' $rank_json` yes_rank=$(jq --argjson index $index '.contents[$index].yes_rank' $rank_json)
yes_rank=`jq --argjson index $index '.contents[$index].yes_rank' $rank_json`
if [ $(((index+1) % 10)) == 1 ] if [ $(((index + 1) % 10)) == 1 ]; then
then start_rank=$rank
start_rank=$rank fi
fi if [ $(((index + 1) % 10)) == 0 ]; then
if [ $(((index+1) % 10)) == 0 ] end_rank=$rank
then fi
end_rank=$rank if [ $yes_rank -eq 0 ]; then
fi rank_info="\#排名$rank \#rank$rank \#首次登场"
if [ $yes_rank -eq 0 ] else
then rank_info="\#排名$rank \#rank$rank 之前 \#排名$yes_rank \#rank$yes_rank"
rank_info="\#排名$rank \#rank$rank \#首次登场" fi
else echo "pid=$pid,artworkLink=$artworkLink,rank_info=$rank_info"
rank_info="\#排名$rank \#rank$rank 之前 \#排名$yes_rank \#rank$yes_rank" png_html_file=$pid.html
fi if [ ! -f $png_html_file ]; then
echo "pid=$pid,artworkLink=$artworkLink,rank_info=$rank_info" echo "get data from $artworkLink"
png_html_file=$pid.html curl -v $artworkLink >$png_html_file
if [ ! -f $png_html_file ] fi
then json_file=$pid.json
echo "get data from $artworkLink" if [ ! -f $json_file ]; then
curl -v $artworkLink >$png_html_file egrep -o "content='{\"timestamp.*" $png_html_file | sed -e "s/content='//" | sed -e "s/..$//" >$json_file
fi fi
json_file=$pid.json pageCount=$(jq --arg pid $pid '.illust[$pid].pageCount' $json_file)
if [ ! -f $json_file ] original_url=$(jq -r --arg pid $pid '.illust[$pid].urls.original' $json_file)
then small_url=$(jq -r --arg pid $pid '.illust[$pid].urls.small' $json_file)
egrep -o "content='{\"timestamp.*" $png_html_file|sed -e "s/content='//"|sed -e "s/..$//" >$json_file title=$(jq -r --arg pid $pid '.illust[$pid].title' $json_file | sed -e 's/\"/\\\"/g' | sed -e 's/&lt;//g' | sed -e 's/&gt;//g')
fi title=$(bash -c "echo '$title'|$rule")
pageCount=`jq --arg pid $pid '.illust[$pid].pageCount' $json_file` description=$(jq -r --arg pid $pid '.illust[$pid].description' $json_file)
original_url=`jq -r --arg pid $pid '.illust[$pid].urls.original' $json_file` userName=$(jq -r --arg pid $pid '.illust[$pid].userName' $json_file | sed -e 's/\"/\\\"/g')
small_url=`jq -r --arg pid $pid '.illust[$pid].urls.small' $json_file` userName=$(bash -c "echo '$userName'|$rule")
title=`jq -r --arg pid $pid '.illust[$pid].title' $json_file|sed -e 's/\"/\\\"/g'|sed -e 's/&lt;//g'|sed -e 's/&gt;//g'` userId=$(jq -r --arg pid $pid '.illust[$pid].userId' $json_file)
title=`bash -c "echo '$title'|$rule"` likeCount=$(jq --arg pid $pid '.illust[$pid].likeCount' $json_file)
description=`jq -r --arg pid $pid '.illust[$pid].description' $json_file` if [ $likeCount -gt $maxLikeCount ]; then
userName=`jq -r --arg pid $pid '.illust[$pid].userName' $json_file|sed -e 's/\"/\\\"/g'` maxLikeCount=$likeCount
userName=`bash -c "echo '$userName'|$rule"` maxLikeCountPid=$pid
userId=`jq -r --arg pid $pid '.illust[$pid].userId' $json_file` fi
likeCount=`jq --arg pid $pid '.illust[$pid].likeCount' $json_file` bookmarkCount=$(jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file)
if [ $likeCount -gt $maxLikeCount ] if [ $bookmarkCount -gt $maxBookmarkCount ]; then
then maxBookmarkCount=$likeCount
maxLikeCount=$likeCount maxBookmarkCountPid=$pid
maxLikeCountPid=$pid fi
fi viewCount=$(jq --arg pid $pid '.illust[$pid].viewCount' $json_file)
bookmarkCount=`jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file` if [ $viewCount -gt $maxViewCount ]; then
if [ $bookmarkCount -gt $maxBookmarkCount ] maxViewCount=$likeCount
then maxViewCountPid=$pid
maxBookmarkCount=$likeCount fi
maxBookmarkCountPid=$pid tag=$(jq -r --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file | sed -e 's/^/\\#/g' | sed ':a;N;s/\n/ /;t a;' | sed 's/"/\\"/g')
fi tag=$(bash -c "echo '$tag'|$rule")
viewCount=`jq --arg pid $pid '.illust[$pid].viewCount' $json_file`
if [ $viewCount -gt $maxViewCount ] echo -e "pageCount=$pageCount,original_url=$original_url,small_url=$small_url\n\
then
maxViewCount=$likeCount
maxViewCountPid=$pid
fi
tag=`jq -r --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file|sed -e 's/^/\\#/g'|sed ':a;N;s/\n/ /;t a;'|sed 's/"/\\"/g'`
tag=`bash -c "echo '$tag'|$rule"`
echo -e "pageCount=$pageCount,original_url=$original_url,small_url=$small_url\n\
title=$title,description=$description,userName=$userName\n\ title=$title,description=$description,userName=$userName\n\
likeCount=$likeCount,bookmarkCount=$bookmarkCount,viewCount=$viewCount\n\ likeCount=$likeCount,bookmarkCount=$bookmarkCount,viewCount=$viewCount\n\
tag=$tag" tag=$tag"
for page in `seq 1 $pageCount` for page in $(seq 1 $pageCount); do
do page=$((page - 1))
page=$((page - 1)) page_original_url=$(echo $original_url | sed -e "s/p0/p$page/")
page_original_url=`echo $original_url | sed -e "s/p0/p$page/"` page_small_url=$(echo $small_url | sed -e"s/p0/p$page/")
page_small_url=`echo $small_url | sed -e"s/p0/p$page/"` original_file_name=$(echo $page_original_url | egrep -o "$pid.*")
original_file_name=`echo $page_original_url | egrep -o "$pid.*"` webp_file_name=$(echo $original_file_name | sed 's/jpg/webp/' | sed 's/png/webp/')
webp_file_name=`echo $original_file_name|sed 's/jpg/webp/'|sed 's/png/webp/'`
if [ "`cat $db_file|grep $original_file_name`" != "$original_file_name" ] if [ "$(cat $db_file | grep $original_file_name)" != "$original_file_name" ]; then
then echo "download image file name=$original_file_name,url=$page_original_url"
echo "download image file name=$original_file_name,url=$page_original_url" if [ ! -f $original_file_name ]; then
if [ ! -f $original_file_name ] curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name
then fi
curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name original_file_size=$(du -b $original_file_name | awk '{print $1}')
fi if [ $original_file_size -gt $maxFileSize ]; then
original_file_size=`du -b $original_file_name|awk '{print $1}'` echo "${original_file_name}文件体积：${original_file_size}字节超过${maxFileSize}字节限制，需要在线压缩"
if [ $original_file_size -gt $maxFileSize ] result=$(curl -v -X POST -d "{\"apikey\": \"$convertio_apikey\", \"input\":\"upload\", \"outputformat\":\"webp\"}" http://api.convertio.co/convert)
then
echo "${original_file_name}文件体积：${original_file_size}字节超过${maxFileSize}字节限制，需要在线压缩"
result=`curl -v -X POST -d "{\"apikey\": \"$convertio_apikey\", \"input\":\"upload\", \"outputformat\":\"webp\"}" http://api.convertio.co/convert`
if [ `echo $result|jq -r .status` = ok ] if [ $(echo $result | jq -r .status) = ok ]; then
then id=$(echo $result | jq -r .data.id)
id=`echo $result|jq -r .data.id` result=$(curl -v -X PUT --upload-file $original_file_name http://api.convertio.co/convert/$id/$original_file_name)
result=`curl -v -X PUT --upload-file $original_file_name http://api.convertio.co/convert/$id/$original_file_name` if [ $(echo $result | jq -r .status) = ok ]; then
if [ `echo $result|jq -r .status` = ok ] while true; do
then wait_second=10
while true sleep $wait_second
do result=$(curl -v -X GET http://api.convertio.co/convert/$id/status)
sleep 10 echo "转换响应：$result"
result=`curl -v -X GET http://api.convertio.co/convert/$id/status` if [[ $(echo $result | jq -r .status) = ok && $(echo $result | jq -r .data.step) = finish ]]; then
echo "转换响应：$result" compress_size=$(echo $result | jq -r .data.output.size)
if [[ `echo $result|jq -r .status` = ok && `echo $result|jq -r .data.step` = finish ]] echo "${original_file_name}文件体积：${original_file_size}字节，压缩后的字节：${compress_size}"
then if [ $compress_size -gt $maxFileSize ]; then
echo $result|jq .data.output.url|xargs curl -v -o $webp_file_name echo "压缩无效，返回异常图片"
PutObject $webp_file_name FileSize=$(fileSizeStr $compress_size)
break url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_$(echo "画作pid：$pid" | base64),g_north/watermark,text_$(echo 因为图片文件体积：${FileSize}不合法 | base64 | sed 's/+/-/g; s,/,_,g'),color_FF0000,y_50,g_north/watermark,text_5omA5Lul57yp55Wl5Zu-5peg5rOV5LiK5Lyg5YiwdGVsZWdyYW0=,color_FF0000,y_100,g_north/watermark,text_5bi46KeB5LqO6LaF6ZW_5p2h54q25ryr55S7,y_150,g_north/watermark,text_5rOo5oSP77ya4oCc6aKE6KeI5Y6f5Zu-4oCd44CB4oCc5Yqg6YCf5LiL6L295Y6f5Zu-4oCd,color_008800,y_200,g_north/watermark,text_5Lul5LiK5Yqf6IO954Wn5bi45L2_55So,color_008800,y_250,g_north"
elif [ `echo $result|jq -r .status` = error ] echo "文字水印请求url：$url"
then curl -v $url -o $webp_file_name
break
else
echo "10s后重新获取转换结果"
fi
done
else
echo "Direct File Upload For Conversion error!!!"
fi
else
echo "Start a New Conversion error!!!"
fi
else
PutObject $original_file_name
fi
fi
if [ ! -f $webp_file_name ]
then
webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam"
echo "download image file name=$webp_file_name,url=$webp_url"
curl -v $webp_url -o $webp_file_name
image_ratio=`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'|sed '/^$/d'|awk '{split($0,a,"x");f=a[1]>a[2]?a[1]/a[2]:a[2]/a[1];print f}'`
if [[ -n $image_ratio && `echo "$image_ratio > $maxRatio"|bc` -eq 1 ]]
then
echo "图片分辨率`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'`比率：$image_ratio>$maxRatio,视为异常，返回异常图片"
watermark=`echo "画作pid：$pid"|base64`
url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
echo "文字水印请求url：$url"
curl -v $url -o $webp_file_name
fi
fi
if [ $page -eq 0 ]
then
info_file=$original_file_name.info
original_image_info=`curl -v --output $info_file https://$bucketname.$Host/$original_file_name?x-oss-process=image/info`
FileSize=$(fileSizeStr $(cat $info_file|jq -r .FileSize.value))
ImageHeight=$(cat $info_file|jq -r .ImageHeight.value)
ImageWidth=$(cat $info_file|jq -r .ImageWidth.value)
media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\n原图分辨率：${ImageWidth}X${ImageHeight}(宽X高)，文件体积：$FileSize，<a href=\\\"https://$bucketname.$Host/$original_file_name\\\">预览原图</a> <a href=\\\"https://$bucketname.$accelerateHost/$original_file_name\\\">加速下载原图</a>\"}"
fileList="$fileList -F $webp_file_name=@$webp_file_name"
fileCount=$((fileCount + 1))
fileSize=`du $webp_file_name | awk '{print $1}'`
echo "fileCount=$fileCount"
fi
if [[ $fileCount -eq $maxFileCount ]]
then
Request "curl -v -F chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup"
if [ `echo $?` -eq 1 ]
then
sleep $sleepImage
text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息."
else else
text="#排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} 作品上传失败，请联系管理员" echo $result | jq .data.output.url | xargs curl -v -o $webp_file_name
fi fi
original_file_name=$webp_file_name
Request "curl -v -d chat_id=$chat_id -d text=\"$text\" $baseApi/sendMessage" break
elif [ $(echo $result | jq -r .status) = error ]; then
fileList='' break
media='' else
fileCount=0 echo "${wait_second}s后重新获取转换结果"
fi fi
done done
done else
echo "Direct File Upload For Conversion error!!!"
fi
else
echo "Start a New Conversion error!!!"
fi
fi
PutObject $original_file_name
fi
if [ ! -f $webp_file_name ]; then
webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam"
echo "download image file name=$webp_file_name,url=$webp_url"
curl -v $webp_url -o $webp_file_name
image_size=$(file $webp_file_name | awk '{print $9}' | sed -e 's/,//')
image_ratio=$(echo $image_size | sed '/^$/d' | awk '{split($0,a,"x");f=a[1]>a[2]?a[1]/a[2]:a[2]/a[1];print f}')
if [[ -n $image_ratio && $(echo "$image_ratio > $maxRatio" | bc) -eq 1 ]]; then
echo "图片分辨率：${image_size},分辨率比例：${image_ratio}>${maxRatio},视为异常，返回异常图片"
url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_$(echo "画作pid：$pid" | base64),g_north/watermark,text_$(echo 因为图片分辨率：${image_size}不合法 | base64 | sed 's/+/-/g; s,/,_,g'),color_FF0000,y_50,g_north/watermark,text_5omA5Lul57yp55Wl5Zu-5peg5rOV5LiK5Lyg5YiwdGVsZWdyYW0=,color_FF0000,y_100,g_north/watermark,text_5bi46KeB5LqO6LaF6ZW_5p2h54q25ryr55S7,y_150,g_north/watermark,text_5rOo5oSP77ya4oCc6aKE6KeI5Y6f5Zu-4oCd44CB4oCc5Yqg6YCf5LiL6L295Y6f5Zu-4oCd,color_008800,y_200,g_north/watermark,text_5Lul5LiK5Yqf6IO954Wn5bi45L2_55So,color_008800,y_250,g_north"
echo "文字水印请求url：$url"
curl -v $url -o $webp_file_name
fi
fi
if [ $page -eq 0 ]; then
info_file=$original_file_name.info
curl -v -o $info_file https://$bucketname.$Host/$original_file_name?x-oss-process=image/info
FileSize=$(fileSizeStr $(cat $info_file | jq -r .FileSize.value))
ImageHeight=$(cat $info_file | jq -r .ImageHeight.value)
ImageWidth=$(cat $info_file | jq -r .ImageWidth.value)
media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\n原图分辨率：${ImageWidth}X${ImageHeight}(宽X高)，文件体积：$FileSize，<a href=\\\"https://$bucketname.$Host/$original_file_name\\\">预览原图</a> <a href=\\\"https://$bucketname.$accelerateHost/$original_file_name\\\">加速下载原图</a>\"}"
fileList="$fileList -F $webp_file_name=@$webp_file_name"
fileCount=$((fileCount + 1))
fileSize=$(du $webp_file_name | awk '{print $1}')
echo "fileCount=$fileCount"
fi
if [[ $fileCount -eq $maxFileCount ]]; then
Request "curl -v -F chat_id=$chat_id $fileList -F media='[$(echo $media | cut -c 2-)]' $baseApi/sendMediaGroup"
if [ $(echo $?) -eq 1 ]; then
sleep $sleepImage
text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息."
else
text="#排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} 作品上传失败，请联系管理员"
fi
Request "curl -v -d chat_id=$chat_id -d text=\"$text\" $baseApi/sendMessage"
fileList=''
media=''
fileCount=0
fi
done
done
next_expected_at=`curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:''|jq .next_expected_at` next_expected_at=$(curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:'' | jq .next_expected_at)
Request "curl -v -d chat_id=$chat_id -d text=\"以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today #日期$_today \" $baseApi/sendMessage" Request "curl -v -d chat_id=$chat_id -d text=\"以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是$(date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S')，如有问题请联系管理员。 #date$_today #日期$_today \" $baseApi/sendMessage"
find -type f -mtime +7 -name "*.webp" -o -name "*.html" -o -name "*.info" -o -name "*.json" -not -name `basename $config_file`|xargs rm -f find -type f -name "*.jpg" -mtime +$backup_time -o -name "*.png" -mtime +$backup_time -o -name "*.webp" -mtime +$backup_time -o -name "*.html" -mtime +$backup_time -o -name "*.info" -mtime +$backup_time -o -name "*.json" -mtime +$backup_time -not -name $(basename $config_file) | xargs rm -f
DeleteMultipleObjects DeleteMultipleObjects

Write Preview
Loading…
Cancel
Save