panqihua
/
shelltool
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

1.图片云存储服务，由anonfiles替换为ali oss

Browse Source 
2.通过ali oss图片处理服务优化缩略图质量
3.完善图片介绍
4.优化推送文本消息
master
Qihua Pan 3 years ago
parent 6440cb8a4f
commit 27db68c13a
2 changed files with 127 additions and 34 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 23
      pixiv/config.json.example
  2. 138
      pixiv/pixiv.sh

23
pixiv/config.json.example
Unescape View File

@ -5,4 +5,25 @@
"anonfilesToken": "https://anonfiles.com/ 申请的API Token",
"mode": "Pixiv排行榜类型，有效值：daily/weekly/monthly/rookie/daily_r18/weekly_r18/male_r18/female_r18/r18g,分别是日榜/周榜/月榜/新人榜/日榜r18/周榜r18/受男性欢迎r18/受女性欢迎r18/r18g",
"content": "Pixiv排行榜类型，有效值：illust/ugoira/manga/,分别是插画/动图/漫画",
}
"rule":{"匹配需要转义的内容":"转换后的值","&amp;":"&","&lt;":"<"},
"sleep":{
"ready":0,
"request":10
},
"aliyun_oss":{
"Host":"Endpoint（地域节点）",
"accelerateHost":"传输加速Endpoint（地域节点）",
"bucketname":"bucket名称",
"AccessKeyId":"OpenAPI AccessKey ID",
"AccessKeySecret":"OpenAPI AccessKey Secret",
"db":"文件上传成功记录数据库",
"imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560"
},
"compress":{
"Host":"图片压缩服务器地址",
"Port":"端口",
"User":"用户",
"Path":"压缩图片输出绝对路径"
},
"CronitorKey":"cronitor api key"
}

138
pixiv/pixiv.sh
Unescape View File

@ -19,6 +19,54 @@ else
echo "jq version:`jq --version`"
fi
# ==================== Config ====================
Host=`cat $config_file | jq -r .aliyun_oss.Host`
accelerateHost=`cat $config_file | jq -r .aliyun_oss.accelerateHost`
bucketname=`cat $config_file | jq -r .aliyun_oss.bucketname`
AccessKeyId=`cat $config_file | jq -r .aliyun_oss.AccessKeyId`
AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret`
db_file=`cat $config_file | jq -r .aliyun_oss.db`
db_file_err=$db_file.err
imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam`
CompressHost=`cat $config_file | jq -r .compress.Host`
CompressPort=`cat $config_file | jq -r .compress.Port`
CompressUser=`cat $config_file | jq -r .compress.User`
CompressPath=`cat $config_file | jq -r .compress.Path`
CronitorKey=`cat $config_file | jq -r .Cronitor.API_KEY`
CronitorJobName=`cat $config_file | jq -r .Cronitor.JOB_NAME`
# ================================================
if [ ! -f $db_file ]
then
touch $db_file
fi
function upload(){
VERB="PUT"
file=$1
Content_MD5=""
Content_Type="application/x-www-form-urlencoded"
Content_Type="text/plain"
Content_Type=`file -b --mime-type $file`
Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'`
CanonicalizedOSSHeaders="x-oss-object-acl:public-read\n"
CanonicalizedResource="/$bucketname/$file"
stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64`
Authorization="OSS $AccessKeyId:$Signature"
http_code=`curl -v -w "%{http_code}" -X PUT -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"`
if [ $http_code -eq "200" ]
then
echo $file>>$db_file
rm -f $file
else
echo $file>>$db_file_err
fi
}
basePath=`cat $config_file|jq -r .basePath`
if [ ! -f $basePath ]
then
@ -33,13 +81,14 @@ mode=`cat $config_file | jq -r .mode`
content=`cat $config_file | jq -r .content`
rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&format=json"
today=`date "+%Y-%m-%d"`
_today=`echo $today|sed -e 's/-/\\\\-/g'`
_today=`date "+%Y%m%d"`
rank_json=$today.json
commands_file=$today.sh
anonfiles_token=`cat $config_file | jq .anonfilesToken`
rule=`cat $config_file | jq .rule|jq to_entries|jq 'map("sed -e \"s/\\\\"+.key+"/\\\\"+.value+"/g\"")'|jq -r '.[]'|sed ':a;N;s/\n/|/;t a;'`
touch $commands_file && chmod +x $commands_file
sleep 30
curl -v -d chat_id=$chat_id -d parse_mode=HTML -d text="Pixiv排行榜已更新，30秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。%0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage
sleep `cat $config_file | jq .sleep.ready`
curl -v -d chat_id=$chat_id -d parse_mode=HTML -d text="Pixiv排行榜已更新，30秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today%0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage
if [ ! -f $rank_json ]
then
@ -48,13 +97,15 @@ then
fi
length=`jq '.contents|length' $rank_json`
fileCountSize=0
maxSize=2500
fileList=''
media=''
fileCount=0
maxFileCount=10
tarFile=$today.tar.gz
originalFileList=''
maxFileSize=20971520
maxFileSize_M="$((maxFileSize/1024/1024))M"
start_rank=''
end_rank=''
for index in `seq 1 $length`
do
index=$((index-1))
@ -62,9 +113,18 @@ do
artworkLink="https://www.pixiv.net/artworks/$pid"
rank=`jq --argjson index $index '.contents[$index].rank' $rank_json`
yes_rank=`jq --argjson index $index '.contents[$index].yes_rank' $rank_json`
if [ $(((index+1) % 10)) == 1 ]
then
start_rank=$rank
fi
if [ $(((index+1) % 10)) == 0 ]
then
end_rank=$rank
fi
if [ $yes_rank -eq 0 ]
then
rank_info="#首次登场"
rank_info="\#rank$rank \#首次登场"
else
rank_info="\#rank$rank 之前 \#rank$yes_rank"
fi
@ -82,21 +142,25 @@ do
egrep -o "content='{\"timestamp.*].{3}" $png_html_file | sed -e "s/content='//" >$json_file
fi
pageCount=`jq --arg pid $pid '.illust[$pid].pageCount' $json_file`
original_url=`jq --arg pid $pid '.illust[$pid].urls.original' $json_file | sed -e 's/\"//g'`
small_url=`jq --arg pid $pid '.illust[$pid].urls.small' $json_file | sed -e 's/\"//g'`
title=`jq -r --arg pid $pid '.illust[$pid].title' $json_file`
original_url=`jq -r --arg pid $pid '.illust[$pid].urls.original' $json_file`
small_url=`jq -r --arg pid $pid '.illust[$pid].urls.small' $json_file`
title=`jq -r --arg pid $pid '.illust[$pid].title' $json_file|sed -e 's/\"/\\\"/g'`
title=`bash -c "echo '$title'|$rule"`
description=`jq -r --arg pid $pid '.illust[$pid].description' $json_file`
userName=`jq -r --arg pid $pid '.illust[$pid].userName' $json_file`
userName=`jq -r --arg pid $pid '.illust[$pid].userName' $json_file|sed -e 's/\"/\\\"/g'`
userName=`bash -c "echo '$userName'|$rule"`
userId=`jq -r --arg pid $pid '.illust[$pid].userId' $json_file`
likeCount=`jq --arg pid $pid '.illust[$pid].likeCount' $json_file`
bookmarkCount=`jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file`
viewCount=`jq --arg pid $pid '.illust[$pid].viewCount' $json_file`
tag=`jq -r --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file|sed -e 's/^/\\#/g'|sed ':a;N;s/\n/ /;t a;'`
tag=`bash -c "echo '$tag'|$rule"`
echo -e "pageCount=$pageCount,original_url=$original_url,small_url=$small_url\n\
title=$title,description=$description,userName=$userName\n\
likeCount=$likeCount,bookmarkCount=$bookmarkCount,viewCount=$viewCount\n\
tag=$tag"
for page in `seq 1 $pageCount`
do
page=$((page - 1))
@ -104,30 +168,50 @@ do
page_small_url=`echo $small_url | sed -e"s/p0/p$page/"`
original_file_name=`echo $page_original_url | egrep -o "$pid.*"`
small_file_name=`echo $page_small_url | egrep -o "$pid.*"`
echo "download image file name=$original_file_name,url=$page_original_url"
if [ ! -f $original_file_name ]
webp_file_name=`echo $original_file_name|sed 's/jpg/webp/'|sed 's/png/webp/'`
if [ "`cat $db_file|grep $original_file_name`" != "$original_file_name" ]
then
echo "download image file name=$original_file_name,url=$page_original_url"
if [ ! -f $original_file_name ]
then
curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name
fi
if [ `du -b $original_file_name|awk '{print $1}'` -gt $maxFileSize ]
then
echo "图片：$original_file_name 体积：`du -h $original_file_name|awk '{print $1}'` 超过 $maxFileSize_M,需要压缩"
scp -i ~/.ssh/$CompressHost -P $CompressPort $original_file_name $CompressUser@$CompressHost:$CompressPath/$original_file_name
ssh -i ~/.ssh/$CompressHost -p $CompressPort $CompressUser@$CompressHost "cd $CompressPath;jpegoptim --size=$maxFileSize_M $original_file_name"
scp -i ~/.ssh/$CompressHost -P $CompressPort $CompressUser@$CompressHost:$CompressPath/$original_file_name $original_file_name
echo "图片：$original_file_name 压缩体积：`du -h $original_file_name`"
fi
upload $original_file_name
fi
if [ ! -f $webp_file_name ]
then
curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name
webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam"
echo "download image file name=$webp_file_name,url=$webp_url"
curl -v $webp_url -o $webp_file_name
fi
originalFileList="$originalFileList $original_file_name"
echo "download image file name=$small_file_name,url=$page_small_url"
if [ ! -f $small_file_name ]
then
echo "download image file name=$small_file_name,url=$page_small_url"
curl -v -H 'referer: https://www.pixiv.net/' $page_small_url -o $small_file_name
fi
if [ $page -eq 0 ]
then
media="$media,{\"type\":\"photo\",\"media\":\"attach://$small_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\"}"
fileList=$fileList' -F '$small_file_name'=@'$basePath'/'$small_file_name
media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\"}"
fileList="$fileList -F $webp_file_name=@$webp_file_name"
fileCount=$((fileCount + 1))
fileSize=`du $small_file_name | awk '{print $1}'`
fileCountSize=$((fileCountSize + fileSize))
echo "fileCountSize=$fileCountSize,fileCount=$fileCount"
fi
if [[ $fileCountSize -gt $maxSize || $fileCount -eq $maxFileCount ]]
if [[ $fileCount -eq $maxFileCount ]]
then
echo "sleep 1m" >>$commands_file
echo "sleep `cat $config_file | jq .sleep.request`" >>$commands_file
echo "curl -v -F chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup" >>$commands_file
echo "curl -v -d chat_id=$chat_id -d text='以上作品日榜排名分别是 #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息' $baseApi/sendMessage" >>$commands_file
echo >>$commands_file
echo >>$commands_file
echo >>$commands_file
@ -148,18 +232,6 @@ if [ ! -f $hasSend ]
then
bash -c ./$commands_file
touch $hasSend
next_expected_at=`curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:''|jq .next_expected_at`
curl -v -d chat_id=$chat_id -d text="以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today" $baseApi/sendMessage
fi
if [ ! -f $tarFile ]
then
tar -zcvf $tarFile $originalFileList
fi
uploadResFile=$today.upload
if [ ! -f $uploadResFile ]
then
curl -v -F "file=@$tarFile" -F token=$anonfiles_token https://api.anonfiles.com/upload -o $uploadResFile
fi
shortUrl=`cat $uploadResFile | jq -r .data.file.url.short`
readable=`cat $uploadResFile | jq -r .data.file.metadata.size.readable|sed -e 's/\./\\\\./'`
echo "shortUrl=$shortUrl,readable=$readable"
curl -v -d text="$_today日榜原图（$readable）已打包上传到服务器，[下载地址]($shortUrl) \#图包下载 \#资源下载 \#下载" -d chat_id=$chat_id -d parse_mode=MarkdownV2 $baseApi/sendMessage

Write Preview
Loading…
Cancel
Save