1.图片云存储服务，由anonfiles替换为ali oss

2.通过ali oss图片处理服务优化缩略图质量
3.完善图片介绍
4.优化推送文本消息

pixiv/config.json.example

@@ -5,4 +5,25 @@
   "anonfilesToken": "https://anonfiles.com/ 申请的API Token",
   "mode": "Pixiv排行榜类型，有效值：daily/weekly/monthly/rookie/daily_r18/weekly_r18/male_r18/female_r18/r18g,分别是日榜/周榜/月榜/新人榜/日榜r18/周榜r18/受男性欢迎r18/受女性欢迎r18/r18g",
   "content": "Pixiv排行榜类型，有效值：illust/ugoira/manga/,分别是插画/动图/漫画",
+  "rule":{"匹配需要转义的内容":"转换后的值","&amp;":"&","&lt;":"<"},
+  "sleep":{
+    "ready":0,
+    "request":10
+  },
+  "aliyun_oss":{
+    "Host":"Endpoint（地域节点）",
+    "accelerateHost":"传输加速Endpoint（地域节点）",
+    "bucketname":"bucket名称",
+    "AccessKeyId":"OpenAPI AccessKey ID",
+    "AccessKeySecret":"OpenAPI AccessKey Secret",
+    "db":"文件上传成功记录数据库",
+    "imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560"
+  },
+  "compress":{
+    "Host":"图片压缩服务器地址",
+    "Port":"端口",
+    "User":"用户",
+    "Path":"压缩图片输出绝对路径"
+  },
+  "CronitorKey":"cronitor api key"
 }

pixiv/pixiv.sh

@@ -19,6 +19,54 @@ else
         echo "jq version:`jq --version`"
 fi
 
+
+# ==================== Config ====================
+Host=`cat $config_file | jq -r .aliyun_oss.Host`
+accelerateHost=`cat $config_file | jq -r .aliyun_oss.accelerateHost`
+bucketname=`cat $config_file | jq -r .aliyun_oss.bucketname`
+AccessKeyId=`cat $config_file | jq -r .aliyun_oss.AccessKeyId`
+AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret`
+db_file=`cat $config_file | jq -r .aliyun_oss.db`
+db_file_err=$db_file.err
+imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam`
+
+CompressHost=`cat $config_file | jq -r .compress.Host`
+CompressPort=`cat $config_file | jq -r .compress.Port`
+CompressUser=`cat $config_file | jq -r .compress.User`
+CompressPath=`cat $config_file | jq -r .compress.Path`
+
+CronitorKey=`cat $config_file | jq -r .Cronitor.API_KEY`
+CronitorJobName=`cat $config_file | jq -r .Cronitor.JOB_NAME`
+# ================================================
+
+if [ ! -f $db_file ]
+then
+   touch $db_file
+fi
+
+function upload(){
+        VERB="PUT"
+        file=$1
+        Content_MD5=""
+        Content_Type="application/x-www-form-urlencoded"
+        Content_Type="text/plain"
+        Content_Type=`file -b --mime-type $file`
+        Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'`
+        CanonicalizedOSSHeaders="x-oss-object-acl:public-read\n"
+        CanonicalizedResource="/$bucketname/$file"
+        stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
+        Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64`
+        Authorization="OSS $AccessKeyId:$Signature"
+        http_code=`curl -v -w "%{http_code}" -X PUT -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read"  -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"`
+        if [ $http_code -eq "200" ]
+        then
+           echo $file>>$db_file     
+           rm -f $file
+        else
+           echo $file>>$db_file_err     
+        fi
+}
+
 basePath=`cat $config_file|jq -r .basePath`
 if [ ! -f $basePath ]
 then
@@ -33,13 +81,14 @@ mode=`cat $config_file | jq -r .mode`
 content=`cat $config_file | jq -r .content`
 rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&format=json"
 today=`date "+%Y-%m-%d"`
-_today=`echo $today|sed -e 's/-/\\\\-/g'`
+_today=`date "+%Y%m%d"`
 rank_json=$today.json
 commands_file=$today.sh
 anonfiles_token=`cat $config_file | jq .anonfilesToken`
+rule=`cat $config_file | jq .rule|jq to_entries|jq 'map("sed -e \"s/\\\\"+.key+"/\\\\"+.value+"/g\"")'|jq -r  '.[]'|sed ':a;N;s/\n/|/;t a;'`
 touch $commands_file && chmod +x $commands_file
-sleep 30
+sleep `cat $config_file | jq .sleep.ready`
-curl -v -d chat_id=$chat_id -d parse_mode=HTML  -d text="Pixiv排行榜已更新，30秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。%0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage
+curl -v -d chat_id=$chat_id -d parse_mode=HTML  -d text="Pixiv排行榜已更新，30秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today%0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage
 
 if [ ! -f $rank_json ]
 then
@@ -48,13 +97,15 @@ then
 fi
 length=`jq '.contents|length' $rank_json`
 fileCountSize=0
-maxSize=2500
 fileList=''
 media=''
 fileCount=0
 maxFileCount=10
 tarFile=$today.tar.gz
-originalFileList=''
+maxFileSize=20971520
+maxFileSize_M="$((maxFileSize/1024/1024))M"
+start_rank=''
+end_rank=''
 for index in `seq 1 $length`
 do
         index=$((index-1))
@@ -62,9 +113,18 @@ do
         artworkLink="https://www.pixiv.net/artworks/$pid"
         rank=`jq  --argjson index $index '.contents[$index].rank' $rank_json`
         yes_rank=`jq  --argjson index $index '.contents[$index].yes_rank' $rank_json`
+
+        if [ $(((index+1) % 10)) == 1 ]
+        then
+           start_rank=$rank
+        fi
+        if [ $(((index+1) % 10)) == 0 ]
+        then
+           end_rank=$rank
+        fi
         if [ $yes_rank -eq 0 ]
         then
-          rank_info="#首次登场"
+          rank_info="\#rank$rank \#首次登场"
         else
           rank_info="\#rank$rank 之前 \#rank$yes_rank"   
         fi
@@ -82,21 +142,25 @@ do
                 egrep -o "content='{\"timestamp.*].{3}" $png_html_file | sed -e "s/content='//" >$json_file
         fi
         pageCount=`jq --arg pid $pid '.illust[$pid].pageCount' $json_file`
-        original_url=`jq --arg pid $pid '.illust[$pid].urls.original' $json_file | sed -e 's/\"//g'`
+        original_url=`jq -r --arg pid $pid '.illust[$pid].urls.original' $json_file`
-        small_url=`jq --arg pid $pid '.illust[$pid].urls.small' $json_file | sed -e 's/\"//g'`
+        small_url=`jq -r --arg pid $pid '.illust[$pid].urls.small' $json_file`
-        title=`jq -r --arg pid $pid '.illust[$pid].title' $json_file`
+        title=`jq -r --arg pid $pid '.illust[$pid].title' $json_file|sed -e 's/\"/\\\"/g'`
+        title=`bash -c "echo '$title'|$rule"`
         description=`jq -r --arg pid $pid '.illust[$pid].description' $json_file`
-        userName=`jq -r --arg pid $pid '.illust[$pid].userName' $json_file`
+        userName=`jq -r --arg pid $pid '.illust[$pid].userName' $json_file|sed -e 's/\"/\\\"/g'`
+        userName=`bash -c "echo '$userName'|$rule"`
         userId=`jq -r --arg pid $pid '.illust[$pid].userId' $json_file`
         likeCount=`jq --arg pid $pid '.illust[$pid].likeCount' $json_file`
         bookmarkCount=`jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file`
         viewCount=`jq --arg pid $pid '.illust[$pid].viewCount' $json_file`
         tag=`jq -r  --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file|sed -e 's/^/\\#/g'|sed ':a;N;s/\n/ /;t a;'`
+        tag=`bash -c "echo '$tag'|$rule"`
         
         echo -e "pageCount=$pageCount,original_url=$original_url,small_url=$small_url\n\
         title=$title,description=$description,userName=$userName\n\
         likeCount=$likeCount,bookmarkCount=$bookmarkCount,viewCount=$viewCount\n\
         tag=$tag"
+
         for page in `seq 1 $pageCount`
         do
                 page=$((page - 1))
@@ -104,30 +168,50 @@ do
                 page_small_url=`echo $small_url | sed -e"s/p0/p$page/"`
                 original_file_name=`echo $page_original_url | egrep -o "$pid.*"`
                 small_file_name=`echo $page_small_url | egrep -o "$pid.*"`
-                echo "download image file name=$original_file_name,url=$page_original_url"
+                webp_file_name=`echo $original_file_name|sed 's/jpg/webp/'|sed 's/png/webp/'`
-                if [ ! -f $original_file_name ]
+                if [ "`cat $db_file|grep $original_file_name`" != "$original_file_name" ]
+                then
+                        echo "download image file name=$original_file_name,url=$page_original_url"
+                        if [ ! -f $original_file_name ]
+                        then
+                                curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name       
+                        fi
+                        if [ `du -b  $original_file_name|awk '{print $1}'` -gt $maxFileSize ]
+                        then
+                           echo "图片：$original_file_name 体积：`du -h $original_file_name|awk '{print $1}'` 超过 $maxFileSize_M,需要压缩"     
+                           scp -i ~/.ssh/$CompressHost  -P $CompressPort $original_file_name $CompressUser@$CompressHost:$CompressPath/$original_file_name
+                           ssh -i ~/.ssh/$CompressHost -p $CompressPort $CompressUser@$CompressHost "cd $CompressPath;jpegoptim --size=$maxFileSize_M $original_file_name"
+                           scp -i ~/.ssh/$CompressHost  -P $CompressPort $CompressUser@$CompressHost:$CompressPath/$original_file_name $original_file_name
+                           echo "图片：$original_file_name 压缩体积：`du -h $original_file_name`"
+                        fi
+                        upload $original_file_name
+                fi
+                if [ ! -f $webp_file_name ]
                 then
-                        curl -v -H 'referer: https://www.pixiv.net/' $page_original_url -o $original_file_name
+                   webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam"
+                   echo "download image file name=$webp_file_name,url=$webp_url"      
+                   curl -v $webp_url -o $webp_file_name
                 fi
-                originalFileList="$originalFileList $original_file_name"
-                echo "download image file name=$small_file_name,url=$page_small_url"
                 if [ ! -f $small_file_name ]
                 then
+                        echo "download image file name=$small_file_name,url=$page_small_url"
                         curl -v -H 'referer: https://www.pixiv.net/' $page_small_url -o $small_file_name
                 fi
+   
                 if [ $page -eq 0 ]
                 then
-                        media="$media,{\"type\":\"photo\",\"media\":\"attach://$small_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\"}"
+                        media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\"}"
-                        fileList=$fileList' -F '$small_file_name'=@'$basePath'/'$small_file_name
+                        fileList="$fileList -F $webp_file_name=@$webp_file_name"
                         fileCount=$((fileCount + 1))
                         fileSize=`du $small_file_name | awk '{print $1}'`
                         fileCountSize=$((fileCountSize + fileSize))
                         echo "fileCountSize=$fileCountSize,fileCount=$fileCount"
                 fi
-                if [[ $fileCountSize -gt $maxSize || $fileCount -eq $maxFileCount ]]
+                if [[ $fileCount -eq $maxFileCount ]]
                 then
-                        echo "sleep 1m" >>$commands_file
+                        echo "sleep `cat $config_file | jq .sleep.request`" >>$commands_file
                         echo "curl -v -F  chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup" >>$commands_file
+                        echo "curl -v -d chat_id=$chat_id -d text='以上作品日榜排名分别是 #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息' $baseApi/sendMessage" >>$commands_file 
                         echo >>$commands_file
                         echo >>$commands_file
                         echo >>$commands_file
@@ -148,18 +232,6 @@ if [ ! -f $hasSend ]
 then
         bash -c ./$commands_file
         touch $hasSend
+        next_expected_at=`curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:''|jq .next_expected_at`
+        curl -v -d chat_id=$chat_id -d text="以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today" $baseApi/sendMessage
 fi
-
-if [ ! -f $tarFile ]
-then
-        tar -zcvf $tarFile $originalFileList
-fi
-uploadResFile=$today.upload
-if [ ! -f $uploadResFile ]
-then
-        curl -v -F "file=@$tarFile" -F token=$anonfiles_token https://api.anonfiles.com/upload -o $uploadResFile     
-fi
-shortUrl=`cat $uploadResFile | jq -r .data.file.url.short`
-readable=`cat $uploadResFile | jq -r .data.file.metadata.size.readable|sed -e 's/\./\\\\./'`
-echo "shortUrl=$shortUrl,readable=$readable"
-curl -v -d text="$_today日榜原图（$readable）已打包上传到服务器，[下载地址]($shortUrl) \#图包下载 \#资源下载 \#下载" -d chat_id=$chat_id -d parse_mode=MarkdownV2 $baseApi/sendMessage