1.异常分辨率比例图片特殊处理

2.记录最多收藏数/喜欢/浏览数画作（暂无用） 3.增加中文标签 4.更新config.json模版配置
2 years ago · 55b042f87e
parent 27db68c13a
commit 55b042f87e
2 changed files with 53 additions and 30 deletions
--- a/pixiv/config.json.example
+++ b/pixiv/config.json.example
@ -7,8 +7,8 @@
  "content": "Pixiv排行榜类型，有效值：illust/ugoira/manga/,分别是插画/动图/漫画",
  "rule":{"匹配需要转义的内容":"转换后的值","&amp;":"&","&lt;":"<"},
  "sleep":{
-    "ready":0,
-    "request":10
+    "text":10,
+    "image":60
  },
  "aliyun_oss":{
    "Host":"Endpoint（地域节点）",
@ -25,5 +25,8 @@
    "User":"用户",
    "Path":"压缩图片输出绝对路径"
  },
-  "CronitorKey":"cronitor api key"
+  "Cronitor":{
+    "API_KEY":"https://cronitor.io/ 申请的Key",
+    "JOB_NAME":"定时任务名称"
+  }
 }
--- a/pixiv/pixiv.sh
+++ b/pixiv/pixiv.sh
@ -83,13 +83,14 @@ rank_url="https://www.pixiv.net/ranking.php?mode=$mode&content=$content&p=1&form
 today=`date "+%Y-%m-%d"`
 _today=`date "+%Y%m%d"`
 rank_json=$today.json
-commands_file=$today.sh
+
 anonfiles_token=`cat $config_file | jq .anonfilesToken`
 rule=`cat $config_file | jq .rule|jq to_entries|jq 'map("sed -e \"s/\\\\"+.key+"/\\\\"+.value+"/g\"")'|jq -r  '.[]'|sed ':a;N;s/\n/|/;t a;'`
-touch $commands_file && chmod +x $commands_file
-sleep `cat $config_file | jq .sleep.ready`
-curl -v -d chat_id=$chat_id -d parse_mode=HTML  -d text="Pixiv排行榜已更新，30秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today%0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage

+sleepText=`cat $config_file | jq .sleep.text`
+sleepImage=`cat $config_file | jq .sleep.image`
+curl -v -d chat_id=$chat_id -d parse_mode=HTML  -d text="Pixiv排行榜已更新，$sleepText秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage
+sleep $sleepText
 if [ ! -f $rank_json ]
 then
        echo "get data from $rank_url"
@ -106,6 +107,13 @@ maxFileSize=20971520
 maxFileSize_M="$((maxFileSize/1024/1024))M"
 start_rank=''
 end_rank=''
+maxLikeCount=0
+maxLikeCountPid=0
+maxBookmarkCount=0
+maxBookmarkCountPid=0
+maxViewCount=0
+maxViewCountPid=0
+maxRatio=10
 for index in `seq 1 $length`
 do
        index=$((index-1))
@ -124,15 +132,14 @@ do
        fi
        if [ $yes_rank -eq 0 ]
        then
-          rank_info="\#rank$rank \#首次登场"
+          rank_info="\#排名$rank \#rank$rank \#首次登场"
        else
-          rank_info="\#rank$rank 之前 \#rank$yes_rank"   
+          rank_info="\#排名$rank \#rank$rank 之前 \#排名$yes_rank \#rank$yes_rank"   
        fi
        echo "pid=$pid,artworkLink=$artworkLink,rank_info=$rank_info"
        png_html_file=$pid.html
        if [ ! -f $png_html_file ]
        then
-                sleep 1
                echo "get data from $artworkLink"
                curl -v $artworkLink >$png_html_file
        fi
@ -151,8 +158,23 @@ do
        userName=`bash -c "echo '$userName'|$rule"`
        userId=`jq -r --arg pid $pid '.illust[$pid].userId' $json_file`
        likeCount=`jq --arg pid $pid '.illust[$pid].likeCount' $json_file`
+        if [ $likeCount -gt $maxLikeCount ]
+        then
+           maxLikeCount=$likeCount
+           maxLikeCountPid=$pid
+        fi
        bookmarkCount=`jq --arg pid $pid '.illust[$pid].bookmarkCount' $json_file`
+        if [ $bookmarkCount -gt $maxBookmarkCount ]
+        then
+           maxBookmarkCount=$likeCount
+           maxBookmarkCountPid=$pid
+        fi
        viewCount=`jq --arg pid $pid '.illust[$pid].viewCount' $json_file`
+        if [ $viewCount -gt $maxViewCount ]
+        then
+           maxViewCount=$likeCount
+           maxViewCountPid=$pid
+        fi
        tag=`jq -r  --arg pid $pid '.illust[$pid].tags.tags[].tag' $json_file|sed -e 's/^/\\#/g'|sed ':a;N;s/\n/ /;t a;'`
        tag=`bash -c "echo '$tag'|$rule"`
        
@ -167,7 +189,6 @@ do
                page_original_url=`echo $original_url | sed -e "s/p0/p$page/"`
                page_small_url=`echo $small_url | sed -e"s/p0/p$page/"`
                original_file_name=`echo $page_original_url | egrep -o "$pid.*"`
-                small_file_name=`echo $page_small_url | egrep -o "$pid.*"`
                webp_file_name=`echo $original_file_name|sed 's/jpg/webp/'|sed 's/png/webp/'`
                if [ "`cat $db_file|grep $original_file_name`" != "$original_file_name" ]
                then
@ -191,11 +212,15 @@ do
                   webp_url="https://$bucketname.$accelerateHost/$original_file_name$imageParam"
                   echo "download image file name=$webp_file_name,url=$webp_url"      
                   curl -v $webp_url -o $webp_file_name
-                fi
-                if [ ! -f $small_file_name ]
-                then
-                        echo "download image file name=$small_file_name,url=$page_small_url"
-                        curl -v -H 'referer: https://www.pixiv.net/' $page_small_url -o $small_file_name
+                   image_ratio=`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'|sed '/^$/d'|awk '{split($0,a,"x");f=a[1]>a[2]?a[1]/a[2]:a[2]/a[1];print f}'`
+                   if [[ -n $image_ratio && $image_ratio > $maxRatio ]]
+                   then
+                     echo "图片分辨率`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'`比率：$image_ratio>$maxRatio,视为异常，返回异常图片"
+                     watermark=`echo "画作pid：$pid"|base64`
+                     url="https://$bucketname.$accelerateHost/error.jpeg?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
+                     echo "文字水印请求url：$url"
+                     curl -v $url -o $webp_file_name
+                   fi
                fi
   
                if [ $page -eq 0 ]
@ -203,18 +228,18 @@ do
                        media="$media,{\"type\":\"photo\",\"media\":\"attach://$webp_file_name\",\"parse_mode\":\"HTML\",\"caption\":\"$rank_info\n<a href=\\\"$artworkLink\\\">$title</a>\n<a href=\\\"https://www.pixiv.net/users/$userId\\\">$userName</a>\n$tag\"}"
                        fileList="$fileList -F $webp_file_name=@$webp_file_name"
                        fileCount=$((fileCount + 1))
-                        fileSize=`du $small_file_name | awk '{print $1}'`
+                        fileSize=`du $webp_file_name | awk '{print $1}'`
                        fileCountSize=$((fileCountSize + fileSize))
                        echo "fileCountSize=$fileCountSize,fileCount=$fileCount"
                fi
                if [[ $fileCount -eq $maxFileCount ]]
                then
-                        echo "sleep `cat $config_file | jq .sleep.request`" >>$commands_file
-                        echo "curl -v -F  chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup" >>$commands_file
-                        echo "curl -v -d chat_id=$chat_id -d text='以上作品日榜排名分别是 #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息' $baseApi/sendMessage" >>$commands_file 
-                        echo >>$commands_file
-                        echo >>$commands_file
-                        echo >>$commands_file
+                        command="curl -v -F  chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup"
+                        echo "上传图片命令：$command"
+                        bash -c "$command"
+                        sleep $sleepImage
+                        curl -v -d chat_id=$chat_id -d text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息." $baseApi/sendMessage
+                        sleep $sleepText
                        fileCountSize=0
                        fileList=''
                        media=''
@ -222,16 +247,11 @@ do
                fi
        done
 done
-if [ $fileCount -gt 0 ]
-then
-        echo "curl -v -F  chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup" >>$commands_file
-fi
-sed -i '1d' $commands_file
+
 hasSend=$today.hasSend
 if [ ! -f $hasSend ]
 then
-        bash -c ./$commands_file
        touch $hasSend
        next_expected_at=`curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:''|jq .next_expected_at`
-        curl -v -d chat_id=$chat_id -d text="以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today" $baseApi/sendMessage
+        curl -v -d chat_id=$chat_id -d text="以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today  #日期$_today " $baseApi/sendMessage
 fi