封装tg请求响应检测逻辑

2.修复自动清理文件逻辑
3.上传图片失败增加错误提示
4.在线压缩格式修改为webp

pixiv/config.json.example

@@ -2,23 +2,18 @@
   "basePath": "本地缓存路径，包括接口数据，图片等",
   "telegramToken": "telegram 机器人申请的API Token",
   "chatId": "推送目标聊天的唯一标识符或目标频道的用户名",
-  "anonfilesToken": "https://anonfiles.com/ 申请的API Token",
   "mode": "Pixiv排行榜类型，有效值：daily/weekly/monthly/rookie/daily_r18/weekly_r18/male_r18/female_r18/r18g,分别是日榜/周榜/月榜/新人榜/日榜r18/周榜r18/受男性欢迎r18/受女性欢迎r18/r18g",
   "content": "Pixiv排行榜类型，有效值：illust/ugoira/manga/,分别是插画/动图/漫画",
-  "rule":{"匹配需要转义的内容":"转换后的值","&amp;":"&","&lt;":"<"},
-  "sleep":{
-    "text":10,
-    "image":60
-  },
   "aliyun_oss":{
     "Host":"Endpoint（地域节点）",
     "accelerateHost":"传输加速Endpoint（地域节点）",
     "bucketname":"bucket名称",
     "AccessKeyId":"OpenAPI AccessKey ID",
     "AccessKeySecret":"OpenAPI AccessKey Secret",
-    "db":"文件上传成功记录数据库",
+    "db":"文件上传成功记录数据库文件名",
     "imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560",
-    "deleteApi":"删除老图片的接口地址"
+    "backup_time":"1 hour",
+    "error_file":"error.jpeg"
   },
   "convertio":{
     "apikey":"https://developers.convertio.co/zh/ 申请的API密钥",
@@ -27,4 +22,4 @@
     "API_KEY":"https://cronitor.io/ 申请的Key",
     "JOB_NAME":"定时任务名称"
   }
-}
+}

pixiv/pixiv.sh

@@ -18,6 +18,13 @@ then
 else
         echo "jq version:`jq --version`"
 fi
+if [ ! -f `which xmlstarlet` ]
+then
+        echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php'
+        exit 1
+else
+        echo "xmlstarlet version:`xmlstarlet --version`"
+fi
 
 
 # ==================== Config ====================
@@ -29,7 +36,8 @@ AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret`
 db_file=`cat $config_file | jq -r .aliyun_oss.db`
 db_file_err=$db_file.err
 imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam`
-deleteApi=`cat $config_file | jq -r .aliyun_oss.deleteApi`
+backup_time=`cat $config_file | jq -r .aliyun_oss.backup_time`
+error_file=`cat $config_file | jq -r .aliyun_oss.error_file`
 
 convertio_apikey=`cat $config_file | jq -r .convertio.apikey`
 
@@ -56,31 +64,77 @@ function PutObject(){
         http_code=`curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read"  -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"`
         if [ $http_code -eq "200" ]
         then
-           echo $file>>$db_file     
            rm -f $file
         else
            echo $file>>$db_file_err     
         fi
 }
 
+function GetBucketV2(){
+   VERB="GET"
+   Content_MD5=""
+   Content_Type=""
+   Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'`
+   CanonicalizedOSSHeaders=""
+   CanonicalizedResource="/$bucketname/"
+   stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
+   Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64`
+   Authorization="OSS $AccessKeyId:$Signature"
+   curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2"
+}
+
 function DeleteMultipleObjects(){
-      temp_file=DeleteMultipleObjects.json
-      while true
+      file="DeleteMultipleObjects.temp"
+      GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n > $file
+      # 获取备份时间之前的日期时间戳
+      oldTimestamp=`date -d "-$backup_time"  +%s`
+      ObjectNode=""
+      while read line
       do
-         curl -v -o $temp_file -d "AccessKeyId=$AccessKeyId&AccessKeySecret=$AccessKeySecret&host=$Host&bucketname=$bucketname&max=50"  $deleteApi/aliyun_oss
-         cat $temp_file
-         count=`cat $temp_file|jq -r .count`
-         echo "剩余需要清理文件个数：$count"
-         if [ $count == 0 ]
+        filename=`echo $line|awk '{print $1}'`
+        date=`echo $line|awk '{print $2}'`
+        timestamp=`date -d "$date" +%s`
+        if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]]
+        then
+          ObjectNode="${ObjectNode}<Object><Key>${filename}</Key></Object>"
+        fi
+      done < $file
+      rm $file
+
+      if [ -n "$ObjectNode" ]
+      then
+         echo "存在${backup_time}以前的图片"
+         compress_xml='<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>'${ObjectNode}'</Delete>'
+         echo $compress_xml|xmlstarlet format
+         VERB="POST"
+         Content_MD5=`echo -n $compress_xml |openssl dgst -md5 -binary|base64`
+         Content_Type="application/xml"
+         Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'`
+         CanonicalizedOSSHeaders=""
+         CanonicalizedResource="/$bucketname/?delete"
+         stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
+         echo "sign:$stringToSign"
+         Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64`
+         Authorization="OSS $AccessKeyId:$Signature"
+         http_code=`curl -X $VERB -v -w "%{http_code}"  -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete"`
+         if [ $http_code -eq "200" ]
          then
-            echo "delete finish"
-            break
+            echo "${backup_time}以前的图片删除成功"    
+         else
+            echo "${backup_time}以前的图片删除失败"  
          fi
-      done
-      rm -f $temp_file
-      curl -v -o $db_file  -d "AccessKeyId=$AccessKeyId&AccessKeySecret=$AccessKeySecret&host=$Host&bucketname=$bucketname" $deleteApi/list_file
-}
-
+         echo ''>$db_file
+         GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n|while read line
+         do
+               if [ $line != "$error_file" ]
+               then
+                  echo $line >> $db_file
+               fi
+         done
+      else
+        echo "没有${backup_time}以前的图片需要删除"  
+      fi      
+   }
 
 function fileSizeStr(){
      fileSize=$1
@@ -92,6 +146,28 @@ function fileSizeStr(){
      fi
 }
 
+function Request(){
+   while true
+   do
+      echo "发送消息命令：$1"
+      result=`bash -c "$1"`
+      echo "请求响应：$result"
+      if [ `echo $result|jq .ok` = true ]
+      then
+         return 1
+      elif [ `echo $result|jq .error_code` -eq 429 ]
+      then
+         second=`echo $result|jq .parameters.retry_after`
+         sleep $second
+         echo "${second}后再发起请求"
+         Request $1
+      else
+         echo "执行命令遇到未知错误！" 
+         return 0
+      fi
+   done
+}
+
 basePath=`cat $config_file|jq -r .basePath`
 if [ ! -f $basePath ]
 then
@@ -109,20 +185,17 @@ today=`date "+%Y-%m-%d"`
 _today=`date "+%Y%m%d"`
 rank_json=$today.json
 
-anonfiles_token=`cat $config_file | jq .anonfilesToken`
-rule=`cat $config_file | jq .rule|jq to_entries|jq 'map("sed -e \"s/\\\\"+.key+"/\\\\"+.value+"/g\"")'|jq -r  '.[]'|sed ':a;N;s/\n/|/;t a;'`
+rule='sed -e "s/\\&amp;/\\&/g"|sed -e "s/\\&lt;/\\</g"|sed -e "s/\\&gt;/\\>/g"'
+
+Request "curl -v -d chat_id=$chat_id -d parse_mode=HTML  -d text=\"Pixiv排行榜已更新，开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>\" $baseApi/sendMessage"
 
-sleepText=`cat $config_file | jq .sleep.text`
-sleepImage=`cat $config_file | jq .sleep.image`
-curl -v -d chat_id=$chat_id -d parse_mode=HTML  -d text="Pixiv排行榜已更新，$sleepText秒后开始处理<a href='https://www.pixiv.net/ranking.php?mode%3D$mode%26content%3D$content'>$today日榜</a>数据。#date$_today #日期$_today %0A%0A<strong>排名是什么？</strong>%0A排名是以pixiv上所有公开作品为对象的统计以及排名。%0A毎日0:00～23时59分59秒的阅览树・「赞！」数等为排名的依据，期结果由pixiv独自的算法「pixiv rank β」决定。统计结果于每日中午12:00公开。%0A<a href='https://www.pixiv.help/hc/zh-cn/categories/360001065093-%E6%9C%89%E5%85%B3%E6%8E%92%E8%A1%8C%E6%A6%9C'>有关排行榜</a>" $baseApi/sendMessage
-sleep $sleepText
 if [ ! -f $rank_json ]
 then
         echo "get data from $rank_url"
         curl -v $rank_url >$rank_json
 fi
 length=`jq '.contents|length' $rank_json`
-fileCountSize=0
+
 fileList=''
 media=''
 fileCount=0
@@ -226,8 +299,8 @@ do
                         original_file_size=`du -b  $original_file_name|awk '{print $1}'`
                         if [ $original_file_size -gt $maxFileSize ]
                         then
-                          echo "${original_file_name}文件体积超过${maxFileSize}字节，需要在线压缩"
-                          result=`curl -v -X POST -d "{\"apikey\": \"$convertio_apikey\", \"input\":\"upload\", \"outputformat\":\"jpeg\"}" http://api.convertio.co/convert`
+                          echo "${original_file_name}文件体积：${original_file_size}字节超过${maxFileSize}字节限制，需要在线压缩"
+                          result=`curl -v -X POST -d "{\"apikey\": \"$convertio_apikey\", \"input\":\"upload\", \"outputformat\":\"webp\"}" http://api.convertio.co/convert`
 
                           if [ `echo $result|jq -r .status` = ok ]
                           then
@@ -239,10 +312,11 @@ do
                                 do
                                    sleep 10
                                    result=`curl -v -X GET http://api.convertio.co/convert/$id/status`
+                                   echo "转换响应：$result"
                                    if [[ `echo $result|jq -r .status` = ok && `echo $result|jq -r .data.step` = finish ]]
                                    then
-                                       echo $result|jq .data.output.url|xargs curl -v -o $original_file_name 
-                                       PutObject $original_file_name
+                                       echo $result|jq .data.output.url|xargs curl -v -o $webp_file_name 
+                                       PutObject $webp_file_name
                                        break
                                    elif [ `echo $result|jq -r .status` = error ]
                                    then
@@ -268,11 +342,11 @@ do
                    echo "download image file name=$webp_file_name,url=$webp_url"      
                    curl -v $webp_url -o $webp_file_name
                    image_ratio=`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'|sed '/^$/d'|awk '{split($0,a,"x");f=a[1]>a[2]?a[1]/a[2]:a[2]/a[1];print f}'`
-                   if [[ -n $image_ratio && $image_ratio > $maxRatio ]]
+                   if [[ -n $image_ratio && `echo "$image_ratio > $maxRatio"|bc` -eq 1 ]]
                    then
                      echo "图片分辨率`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'`比率：$image_ratio>$maxRatio,视为异常，返回异常图片"
                      watermark=`echo "画作pid：$pid"|base64`
-                     url="https://$bucketname.$accelerateHost/error.jpeg?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
+                     url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
                      echo "文字水印请求url：$url"
                      curl -v $url -o $webp_file_name
                    fi
@@ -289,18 +363,21 @@ do
                         fileList="$fileList -F $webp_file_name=@$webp_file_name"
                         fileCount=$((fileCount + 1))
                         fileSize=`du $webp_file_name | awk '{print $1}'`
-                        fileCountSize=$((fileCountSize + fileSize))
-                        echo "fileCountSize=$fileCountSize,fileCount=$fileCount"
+                        echo "fileCount=$fileCount"
                 fi
                 if [[ $fileCount -eq $maxFileCount ]]
                 then
-                        command="curl -v -F  chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup"
-                        echo "上传图片命令：$command"
-                        bash -c "$command"
-                        sleep $sleepImage
-                        curl -v -d chat_id=$chat_id -d text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息." $baseApi/sendMessage
-                        sleep $sleepText
-                        fileCountSize=0
+                        Request "curl -v -F  chat_id=$chat_id $fileList -F media='[`echo $media | cut -c 2-`]' $baseApi/sendMediaGroup"
+                        if [ `echo $?` -eq 1 ]
+                        then
+                           sleep $sleepImage
+                           text="以上作品日榜排名分别是 #排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank} ，点击作品可以查看pid/标题/画师/tag信息."
+                        else
+                           text="#排名${start_rank}_${end_rank} #rank${start_rank}_${end_rank}作品上传失败，请联系管理员"
+                        fi
+                         
+                        Request "curl -v -d chat_id=$chat_id -d text=\"$text\" $baseApi/sendMessage"
+                        
                         fileList=''
                         media=''
                         fileCount=0
@@ -310,9 +387,10 @@ done
 
 
 next_expected_at=`curl -v https://cronitor.io/api/monitors/$CronitorJobName -u $CronitorKey:''|jq .next_expected_at`
-curl -v -d chat_id=$chat_id -d text="以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today  #日期$_today " $baseApi/sendMessage
+Request "curl -v -d chat_id=$chat_id -d text=\"以上就是$today日榜前${length}名作品，本次推送完毕，下次推送时间预计是`date -d @$next_expected_at '+%Y-%m-%d %H:%M:%S'`，如有问题请联系管理员。 #date$_today  #日期$_today \" $baseApi/sendMessage"
 
-find -type f  -mtime +7|grep html
-find -type f  -mtime +7|grep json
-find -type f  -mtime +7|grep webp
+find -type f  -mtime +7|grep html|xargs rm -f
+find -type f  -mtime +7|grep json|xargs rm -f
+find -type f  -mtime +7|grep webp|xargs rm -f
+find -type f  -mtime +7|grep info|xargs rm -f
 DeleteMultipleObjects