删除阿里云函数实现清空文件逻辑，修改为shell实现。

2 years ago · 622520ceb2
parent 21ecdfe138
commit 622520ceb2
2 changed files with 75 additions and 22 deletions
--- a/pixiv/config.json.example
+++ b/pixiv/config.json.example
@ -2,7 +2,6 @@
  "basePath": "本地缓存路径，包括接口数据，图片等",
  "telegramToken": "telegram 机器人申请的API Token",
  "chatId": "推送目标聊天的唯一标识符或目标频道的用户名",
-  "anonfilesToken": "https://anonfiles.com/ 申请的API Token",
  "mode": "Pixiv排行榜类型，有效值：daily/weekly/monthly/rookie/daily_r18/weekly_r18/male_r18/female_r18/r18g,分别是日榜/周榜/月榜/新人榜/日榜r18/周榜r18/受男性欢迎r18/受女性欢迎r18/r18g",
  "content": "Pixiv排行榜类型，有效值：illust/ugoira/manga/,分别是插画/动图/漫画",
  "rule":{"匹配需要转义的内容":"转换后的值","&amp;":"&","&lt;":"<"},
@ -16,9 +15,10 @@
    "bucketname":"bucket名称",
    "AccessKeyId":"OpenAPI AccessKey ID",
    "AccessKeySecret":"OpenAPI AccessKey Secret",
-    "db":"文件上传成功记录数据库",
+    "db":"文件上传成功记录数据库文件名",
    "imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560",
-    "deleteApi":"删除老图片的接口地址"
+    "backup_time":"1 hour",
+    "error_file":"error.jpeg"
  },
  "convertio":{
    "apikey":"https://developers.convertio.co/zh/ 申请的API密钥",
@ -27,4 +27,4 @@
    "API_KEY":"https://cronitor.io/ 申请的Key",
    "JOB_NAME":"定时任务名称"
  }
-}
+}
--- a/pixiv/pixiv.sh
+++ b/pixiv/pixiv.sh
@ -18,6 +18,13 @@ then
 else
        echo "jq version:`jq --version`"
 fi
+if [ ! -f `which xmlstarlet` ]
+then
+        echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php'
+        exit 1
+else
+        echo "xmlstarlet version:`xmlstarlet --version`"
+fi


 # ==================== Config ====================
@ -29,7 +36,8 @@ AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret`
 db_file=`cat $config_file | jq -r .aliyun_oss.db`
 db_file_err=$db_file.err
 imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam`
-deleteApi=`cat $config_file | jq -r .aliyun_oss.deleteApi`
+backup_time=`cat $config_file | jq -r .aliyun_oss.backup_time`
+error_file=`cat $config_file | jq -r .aliyun_oss.error_file`

 convertio_apikey=`cat $config_file | jq -r .convertio.apikey`

@ -56,31 +64,77 @@ function PutObject(){
        http_code=`curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read"  -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"`
        if [ $http_code -eq "200" ]
        then
-           echo $file>>$db_file     
           rm -f $file
        else
           echo $file>>$db_file_err     
        fi
 }

+function GetBucketV2(){
+   VERB="GET"
+   Content_MD5=""
+   Content_Type=""
+   Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'`
+   CanonicalizedOSSHeaders=""
+   CanonicalizedResource="/$bucketname/"
+   stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
+   Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64`
+   Authorization="OSS $AccessKeyId:$Signature"
+   curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2"
+}
+
 function DeleteMultipleObjects(){
-      temp_file=DeleteMultipleObjects.json
-      while true
+      file="DeleteMultipleObjects.temp"
+      GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n > $file
+      # 获取备份时间之前的日期时间戳
+      oldTimestamp=`date -d "-$backup_time"  +%s`
+      ObjectNode=""
+      while read line
      do
-         curl -v -o $temp_file -d "AccessKeyId=$AccessKeyId&AccessKeySecret=$AccessKeySecret&host=$Host&bucketname=$bucketname&max=50"  $deleteApi/aliyun_oss
-         cat $temp_file
-         count=`cat $temp_file|jq -r .count`
-         echo "剩余需要清理文件个数：$count"
-         if [ $count == 0 ]
+        filename=`echo $line|awk '{print $1}'`
+        date=`echo $line|awk '{print $2}'`
+        timestamp=`date -d "$date" +%s`
+        if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]]
+        then
+          ObjectNode="${ObjectNode}<Object><Key>${filename}</Key></Object>"
+        fi
+      done < $file
+      rm $file
+
+      if [ -n "$ObjectNode" ]
+      then
+         echo "存在${backup_time}以前的图片"
+         compress_xml='<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>'${ObjectNode}'</Delete>'
+         echo $compress_xml|xmlstarlet format
+         VERB="POST"
+         Content_MD5=`echo -n $compress_xml |openssl dgst -md5 -binary|base64`
+         Content_Type="application/xml"
+         Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'`
+         CanonicalizedOSSHeaders=""
+         CanonicalizedResource="/$bucketname/?delete"
+         stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource"
+         echo "sign:$stringToSign"
+         Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64`
+         Authorization="OSS $AccessKeyId:$Signature"
+         http_code=`curl -X $VERB -v -w "%{http_code}"  -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete"`
+         if [ $http_code -eq "200" ]
         then
-            echo "delete finish"
-            break
+            echo "${backup_time}以前的图片删除成功"    
+         else
+            echo "${backup_time}以前的图片删除失败"  
         fi
-      done
-      rm -f $temp_file
-      curl -v -o $db_file  -d "AccessKeyId=$AccessKeyId&AccessKeySecret=$AccessKeySecret&host=$Host&bucketname=$bucketname" $deleteApi/list_file
-}
-
+         echo ''>$db_file
+         GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n|while read line
+         do
+               if [ $line != "$error_file" ]
+               then
+                  echo $line >> $db_file
+               fi
+         done
+      else
+        echo "没有${backup_time}以前的图片需要删除"  
+      fi      
+   }

 function fileSizeStr(){
     fileSize=$1
@ -109,7 +163,6 @@ today=`date "+%Y-%m-%d"`
 _today=`date "+%Y%m%d"`
 rank_json=$today.json

-anonfiles_token=`cat $config_file | jq .anonfilesToken`
 rule=`cat $config_file | jq .rule|jq to_entries|jq 'map("sed -e \"s/\\\\"+.key+"/\\\\"+.value+"/g\"")'|jq -r  '.[]'|sed ':a;N;s/\n/|/;t a;'`

 sleepText=`cat $config_file | jq .sleep.text`
@ -272,7 +325,7 @@ do
                   then
                     echo "图片分辨率`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'`比率：$image_ratio>$maxRatio,视为异常，返回异常图片"
                     watermark=`echo "画作pid：$pid"|base64`
-                     url="https://$bucketname.$accelerateHost/error.jpeg?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
+                     url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south"
                     echo "文字水印请求url：$url"
                     curl -v $url -o $webp_file_name
                   fi