diff --git a/pixiv/config.json.example b/pixiv/config.json.example index bbf51f7..13a8722 100644 --- a/pixiv/config.json.example +++ b/pixiv/config.json.example @@ -2,7 +2,6 @@ "basePath": "本地缓存路径，包括接口数据，图片等", "telegramToken": "telegram 机器人申请的API Token", "chatId": "推送目标聊天的唯一标识符或目标频道的用户名", - "anonfilesToken": "https://anonfiles.com/ 申请的API Token", "mode": "Pixiv排行榜类型，有效值：daily/weekly/monthly/rookie/daily_r18/weekly_r18/male_r18/female_r18/r18g,分别是日榜/周榜/月榜/新人榜/日榜r18/周榜r18/受男性欢迎r18/受女性欢迎r18/r18g", "content": "Pixiv排行榜类型，有效值：illust/ugoira/manga/,分别是插画/动图/漫画", "rule":{"匹配需要转义的内容":"转换后的值","&":"&","<":"<"}, @@ -16,9 +15,10 @@ "bucketname":"bucket名称", "AccessKeyId":"OpenAPI AccessKey ID", "AccessKeySecret":"OpenAPI AccessKey Secret", - "db":"文件上传成功记录数据库", + "db":"文件上传成功记录数据库文件名", "imageParam":"阿里云图片处理参数 比如：?x-oss-process=image/format,webp/resize,w_2560,h_2560", - "deleteApi":"删除老图片的接口地址" + "backup_time":"1 hour", + "error_file":"error.jpeg" }, "convertio":{ "apikey":"https://developers.convertio.co/zh/ 申请的API密钥", @@ -27,4 +27,4 @@ "API_KEY":"https://cronitor.io/ 申请的Key", "JOB_NAME":"定时任务名称" } -} +} \ No newline at end of file diff --git a/pixiv/pixiv.sh b/pixiv/pixiv.sh index d981f03..ccc19ca 100644 --- a/pixiv/pixiv.sh +++ b/pixiv/pixiv.sh @@ -18,6 +18,13 @@ then else echo "jq version:`jq --version`" fi +if [ ! -f `which xmlstarlet` ] +then + echo 'To run this script, you need to install xmlstarlet，http://xmlstar.sourceforge.net/docs.php' + exit 1 +else + echo "xmlstarlet version:`xmlstarlet --version`" +fi # ==================== Config ==================== @@ -29,7 +36,8 @@ AccessKeySecret=`cat $config_file | jq -r .aliyun_oss.AccessKeySecret` db_file=`cat $config_file | jq -r .aliyun_oss.db` db_file_err=$db_file.err imageParam=`cat $config_file | jq -r .aliyun_oss.imageParam` -deleteApi=`cat $config_file | jq -r .aliyun_oss.deleteApi` +backup_time=`cat $config_file | jq -r .aliyun_oss.backup_time` +error_file=`cat $config_file | jq -r .aliyun_oss.error_file` convertio_apikey=`cat $config_file | jq -r .convertio.apikey` @@ -56,31 +64,77 @@ function PutObject(){ http_code=`curl -v -w "%{http_code}" -X $VERB -H "HOST:$bucketname.$Host" -H "x-oss-object-acl:public-read" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Authorization:$Authorization" --data-binary "@$file" "https://$bucketname.$Host/$file"` if [ $http_code -eq "200" ] then - echo $file>>$db_file rm -f $file else echo $file>>$db_file_err fi } +function GetBucketV2(){ + VERB="GET" + Content_MD5="" + Content_Type="" + Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` + CanonicalizedOSSHeaders="" + CanonicalizedResource="/$bucketname/" + stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" + Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` + Authorization="OSS $AccessKeyId:$Signature" + curl -v -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Authorization:$Authorization" "https://$bucketname.$Host/?list-type=2" +} + function DeleteMultipleObjects(){ - temp_file=DeleteMultipleObjects.json - while true + file="DeleteMultipleObjects.temp" + GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -o ' ' -v 'LastModified' -n > $file + # 获取备份时间之前的日期时间戳 + oldTimestamp=`date -d "-$backup_time" +%s` + ObjectNode="" + while read line do - curl -v -o $temp_file -d "AccessKeyId=$AccessKeyId&AccessKeySecret=$AccessKeySecret&host=$Host&bucketname=$bucketname&max=50" $deleteApi/aliyun_oss - cat $temp_file - count=`cat $temp_file|jq -r .count` - echo "剩余需要清理文件个数：$count" - if [ $count == 0 ] + filename=`echo $line|awk '{print $1}'` + date=`echo $line|awk '{print $2}'` + timestamp=`date -d "$date" +%s` + if [[ $filename != "$error_file" && $timestamp -lt $oldTimestamp ]] + then + ObjectNode="${ObjectNode}${filename}" + fi + done < $file + rm $file + + if [ -n "$ObjectNode" ] + then + echo "存在${backup_time}以前的图片" + compress_xml='true'${ObjectNode}'' + echo $compress_xml|xmlstarlet format + VERB="POST" + Content_MD5=`echo -n $compress_xml |openssl dgst -md5 -binary|base64` + Content_Type="application/xml" + Date=`TZ=GMT env LANG=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S GMT'` + CanonicalizedOSSHeaders="" + CanonicalizedResource="/$bucketname/?delete" + stringToSign="$VERB\n$Content_MD5\n$Content_Type\n$Date\n$CanonicalizedOSSHeaders$CanonicalizedResource" + echo "sign:$stringToSign" + Signature=`echo -en $stringToSign | openssl sha1 -hmac $AccessKeySecret -binary | base64` + Authorization="OSS $AccessKeyId:$Signature" + http_code=`curl -X $VERB -v -w "%{http_code}" -H "HOST:$bucketname.$Host" -H "Date:$Date" -H "Content-Type:$Content_Type" -H "Content-MD5:$Content_MD5" -H "Authorization:$Authorization" -d "$compress_xml" "https://$bucketname.$Host/?delete"` + if [ $http_code -eq "200" ] then - echo "delete finish" - break + echo "${backup_time}以前的图片删除成功" + else + echo "${backup_time}以前的图片删除失败" fi - done - rm -f $temp_file - curl -v -o $db_file -d "AccessKeyId=$AccessKeyId&AccessKeySecret=$AccessKeySecret&host=$Host&bucketname=$bucketname" $deleteApi/list_file -} - + echo ''>$db_file + GetBucketV2|xmlstarlet select -t -m '/ListBucketResult//Contents' -v 'Key' -n|while read line + do + if [ $line != "$error_file" ] + then + echo $line >> $db_file + fi + done + else + echo "没有${backup_time}以前的图片需要删除" + fi + } function fileSizeStr(){ fileSize=$1 @@ -109,7 +163,6 @@ today=`date "+%Y-%m-%d"` _today=`date "+%Y%m%d"` rank_json=$today.json -anonfiles_token=`cat $config_file | jq .anonfilesToken` rule=`cat $config_file | jq .rule|jq to_entries|jq 'map("sed -e \"s/\\\\"+.key+"/\\\\"+.value+"/g\"")'|jq -r '.[]'|sed ':a;N;s/\n/|/;t a;'` sleepText=`cat $config_file | jq .sleep.text` @@ -272,7 +325,7 @@ do then echo "图片分辨率`file $webp_file_name|awk '{print $9}'|sed -e 's/,//'`比率：$image_ratio>$maxRatio,视为异常，返回异常图片" watermark=`echo "画作pid：$pid"|base64` - url="https://$bucketname.$accelerateHost/error.jpeg?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south" + url="https://$bucketname.$accelerateHost/$error_file?x-oss-process=image/watermark,text_`echo "画作pid：$pid"|base64`,g_north/watermark,text_5YiG6L6o546H5q%2BU5L6L5byC5bi4Cg==,g_center/watermark,text_5bi46KeB5LqO6LaF6ZW_5Zu-,g_south" echo "文字水印请求url：$url" curl -v $url -o $webp_file_name fi