From 20eee83f85402a345ad88ff213ebdd873be54045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=A0=E9=99=90=E8=B6=85=E9=A2=91?= <1029559041@qq.com> Date: Sun, 18 Mar 2018 04:02:22 +0800 Subject: [PATCH 01/22] Initial commit --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..af78c66 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# PixivSearch From bc06a1a5fee6077314387e878c5729e929b4cc55 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Sun, 18 Mar 2018 04:15:03 +0800 Subject: [PATCH 02/22] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/__init__.py | 0 PixivSearch/logging.conf | 32 ++++++ PixivSearch/module/__init__.py | 0 PixivSearch/module/config.py | 5 + PixivSearch/pixiv.py | 132 ++++++++++++++++++++++ PixivSearch/settings.py | 124 ++++++++++++++++++++ PixivSearch/static/js/jquery-3.2.1.min.js | 4 + PixivSearch/templates/test.html | 63 +++++++++++ PixivSearch/tsdm.py | 0 PixivSearch/urls.py | 24 ++++ PixivSearch/view.py | 72 ++++++++++++ PixivSearch/wsgi.py | 16 +++ manage.py | 22 ++++ 13 files changed, 494 insertions(+) create mode 100644 PixivSearch/__init__.py create mode 100644 PixivSearch/logging.conf create mode 100644 PixivSearch/module/__init__.py create mode 100644 PixivSearch/module/config.py create mode 100644 PixivSearch/pixiv.py create mode 100644 PixivSearch/settings.py create mode 100644 PixivSearch/static/js/jquery-3.2.1.min.js create mode 100644 PixivSearch/templates/test.html create mode 100644 PixivSearch/tsdm.py create mode 100644 PixivSearch/urls.py create mode 100644 PixivSearch/view.py create mode 100644 PixivSearch/wsgi.py create mode 100644 manage.py diff --git a/PixivSearch/__init__.py b/PixivSearch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/logging.conf b/PixivSearch/logging.conf new file mode 100644 index 0000000..9203d73 --- /dev/null +++ b/PixivSearch/logging.conf @@ -0,0 +1,32 @@ +[loggers] +keys=root,main + +[handlers] +keys=consoleHandler,fileHandler + +[formatters] +keys=fmt + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_main] +level=DEBUG +qualname=file +handlers=fileHandler + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=fmt +args=(sys.stdout,) + +[handler_fileHandler] +class=FileHandler +level=DEBUG +formatter=fmt +args=('pixiv.log','a','utf-8',False) + +[formatter_fmt] +format=%(asctime)s - %(module)s:%(lineno)d - %(levelname)s - %(message)s \ No newline at end of file diff --git a/PixivSearch/module/__init__.py b/PixivSearch/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/module/config.py b/PixivSearch/module/config.py new file mode 100644 index 0000000..2eb0653 --- /dev/null +++ b/PixivSearch/module/config.py @@ -0,0 +1,5 @@ +from django.db import models + +class param(models.Model): + param_name = models.CharField(max_length=10,primary_key=True) + param_value = models.CharField(max_length=128) diff --git a/PixivSearch/pixiv.py b/PixivSearch/pixiv.py new file mode 100644 index 0000000..4a11cd5 --- /dev/null +++ b/PixivSearch/pixiv.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +#coding:utf-8 +from concurrent import futures +import threading +import json +import requests +from bs4 import BeautifulSoup +import sys +from datetime import datetime +import os +import zipfile +import logging.config + +headers = { + 'X-Requested-With': 'XMLHttpRequest', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/56.0.2924.87 Safari/537.36' +} + +lock = threading.Lock() # 多线程全局资源锁 +total = 1 +logging.config.fileConfig('PixivSearch/logging.conf') +logger = logging.getLogger('file') + +def get_cookies(): + _cookies = {} + array = "p_ab_id=5; p_ab_id_2=9; login_ever=yes; device_token=c8c37fdf24b917b4e7fb191fe11c5ca5; search_tools_toggle=1; _ga=GA1.2.887334537.1498628532; PHPSESSID=25745470_75a76e86ff3145b53e21b440183b4822; a_type=0; is_sensei_service_user=1; module_orders_mypage=%5B%7B%22name%22%3A%22recommended_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22everyone_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22following_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22mypixiv_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22fanbox%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22featured_tags%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22contests%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22sensei_courses%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22spotlight%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22booth_follow_items%22%2C%22visible%22%3Atrue%7D%5D; __utmt=1; __utma=235335808.887334537.1498628532.1498720739.1498819600.5; __utmb=235335808.1.10.1498819600; __utmc=235335808; __utmz=235335808.1498713152.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=235335808.|2=login%20ever=yes=1^3=plan=normal=1^5=gender=male=1^6=user_id=25745470=1^9=p_ab_id=5=1^10=p_ab_id_2=9=1^11=lang=zh_tw=1".split(';') + for row in array: + k, v = row.strip().split('=', 1) + _cookies[k] = v + return _cookies + + + + +def crawl(url): + global total + req = requests.get(url, headers=headers, cookies=get_cookies()).text + tag=BeautifulSoup(req, 'lxml').select('#js-mount-point-search-result-list')[0].attrs['data-items'] + imageNodes=json.loads(tag) + for imageNode in imageNodes: + with lock: + nodes.append(imageNode) + +def get_urls(search, page): + fmt = 'https://www.pixiv.net/search.php?word={}&order=date_d&p={}' + return [fmt.format(search, p) for p in range(1, page)] + +def get_Img(params): + params[1]['imgUrl']='https://i.pximg.net/img-original/img/'+params[1]['url'][-46:-15] + + headers['referer']='https://www.pixiv.net/member_illust.php?mode=medium&illust_id='+params[1]['illustId'] + + suffix=".jpg" + logger.info('开始下载图片：%s%s' % (params[1]['imgUrl'], suffix)) + + s=requests.get(params[1]['imgUrl']+suffix, headers=headers, cookies=get_cookies()) + if(s.status_code==404): + suffix='.png' + s=requests.get(params[1]['imgUrl']+suffix,headers=headers, cookies=get_cookies()) + if(s.status_code==404): + logger.error('无法下载图片：%s' % (params[1]['illustTitle'])) + return + + logger.info('下载图片："%s"到%s'%(params[1]['illustTitle'],os.getcwd().replace('\\','/')+'/'+imgPath+params[1]['illustId']+suffix)) + f = open(imgPath+params[1]['illustId']+suffix, 'wb') #写入多媒体文件要 b 这个参数 + f.write(s.content) #多媒体文件要是用conctent + f.close() + + params[1]['localName']=params[1]['illustId']+suffix + logger.info('排行第%d名，收藏数%d，标题：%s，标签：%s，(%s)前投稿，链接：%s'%(params[0],params[1]['bookmarkCount'],params[1]['illustTitle'], ','.join(params[1]['tags']),'',params[1]['imgUrl'])) + +def zip(inputFile,outFile): + f = zipfile.ZipFile(outFile,'w',zipfile.ZIP_DEFLATED) + for dirpath, dirnames, filenames in os.walk(inputFile): + for filename in filenames: + f.write(os.path.join(dirpath,filename),filename) + f.close() + +fsize='' +def get_nodes(param): + global nodes,fsize,imgPath + nodes=[] + start = datetime.now() + urls = get_urls(param[1], int(param[2])+1) + logger.info('开始从P站获取图片数据') + with futures.ThreadPoolExecutor(32) as executor: + executor.map(crawl, urls) + + # for url in urls: + # crawl(url) + + length=len(nodes) + logger.info('获取到%d张图片'%(length)) + logger.info('对图片收藏数进行排序') + nodes=sorted(nodes, key=lambda v: v.get('bookmarkCount'), reverse=True)[:int(param[3])] # 按star数降序排序 + if(param[4]!=None and param[4]=='img'): + imgPath='PixivSearch/static/images/' + for file in os.listdir(imgPath): + os.remove(imgPath+file) + nodes_tup=[] + start_d=datetime.now() + for index,img in enumerate(nodes): + nodes_tup.append((index+1,img)) + # get_Img((index+1,img)) + + with futures.ThreadPoolExecutor(32) as executor: + executor.map(get_Img, nodes_tup) + print('下载图片花费时间：%s' % (datetime.now() - start_d)) + logger.info('%s张图片下载完毕'%(len(os.listdir(imgPath)))) + + zipPath='PixivSearch/static/download/' + param[1] + '.zip' + logger.info('图片打包到：%s' % (zipPath)) + zip(imgPath,zipPath) + fsize = str(round(os.path.getsize(zipPath)/float(1024*1024),2))+'MB' + logger.info('图包大小：%s'%(fsize)) + + tip='从%d张图片中筛选出收藏数前%s的图片,处理耗时：%s'%(length,param[3],datetime.now()-start) + logger.info(tip) + return [nodes,tip,fsize] + + + +if __name__ == "__main__": + if (len(sys.argv))==5 and sys.argv[2].isdigit() and sys.argv[3].isdigit(): + try: + get_nodes(sys.argv) + except BaseException as e: + repr(e) + + else : + logger.error('参数不合法') \ No newline at end of file diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py new file mode 100644 index 0000000..8dc8ad5 --- /dev/null +++ b/PixivSearch/settings.py @@ -0,0 +1,124 @@ +""" +Django settings for PixivSearch project. + +Generated by 'django-admin startproject' using Django 1.11.7. + +For more information on this file, see +https://docs.djangoproject.com/en/1.11/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.11/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'dh3^+=iugoo*+p_ea4u3dh&b!_zlgs8*m9kc+#*f2eozglsqjh' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'PixivSearch.module', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'PixivSearch.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [os.path.join(os.path.dirname(__file__), 'templates').replace('\\','/')], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'PixivSearch.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.11/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.11/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.11/howto/static-files/ + +STATIC_URL = '/static/' +STATICFILES_DIRS = [ + os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'), +] diff --git a/PixivSearch/static/js/jquery-3.2.1.min.js b/PixivSearch/static/js/jquery-3.2.1.min.js new file mode 100644 index 0000000..644d35e --- /dev/null +++ b/PixivSearch/static/js/jquery-3.2.1.min.js @@ -0,0 +1,4 @@ +/*! jQuery v3.2.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var q="3.2.1",r=function(a,b){return new r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return f.call(this)},get:function(a){return null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var b=r.merge(this.constructor(),a);return b.prevObject=this,b},each:function(a){return r.each(this,a)},map:function(a){return this.pushStack(r.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(f.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(a<0?b:0);return this.pushStack(c>=0&&c0&&b-1 in a)}var x=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var c=0,d=a.length;c+~]|"+K+")"+K+"*"),S=new RegExp("="+K+"*([^\\]'\"]*?)"+K+"*\\]","g"),T=new RegExp(N),U=new RegExp("^"+L+"$"),V={ID:new RegExp("^#("+L+")"),CLASS:new RegExp("^\\.("+L+")"),TAG:new RegExp("^("+L+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+N),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+K+"*(even|odd|(([+-]|)(\\d*)n|)"+K+"*(?:([+-]|)"+K+"*(\\d+)|))"+K+"*\\)|)","i"),bool:new RegExp("^(?:"+J+")$","i"),needsContext:new RegExp("^"+K+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+K+"*((?:-\\d)?\\d*)"+K+"*\\)|)(?=[^-]|$)","i")},W=/^(?:input|select|textarea|button)$/i,X=/^h\d$/i,Y=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,$=/[+~]/,_=new RegExp("\\\\([\\da-f]{1,6}"+K+"?|("+K+")|.)","ig"),aa=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:d<0?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},ba=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ca=function(a,b){return b?"\0"===a?"\ufffd":a.slice(0,-1)+"\\"+a.charCodeAt(a.length-1).toString(16)+" ":"\\"+a},da=function(){m()},ea=ta(function(a){return a.disabled===!0&&("form"in a||"label"in a)},{dir:"parentNode",next:"legend"});try{G.apply(D=H.call(v.childNodes),v.childNodes),D[v.childNodes.length].nodeType}catch(fa){G={apply:D.length?function(a,b){F.apply(a,H.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function ga(a,b,d,e){var f,h,j,k,l,o,r,s=b&&b.ownerDocument,w=b?b.nodeType:9;if(d=d||[],"string"!=typeof a||!a||1!==w&&9!==w&&11!==w)return d;if(!e&&((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,p)){if(11!==w&&(l=Z.exec(a)))if(f=l[1]){if(9===w){if(!(j=b.getElementById(f)))return d;if(j.id===f)return d.push(j),d}else if(s&&(j=s.getElementById(f))&&t(b,j)&&j.id===f)return d.push(j),d}else{if(l[2])return G.apply(d,b.getElementsByTagName(a)),d;if((f=l[3])&&c.getElementsByClassName&&b.getElementsByClassName)return G.apply(d,b.getElementsByClassName(f)),d}if(c.qsa&&!A[a+" "]&&(!q||!q.test(a))){if(1!==w)s=b,r=a;else if("object"!==b.nodeName.toLowerCase()){(k=b.getAttribute("id"))?k=k.replace(ba,ca):b.setAttribute("id",k=u),o=g(a),h=o.length;while(h--)o[h]="#"+k+" "+sa(o[h]);r=o.join(","),s=$.test(a)&&qa(b.parentNode)||b}if(r)try{return G.apply(d,s.querySelectorAll(r)),d}catch(x){}finally{k===u&&b.removeAttribute("id")}}}return i(a.replace(P,"$1"),b,d,e)}function ha(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ia(a){return a[u]=!0,a}function ja(a){var b=n.createElement("fieldset");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function ka(a,b){var c=a.split("|"),e=c.length;while(e--)d.attrHandle[c[e]]=b}function la(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&a.sourceIndex-b.sourceIndex;if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function ma(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function na(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function oa(a){return function(b){return"form"in b?b.parentNode&&b.disabled===!1?"label"in b?"label"in b.parentNode?b.parentNode.disabled===a:b.disabled===a:b.isDisabled===a||b.isDisabled!==!a&&ea(b)===a:b.disabled===a:"label"in b&&b.disabled===a}}function pa(a){return ia(function(b){return b=+b,ia(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function qa(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=ga.support={},f=ga.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return!!b&&"HTML"!==b.nodeName},m=ga.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=n.documentElement,p=!f(n),v!==n&&(e=n.defaultView)&&e.top!==e&&(e.addEventListener?e.addEventListener("unload",da,!1):e.attachEvent&&e.attachEvent("onunload",da)),c.attributes=ja(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ja(function(a){return a.appendChild(n.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Y.test(n.getElementsByClassName),c.getById=ja(function(a){return o.appendChild(a).id=u,!n.getElementsByName||!n.getElementsByName(u).length}),c.getById?(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){return a.getAttribute("id")===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c?[c]:[]}}):(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c,d,e,f=b.getElementById(a);if(f){if(c=f.getAttributeNode("id"),c&&c.value===a)return[f];e=b.getElementsByName(a),d=0;while(f=e[d++])if(c=f.getAttributeNode("id"),c&&c.value===a)return[f]}return[]}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){if("undefined"!=typeof b.getElementsByClassName&&p)return b.getElementsByClassName(a)},r=[],q=[],(c.qsa=Y.test(n.querySelectorAll))&&(ja(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+K+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+K+"*(?:value|"+J+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),ja(function(a){a.innerHTML="";var b=n.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+K+"*[*^$|!~]?="),2!==a.querySelectorAll(":enabled").length&&q.push(":enabled",":disabled"),o.appendChild(a).disabled=!0,2!==a.querySelectorAll(":disabled").length&&q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=Y.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ja(function(a){c.disconnectedMatch=s.call(a,"*"),s.call(a,"[s!='']:x"),r.push("!=",N)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=Y.test(o.compareDocumentPosition),t=b||Y.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===n||a.ownerDocument===v&&t(v,a)?-1:b===n||b.ownerDocument===v&&t(v,b)?1:k?I(k,a)-I(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,g=[a],h=[b];if(!e||!f)return a===n?-1:b===n?1:e?-1:f?1:k?I(k,a)-I(k,b):0;if(e===f)return la(a,b);c=a;while(c=c.parentNode)g.unshift(c);c=b;while(c=c.parentNode)h.unshift(c);while(g[d]===h[d])d++;return d?la(g[d],h[d]):g[d]===v?-1:h[d]===v?1:0},n):n},ga.matches=function(a,b){return ga(a,null,null,b)},ga.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(S,"='$1']"),c.matchesSelector&&p&&!A[b+" "]&&(!r||!r.test(b))&&(!q||!q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return ga(b,n,null,[a]).length>0},ga.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},ga.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&C.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},ga.escape=function(a){return(a+"").replace(ba,ca)},ga.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},ga.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=ga.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=ga.selectors={cacheLength:50,createPseudo:ia,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(_,aa),a[3]=(a[3]||a[4]||a[5]||"").replace(_,aa),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||ga.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&ga.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return V.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&T.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(_,aa).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+K+")"+a+"("+K+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=ga.attr(d,a);return null==e?"!="===b:!b||(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(O," ")+" ").indexOf(c)>-1:"|="===b&&(e===c||e.slice(0,c.length+1)===c+"-"))}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h,t=!1;if(q){if(f){while(p){m=b;while(m=m[p])if(h?m.nodeName.toLowerCase()===r:1===m.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){m=q,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n&&j[2],m=n&&q.childNodes[n];while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if(1===m.nodeType&&++t&&m===b){k[a]=[w,n,t];break}}else if(s&&(m=b,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n),t===!1)while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if((h?m.nodeName.toLowerCase()===r:1===m.nodeType)&&++t&&(s&&(l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),k[a]=[w,t]),m===b))break;return t-=e,t===d||t%d===0&&t/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||ga.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ia(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=I(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ia(function(a){var b=[],c=[],d=h(a.replace(P,"$1"));return d[u]?ia(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ia(function(a){return function(b){return ga(a,b).length>0}}),contains:ia(function(a){return a=a.replace(_,aa),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ia(function(a){return U.test(a||"")||ga.error("unsupported lang: "+a),a=a.replace(_,aa).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:oa(!1),disabled:oa(!0),checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return X.test(a.nodeName)},input:function(a){return W.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:pa(function(){return[0]}),last:pa(function(a,b){return[b-1]}),eq:pa(function(a,b,c){return[c<0?c+b:c]}),even:pa(function(a,b){for(var c=0;c=0;)a.push(d);return a}),gt:pa(function(a,b,c){for(var d=c<0?c+b:c;++d1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function va(a,b,c){for(var d=0,e=b.length;d-1&&(f[j]=!(g[j]=l))}}else r=wa(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):G.apply(g,r)})}function ya(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=ta(function(a){return a===b},h,!0),l=ta(function(a){return I(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];i1&&ua(m),i>1&&sa(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(P,"$1"),c,i0,e=a.length>0,f=function(f,g,h,i,k){var l,o,q,r=0,s="0",t=f&&[],u=[],v=j,x=f||e&&d.find.TAG("*",k),y=w+=null==v?1:Math.random()||.1,z=x.length;for(k&&(j=g===n||g||k);s!==z&&null!=(l=x[s]);s++){if(e&&l){o=0,g||l.ownerDocument===n||(m(l),h=!p);while(q=a[o++])if(q(l,g||n,h)){i.push(l);break}k&&(w=y)}c&&((l=!q&&l)&&r--,f&&t.push(l))}if(r+=s,c&&s!==r){o=0;while(q=b[o++])q(t,u,g,h);if(f){if(r>0)while(s--)t[s]||u[s]||(u[s]=E.call(i));u=wa(u)}G.apply(i,u),k&&!f&&u.length>0&&r+b.length>1&&ga.uniqueSort(i)}return k&&(w=y,j=v),t};return c?ia(f):f}return h=ga.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=ya(b[c]),f[u]?d.push(f):e.push(f);f=A(a,za(e,d)),f.selector=a}return f},i=ga.select=function(a,b,c,e){var f,i,j,k,l,m="function"==typeof a&&a,n=!e&&g(a=m.selector||a);if(c=c||[],1===n.length){if(i=n[0]=n[0].slice(0),i.length>2&&"ID"===(j=i[0]).type&&9===b.nodeType&&p&&d.relative[i[1].type]){if(b=(d.find.ID(j.matches[0].replace(_,aa),b)||[])[0],!b)return c;m&&(b=b.parentNode),a=a.slice(i.shift().value.length)}f=V.needsContext.test(a)?0:i.length;while(f--){if(j=i[f],d.relative[k=j.type])break;if((l=d.find[k])&&(e=l(j.matches[0].replace(_,aa),$.test(i[0].type)&&qa(b.parentNode)||b))){if(i.splice(f,1),a=e.length&&sa(i),!a)return G.apply(c,e),c;break}}}return(m||h(a,n))(e,b,!p,c,!b||$.test(a)&&qa(b.parentNode)||b),c},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ja(function(a){return 1&a.compareDocumentPosition(n.createElement("fieldset"))}),ja(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||ka("type|href|height|width",function(a,b,c){if(!c)return a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ja(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||ka("value",function(a,b,c){if(!c&&"input"===a.nodeName.toLowerCase())return a.defaultValue}),ja(function(a){return null==a.getAttribute("disabled")})||ka(J,function(a,b,c){var d;if(!c)return a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),ga}(a);r.find=x,r.expr=x.selectors,r.expr[":"]=r.expr.pseudos,r.uniqueSort=r.unique=x.uniqueSort,r.text=x.getText,r.isXMLDoc=x.isXML,r.contains=x.contains,r.escapeSelector=x.escape;var y=function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&r(a).is(c))break;d.push(a)}return d},z=function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c},A=r.expr.match.needsContext;function B(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()}var C=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i,D=/^.[^:#\[\.,]*$/;function E(a,b,c){return r.isFunction(b)?r.grep(a,function(a,d){return!!b.call(a,d,a)!==c}):b.nodeType?r.grep(a,function(a){return a===b!==c}):"string"!=typeof b?r.grep(a,function(a){return i.call(b,a)>-1!==c}):D.test(b)?r.filter(b,a,c):(b=r.filter(b,a),r.grep(a,function(a){return i.call(b,a)>-1!==c&&1===a.nodeType}))}r.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?r.find.matchesSelector(d,a)?[d]:[]:r.find.matches(a,r.grep(b,function(a){return 1===a.nodeType}))},r.fn.extend({find:function(a){var b,c,d=this.length,e=this;if("string"!=typeof a)return this.pushStack(r(a).filter(function(){for(b=0;b1?r.uniqueSort(c):c},filter:function(a){return this.pushStack(E(this,a||[],!1))},not:function(a){return this.pushStack(E(this,a||[],!0))},is:function(a){return!!E(this,"string"==typeof a&&A.test(a)?r(a):a||[],!1).length}});var F,G=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/,H=r.fn.init=function(a,b,c){var e,f;if(!a)return this;if(c=c||F,"string"==typeof a){if(e="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:G.exec(a),!e||!e[1]&&b)return!b||b.jquery?(b||c).find(a):this.constructor(b).find(a);if(e[1]){if(b=b instanceof r?b[0]:b,r.merge(this,r.parseHTML(e[1],b&&b.nodeType?b.ownerDocument||b:d,!0)),C.test(e[1])&&r.isPlainObject(b))for(e in b)r.isFunction(this[e])?this[e](b[e]):this.attr(e,b[e]);return this}return f=d.getElementById(e[2]),f&&(this[0]=f,this.length=1),this}return a.nodeType?(this[0]=a,this.length=1,this):r.isFunction(a)?void 0!==c.ready?c.ready(a):a(r):r.makeArray(a,this)};H.prototype=r.fn,F=r(d);var I=/^(?:parents|prev(?:Until|All))/,J={children:!0,contents:!0,next:!0,prev:!0};r.fn.extend({has:function(a){var b=r(a,this),c=b.length;return this.filter(function(){for(var a=0;a-1:1===c.nodeType&&r.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?r.uniqueSort(f):f)},index:function(a){return a?"string"==typeof a?i.call(r(a),this[0]):i.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(r.uniqueSort(r.merge(this.get(),r(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function K(a,b){while((a=a[b])&&1!==a.nodeType);return a}r.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return y(a,"parentNode")},parentsUntil:function(a,b,c){return y(a,"parentNode",c)},next:function(a){return K(a,"nextSibling")},prev:function(a){return K(a,"previousSibling")},nextAll:function(a){return y(a,"nextSibling")},prevAll:function(a){return y(a,"previousSibling")},nextUntil:function(a,b,c){return y(a,"nextSibling",c)},prevUntil:function(a,b,c){return y(a,"previousSibling",c)},siblings:function(a){return z((a.parentNode||{}).firstChild,a)},children:function(a){return z(a.firstChild)},contents:function(a){return B(a,"iframe")?a.contentDocument:(B(a,"template")&&(a=a.content||a),r.merge([],a.childNodes))}},function(a,b){r.fn[a]=function(c,d){var e=r.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=r.filter(d,e)),this.length>1&&(J[a]||r.uniqueSort(e),I.test(a)&&e.reverse()),this.pushStack(e)}});var L=/[^\x20\t\r\n\f]+/g;function M(a){var b={};return r.each(a.match(L)||[],function(a,c){b[c]=!0}),b}r.Callbacks=function(a){a="string"==typeof a?M(a):r.extend({},a);var b,c,d,e,f=[],g=[],h=-1,i=function(){for(e=e||a.once,d=b=!0;g.length;h=-1){c=g.shift();while(++h-1)f.splice(c,1),c<=h&&h--}),this},has:function(a){return a?r.inArray(a,f)>-1:f.length>0},empty:function(){return f&&(f=[]),this},disable:function(){return e=g=[],f=c="",this},disabled:function(){return!f},lock:function(){return e=g=[],c||b||(f=c=""),this},locked:function(){return!!e},fireWith:function(a,c){return e||(c=c||[],c=[a,c.slice?c.slice():c],g.push(c),b||i()),this},fire:function(){return j.fireWith(this,arguments),this},fired:function(){return!!d}};return j};function N(a){return a}function O(a){throw a}function P(a,b,c,d){var e;try{a&&r.isFunction(e=a.promise)?e.call(a).done(b).fail(c):a&&r.isFunction(e=a.then)?e.call(a,b,c):b.apply(void 0,[a].slice(d))}catch(a){c.apply(void 0,[a])}}r.extend({Deferred:function(b){var c=[["notify","progress",r.Callbacks("memory"),r.Callbacks("memory"),2],["resolve","done",r.Callbacks("once memory"),r.Callbacks("once memory"),0,"resolved"],["reject","fail",r.Callbacks("once memory"),r.Callbacks("once memory"),1,"rejected"]],d="pending",e={state:function(){return d},always:function(){return f.done(arguments).fail(arguments),this},"catch":function(a){return e.then(null,a)},pipe:function(){var a=arguments;return r.Deferred(function(b){r.each(c,function(c,d){var e=r.isFunction(a[d[4]])&&a[d[4]];f[d[1]](function(){var a=e&&e.apply(this,arguments);a&&r.isFunction(a.promise)?a.promise().progress(b.notify).done(b.resolve).fail(b.reject):b[d[0]+"With"](this,e?[a]:arguments)})}),a=null}).promise()},then:function(b,d,e){var f=0;function g(b,c,d,e){return function(){var h=this,i=arguments,j=function(){var a,j;if(!(b=f&&(d!==O&&(h=void 0,i=[a]),c.rejectWith(h,i))}};b?k():(r.Deferred.getStackHook&&(k.stackTrace=r.Deferred.getStackHook()),a.setTimeout(k))}}return r.Deferred(function(a){c[0][3].add(g(0,a,r.isFunction(e)?e:N,a.notifyWith)),c[1][3].add(g(0,a,r.isFunction(b)?b:N)),c[2][3].add(g(0,a,r.isFunction(d)?d:O))}).promise()},promise:function(a){return null!=a?r.extend(a,e):e}},f={};return r.each(c,function(a,b){var g=b[2],h=b[5];e[b[1]]=g.add,h&&g.add(function(){d=h},c[3-a][2].disable,c[0][2].lock),g.add(b[3].fire),f[b[0]]=function(){return f[b[0]+"With"](this===f?void 0:this,arguments),this},f[b[0]+"With"]=g.fireWith}),e.promise(f),b&&b.call(f,f),f},when:function(a){var b=arguments.length,c=b,d=Array(c),e=f.call(arguments),g=r.Deferred(),h=function(a){return function(c){d[a]=this,e[a]=arguments.length>1?f.call(arguments):c,--b||g.resolveWith(d,e)}};if(b<=1&&(P(a,g.done(h(c)).resolve,g.reject,!b),"pending"===g.state()||r.isFunction(e[c]&&e[c].then)))return g.then();while(c--)P(e[c],h(c),g.reject);return g.promise()}});var Q=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;r.Deferred.exceptionHook=function(b,c){a.console&&a.console.warn&&b&&Q.test(b.name)&&a.console.warn("jQuery.Deferred exception: "+b.message,b.stack,c)},r.readyException=function(b){a.setTimeout(function(){throw b})};var R=r.Deferred();r.fn.ready=function(a){return R.then(a)["catch"](function(a){r.readyException(a)}),this},r.extend({isReady:!1,readyWait:1,ready:function(a){(a===!0?--r.readyWait:r.isReady)||(r.isReady=!0,a!==!0&&--r.readyWait>0||R.resolveWith(d,[r]))}}),r.ready.then=R.then;function S(){d.removeEventListener("DOMContentLoaded",S), +a.removeEventListener("load",S),r.ready()}"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll?a.setTimeout(r.ready):(d.addEventListener("DOMContentLoaded",S),a.addEventListener("load",S));var T=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===r.type(c)){e=!0;for(h in c)T(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,r.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(r(a),c)})),b))for(;h1,null,!0)},removeData:function(a){return this.each(function(){X.remove(this,a)})}}),r.extend({queue:function(a,b,c){var d;if(a)return b=(b||"fx")+"queue",d=W.get(a,b),c&&(!d||Array.isArray(c)?d=W.access(a,b,r.makeArray(c)):d.push(c)),d||[]},dequeue:function(a,b){b=b||"fx";var c=r.queue(a,b),d=c.length,e=c.shift(),f=r._queueHooks(a,b),g=function(){r.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return W.get(a,c)||W.access(a,c,{empty:r.Callbacks("once memory").add(function(){W.remove(a,[b+"queue",c])})})}}),r.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.length\x20\t\r\n\f]+)/i,la=/^$|\/(?:java|ecma)script/i,ma={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};ma.optgroup=ma.option,ma.tbody=ma.tfoot=ma.colgroup=ma.caption=ma.thead,ma.th=ma.td;function na(a,b){var c;return c="undefined"!=typeof a.getElementsByTagName?a.getElementsByTagName(b||"*"):"undefined"!=typeof a.querySelectorAll?a.querySelectorAll(b||"*"):[],void 0===b||b&&B(a,b)?r.merge([a],c):c}function oa(a,b){for(var c=0,d=a.length;c-1)e&&e.push(f);else if(j=r.contains(f.ownerDocument,f),g=na(l.appendChild(f),"script"),j&&oa(g),c){k=0;while(f=g[k++])la.test(f.type||"")&&c.push(f)}return l}!function(){var a=d.createDocumentFragment(),b=a.appendChild(d.createElement("div")),c=d.createElement("input");c.setAttribute("type","radio"),c.setAttribute("checked","checked"),c.setAttribute("name","t"),b.appendChild(c),o.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,b.innerHTML="",o.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var ra=d.documentElement,sa=/^key/,ta=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,ua=/^([^.]*)(?:\.(.+)|)/;function va(){return!0}function wa(){return!1}function xa(){try{return d.activeElement}catch(a){}}function ya(a,b,c,d,e,f){var g,h;if("object"==typeof b){"string"!=typeof c&&(d=d||c,c=void 0);for(h in b)ya(a,h,c,d,b[h],f);return a}if(null==d&&null==e?(e=c,d=c=void 0):null==e&&("string"==typeof c?(e=d,d=void 0):(e=d,d=c,c=void 0)),e===!1)e=wa;else if(!e)return a;return 1===f&&(g=e,e=function(a){return r().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=r.guid++)),a.each(function(){r.event.add(this,b,e,d,c)})}r.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=W.get(a);if(q){c.handler&&(f=c,c=f.handler,e=f.selector),e&&r.find.matchesSelector(ra,e),c.guid||(c.guid=r.guid++),(i=q.events)||(i=q.events={}),(g=q.handle)||(g=q.handle=function(b){return"undefined"!=typeof r&&r.event.triggered!==b.type?r.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(L)||[""],j=b.length;while(j--)h=ua.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n&&(l=r.event.special[n]||{},n=(e?l.delegateType:l.bindType)||n,l=r.event.special[n]||{},k=r.extend({type:n,origType:p,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&r.expr.match.needsContext.test(e),namespace:o.join(".")},f),(m=i[n])||(m=i[n]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,o,g)!==!1||a.addEventListener&&a.addEventListener(n,g)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),r.event.global[n]=!0)}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=W.hasData(a)&&W.get(a);if(q&&(i=q.events)){b=(b||"").match(L)||[""],j=b.length;while(j--)if(h=ua.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n){l=r.event.special[n]||{},n=(d?l.delegateType:l.bindType)||n,m=i[n]||[],h=h[2]&&new RegExp("(^|\\.)"+o.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&p!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,o,q.handle)!==!1||r.removeEvent(a,n,q.handle),delete i[n])}else for(n in i)r.event.remove(a,n+b[j],c,d,!0);r.isEmptyObject(i)&&W.remove(a,"handle events")}},dispatch:function(a){var b=r.event.fix(a),c,d,e,f,g,h,i=new Array(arguments.length),j=(W.get(this,"events")||{})[b.type]||[],k=r.event.special[b.type]||{};for(i[0]=b,c=1;c=1))for(;j!==this;j=j.parentNode||this)if(1===j.nodeType&&("click"!==a.type||j.disabled!==!0)){for(f=[],g={},c=0;c-1:r.find(e,this,null,[j]).length),g[e]&&f.push(d);f.length&&h.push({elem:j,handlers:f})}return j=this,i\x20\t\r\n\f]*)[^>]*)\/>/gi,Aa=/\s*$/g;function Ea(a,b){return B(a,"table")&&B(11!==b.nodeType?b:b.firstChild,"tr")?r(">tbody",a)[0]||a:a}function Fa(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function Ga(a){var b=Ca.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Ha(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(W.hasData(a)&&(f=W.access(a),g=W.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;c1&&"string"==typeof q&&!o.checkClone&&Ba.test(q))return a.each(function(e){var f=a.eq(e);s&&(b[0]=q.call(this,e,f.html())),Ja(f,b,c,d)});if(m&&(e=qa(b,a[0].ownerDocument,!1,a,d),f=e.firstChild,1===e.childNodes.length&&(e=f),f||d)){for(h=r.map(na(e,"script"),Fa),i=h.length;l")},clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=r.contains(a.ownerDocument,a);if(!(o.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||r.isXMLDoc(a)))for(g=na(h),f=na(a),d=0,e=f.length;d0&&oa(g,!i&&na(a,"script")),h},cleanData:function(a){for(var b,c,d,e=r.event.special,f=0;void 0!==(c=a[f]);f++)if(U(c)){if(b=c[W.expando]){if(b.events)for(d in b.events)e[d]?r.event.remove(c,d):r.removeEvent(c,d,b.handle);c[W.expando]=void 0}c[X.expando]&&(c[X.expando]=void 0)}}}),r.fn.extend({detach:function(a){return Ka(this,a,!0)},remove:function(a){return Ka(this,a)},text:function(a){return T(this,function(a){return void 0===a?r.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=a)})},null,a,arguments.length)},append:function(){return Ja(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ea(this,a);b.appendChild(a)}})},prepend:function(){return Ja(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ea(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return Ja(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return Ja(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(r.cleanData(na(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null!=a&&a,b=null==b?a:b,this.map(function(){return r.clone(this,a,b)})},html:function(a){return T(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!Aa.test(a)&&!ma[(ka.exec(a)||["",""])[1].toLowerCase()]){a=r.htmlPrefilter(a);try{for(;c1)}});function _a(a,b,c,d,e){return new _a.prototype.init(a,b,c,d,e)}r.Tween=_a,_a.prototype={constructor:_a,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||r.easing._default,this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(r.cssNumber[c]?"":"px")},cur:function(){var a=_a.propHooks[this.prop];return a&&a.get?a.get(this):_a.propHooks._default.get(this)},run:function(a){var b,c=_a.propHooks[this.prop];return this.options.duration?this.pos=b=r.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):this.pos=b=a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):_a.propHooks._default.set(this),this}},_a.prototype.init.prototype=_a.prototype,_a.propHooks={_default:{get:function(a){var b;return 1!==a.elem.nodeType||null!=a.elem[a.prop]&&null==a.elem.style[a.prop]?a.elem[a.prop]:(b=r.css(a.elem,a.prop,""),b&&"auto"!==b?b:0)},set:function(a){r.fx.step[a.prop]?r.fx.step[a.prop](a):1!==a.elem.nodeType||null==a.elem.style[r.cssProps[a.prop]]&&!r.cssHooks[a.prop]?a.elem[a.prop]=a.now:r.style(a.elem,a.prop,a.now+a.unit)}}},_a.propHooks.scrollTop=_a.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},r.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2},_default:"swing"},r.fx=_a.prototype.init,r.fx.step={};var ab,bb,cb=/^(?:toggle|show|hide)$/,db=/queueHooks$/;function eb(){bb&&(d.hidden===!1&&a.requestAnimationFrame?a.requestAnimationFrame(eb):a.setTimeout(eb,r.fx.interval),r.fx.tick())}function fb(){return a.setTimeout(function(){ab=void 0}),ab=r.now()}function gb(a,b){var c,d=0,e={height:a};for(b=b?1:0;d<4;d+=2-b)c=ca[d],e["margin"+c]=e["padding"+c]=a;return b&&(e.opacity=e.width=a),e}function hb(a,b,c){for(var d,e=(kb.tweeners[b]||[]).concat(kb.tweeners["*"]),f=0,g=e.length;f1)},removeAttr:function(a){return this.each(function(){r.removeAttr(this,a)})}}),r.extend({attr:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return"undefined"==typeof a.getAttribute?r.prop(a,b,c):(1===f&&r.isXMLDoc(a)||(e=r.attrHooks[b.toLowerCase()]||(r.expr.match.bool.test(b)?lb:void 0)),void 0!==c?null===c?void r.removeAttr(a,b):e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:(a.setAttribute(b,c+""),c):e&&"get"in e&&null!==(d=e.get(a,b))?d:(d=r.find.attr(a,b), +null==d?void 0:d))},attrHooks:{type:{set:function(a,b){if(!o.radioValue&&"radio"===b&&B(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}}},removeAttr:function(a,b){var c,d=0,e=b&&b.match(L);if(e&&1===a.nodeType)while(c=e[d++])a.removeAttribute(c)}}),lb={set:function(a,b,c){return b===!1?r.removeAttr(a,c):a.setAttribute(c,c),c}},r.each(r.expr.match.bool.source.match(/\w+/g),function(a,b){var c=mb[b]||r.find.attr;mb[b]=function(a,b,d){var e,f,g=b.toLowerCase();return d||(f=mb[g],mb[g]=e,e=null!=c(a,b,d)?g:null,mb[g]=f),e}});var nb=/^(?:input|select|textarea|button)$/i,ob=/^(?:a|area)$/i;r.fn.extend({prop:function(a,b){return T(this,r.prop,a,b,arguments.length>1)},removeProp:function(a){return this.each(function(){delete this[r.propFix[a]||a]})}}),r.extend({prop:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return 1===f&&r.isXMLDoc(a)||(b=r.propFix[b]||b,e=r.propHooks[b]),void 0!==c?e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:a[b]=c:e&&"get"in e&&null!==(d=e.get(a,b))?d:a[b]},propHooks:{tabIndex:{get:function(a){var b=r.find.attr(a,"tabindex");return b?parseInt(b,10):nb.test(a.nodeName)||ob.test(a.nodeName)&&a.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),o.optSelected||(r.propHooks.selected={get:function(a){var b=a.parentNode;return b&&b.parentNode&&b.parentNode.selectedIndex,null},set:function(a){var b=a.parentNode;b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex)}}),r.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){r.propFix[this.toLowerCase()]=this});function pb(a){var b=a.match(L)||[];return b.join(" ")}function qb(a){return a.getAttribute&&a.getAttribute("class")||""}r.fn.extend({addClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).addClass(a.call(this,b,qb(this)))});if("string"==typeof a&&a){b=a.match(L)||[];while(c=this[i++])if(e=qb(c),d=1===c.nodeType&&" "+pb(e)+" "){g=0;while(f=b[g++])d.indexOf(" "+f+" ")<0&&(d+=f+" ");h=pb(d),e!==h&&c.setAttribute("class",h)}}return this},removeClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).removeClass(a.call(this,b,qb(this)))});if(!arguments.length)return this.attr("class","");if("string"==typeof a&&a){b=a.match(L)||[];while(c=this[i++])if(e=qb(c),d=1===c.nodeType&&" "+pb(e)+" "){g=0;while(f=b[g++])while(d.indexOf(" "+f+" ")>-1)d=d.replace(" "+f+" "," ");h=pb(d),e!==h&&c.setAttribute("class",h)}}return this},toggleClass:function(a,b){var c=typeof a;return"boolean"==typeof b&&"string"===c?b?this.addClass(a):this.removeClass(a):r.isFunction(a)?this.each(function(c){r(this).toggleClass(a.call(this,c,qb(this),b),b)}):this.each(function(){var b,d,e,f;if("string"===c){d=0,e=r(this),f=a.match(L)||[];while(b=f[d++])e.hasClass(b)?e.removeClass(b):e.addClass(b)}else void 0!==a&&"boolean"!==c||(b=qb(this),b&&W.set(this,"__className__",b),this.setAttribute&&this.setAttribute("class",b||a===!1?"":W.get(this,"__className__")||""))})},hasClass:function(a){var b,c,d=0;b=" "+a+" ";while(c=this[d++])if(1===c.nodeType&&(" "+pb(qb(c))+" ").indexOf(b)>-1)return!0;return!1}});var rb=/\r/g;r.fn.extend({val:function(a){var b,c,d,e=this[0];{if(arguments.length)return d=r.isFunction(a),this.each(function(c){var e;1===this.nodeType&&(e=d?a.call(this,c,r(this).val()):a,null==e?e="":"number"==typeof e?e+="":Array.isArray(e)&&(e=r.map(e,function(a){return null==a?"":a+""})),b=r.valHooks[this.type]||r.valHooks[this.nodeName.toLowerCase()],b&&"set"in b&&void 0!==b.set(this,e,"value")||(this.value=e))});if(e)return b=r.valHooks[e.type]||r.valHooks[e.nodeName.toLowerCase()],b&&"get"in b&&void 0!==(c=b.get(e,"value"))?c:(c=e.value,"string"==typeof c?c.replace(rb,""):null==c?"":c)}}}),r.extend({valHooks:{option:{get:function(a){var b=r.find.attr(a,"value");return null!=b?b:pb(r.text(a))}},select:{get:function(a){var b,c,d,e=a.options,f=a.selectedIndex,g="select-one"===a.type,h=g?null:[],i=g?f+1:e.length;for(d=f<0?i:g?f:0;d-1)&&(c=!0);return c||(a.selectedIndex=-1),f}}}}),r.each(["radio","checkbox"],function(){r.valHooks[this]={set:function(a,b){if(Array.isArray(b))return a.checked=r.inArray(r(a).val(),b)>-1}},o.checkOn||(r.valHooks[this].get=function(a){return null===a.getAttribute("value")?"on":a.value})});var sb=/^(?:focusinfocus|focusoutblur)$/;r.extend(r.event,{trigger:function(b,c,e,f){var g,h,i,j,k,m,n,o=[e||d],p=l.call(b,"type")?b.type:b,q=l.call(b,"namespace")?b.namespace.split("."):[];if(h=i=e=e||d,3!==e.nodeType&&8!==e.nodeType&&!sb.test(p+r.event.triggered)&&(p.indexOf(".")>-1&&(q=p.split("."),p=q.shift(),q.sort()),k=p.indexOf(":")<0&&"on"+p,b=b[r.expando]?b:new r.Event(p,"object"==typeof b&&b),b.isTrigger=f?2:3,b.namespace=q.join("."),b.rnamespace=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=e),c=null==c?[b]:r.makeArray(c,[b]),n=r.event.special[p]||{},f||!n.trigger||n.trigger.apply(e,c)!==!1)){if(!f&&!n.noBubble&&!r.isWindow(e)){for(j=n.delegateType||p,sb.test(j+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),i=h;i===(e.ownerDocument||d)&&o.push(i.defaultView||i.parentWindow||a)}g=0;while((h=o[g++])&&!b.isPropagationStopped())b.type=g>1?j:n.bindType||p,m=(W.get(h,"events")||{})[b.type]&&W.get(h,"handle"),m&&m.apply(h,c),m=k&&h[k],m&&m.apply&&U(h)&&(b.result=m.apply(h,c),b.result===!1&&b.preventDefault());return b.type=p,f||b.isDefaultPrevented()||n._default&&n._default.apply(o.pop(),c)!==!1||!U(e)||k&&r.isFunction(e[p])&&!r.isWindow(e)&&(i=e[k],i&&(e[k]=null),r.event.triggered=p,e[p](),r.event.triggered=void 0,i&&(e[k]=i)),b.result}},simulate:function(a,b,c){var d=r.extend(new r.Event,c,{type:a,isSimulated:!0});r.event.trigger(d,null,b)}}),r.fn.extend({trigger:function(a,b){return this.each(function(){r.event.trigger(a,b,this)})},triggerHandler:function(a,b){var c=this[0];if(c)return r.event.trigger(a,b,c,!0)}}),r.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(a,b){r.fn[b]=function(a,c){return arguments.length>0?this.on(b,null,a,c):this.trigger(b)}}),r.fn.extend({hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)}}),o.focusin="onfocusin"in a,o.focusin||r.each({focus:"focusin",blur:"focusout"},function(a,b){var c=function(a){r.event.simulate(b,a.target,r.event.fix(a))};r.event.special[b]={setup:function(){var d=this.ownerDocument||this,e=W.access(d,b);e||d.addEventListener(a,c,!0),W.access(d,b,(e||0)+1)},teardown:function(){var d=this.ownerDocument||this,e=W.access(d,b)-1;e?W.access(d,b,e):(d.removeEventListener(a,c,!0),W.remove(d,b))}}});var tb=a.location,ub=r.now(),vb=/\?/;r.parseXML=function(b){var c;if(!b||"string"!=typeof b)return null;try{c=(new a.DOMParser).parseFromString(b,"text/xml")}catch(d){c=void 0}return c&&!c.getElementsByTagName("parsererror").length||r.error("Invalid XML: "+b),c};var wb=/\[\]$/,xb=/\r?\n/g,yb=/^(?:submit|button|image|reset|file)$/i,zb=/^(?:input|select|textarea|keygen)/i;function Ab(a,b,c,d){var e;if(Array.isArray(b))r.each(b,function(b,e){c||wb.test(a)?d(a,e):Ab(a+"["+("object"==typeof e&&null!=e?b:"")+"]",e,c,d)});else if(c||"object"!==r.type(b))d(a,b);else for(e in b)Ab(a+"["+e+"]",b[e],c,d)}r.param=function(a,b){var c,d=[],e=function(a,b){var c=r.isFunction(b)?b():b;d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(null==c?"":c)};if(Array.isArray(a)||a.jquery&&!r.isPlainObject(a))r.each(a,function(){e(this.name,this.value)});else for(c in a)Ab(c,a[c],b,e);return d.join("&")},r.fn.extend({serialize:function(){return r.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var a=r.prop(this,"elements");return a?r.makeArray(a):this}).filter(function(){var a=this.type;return this.name&&!r(this).is(":disabled")&&zb.test(this.nodeName)&&!yb.test(a)&&(this.checked||!ja.test(a))}).map(function(a,b){var c=r(this).val();return null==c?null:Array.isArray(c)?r.map(c,function(a){return{name:b.name,value:a.replace(xb,"\r\n")}}):{name:b.name,value:c.replace(xb,"\r\n")}}).get()}});var Bb=/%20/g,Cb=/#.*$/,Db=/([?&])_=[^&]*/,Eb=/^(.*?):[ \t]*([^\r\n]*)$/gm,Fb=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Gb=/^(?:GET|HEAD)$/,Hb=/^\/\//,Ib={},Jb={},Kb="*/".concat("*"),Lb=d.createElement("a");Lb.href=tb.href;function Mb(a){return function(b,c){"string"!=typeof b&&(c=b,b="*");var d,e=0,f=b.toLowerCase().match(L)||[];if(r.isFunction(c))while(d=f[e++])"+"===d[0]?(d=d.slice(1)||"*",(a[d]=a[d]||[]).unshift(c)):(a[d]=a[d]||[]).push(c)}}function Nb(a,b,c,d){var e={},f=a===Jb;function g(h){var i;return e[h]=!0,r.each(a[h]||[],function(a,h){var j=h(b,c,d);return"string"!=typeof j||f||e[j]?f?!(i=j):void 0:(b.dataTypes.unshift(j),g(j),!1)}),i}return g(b.dataTypes[0])||!e["*"]&&g("*")}function Ob(a,b){var c,d,e=r.ajaxSettings.flatOptions||{};for(c in b)void 0!==b[c]&&((e[c]?a:d||(d={}))[c]=b[c]);return d&&r.extend(!0,a,d),a}function Pb(a,b,c){var d,e,f,g,h=a.contents,i=a.dataTypes;while("*"===i[0])i.shift(),void 0===d&&(d=a.mimeType||b.getResponseHeader("Content-Type"));if(d)for(e in h)if(h[e]&&h[e].test(d)){i.unshift(e);break}if(i[0]in c)f=i[0];else{for(e in c){if(!i[0]||a.converters[e+" "+i[0]]){f=e;break}g||(g=e)}f=f||g}if(f)return f!==i[0]&&i.unshift(f),c[f]}function Qb(a,b,c,d){var e,f,g,h,i,j={},k=a.dataTypes.slice();if(k[1])for(g in a.converters)j[g.toLowerCase()]=a.converters[g];f=k.shift();while(f)if(a.responseFields[f]&&(c[a.responseFields[f]]=b),!i&&d&&a.dataFilter&&(b=a.dataFilter(b,a.dataType)),i=f,f=k.shift())if("*"===f)f=i;else if("*"!==i&&i!==f){if(g=j[i+" "+f]||j["* "+f],!g)for(e in j)if(h=e.split(" "),h[1]===f&&(g=j[i+" "+h[0]]||j["* "+h[0]])){g===!0?g=j[e]:j[e]!==!0&&(f=h[0],k.unshift(h[1]));break}if(g!==!0)if(g&&a["throws"])b=g(b);else try{b=g(b)}catch(l){return{state:"parsererror",error:g?l:"No conversion from "+i+" to "+f}}}return{state:"success",data:b}}r.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:tb.href,type:"GET",isLocal:Fb.test(tb.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Kb,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":r.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(a,b){return b?Ob(Ob(a,r.ajaxSettings),b):Ob(r.ajaxSettings,a)},ajaxPrefilter:Mb(Ib),ajaxTransport:Mb(Jb),ajax:function(b,c){"object"==typeof b&&(c=b,b=void 0),c=c||{};var e,f,g,h,i,j,k,l,m,n,o=r.ajaxSetup({},c),p=o.context||o,q=o.context&&(p.nodeType||p.jquery)?r(p):r.event,s=r.Deferred(),t=r.Callbacks("once memory"),u=o.statusCode||{},v={},w={},x="canceled",y={readyState:0,getResponseHeader:function(a){var b;if(k){if(!h){h={};while(b=Eb.exec(g))h[b[1].toLowerCase()]=b[2]}b=h[a.toLowerCase()]}return null==b?null:b},getAllResponseHeaders:function(){return k?g:null},setRequestHeader:function(a,b){return null==k&&(a=w[a.toLowerCase()]=w[a.toLowerCase()]||a,v[a]=b),this},overrideMimeType:function(a){return null==k&&(o.mimeType=a),this},statusCode:function(a){var b;if(a)if(k)y.always(a[y.status]);else for(b in a)u[b]=[u[b],a[b]];return this},abort:function(a){var b=a||x;return e&&e.abort(b),A(0,b),this}};if(s.promise(y),o.url=((b||o.url||tb.href)+"").replace(Hb,tb.protocol+"//"),o.type=c.method||c.type||o.method||o.type,o.dataTypes=(o.dataType||"*").toLowerCase().match(L)||[""],null==o.crossDomain){j=d.createElement("a");try{j.href=o.url,j.href=j.href,o.crossDomain=Lb.protocol+"//"+Lb.host!=j.protocol+"//"+j.host}catch(z){o.crossDomain=!0}}if(o.data&&o.processData&&"string"!=typeof o.data&&(o.data=r.param(o.data,o.traditional)),Nb(Ib,o,c,y),k)return y;l=r.event&&o.global,l&&0===r.active++&&r.event.trigger("ajaxStart"),o.type=o.type.toUpperCase(),o.hasContent=!Gb.test(o.type),f=o.url.replace(Cb,""),o.hasContent?o.data&&o.processData&&0===(o.contentType||"").indexOf("application/x-www-form-urlencoded")&&(o.data=o.data.replace(Bb,"+")):(n=o.url.slice(f.length),o.data&&(f+=(vb.test(f)?"&":"?")+o.data,delete o.data),o.cache===!1&&(f=f.replace(Db,"$1"),n=(vb.test(f)?"&":"?")+"_="+ub++ +n),o.url=f+n),o.ifModified&&(r.lastModified[f]&&y.setRequestHeader("If-Modified-Since",r.lastModified[f]),r.etag[f]&&y.setRequestHeader("If-None-Match",r.etag[f])),(o.data&&o.hasContent&&o.contentType!==!1||c.contentType)&&y.setRequestHeader("Content-Type",o.contentType),y.setRequestHeader("Accept",o.dataTypes[0]&&o.accepts[o.dataTypes[0]]?o.accepts[o.dataTypes[0]]+("*"!==o.dataTypes[0]?", "+Kb+"; q=0.01":""):o.accepts["*"]);for(m in o.headers)y.setRequestHeader(m,o.headers[m]);if(o.beforeSend&&(o.beforeSend.call(p,y,o)===!1||k))return y.abort();if(x="abort",t.add(o.complete),y.done(o.success),y.fail(o.error),e=Nb(Jb,o,c,y)){if(y.readyState=1,l&&q.trigger("ajaxSend",[y,o]),k)return y;o.async&&o.timeout>0&&(i=a.setTimeout(function(){y.abort("timeout")},o.timeout));try{k=!1,e.send(v,A)}catch(z){if(k)throw z;A(-1,z)}}else A(-1,"No Transport");function A(b,c,d,h){var j,m,n,v,w,x=c;k||(k=!0,i&&a.clearTimeout(i),e=void 0,g=h||"",y.readyState=b>0?4:0,j=b>=200&&b<300||304===b,d&&(v=Pb(o,y,d)),v=Qb(o,v,y,j),j?(o.ifModified&&(w=y.getResponseHeader("Last-Modified"),w&&(r.lastModified[f]=w),w=y.getResponseHeader("etag"),w&&(r.etag[f]=w)),204===b||"HEAD"===o.type?x="nocontent":304===b?x="notmodified":(x=v.state,m=v.data,n=v.error,j=!n)):(n=x,!b&&x||(x="error",b<0&&(b=0))),y.status=b,y.statusText=(c||x)+"",j?s.resolveWith(p,[m,x,y]):s.rejectWith(p,[y,x,n]),y.statusCode(u),u=void 0,l&&q.trigger(j?"ajaxSuccess":"ajaxError",[y,o,j?m:n]),t.fireWith(p,[y,x]),l&&(q.trigger("ajaxComplete",[y,o]),--r.active||r.event.trigger("ajaxStop")))}return y},getJSON:function(a,b,c){return r.get(a,b,c,"json")},getScript:function(a,b){return r.get(a,void 0,b,"script")}}),r.each(["get","post"],function(a,b){r[b]=function(a,c,d,e){return r.isFunction(c)&&(e=e||d,d=c,c=void 0),r.ajax(r.extend({url:a,type:b,dataType:e,data:c,success:d},r.isPlainObject(a)&&a))}}),r._evalUrl=function(a){return r.ajax({url:a,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},r.fn.extend({wrapAll:function(a){var b;return this[0]&&(r.isFunction(a)&&(a=a.call(this[0])),b=r(a,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstElementChild)a=a.firstElementChild;return a}).append(this)),this},wrapInner:function(a){return r.isFunction(a)?this.each(function(b){r(this).wrapInner(a.call(this,b))}):this.each(function(){var b=r(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=r.isFunction(a);return this.each(function(c){r(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(a){return this.parent(a).not("body").each(function(){r(this).replaceWith(this.childNodes)}),this}}),r.expr.pseudos.hidden=function(a){return!r.expr.pseudos.visible(a)},r.expr.pseudos.visible=function(a){return!!(a.offsetWidth||a.offsetHeight||a.getClientRects().length)},r.ajaxSettings.xhr=function(){try{return new a.XMLHttpRequest}catch(b){}};var Rb={0:200,1223:204},Sb=r.ajaxSettings.xhr();o.cors=!!Sb&&"withCredentials"in Sb,o.ajax=Sb=!!Sb,r.ajaxTransport(function(b){var c,d;if(o.cors||Sb&&!b.crossDomain)return{send:function(e,f){var g,h=b.xhr();if(h.open(b.type,b.url,b.async,b.username,b.password),b.xhrFields)for(g in b.xhrFields)h[g]=b.xhrFields[g];b.mimeType&&h.overrideMimeType&&h.overrideMimeType(b.mimeType),b.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest");for(g in e)h.setRequestHeader(g,e[g]);c=function(a){return function(){c&&(c=d=h.onload=h.onerror=h.onabort=h.onreadystatechange=null,"abort"===a?h.abort():"error"===a?"number"!=typeof h.status?f(0,"error"):f(h.status,h.statusText):f(Rb[h.status]||h.status,h.statusText,"text"!==(h.responseType||"text")||"string"!=typeof h.responseText?{binary:h.response}:{text:h.responseText},h.getAllResponseHeaders()))}},h.onload=c(),d=h.onerror=c("error"),void 0!==h.onabort?h.onabort=d:h.onreadystatechange=function(){4===h.readyState&&a.setTimeout(function(){c&&d()})},c=c("abort");try{h.send(b.hasContent&&b.data||null)}catch(i){if(c)throw i}},abort:function(){c&&c()}}}),r.ajaxPrefilter(function(a){a.crossDomain&&(a.contents.script=!1)}),r.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(a){return r.globalEval(a),a}}}),r.ajaxPrefilter("script",function(a){void 0===a.cache&&(a.cache=!1),a.crossDomain&&(a.type="GET")}),r.ajaxTransport("script",function(a){if(a.crossDomain){var b,c;return{send:function(e,f){b=r(" + + + +
+ {% csrf_token %} + + + + + + + + {% if download %} + 图包提取(文件大小:{{ size}}) +{# #} + {% endif %} +
+ +
+

{{ msg}}

+
+
    + {% for imageNode in imageNodes %} +
  1. +
    +

    标题：{{imageNode.illustTitle}}，画师：{{imageNode.userName}}，收藏数：{{imageNode.bookmarkCount}}

    + {% if imageNode.localName %} + + {% endif %} +
    +
    2. + {% empty %} +

    {{tip}}

    + {% endfor %} + +
+ + \ No newline at end of file diff --git a/PixivSearch/tsdm.py b/PixivSearch/tsdm.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/urls.py b/PixivSearch/urls.py new file mode 100644 index 0000000..db9a4b9 --- /dev/null +++ b/PixivSearch/urls.py @@ -0,0 +1,24 @@ +"""PixivSearch URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.11/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf.urls import url +from PixivSearch.view import search, index, download, haha + +urlpatterns = [ + url(r'^$', index), + url(r'^pixiv/search', search), + url(r'^pixiv/download', download), + url(r'^tsdm',haha) +] diff --git a/PixivSearch/view.py b/PixivSearch/view.py new file mode 100644 index 0000000..a1f227d --- /dev/null +++ b/PixivSearch/view.py @@ -0,0 +1,72 @@ +# coding=utf-8 +import os + +from django.http import Http404, StreamingHttpResponse, HttpResponse +from django.shortcuts import render + +from PixivSearch import pixiv +from PixivSearch.module import config +from PixivSearch.pixiv import get_nodes + +logger = pixiv.logging.getLogger('file') + + +def search(request): + if request.POST.get('word') != '': + word = request.POST.get('word') + else: + word = 'R-18' + if request.POST.get('pageSize') != '': + pageSize = request.POST.get('pageSize') + else: + pageSize = 10 + if request.POST.get('order') != '': + order = request.POST.get('order') + else: + order = 10 + if request.POST.get('type') != '': + type = request.POST.get('type') + else: + type = None + try: + logger.info("word:%s,pageSize:%d,order:%d,type:%s", word, pageSize, order, type) + array = get_nodes([0, word, pageSize, order, type]) + if len(array[0]) > 0: + if request.POST.get('type') == 'img': + nodes = {'imageNodes': array[0], 'msg': array[1], 'download': word + '.zip', 'size': array[2]} + else: + nodes = {'imageNodes': array[0], 'msg': array[1]} + else: + nodes = {'tip': '没有返回结果'} + return render(request, 'test.html', nodes) + except ValueError: + raise Http404() + + +def index(request): + return render(request, 'test.html', {'tip': '输入参数进行搜索'}) + + +def download(request): + # do something... + def file_iterator(file_name, chunk_size=512): + with open(file_name) as f: + while True: + c = f.read(chunk_size) + if c: + yield c + else: + break + + fileName = os.path.join(os.path.dirname(__file__), 'static/download/').replace('\\', '/') + request.GET.get( + 'fileName').encode('utf-8') + response = StreamingHttpResponse(file_iterator(fileName)) + response['Content-Type'] = 'application/octet-stream' + response['Content-Disposition'] = 'attachment;filename="{0}"'.format(request.GET.get('fileName').encode('utf-8')) + return response + + +def haha(request): + p=config.param(param_name='123',param_value='456') + p.save() + return HttpResponse("%s" % (config.param.param_name)) diff --git a/PixivSearch/wsgi.py b/PixivSearch/wsgi.py new file mode 100644 index 0000000..f079af7 --- /dev/null +++ b/PixivSearch/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for PixivSearch project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..7df6bab --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") + try: + from django.core.management import execute_from_command_line + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise + execute_from_command_line(sys.argv) From bad777ad6c320e8a98e85fd7db77eba0ea49862a Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Thu, 22 Mar 2018 03:40:37 +0800 Subject: [PATCH 03/22] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 61 +++++++++ PixivSearch/model/config.py | 44 +++++++ PixivSearch/module/__init__.py | 0 PixivSearch/module/config.py | 5 - PixivSearch/pixiv.py | 4 +- PixivSearch/settings.py | 9 +- PixivSearch/templates/addConfig.html | 27 ++++ .../templates/{test.html => index.html} | 124 +++++++++--------- PixivSearch/urls.py | 7 +- PixivSearch/view.py | 48 +++++-- PixivSearch/wsgi.py | 2 +- 11 files changed, 248 insertions(+), 83 deletions(-) create mode 100644 PixivSearch/dao/bangumi.py create mode 100644 PixivSearch/model/config.py delete mode 100644 PixivSearch/module/__init__.py delete mode 100644 PixivSearch/module/config.py create mode 100644 PixivSearch/templates/addConfig.html rename PixivSearch/templates/{test.html => index.html} (96%) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py new file mode 100644 index 0000000..471f1c1 --- /dev/null +++ b/PixivSearch/dao/bangumi.py @@ -0,0 +1,61 @@ +import json +import os +import time + +import django +import requests +import threading +from bs4 import BeautifulSoup + +from PixivSearch.settings import logger + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") +django.setup() +from PixivSearch.model.config import mediaInfo, stat + +current_mediaInfo = mediaInfo(id=0, chn_name='null') +flag = True + + +class bangumi(threading.Thread): + begin = 0 + end = 0 + id = 0 + + def __init__(self, begin, end): + threading.Thread.__init__(self) + self.begin = begin + self.end = end + + def save(self): + req = requests.get("https://www.bilibili.com/bangumi/media/md%d" % self.id) + if (req.status_code == 200): + tag = BeautifulSoup(req.text, 'lxml') + script = tag.select("script")[3].text + json_str = script[script.index("=") + 1:script.index("function") - 2] + json_obj = json.loads(json_str) + stat_info = json_obj['mediaInfo']['stat'] + print(json_obj['mediaInfo']['chn_name']) + print(stat_info) + mediaInfo(id=self.id, chn_name=json_obj['mediaInfo']['chn_name']).save() + global current_mediaInfo + current_mediaInfo = mediaInfo.objects.get(pk=self.id) + stat(id=self.id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], + views=stat_info['views']).save() + + def run(self) -> None: + self.go(self.begin, self.end) + + def get(self): + return current_mediaInfo + + def go(self, start, end): + for num in range(start, end): + if (flag): + time.sleep(1) + logger.info("爬虫进度：%d" % num) + self.id = num + self.save() + else: + logger.info("停止爬虫") + break diff --git a/PixivSearch/model/config.py b/PixivSearch/model/config.py new file mode 100644 index 0000000..2b8e7b0 --- /dev/null +++ b/PixivSearch/model/config.py @@ -0,0 +1,44 @@ +from django.db import models +import json + + +class param(models.Model): + param_name = models.CharField(max_length=10, primary_key=True) + param_value = models.CharField(max_length=128) + + +class stat(models.Model): + id = models.IntegerField(primary_key=True) + danmakus = models.IntegerField() + favorites = models.IntegerField() + views = models.IntegerField() + + # def json(self): + # return bangumi_stat(self['danmakus'],self['favorites'],self['views']) + + +class mediaInfo(models.Model): + id = models.IntegerField(primary_key=True) + chn_name = models.CharField(max_length=128) + + def __str__(self) -> str: + i = {} + i['media_id'] = self.id + i['chn_name'] = self.chn_name + return i + + +class bangumi_list(models.Model): + season_id = models.IntegerField(primary_key=True) + badge = models.CharField(max_length=128) + brief = models.CharField(max_length=128) + copyright = models.CharField(max_length=128) + cover = models.CharField(max_length=128) + favorites = models.IntegerField() + is_finish = models.IntegerField() + newest_ep_index = models.IntegerField() + pub_time = models.DateTimeField() + season_status = models.IntegerField() + title = models.CharField(max_length=128) + total_count = models.IntegerField() + trailer_aid = models.IntegerField() diff --git a/PixivSearch/module/__init__.py b/PixivSearch/module/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/PixivSearch/module/config.py b/PixivSearch/module/config.py deleted file mode 100644 index 2eb0653..0000000 --- a/PixivSearch/module/config.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.db import models - -class param(models.Model): - param_name = models.CharField(max_length=10,primary_key=True) - param_value = models.CharField(max_length=128) diff --git a/PixivSearch/pixiv.py b/PixivSearch/pixiv.py index 4a11cd5..1e3822d 100644 --- a/PixivSearch/pixiv.py +++ b/PixivSearch/pixiv.py @@ -11,6 +11,8 @@ import os import zipfile import logging.config +from PixivSearch.model import config + headers = { 'X-Requested-With': 'XMLHttpRequest', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' @@ -24,7 +26,7 @@ logger = logging.getLogger('file') def get_cookies(): _cookies = {} - array = "p_ab_id=5; p_ab_id_2=9; login_ever=yes; device_token=c8c37fdf24b917b4e7fb191fe11c5ca5; search_tools_toggle=1; _ga=GA1.2.887334537.1498628532; PHPSESSID=25745470_75a76e86ff3145b53e21b440183b4822; a_type=0; is_sensei_service_user=1; module_orders_mypage=%5B%7B%22name%22%3A%22recommended_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22everyone_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22following_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22mypixiv_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22fanbox%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22featured_tags%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22contests%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22sensei_courses%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22spotlight%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22booth_follow_items%22%2C%22visible%22%3Atrue%7D%5D; __utmt=1; __utma=235335808.887334537.1498628532.1498720739.1498819600.5; __utmb=235335808.1.10.1498819600; __utmc=235335808; __utmz=235335808.1498713152.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=235335808.|2=login%20ever=yes=1^3=plan=normal=1^5=gender=male=1^6=user_id=25745470=1^9=p_ab_id=5=1^10=p_ab_id_2=9=1^11=lang=zh_tw=1".split(';') + array = config.param.objects.get(param_name='pixiv_cookie').param_value.split(';') for row in array: k, v = row.strip().split('=', 1) _cookies[k] = v diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index 8dc8ad5..928f83d 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -9,7 +9,7 @@ https://docs.djangoproject.com/en/1.11/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ """ - +import logging.config import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -37,7 +37,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'PixivSearch.module', + 'PixivSearch.model' ] MIDDLEWARE = [ @@ -122,3 +122,8 @@ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'), ] + +curr_dir = os.path.dirname(os.path.realpath(__file__)) +logging.config.fileConfig('%s%slogging.conf' % (curr_dir,os.sep)) +logger = logging.getLogger('file') + diff --git a/PixivSearch/templates/addConfig.html b/PixivSearch/templates/addConfig.html new file mode 100644 index 0000000..63d9962 --- /dev/null +++ b/PixivSearch/templates/addConfig.html @@ -0,0 +1,27 @@ + + + + 参数管理 + {% load staticfiles %} + + + + +
+ {% csrf_token %} + + +
+ + +
+
+ + \ No newline at end of file diff --git a/PixivSearch/templates/test.html b/PixivSearch/templates/index.html similarity index 96% rename from PixivSearch/templates/test.html rename to PixivSearch/templates/index.html index 3561152..9d2d15b 100644 --- a/PixivSearch/templates/test.html +++ b/PixivSearch/templates/index.html @@ -1,63 +1,63 @@ - - - - {% load staticfiles %} - - - - -
- {% csrf_token %} - - - - - - - - {% if download %} - 图包提取(文件大小:{{ size}}) -{# #} - {% endif %} -
- -
-

{{ msg}}

-
-
    - {% for imageNode in imageNodes %} -
  1. -
    -

    标题：{{imageNode.illustTitle}}，画师：{{imageNode.userName}}，收藏数：{{imageNode.bookmarkCount}}

    - {% if imageNode.localName %} - - {% endif %} -
    -
    2. - {% empty %} -

    {{tip}}

    - {% endfor %} - -
- + + + + {% load staticfiles %} + + + + +
+ {% csrf_token %} + + + + + + + + {% if download %} + 图包提取(文件大小:{{ size}}) +{# #} + {% endif %} +
+ +
+

{{ msg}}

+
+
    + {% for imageNode in imageNodes %} +
  1. +
    +

    标题：{{imageNode.illustTitle}}，画师：{{imageNode.userName}}，收藏数：{{imageNode.bookmarkCount}}

    + {% if imageNode.localName %} + + {% endif %} +
    +
    2. + {% empty %} +

    {{tip}}

    + {% endfor %} + +
+ \ No newline at end of file diff --git a/PixivSearch/urls.py b/PixivSearch/urls.py index db9a4b9..5808d3f 100644 --- a/PixivSearch/urls.py +++ b/PixivSearch/urls.py @@ -14,11 +14,14 @@ Including another URLconf 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ from django.conf.urls import url -from PixivSearch.view import search, index, download, haha +from PixivSearch.view import search, index, download, saveConfig, save, get, start urlpatterns = [ url(r'^$', index), url(r'^pixiv/search', search), url(r'^pixiv/download', download), - url(r'^tsdm',haha) + url(r'^tsdm', saveConfig), + url(r'^bangumi/save', save), + url(r'^bangumi/get', get), + url(r'^bangumi/start', start), ] diff --git a/PixivSearch/view.py b/PixivSearch/view.py index a1f227d..b89f988 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -1,14 +1,18 @@ # coding=utf-8 import os +import django from django.http import Http404, StreamingHttpResponse, HttpResponse from django.shortcuts import render -from PixivSearch import pixiv -from PixivSearch.module import config -from PixivSearch.pixiv import get_nodes +from PixivSearch.dao.bangumi import bangumi +from PixivSearch.settings import logger + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") +django.setup() # 添加的代码 -logger = pixiv.logging.getLogger('file') +from PixivSearch.model import config +from PixivSearch.pixiv import get_nodes def search(request): @@ -38,13 +42,13 @@ def search(request): nodes = {'imageNodes': array[0], 'msg': array[1]} else: nodes = {'tip': '没有返回结果'} - return render(request, 'test.html', nodes) + return render(request, 'index.html', nodes) except ValueError: raise Http404() def index(request): - return render(request, 'test.html', {'tip': '输入参数进行搜索'}) + return render(request, 'index.html', {'tip': '输入参数进行搜索'}) def download(request): @@ -66,7 +70,31 @@ def download(request): return response -def haha(request): - p=config.param(param_name='123',param_value='456') - p.save() - return HttpResponse("%s" % (config.param.param_name)) +def saveConfig(request): + if 'GET' == request.method and request.GET.get('param_name'): + p = config.param.objects.get(param_name=request.GET.get('param_name')) + return render(request, 'addConfig.html', p.__dict__) + elif 'POST' == request.method and request.POST.get('param_name') and request.POST.get('param_value'): + p = config.param(param_name=request.POST.get('param_name'), param_value=request.POST.get('param_value')) + p.save() + return render(request, 'addConfig.html', p.__dict__) + else: + return render(request, 'addConfig.html') + + +def save(request): + media_id = int(request.GET.get('id')) + bangumi.save(media_id) + return HttpResponse("save success") + + +def get(request): + return HttpResponse(str(bangumi.get(None).__str__())) + + +def start(request): + begin = int(request.GET.get('start')) + end = int(request.GET.get('end')) + bangumi(begin=begin, end=end).start() + + return HttpResponse("start success") diff --git a/PixivSearch/wsgi.py b/PixivSearch/wsgi.py index f079af7..59e2109 100644 --- a/PixivSearch/wsgi.py +++ b/PixivSearch/wsgi.py @@ -1,7 +1,7 @@ """ WSGI config for PixivSearch project. -It exposes the WSGI callable as a module-level variable named ``application``. +It exposes the WSGI callable as a model-level variable named ``application``. For more information on this file, see https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ From bd556a441d62f8de6843d37887519bfcba9d13f5 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Thu, 22 Mar 2018 03:55:41 +0800 Subject: [PATCH 04/22] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 11 ++++++++++- PixivSearch/urls.py | 3 ++- PixivSearch/view.py | 6 +++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 471f1c1..6a3c3eb 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -14,6 +14,7 @@ django.setup() from PixivSearch.model.config import mediaInfo, stat current_mediaInfo = mediaInfo(id=0, chn_name='null') + flag = True @@ -21,6 +22,7 @@ class bangumi(threading.Thread): begin = 0 end = 0 id = 0 + flag = True def __init__(self, begin, end): threading.Thread.__init__(self) @@ -47,11 +49,18 @@ class bangumi(threading.Thread): self.go(self.begin, self.end) def get(self): + global current_mediaInfo return current_mediaInfo + def stop(self): + global flag + flag = False + def go(self, start, end): + global flag + flag = True for num in range(start, end): - if (flag): + if flag: time.sleep(1) logger.info("爬虫进度：%d" % num) self.id = num diff --git a/PixivSearch/urls.py b/PixivSearch/urls.py index 5808d3f..0b174fd 100644 --- a/PixivSearch/urls.py +++ b/PixivSearch/urls.py @@ -14,7 +14,7 @@ Including another URLconf 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ from django.conf.urls import url -from PixivSearch.view import search, index, download, saveConfig, save, get, start +from PixivSearch.view import search, index, download, saveConfig, save, get, start, stop urlpatterns = [ url(r'^$', index), @@ -24,4 +24,5 @@ urlpatterns = [ url(r'^bangumi/save', save), url(r'^bangumi/get', get), url(r'^bangumi/start', start), + url(r'^bangumi/stop', stop) ] diff --git a/PixivSearch/view.py b/PixivSearch/view.py index b89f988..b2baba1 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -89,7 +89,7 @@ def save(request): def get(request): - return HttpResponse(str(bangumi.get(None).__str__())) + return HttpResponse(str(bangumi.get(bangumi).__str__())) def start(request): @@ -98,3 +98,7 @@ def start(request): bangumi(begin=begin, end=end).start() return HttpResponse("start success") + +def stop(request): + bangumi.stop(bangumi) + return HttpResponse("stop success") \ No newline at end of file From 7312fa23153ea468c07541f06103e85d381ff3a9 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Sat, 24 Mar 2018 05:28:57 +0800 Subject: [PATCH 05/22] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 106 ++++++++++++++++++++++--------------- PixivSearch/logging.conf | 3 +- PixivSearch/settings.py | 12 +++-- PixivSearch/urls.py | 3 +- PixivSearch/view.py | 20 +++---- 5 files changed, 81 insertions(+), 63 deletions(-) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 6a3c3eb..863709f 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -1,10 +1,12 @@ +import _thread import json import os +import random import time +from concurrent import futures import django import requests -import threading from bs4 import BeautifulSoup from PixivSearch.settings import logger @@ -15,56 +17,72 @@ from PixivSearch.model.config import mediaInfo, stat current_mediaInfo = mediaInfo(id=0, chn_name='null') -flag = True +isStop = False +executors = None -class bangumi(threading.Thread): - begin = 0 - end = 0 - id = 0 - flag = True - def __init__(self, begin, end): - threading.Thread.__init__(self) - self.begin = begin - self.end = end +def check(): + while True: + if isStop: + logger.info('停止多线程爬虫') + executors.shutdown() + break + time.sleep(1) - def save(self): - req = requests.get("https://www.bilibili.com/bangumi/media/md%d" % self.id) - if (req.status_code == 200): - tag = BeautifulSoup(req.text, 'lxml') - script = tag.select("script")[3].text - json_str = script[script.index("=") + 1:script.index("function") - 2] - json_obj = json.loads(json_str) + + + + +def save(md): + if isStop: + return + time.sleep(random.randint(1, 3)) + url = "https://www.bilibili.com/bangumi/media/md%d" % md + req = requests.get(url) + + logger.info("request_url=%s,status_code=%d" % (url,req.status_code)) + if (req.status_code == 200): + tag = BeautifulSoup(req.text, 'lxml') + script = tag.select("script")[3].text + json_str = script[script.index("=") + 1:script.index("function") - 2] + json_obj = json.loads(json_str) + try: stat_info = json_obj['mediaInfo']['stat'] print(json_obj['mediaInfo']['chn_name']) print(stat_info) - mediaInfo(id=self.id, chn_name=json_obj['mediaInfo']['chn_name']).save() + mediaInfo(id=md, chn_name=json_obj['mediaInfo']['chn_name']).save() global current_mediaInfo - current_mediaInfo = mediaInfo.objects.get(pk=self.id) - stat(id=self.id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], + current_mediaInfo = mediaInfo.objects.get(pk=md) + stat(id=md, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], views=stat_info['views']).save() + except BaseException as e: + logger.error("发生异常") + logger.error(e) + +#asdasd +def get_(): + global current_mediaInfo + return current_mediaInfo + + +def threadSave(start, end): + ids = [] + + for id in range(start, end): + ids.append(id) + try: + global executors + executors = futures.ThreadPoolExecutor(32) + global isStop + isStop = False + with executors as executor: + executor.map(save, ids) + logger.info('结束爬虫') + except BaseException as e: + logger.error(e) + - def run(self) -> None: - self.go(self.begin, self.end) - - def get(self): - global current_mediaInfo - return current_mediaInfo - - def stop(self): - global flag - flag = False - - def go(self, start, end): - global flag - flag = True - for num in range(start, end): - if flag: - time.sleep(1) - logger.info("爬虫进度：%d" % num) - self.id = num - self.save() - else: - logger.info("停止爬虫") - break +def stop_(): + global isStop + isStop = True diff --git a/PixivSearch/logging.conf b/PixivSearch/logging.conf index 9203d73..005e412 100644 --- a/PixivSearch/logging.conf +++ b/PixivSearch/logging.conf @@ -29,4 +29,5 @@ formatter=fmt args=('pixiv.log','a','utf-8',False) [formatter_fmt] -format=%(asctime)s - %(module)s:%(lineno)d - %(levelname)s - %(message)s \ No newline at end of file +format=%(asctime)s - %(name)s - %(levelname)s - %(module)s :%(message)s +datefmt=%Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index 928f83d..c1f0004 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -76,8 +76,12 @@ WSGI_APPLICATION = 'PixivSearch.wsgi.application' DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'bangumi', + 'USER': 'bilibili', + 'PASSWORD': '2233', + 'HOST': '127.0.0.1', + 'PORT': '3306', } } @@ -106,13 +110,13 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = 'Asia/Shanghai' USE_I18N = True USE_L10N = True -USE_TZ = True +USE_TZ = False # Static files (CSS, JavaScript, Images) diff --git a/PixivSearch/urls.py b/PixivSearch/urls.py index 0b174fd..ea77a9d 100644 --- a/PixivSearch/urls.py +++ b/PixivSearch/urls.py @@ -14,14 +14,13 @@ Including another URLconf 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ from django.conf.urls import url -from PixivSearch.view import search, index, download, saveConfig, save, get, start, stop +from PixivSearch.view import search, index, download, saveConfig, get, start, stop urlpatterns = [ url(r'^$', index), url(r'^pixiv/search', search), url(r'^pixiv/download', download), url(r'^tsdm', saveConfig), - url(r'^bangumi/save', save), url(r'^bangumi/get', get), url(r'^bangumi/start', start), url(r'^bangumi/stop', stop) diff --git a/PixivSearch/view.py b/PixivSearch/view.py index b2baba1..e34dc73 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -1,11 +1,12 @@ # coding=utf-8 +import _thread import os import django from django.http import Http404, StreamingHttpResponse, HttpResponse from django.shortcuts import render -from PixivSearch.dao.bangumi import bangumi +from PixivSearch.dao.bangumi import threadSave, get_, stop_ from PixivSearch.settings import logger os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") @@ -82,23 +83,18 @@ def saveConfig(request): return render(request, 'addConfig.html') -def save(request): - media_id = int(request.GET.get('id')) - bangumi.save(media_id) - return HttpResponse("save success") - - def get(request): - return HttpResponse(str(bangumi.get(bangumi).__str__())) + return HttpResponse(str(get_().__str__())) +# 测试方法 def start(request): begin = int(request.GET.get('start')) end = int(request.GET.get('end')) - bangumi(begin=begin, end=end).start() - + _thread.start_new_thread(threadSave, (begin, end)) return HttpResponse("start success") + def stop(request): - bangumi.stop(bangumi) - return HttpResponse("stop success") \ No newline at end of file + stop_() + return HttpResponse("stop success") From d210153bdb3b6151b781b271edbf200a28b1ee0b Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Sat, 24 Mar 2018 15:03:40 +0800 Subject: [PATCH 06/22] =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B=E7=88=AC?= =?UTF-8?q?=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 53 +++++++++++++----- PixivSearch/logging.conf | 2 +- PixivSearch/pixiv.py | 107 +++++++++++++++++++------------------ PixivSearch/settings.py | 7 +-- PixivSearch/view.py | 4 +- 5 files changed, 103 insertions(+), 70 deletions(-) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 863709f..5915739 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -1,5 +1,6 @@ import _thread import json +import math import os import random import time @@ -31,17 +32,17 @@ def check(): time.sleep(1) - - - def save(md): if isStop: return - time.sleep(random.randint(1, 3)) - url = "https://www.bilibili.com/bangumi/media/md%d" % md - req = requests.get(url) - logger.info("request_url=%s,status_code=%d" % (url,req.status_code)) + url = "https://www.bilibili.com/bangumi/media/md%d" % md + try: + req = requests.get(url, timeout=10) + except BaseException as e: + logger.error(e) + save(md) + logger.info("request_url=%s,status_code=%d" % (url, req.status_code)) if (req.status_code == 200): tag = BeautifulSoup(req.text, 'lxml') script = tag.select("script")[3].text @@ -60,17 +61,45 @@ def save(md): logger.error("发生异常") logger.error(e) -#asdasd + +# asdasd def get_(): global current_mediaInfo return current_mediaInfo -def threadSave(start, end): - ids = [] +page_size = 100 +pages = None +ids = None + + +def getIds(): + global ids + global pages + if ids is None or len(ids) != 0: + ids = [] + page = 1 + while pages is None or page <= pages: + url = 'https://bangumi.bilibili.com/web_api/season/index_global?page=%d&page_size=%d' % (page, page_size) + logger.info(url) + try: + req = requests.get(url, timeout=10) + json_obj = json.loads(req.text) + bangumiList = json_obj['result']['list'] + for bangumi in bangumiList: + ids.append(int(bangumi['season_id'])) + if pages is None: + pages = int(math.ceil(int(json_obj['result']['count']) / page_size)) + except BaseException as e: + logger.error('连接超时') + logger(e) + continue + page = page + 1 + - for id in range(start, end): - ids.append(id) +def threadSave(): + getIds() + logger.info(len(ids)) try: global executors executors = futures.ThreadPoolExecutor(32) diff --git a/PixivSearch/logging.conf b/PixivSearch/logging.conf index 005e412..dc3325b 100644 --- a/PixivSearch/logging.conf +++ b/PixivSearch/logging.conf @@ -29,5 +29,5 @@ formatter=fmt args=('pixiv.log','a','utf-8',False) [formatter_fmt] -format=%(asctime)s - %(name)s - %(levelname)s - %(module)s :%(message)s +format=%(asctime)s - %(name)s - %(levelname)s - %(module)s:line:%(lineno)d - %(message)s datefmt=%Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/PixivSearch/pixiv.py b/PixivSearch/pixiv.py index 1e3822d..a3312b5 100644 --- a/PixivSearch/pixiv.py +++ b/PixivSearch/pixiv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -#coding:utf-8 +# coding:utf-8 from concurrent import futures import threading import json @@ -9,9 +9,9 @@ import sys from datetime import datetime import os import zipfile -import logging.config from PixivSearch.model import config +from PixivSearch.settings import logger headers = { 'X-Requested-With': 'XMLHttpRequest', @@ -19,10 +19,9 @@ headers = { 'Chrome/56.0.2924.87 Safari/537.36' } -lock = threading.Lock() # 多线程全局资源锁 +lock = threading.Lock() # 多线程全局资源锁 total = 1 -logging.config.fileConfig('PixivSearch/logging.conf') -logger = logging.getLogger('file') + def get_cookies(): _cookies = {} @@ -33,58 +32,65 @@ def get_cookies(): return _cookies - - def crawl(url): global total req = requests.get(url, headers=headers, cookies=get_cookies()).text - tag=BeautifulSoup(req, 'lxml').select('#js-mount-point-search-result-list')[0].attrs['data-items'] - imageNodes=json.loads(tag) + tag = BeautifulSoup(req, 'lxml').select('#js-mount-point-search-result-list')[0].attrs['data-items'] + imageNodes = json.loads(tag) for imageNode in imageNodes: with lock: nodes.append(imageNode) + def get_urls(search, page): fmt = 'https://www.pixiv.net/search.php?word={}&order=date_d&p={}' return [fmt.format(search, p) for p in range(1, page)] + def get_Img(params): - params[1]['imgUrl']='https://i.pximg.net/img-original/img/'+params[1]['url'][-46:-15] + params[1]['imgUrl'] = 'https://i.pximg.net/img-original/img/' + params[1]['url'][-46:-15] - headers['referer']='https://www.pixiv.net/member_illust.php?mode=medium&illust_id='+params[1]['illustId'] + headers['referer'] = 'https://www.pixiv.net/member_illust.php?mode=medium&illust_id=' + params[1]['illustId'] - suffix=".jpg" + suffix = ".jpg" logger.info('开始下载图片：%s%s' % (params[1]['imgUrl'], suffix)) - s=requests.get(params[1]['imgUrl']+suffix, headers=headers, cookies=get_cookies()) - if(s.status_code==404): - suffix='.png' - s=requests.get(params[1]['imgUrl']+suffix,headers=headers, cookies=get_cookies()) - if(s.status_code==404): + s = requests.get(params[1]['imgUrl'] + suffix, headers=headers, cookies=get_cookies()) + if (s.status_code == 404): + suffix = '.png' + s = requests.get(params[1]['imgUrl'] + suffix, headers=headers, cookies=get_cookies()) + if (s.status_code == 404): logger.error('无法下载图片：%s' % (params[1]['illustTitle'])) return - logger.info('下载图片："%s"到%s'%(params[1]['illustTitle'],os.getcwd().replace('\\','/')+'/'+imgPath+params[1]['illustId']+suffix)) - f = open(imgPath+params[1]['illustId']+suffix, 'wb') #写入多媒体文件要 b 这个参数 - f.write(s.content) #多媒体文件要是用conctent + logger.info('下载图片："%s"到%s' % ( + params[1]['illustTitle'], os.getcwd().replace('\\', '/') + '/' + imgPath + params[1]['illustId'] + suffix)) + f = open(imgPath + params[1]['illustId'] + suffix, 'wb') # 写入多媒体文件要 b 这个参数 + f.write(s.content) # 多媒体文件要是用conctent f.close() - params[1]['localName']=params[1]['illustId']+suffix - logger.info('排行第%d名，收藏数%d，标题：%s，标签：%s，(%s)前投稿，链接：%s'%(params[0],params[1]['bookmarkCount'],params[1]['illustTitle'], ','.join(params[1]['tags']),'',params[1]['imgUrl'])) + params[1]['localName'] = params[1]['illustId'] + suffix + logger.info('排行第%d名，收藏数%d，标题：%s，标签：%s，(%s)前投稿，链接：%s' % ( + params[0], params[1]['bookmarkCount'], params[1]['illustTitle'], ','.join(params[1]['tags']), '', + params[1]['imgUrl'])) -def zip(inputFile,outFile): - f = zipfile.ZipFile(outFile,'w',zipfile.ZIP_DEFLATED) + +def zip(inputFile, outFile): + f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED) for dirpath, dirnames, filenames in os.walk(inputFile): for filename in filenames: - f.write(os.path.join(dirpath,filename),filename) + f.write(os.path.join(dirpath, filename), filename) f.close() -fsize='' + +fsize = '' + + def get_nodes(param): - global nodes,fsize,imgPath - nodes=[] + global nodes, fsize, imgPath + nodes = [] start = datetime.now() - urls = get_urls(param[1], int(param[2])+1) + urls = get_urls(param[1], int(param[2]) + 1) logger.info('开始从P站获取图片数据') with futures.ThreadPoolExecutor(32) as executor: executor.map(crawl, urls) @@ -92,43 +98,42 @@ def get_nodes(param): # for url in urls: # crawl(url) - length=len(nodes) - logger.info('获取到%d张图片'%(length)) + length = len(nodes) + logger.info('获取到%d张图片' % (length)) logger.info('对图片收藏数进行排序') - nodes=sorted(nodes, key=lambda v: v.get('bookmarkCount'), reverse=True)[:int(param[3])] # 按star数降序排序 - if(param[4]!=None and param[4]=='img'): - imgPath='PixivSearch/static/images/' + nodes = sorted(nodes, key=lambda v: v.get('bookmarkCount'), reverse=True)[:int(param[3])] # 按star数降序排序 + if (param[4] != None and param[4] == 'img'): + imgPath = 'PixivSearch/static/images/' for file in os.listdir(imgPath): - os.remove(imgPath+file) - nodes_tup=[] - start_d=datetime.now() - for index,img in enumerate(nodes): - nodes_tup.append((index+1,img)) + os.remove(imgPath + file) + nodes_tup = [] + start_d = datetime.now() + for index, img in enumerate(nodes): + nodes_tup.append((index + 1, img)) # get_Img((index+1,img)) with futures.ThreadPoolExecutor(32) as executor: - executor.map(get_Img, nodes_tup) + executor.map(get_Img, nodes_tup) print('下载图片花费时间：%s' % (datetime.now() - start_d)) - logger.info('%s张图片下载完毕'%(len(os.listdir(imgPath)))) + logger.info('%s张图片下载完毕' % (len(os.listdir(imgPath)))) - zipPath='PixivSearch/static/download/' + param[1] + '.zip' + zipPath = 'PixivSearch/static/download/' + param[1] + '.zip' logger.info('图片打包到：%s' % (zipPath)) - zip(imgPath,zipPath) - fsize = str(round(os.path.getsize(zipPath)/float(1024*1024),2))+'MB' - logger.info('图包大小：%s'%(fsize)) + zip(imgPath, zipPath) + fsize = str(round(os.path.getsize(zipPath) / float(1024 * 1024), 2)) + 'MB' + logger.info('图包大小：%s' % (fsize)) - tip='从%d张图片中筛选出收藏数前%s的图片,处理耗时：%s'%(length,param[3],datetime.now()-start) + tip = '从%d张图片中筛选出收藏数前%s的图片,处理耗时：%s' % (length, param[3], datetime.now() - start) logger.info(tip) - return [nodes,tip,fsize] - + return [nodes, tip, fsize] if __name__ == "__main__": - if (len(sys.argv))==5 and sys.argv[2].isdigit() and sys.argv[3].isdigit(): + if (len(sys.argv)) == 5 and sys.argv[2].isdigit() and sys.argv[3].isdigit(): try: get_nodes(sys.argv) except BaseException as e: repr(e) - else : - logger.error('参数不合法') \ No newline at end of file + else: + logger.error('参数不合法') diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index c1f0004..717f957 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -80,7 +80,8 @@ DATABASES = { 'NAME': 'bangumi', 'USER': 'bilibili', 'PASSWORD': '2233', - 'HOST': '127.0.0.1', + # 'HOST': '127.0.0.1', + 'HOST': 'mikuhime.xyz', 'PORT': '3306', } } @@ -127,7 +128,7 @@ STATICFILES_DIRS = [ os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'), ] -curr_dir = os.path.dirname(os.path.realpath(__file__)) -logging.config.fileConfig('%s%slogging.conf' % (curr_dir,os.sep)) +configPath = '%s/logging.conf' % os.path.dirname(__file__).replace('\\', '/') +logging.config.fileConfig(configPath) logger = logging.getLogger('file') diff --git a/PixivSearch/view.py b/PixivSearch/view.py index e34dc73..e0659b2 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -89,9 +89,7 @@ def get(request): # 测试方法 def start(request): - begin = int(request.GET.get('start')) - end = int(request.GET.get('end')) - _thread.start_new_thread(threadSave, (begin, end)) + _thread.start_new_thread(threadSave, ()) return HttpResponse("start success") From 09b53c1bb5d558391b73c8a8ae1e687d83c7456e Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Sun, 25 Mar 2018 00:55:21 +0800 Subject: [PATCH 07/22] =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B=E7=88=AC?= =?UTF-8?q?=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 146 +++++++++++++++++++++--------------- PixivSearch/model/config.py | 9 +-- PixivSearch/view.py | 4 +- 3 files changed, 90 insertions(+), 69 deletions(-) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 5915739..a66c738 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -2,9 +2,8 @@ import _thread import json import math import os -import random -import time from concurrent import futures +from queue import Queue import django import requests @@ -16,102 +15,125 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") django.setup() from PixivSearch.model.config import mediaInfo, stat -current_mediaInfo = mediaInfo(id=0, chn_name='null') +current_mediaInfo = None -isStop = False +isStop = None -executors = None - -def check(): - while True: - if isStop: - logger.info('停止多线程爬虫') - executors.shutdown() - break - time.sleep(1) +def stop_(): + global isStop + isStop = True -def save(md): +def save(params): if isStop: return + logger.info(params) + bangumi_id = params[0] + season_id = params[1] + media_id = params[2] - url = "https://www.bilibili.com/bangumi/media/md%d" % md + url = "https://www.bilibili.com/bangumi/media/md%d" % media_id try: req = requests.get(url, timeout=10) except BaseException as e: - logger.error(e) - save(md) + logger.error(repr(e)) + save(media_id) logger.info("request_url=%s,status_code=%d" % (url, req.status_code)) - if (req.status_code == 200): + if req.status_code == 200: tag = BeautifulSoup(req.text, 'lxml') script = tag.select("script")[3].text json_str = script[script.index("=") + 1:script.index("function") - 2] json_obj = json.loads(json_str) try: - stat_info = json_obj['mediaInfo']['stat'] - print(json_obj['mediaInfo']['chn_name']) - print(stat_info) - mediaInfo(id=md, chn_name=json_obj['mediaInfo']['chn_name']).save() - global current_mediaInfo - current_mediaInfo = mediaInfo.objects.get(pk=md) - stat(id=md, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], - views=stat_info['views']).save() + if 'mediaInfo' in json_obj and 'stat' in json_obj['mediaInfo'] and 'chn_name' in json_obj['mediaInfo']: + stat_info = json_obj['mediaInfo']['stat'] + print(stat_info) + mediaInfo(bangumi_id=bangumi_id, season_id=season_id, media_id=media_id, + chn_name=json_obj['mediaInfo']['chn_name']).save() + global current_mediaInfo + current_mediaInfo = mediaInfo.objects.get(pk=season_id) + stat(id=season_id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], + views=stat_info['views']).save() except BaseException as e: - logger.error("发生异常") - logger.error(e) + logger.error(repr(e)) -# asdasd def get_(): global current_mediaInfo return current_mediaInfo -page_size = 100 -pages = None -ids = None +page_size = 10 +queue = Queue(page_size) + + +def listen(): + while True: + ids = queue.get() + try: + executors = futures.ThreadPoolExecutor(page_size) + with executors as executor: + executor.map(save, ids) + logger.info('结束爬虫') + except BaseException as e: + logger.error(repr(e)) + + +_thread.start_new_thread(listen, ()) def getIds(): - global ids - global pages - if ids is None or len(ids) != 0: - ids = [] + seasonIdList = [] page = 1 - while pages is None or page <= pages: + pages = None + name = 'seasonListCallback' + global isStop + isStop = False + while isStop == False and (pages is None or page <= pages): + url = 'https://bangumi.bilibili.com/web_api/season/index_global?page=%d&page_size=%d' % (page, page_size) logger.info(url) try: req = requests.get(url, timeout=10) - json_obj = json.loads(req.text) - bangumiList = json_obj['result']['list'] - for bangumi in bangumiList: - ids.append(int(bangumi['season_id'])) - if pages is None: - pages = int(math.ceil(int(json_obj['result']['count']) / page_size)) + if req.status_code == 200: + json_obj = json.loads(req.text) + if 'result' in json_obj and 'list' in json_obj['result']: + bangumiList = json_obj['result']['list'] + ids = [] + for bangumi in bangumiList: + if isStop: + break + if 'season_id' in bangumi: + season_id = int(bangumi['season_id']) + if season_id in seasonIdList: + continue + url = 'https://bangumi.bilibili.com/jsonp/seasoninfo/%d.ver?callback=%s&jsonp=jsonp' % ( + season_id, name) + logger.info(url) + req = requests.get(url, timeout=10) + if req.status_code == 200: + child_json_obj = json.loads( + req.text.replace('seasonListCallback(', '').replace(');', '')) + if 'result' in child_json_obj and 'bangumi_id' in child_json_obj['result']: + bangumi_id = int(child_json_obj['result']['bangumi_id']) + if 'media' in child_json_obj['result']: + media_id = int(child_json_obj['result']['media']['media_id']) + ids.append((bangumi_id, season_id, media_id)) + seasonIdList.append(season_id) + if pages is None and 'count' in json_obj['result']: + pages = int(math.ceil(int(json_obj['result']['count']) / page_size)) + page = page + 1 + logger.info('获取id数量%d' % len(ids)) + queue.put(ids) except BaseException as e: - logger.error('连接超时') - logger(e) + logger.error(repr(e)) continue - page = page + 1 -def threadSave(): - getIds() - logger.info(len(ids)) - try: - global executors - executors = futures.ThreadPoolExecutor(32) - global isStop - isStop = False - with executors as executor: - executor.map(save, ids) - logger.info('结束爬虫') - except BaseException as e: - logger.error(e) +def A(e): + logger.info(e) -def stop_(): - global isStop - isStop = True +if __name__ == '__main__': + getIds() diff --git a/PixivSearch/model/config.py b/PixivSearch/model/config.py index 2b8e7b0..315d937 100644 --- a/PixivSearch/model/config.py +++ b/PixivSearch/model/config.py @@ -1,5 +1,4 @@ from django.db import models -import json class param(models.Model): @@ -18,13 +17,13 @@ class stat(models.Model): class mediaInfo(models.Model): - id = models.IntegerField(primary_key=True) + bangumi_id = models.IntegerField() + season_id = models.IntegerField(primary_key=True) + media_id = models.IntegerField() chn_name = models.CharField(max_length=128) def __str__(self) -> str: - i = {} - i['media_id'] = self.id - i['chn_name'] = self.chn_name + i = {'media_id': self.id, 'chn_name': self.chn_name} return i diff --git a/PixivSearch/view.py b/PixivSearch/view.py index e0659b2..d01f309 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -6,7 +6,7 @@ import django from django.http import Http404, StreamingHttpResponse, HttpResponse from django.shortcuts import render -from PixivSearch.dao.bangumi import threadSave, get_, stop_ +from PixivSearch.dao.bangumi import get_, stop_, getIds from PixivSearch.settings import logger os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") @@ -89,7 +89,7 @@ def get(request): # 测试方法 def start(request): - _thread.start_new_thread(threadSave, ()) + _thread.start_new_thread(getIds, ()) return HttpResponse("start success") From e7ae75acbfdf6dccda8e6e3364deee5f03e15ee7 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Sat, 31 Mar 2018 22:00:08 +0800 Subject: [PATCH 08/22] =?UTF-8?q?=E5=BC=B9=E5=B9=95=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E8=AF=8D=E6=8E=92=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 124 +++++++++++++++++-- PixivSearch/model/migrations/0001_initial.py | 57 +++++++++ PixivSearch/model/migrations/__init__.py | 0 PixivSearch/settings.py | 4 +- 4 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 PixivSearch/model/migrations/0001_initial.py create mode 100644 PixivSearch/model/migrations/__init__.py diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index a66c738..40e4694 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -2,9 +2,15 @@ import _thread import json import math import os +import random +import threading + +import zlib + from concurrent import futures from queue import Queue +from lxml import etree import django import requests from bs4 import BeautifulSoup @@ -38,13 +44,10 @@ def save(params): req = requests.get(url, timeout=10) except BaseException as e: logger.error(repr(e)) - save(media_id) + save(params) logger.info("request_url=%s,status_code=%d" % (url, req.status_code)) if req.status_code == 200: - tag = BeautifulSoup(req.text, 'lxml') - script = tag.select("script")[3].text - json_str = script[script.index("=") + 1:script.index("function") - 2] - json_obj = json.loads(json_str) + json_obj = getJsonText(req, 3) try: if 'mediaInfo' in json_obj and 'stat' in json_obj['mediaInfo'] and 'chn_name' in json_obj['mediaInfo']: stat_info = json_obj['mediaInfo']['stat'] @@ -59,6 +62,13 @@ def save(params): logger.error(repr(e)) +def getJsonText(req, index): + tag = BeautifulSoup(req.text, 'lxml') + script = tag.select("script")[index].text + json_str = script[script.index("=") + 1:script.index("function") - 2] + return json.loads(json_str) + + def get_(): global current_mediaInfo return current_mediaInfo @@ -131,9 +141,107 @@ def getIds(): continue -def A(e): - logger.info(e) +# def testA(): +# req = requests.post('https://api.bilibili.com/x/report/web/heartbeat', +# data={"aid": 29416,"cid":49052,"csrf": "c0d296db7e33085f9f4730cfee66660b"}, +# cookies=_cookies) +# print(req.status_code) + +_cookies = {'DedeUserID': '4372744', 'DedeUserID__ckMd5': 'e8179b74444cae8e', + 'SESSDATA': '919b17d2%2C1524917631%2C3eede719'} + + +def getCid(aid, type=True): + while True and aid > 0: + url = "https://api.bilibili.com/x/web-interface/archive/stat?aid=%d" % aid + print(url) + req = requests.get(url, cookies=_cookies) + code = json.loads(req.text)["code"] + if code == 0: + req = requests.get("https://www.bilibili.com/video/av%d" % aid) + if req.status_code == 200: + json_obj = getJsonText(req, 9) + if "videoData" in json_obj and "pages" in json_obj['videoData'] and len( + json_obj['videoData']['pages']) > 0 and "cid" in json_obj['videoData']['pages'][0]: + cid = json_obj['videoData']['pages'][0]['cid'] + print('cid=%s' % cid) + return cid + if type: + aid = aid - 1 + else: + aid = aid + 1 + + +def getCids(aid): + s = {"min": getCid(aid, True), "max": getCid(aid, False)} + return s + + +def episodeIdToCid(episode_id): + cids = [] + url = "https://www.bilibili.com/bangumi/play/ep%d" % episode_id + print("url=%s" % url) + req = requests.get(url) + json_obj = getJsonText(req, 8) + if "epList" in json_obj: + for i in json_obj["epList"]: + cids.append(i['cid']) + return cids + + +def parseXml(url): + print("url=%s" % url) + comment_selector = etree.HTML(requests.get(url).content) + comment_content = comment_selector.xpath('//i') + for comment_each in comment_content: + comments = comment_each.xpath('//d/text()') + if comments: + for comment in comments: + if comment in obj["data"]: + with lock: + obj["data"][comment] = obj["data"][comment] + 1 + else: + with lock: + obj["data"][comment] = 1 + if not obj["flag"]: + for keyword in keywords: + if keyword in comment: + obj["flag"] = True + + +lock = threading.Lock() # 多线程全局资源锁 + + +def loadData(cids): + params = [] + for cid in cids: + url = "https://comment.bilibili.com/rolldate,%d" % cid + req = requests.get(url) + urls = ["https://comment.bilibili.com/%d.xml" % cid] + if len(req.text) > 0: + for i in json.loads(req.text): + urls.append("https://comment.bilibili.com/dmroll,%s,%d" % (i['timestamp'], cid)) + for url in urls: + params.append(url) + with futures.ThreadPoolExecutor(32) as executor: + executor.map(parseXml, params) + return obj + + +def getCommentSort(cids, keywords_): + global keywords, obj + keywords = keywords_ + obj = {"data": {}, "flag": False} + return loadData(cids) if __name__ == '__main__': - getIds() + # print(getCids(29416)) + # obj = loadData( + # [49052, 49053, 51525, 51526, 53407, 54180, 55295, 55296, 57255, 57256, 59288, 59289, 61559, 61560, 64034, 64035, + # 67024, 67025, 69284, 73333, 73334, 74024, 74025], ['穹']) + f = getCommentSort(episodeIdToCid(172129), ['小樱']) + + # obj = loadData([34807341], []) + for i in sorted(f["data"].items(), key=lambda d: d[1], reverse=True)[:50]: + print(i) diff --git a/PixivSearch/model/migrations/0001_initial.py b/PixivSearch/model/migrations/0001_initial.py new file mode 100644 index 0000000..807f4da --- /dev/null +++ b/PixivSearch/model/migrations/0001_initial.py @@ -0,0 +1,57 @@ +# Generated by Django 2.0 on 2018-03-24 17:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='bangumi_list', + fields=[ + ('season_id', models.IntegerField(primary_key=True, serialize=False)), + ('badge', models.CharField(max_length=128)), + ('brief', models.CharField(max_length=128)), + ('copyright', models.CharField(max_length=128)), + ('cover', models.CharField(max_length=128)), + ('favorites', models.IntegerField()), + ('is_finish', models.IntegerField()), + ('newest_ep_index', models.IntegerField()), + ('pub_time', models.DateTimeField()), + ('season_status', models.IntegerField()), + ('title', models.CharField(max_length=128)), + ('total_count', models.IntegerField()), + ('trailer_aid', models.IntegerField()), + ], + ), + migrations.CreateModel( + name='mediaInfo', + fields=[ + ('bangumi_id', models.IntegerField(primary_key=True, serialize=False)), + ('season_id', models.IntegerField()), + ('media_id', models.IntegerField()), + ('chn_name', models.CharField(max_length=128)), + ], + ), + migrations.CreateModel( + name='param', + fields=[ + ('param_name', models.CharField(max_length=10, primary_key=True, serialize=False)), + ('param_value', models.CharField(max_length=128)), + ], + ), + migrations.CreateModel( + name='stat', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('danmakus', models.IntegerField()), + ('favorites', models.IntegerField()), + ('views', models.IntegerField()), + ], + ), + ] diff --git a/PixivSearch/model/migrations/__init__.py b/PixivSearch/model/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index 717f957..f584b76 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -78,8 +78,8 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'bangumi', - 'USER': 'bilibili', - 'PASSWORD': '2233', + 'USER': 'root', + 'PASSWORD': 'Luffy9412!', # 'HOST': '127.0.0.1', 'HOST': 'mikuhime.xyz', 'PORT': '3306', From 5eab62491f2e9f568a3d8ca0f7bc5cb2e3861c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=A0=E9=99=90=E8=B6=85=E9=A2=91?= <1029559041@qq.com> Date: Sun, 22 Apr 2018 23:29:36 +0800 Subject: [PATCH 09/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BA=91=E7=9B=B8=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/__init__.py | 0 PixivSearch/aliyun/photo/AliyunPhoto.py | 79 +++++++++++++++++++++++++ PixivSearch/baidu/__init__.py | 0 PixivSearch/dao/bangumi.py | 2 +- 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 PixivSearch/aliyun/__init__.py create mode 100644 PixivSearch/aliyun/photo/AliyunPhoto.py create mode 100644 PixivSearch/baidu/__init__.py diff --git a/PixivSearch/aliyun/__init__.py b/PixivSearch/aliyun/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py new file mode 100644 index 0000000..e5f053c --- /dev/null +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -0,0 +1,79 @@ +import base64 +import calendar +import datetime +import hashlib +import hmac +import os +import random +import time +from urllib import parse + +import requests + + +def get_md5_01(file_path): + md5 = None + if os.path.isfile(file_path): + f = open(file_path, 'rb') + md5_obj = hashlib.md5() + md5_obj.update(f.read()) + hash_code = md5_obj.hexdigest() + f.close() + md5 = str(hash_code).lower() + return md5 + + +def createTransaction(HTTPMethod, filePath): + fileName = filePath.split('/')[-1] + + for Version in getBetweenDay('2017-01-01'): + print("Version={Version}".format(Version=Version)) + param = { + "Format": "JSON", + "Version": Version, + "AccessKeyId": "LTAIeS8aBuPBZxV2", + "SignatureMethod": "HMAC-SHA1", + "Timestamp": int(time.time()), + "SignatureVersion": "1.0", + "SignatureNonce": random.randint(0, 9), + "Action": "CreateTransaction", + "Ext": fileName[-fileName[::-1].index('.'):], + "Md5": get_md5_01(filePath), + "Size": os.path.getsize(filePath) + } + AccessKeySecret = b'hyPeTaDQBQs6jetYcqY0BUdpacXTH3' + canonicalQueryString = '' + for i in sorted(param.items(), key=lambda d: d[0]): + canonicalQueryString += '&' + i[0] + parse.quote('=' + str(i[1])) + # print(canonicalQueryString[1:]) + + strUrlEncoding = HTTPMethod + '&%2F&' + canonicalQueryString[1:] + print(strUrlEncoding) + stringToSign = base64.b64encode(hmac.new(AccessKeySecret, strUrlEncoding.encode('UTF-8'), 'sha1').digest()) + + server = 'https://cloudphoto.cn-shanghai.aliyuncs.com' + url = server + "?" + canonicalQueryString[1:] + '&Signature=' + stringToSign.decode('utf-8').replace('+','%20').replace('/','%2F').replace('=','%3D') + print(url) + result = requests.get(url) + + if result.status_code == 200: + break + else: + print(result.content) + time.sleep(1) + + +def getBetweenDay(begin_date): + date_list = [] + begin_date = datetime.datetime.strptime(begin_date, "%Y-%m-%d") + end_date = datetime.datetime.strptime(time.strftime('%Y-%m-%d', time.localtime(time.time())), "%Y-%m-%d") + while begin_date <= end_date: + date_str = begin_date.strftime("%Y-%m-%d") + date_list.append(date_str) + begin_date += datetime.timedelta(days=1) + return date_list + + +if __name__ == '__main__': + createTransaction("GET", "C:/Users/10295/Desktop/灵梦.png") + # print(getBetweenDay('2017-01-01')) diff --git a/PixivSearch/baidu/__init__.py b/PixivSearch/baidu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 40e4694..7fc523b 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -240,7 +240,7 @@ if __name__ == '__main__': # obj = loadData( # [49052, 49053, 51525, 51526, 53407, 54180, 55295, 55296, 57255, 57256, 59288, 59289, 61559, 61560, 64034, 64035, # 67024, 67025, 69284, 73333, 73334, 74024, 74025], ['穹']) - f = getCommentSort(episodeIdToCid(172129), ['小樱']) + f = getCommentSort(episodeIdToCid(172095), []) # obj = loadData([34807341], []) for i in sorted(f["data"].items(), key=lambda d: d[1], reverse=True)[:50]: From d035e5e206cda8bee19bbf9fa9ae45401b4ea516 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Mon, 23 Apr 2018 01:30:37 +0800 Subject: [PATCH 10/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BA=91=E7=9B=B8=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/photo/AliyunPhoto.py | 80 ++++++++++--------------- 1 file changed, 32 insertions(+), 48 deletions(-) diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index e5f053c..00df17d 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -5,7 +5,6 @@ import hashlib import hmac import os import random -import time from urllib import parse import requests @@ -26,54 +25,39 @@ def get_md5_01(file_path): def createTransaction(HTTPMethod, filePath): fileName = filePath.split('/')[-1] - for Version in getBetweenDay('2017-01-01'): - print("Version={Version}".format(Version=Version)) - param = { - "Format": "JSON", - "Version": Version, - "AccessKeyId": "LTAIeS8aBuPBZxV2", - "SignatureMethod": "HMAC-SHA1", - "Timestamp": int(time.time()), - "SignatureVersion": "1.0", - "SignatureNonce": random.randint(0, 9), - "Action": "CreateTransaction", - "Ext": fileName[-fileName[::-1].index('.'):], - "Md5": get_md5_01(filePath), - "Size": os.path.getsize(filePath) - } - AccessKeySecret = b'hyPeTaDQBQs6jetYcqY0BUdpacXTH3' - canonicalQueryString = '' - for i in sorted(param.items(), key=lambda d: d[0]): - canonicalQueryString += '&' + i[0] + parse.quote('=' + str(i[1])) - # print(canonicalQueryString[1:]) - - strUrlEncoding = HTTPMethod + '&%2F&' + canonicalQueryString[1:] - print(strUrlEncoding) - stringToSign = base64.b64encode(hmac.new(AccessKeySecret, strUrlEncoding.encode('UTF-8'), 'sha1').digest()) - - server = 'https://cloudphoto.cn-shanghai.aliyuncs.com' - url = server + "?" + canonicalQueryString[1:] + '&Signature=' + stringToSign.decode('utf-8').replace('+','%20').replace('/','%2F').replace('=','%3D') - print(url) - result = requests.get(url) - - if result.status_code == 200: - break - else: - print(result.content) - time.sleep(1) - - -def getBetweenDay(begin_date): - date_list = [] - begin_date = datetime.datetime.strptime(begin_date, "%Y-%m-%d") - end_date = datetime.datetime.strptime(time.strftime('%Y-%m-%d', time.localtime(time.time())), "%Y-%m-%d") - while begin_date <= end_date: - date_str = begin_date.strftime("%Y-%m-%d") - date_list.append(date_str) - begin_date += datetime.timedelta(days=1) - return date_list + param = { + # "Format": "JSON", + "Version": '2017-07-11', + "AccessKeyId": "LTAIeS8aBuPBZxV2", + "SignatureMethod": "HMAC-SHA1", + "Timestamp": datetime.datetime.utcnow().isoformat()[:19] + 'Z', + "SignatureVersion": "1.0", + "SignatureNonce": '9166ab59-f445-' + str(random.randint(1000, 9999)) + '-911d-664c1570df0f', + "Action": "CreateTransaction" + # "Ext": fileName[-fileName[::-1].index('.'):], + # "Md5": get_md5_01(filePath), + # "Size": os.path.getsize(filePath) + } + AccessKeySecret = b'hyPeTaDQBQs6jetYcqY0BUdpacXTH3&' + canonicalQueryString = '' + + for i in sorted(param.items(), key=lambda d: d[0]): + canonicalQueryString += '&' + i[0] + '=' + str(i[1]) + print(canonicalQueryString[1:]) + + strUrlEncoding = HTTPMethod + '&%2F&' + parse.quote(canonicalQueryString[1:]) + print(strUrlEncoding) + stringToSign = base64.b64encode(hmac.new(AccessKeySecret, strUrlEncoding.encode('UTF-8'), 'sha1').digest()) + print(stringToSign) + server = 'https://cloudphoto.cn-shanghai.aliyuncs.com' + url = server + "?" + canonicalQueryString[1:] + '&Signature=' + parse.quote(stringToSign.decode('utf-8')).replace( + '/', '%2F') + print(url) + result = requests.get(url) + + print(result.content) if __name__ == '__main__': createTransaction("GET", "C:/Users/10295/Desktop/灵梦.png") - # print(getBetweenDay('2017-01-01')) + # print(datetime.datetime.utcnow()) From a8962d052dd22acb0b861c2bea163993bf36be68 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Wed, 25 Apr 2018 00:05:20 +0800 Subject: [PATCH 11/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BA=91=E7=9B=B8=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/photo/AliyunPhoto.py | 161 +++++++++++++++--------- 1 file changed, 103 insertions(+), 58 deletions(-) diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index 00df17d..51381aa 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -1,63 +1,108 @@ -import base64 -import calendar -import datetime import hashlib -import hmac +import json import os -import random -from urllib import parse - -import requests - - -def get_md5_01(file_path): - md5 = None - if os.path.isfile(file_path): - f = open(file_path, 'rb') - md5_obj = hashlib.md5() - md5_obj.update(f.read()) - hash_code = md5_obj.hexdigest() - f.close() - md5 = str(hash_code).lower() - return md5 - - -def createTransaction(HTTPMethod, filePath): - fileName = filePath.split('/')[-1] - - param = { - # "Format": "JSON", - "Version": '2017-07-11', - "AccessKeyId": "LTAIeS8aBuPBZxV2", - "SignatureMethod": "HMAC-SHA1", - "Timestamp": datetime.datetime.utcnow().isoformat()[:19] + 'Z', - "SignatureVersion": "1.0", - "SignatureNonce": '9166ab59-f445-' + str(random.randint(1000, 9999)) + '-911d-664c1570df0f', - "Action": "CreateTransaction" - # "Ext": fileName[-fileName[::-1].index('.'):], - # "Md5": get_md5_01(filePath), - # "Size": os.path.getsize(filePath) - } - AccessKeySecret = b'hyPeTaDQBQs6jetYcqY0BUdpacXTH3&' - canonicalQueryString = '' - - for i in sorted(param.items(), key=lambda d: d[0]): - canonicalQueryString += '&' + i[0] + '=' + str(i[1]) - print(canonicalQueryString[1:]) - - strUrlEncoding = HTTPMethod + '&%2F&' + parse.quote(canonicalQueryString[1:]) - print(strUrlEncoding) - stringToSign = base64.b64encode(hmac.new(AccessKeySecret, strUrlEncoding.encode('UTF-8'), 'sha1').digest()) - print(stringToSign) - server = 'https://cloudphoto.cn-shanghai.aliyuncs.com' - url = server + "?" + canonicalQueryString[1:] + '&Signature=' + parse.quote(stringToSign.decode('utf-8')).replace( - '/', '%2F') - print(url) - result = requests.get(url) - - print(result.content) +import aliyunsdkcore +import oss2 as oss2 +from aliyunsdkcore.client import AcsClient, DEFAULT_SDK_CONNECTION_TIMEOUT_IN_SECONDS +from aliyunsdkcore.request import CommonRequest + + +class UploadPhoto(AcsClient): + + def __init__(self, ak='LTAIeS8aBuPBZxV2', secret='hyPeTaDQBQs6jetYcqY0BUdpacXTH3', region_id="cn-hongkong", + auto_retry=True, max_retry_time=3, + user_agent=None, port=80, timeout=DEFAULT_SDK_CONNECTION_TIMEOUT_IN_SECONDS, public_key_id=None, + private_key=None, session_period=3600, credential=None, debug=False): + super().__init__(ak, secret, region_id, auto_retry, max_retry_time, user_agent, port, timeout, public_key_id, + private_key, session_period, credential, debug) + aliyunsdkcore.request.set_default_protocol_type("https") + self.domain='cloudphoto.cn-shanghai.aliyuncs.com' + self.version='2017-07-11' + + def get_md5_01(self, file_path): + md5 = None + if os.path.isfile(file_path): + f = open(file_path, 'rb') + md5_obj = hashlib.md5() + md5_obj.update(f.read()) + hash_code = md5_obj.hexdigest() + f.close() + md5 = str(hash_code).lower() + return md5 + + def assumeRole(self): + request = CommonRequest(domain='sts.aliyuncs.com', version='2015-04-01', + action_name='AssumeRole') + RoleSessionName = 'pqh' + request.add_query_param('RoleArn', 'acs:ram::1098806312754985:role/aliyuncloudphotodefaultrole') + request.add_query_param('RoleSessionName', RoleSessionName) + response = json.loads(self.do_action_with_exception(request).decode()) + print(response['Credentials']['SecurityToken']) + + def createAction(self,action): + return CommonRequest(domain=self.domain, version=self.version, + action_name=action) if __name__ == '__main__': - createTransaction("GET", "C:/Users/10295/Desktop/灵梦.png") - # print(datetime.datetime.utcnow()) + # createTransaction("GET", "C:/Users/10295/Desktop/灵梦.png") + client = UploadPhoto() + + request = client.createAction('ListPhotoStores') + response = json.loads(client.do_action_with_exception(request).decode()) + print(response) + + print('PhotoStores:') + photoStores = response['PhotoStores'] + for index, photoStore in enumerate(photoStores): + print('{index}:{name}'.format(index=index + 1, name=photoStore['Name'])) + StoreName = input('输入StoreName\n') + + # request = client.createAction('GetPhotoStore') + # request.set_accept_format('json') + # request.add_query_param('StoreName', StoreName) + # response = json.loads(client.do_action_with_exception(request).decode()) + # print(response) + # buckets = response['PhotoStore']['Buckets'] + # + # request = client.createAction('CreateTransaction') + # filePath = 'C:/Users/10295/Desktop/灵梦.png' + # fileName = filePath.split('/')[-1] + # request.add_query_param('Size', os.path.getsize(filePath)) + # request.add_query_param('Ext', fileName[-fileName[::-1].index('.'):]) + # request.add_query_param('Md5', client.get_md5_01(filePath)) + # request.add_query_param('StoreName', StoreName) + # + # liberaryId = input('输入liberaryId\n') + # request.add_query_param('LibraryId', liberaryId) + # response = json.loads(client.do_action_with_exception(request)) + # print(response) + # + # Upload=response['Transaction']['Upload'] + # FileId = Upload['FileId'] + # SessionId = Upload['SessionId'] + # Bucket=Upload['Bucket'] + # OssEndpoint=Upload['OssEndpoint'] + # ObjectKey=Upload['ObjectKey'] + # + # auth = oss2.Auth(client.get_access_key(), client.get_access_secret()) + # bucket = oss2.Bucket(auth, OssEndpoint,Bucket) + # with open(filePath, 'rb') as fileobj: + # result=bucket.put_object(ObjectKey,fileobj) + # print('文件上传状态{status}'.format(status=result.status)) + # + # request = client.createAction('CreatePhoto') + # request.add_query_param('FileId', FileId) + # request.add_query_param('PhotoTitle', fileName) + # request.add_query_param('SessionId', SessionId) + # request.add_query_param('StoreName', StoreName) + # request.add_query_param('UploadType', 'manual') + # request.add_query_param('LibraryId', liberaryId) + # response = json.loads(client.do_action_with_exception(request)) + # print(response) + + request=client.createAction('FetchLibraries') + request.add_query_param('StoreName', StoreName) + request.add_query_param('LibraryId', '') + response = json.loads(client.do_action_with_exception(request)) + print(response) \ No newline at end of file From a777f835ec488b918dbaac92785bff03354a8140 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Wed, 25 Apr 2018 00:10:12 +0800 Subject: [PATCH 12/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BA=91=E7=9B=B8=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/baidu/downLoadBduImg.py | 112 ++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 PixivSearch/baidu/downLoadBduImg.py diff --git a/PixivSearch/baidu/downLoadBduImg.py b/PixivSearch/baidu/downLoadBduImg.py new file mode 100644 index 0000000..5085c44 --- /dev/null +++ b/PixivSearch/baidu/downLoadBduImg.py @@ -0,0 +1,112 @@ +import json +import os +import re +from concurrent import futures +from datetime import datetime + +import requests + + +def get_cookies(): + _cookies = {} + array = "BDqhfp=fate%26%26NaN%26%260%26%261; BIDUPSID=8689C23BFD1526702A4EF173F3A809DD; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; userFrom=null; BAIDUID=8689C23BFD152670722FAAEB4DDC55FA:FG=1; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm".split( + ';') + for row in array: + k, v = row.strip().split('=', 1) + _cookies[k] = v + return _cookies + + +# 图片保存路径 +savePath = None +threadNum = 10 +startTime = None + + +def getBaiduImage(word): + global startTime, savePath + params = [] + startTime = datetime.now() + start = threadNum + i = 0 + filepath = None + savePath = r'{savePath}/{word}'.format(savePath=savePath, word=word) + if not os.path.exists(savePath): + os.makedirs(savePath) + while True: + try: + url = r"https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord={" \ + r"queryWord}&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word={" \ + r"word}&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&pn={pn}&rn={rn}&gsm=3c&1523890541764= " + url = url.format(queryWord=word, word=word, pn=start, rn=threadNum) + print('request url:%s' % url) + req = requests.get(url) + if req.status_code == 200: + req.encoding = 'utf-8' + obj = json.loads(req.text.replace('\\\'', '')) + if len(obj['data']) == 1: + break + for img in obj['data']: + if 'fromPageTitle' in img: + print('图片:%s\t添加到下载队列' % img['fromPageTitleEnc']) + if 'replaceUrl' in img: + url = img['replaceUrl'][0]['ObjURL'] + params.append((url, i)) + i += 1 + if not filepath is None and os.path.exists(filepath): + os.remove(filepath) + filepath = r'{savePath}/图片下载队列填充：{i}'.format(savePath=savePath, word=word, i=i) + file = open(filepath, 'w') + file.close() + start += threadNum + except BaseException as e: + print(repr(e)) + if not filepath is None and os.path.exists(filepath): + os.remove(filepath) + executors = futures.ThreadPoolExecutor(threadNum) + try: + with executors as executor: + executor.map(downImage, params) + except BaseException as e: + print(repr(e)) + + +def downImage(params): + try: + url = params[0] + index = params[1] + print(r'开始下载图片{url}'.format(url=url)) + imgurl = requests.get(url, headers={"Referer": "image.baidu.com"}) + if imgurl.status_code == 200: + format = url[-url[::-1].index('.'):] + imgPath = r'{savePath}/fate_{index}.{format}'.format(savePath=savePath, + index=index, + format=format) + f = open(imgPath, 'wb') + f.write(imgurl.content) + f.close() + print(r'图片{url}成功下载到{imgPath}'.format(url=url, imgPath=imgPath)) + except BaseException as e: + print(repr(e)) + + +if __name__ == '__main__': + str = '' + while True: + str = input('输入要下载图片的关键字，输入 exit 退出程序\n') + if not str == 'exit': + while True: + savePath = input('输入图片存放目录：例如 E:/图片,注意目录之间使用正斜杠隔开"/"\n') + if re.fullmatch(r"[a-zA-z]:(/[\u4e00-\u9fa5_a-zA-Z0-9]+)+", savePath) is None: + print(r'图片目录{savePath}不合法请重新输入'.format(savePath=savePath)) + else: + break + getBaiduImage(str) + print(r'使用{threadNum}线程成功下载{count}张图片到目录{path}下，耗时：{second}'.format(threadNum=threadNum, + count=len(os.listdir(savePath)), + path=savePath, + second=datetime.now() - startTime)) + flag = False + else: + print('exit') + break From 8a3c9033d1c42e9f96e693b2aa4493a52c19c59b Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Thu, 26 Apr 2018 18:44:16 +0800 Subject: [PATCH 13/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E7=9B=B8?= =?UTF-8?q?=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/photo/AliyunPhoto.py | 75 ++++++++++++++----------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index 51381aa..6d8a534 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -17,8 +17,8 @@ class UploadPhoto(AcsClient): super().__init__(ak, secret, region_id, auto_retry, max_retry_time, user_agent, port, timeout, public_key_id, private_key, session_period, credential, debug) aliyunsdkcore.request.set_default_protocol_type("https") - self.domain='cloudphoto.cn-shanghai.aliyuncs.com' - self.version='2017-07-11' + self.domain = 'cloudphoto.cn-shanghai.aliyuncs.com' + self.version = '2017-07-11' def get_md5_01(self, file_path): md5 = None @@ -40,40 +40,55 @@ class UploadPhoto(AcsClient): response = json.loads(self.do_action_with_exception(request).decode()) print(response['Credentials']['SecurityToken']) - def createAction(self,action): + def createAction(self, action): return CommonRequest(domain=self.domain, version=self.version, - action_name=action) + action_name=action) + + def do_action_with_exception(self, acs_request): + return json.loads(super().do_action_with_exception(acs_request).decode()) + + def showOption(self, options, key, desc): + choose = [] + for index, option in enumerate(options): + choose.append(option[key]) + print('{index}:{name}'.format(index=index + 1, name=option[key])) + return choose[(int)(input('输入{desc}\n'.format(desc=desc))) - 1] + if __name__ == '__main__': - # createTransaction("GET", "C:/Users/10295/Desktop/灵梦.png") client = UploadPhoto() request = client.createAction('ListPhotoStores') - response = json.loads(client.do_action_with_exception(request).decode()) - print(response) - + response = client.do_action_with_exception(request) print('PhotoStores:') photoStores = response['PhotoStores'] - for index, photoStore in enumerate(photoStores): - print('{index}:{name}'.format(index=index + 1, name=photoStore['Name'])) - StoreName = input('输入StoreName\n') + StoreName = client.showOption(photoStores, 'Name', 'StoreName') + + request = client.createAction('GetPhotoStore') + request.add_query_param('StoreName', StoreName) + response = client.do_action_with_exception(request) + print(response) + + request = client.createAction('FetchLibraries') + request.add_query_param('StoreName', StoreName) + response = client.do_action_with_exception(request) + Libraries = response['Libraries'] + LibraryId = client.showOption(Libraries, 'LibraryId', 'LibraryId') + + request = client.createAction('GetLibrary') + request.add_body_params('StoreName', StoreName) + request.add_body_params('LibraryId', LibraryId) + response = client.do_action_with_exception(request) + print(response) + + request = client.createAction('CreateTransaction') + filePath = input('输入上传文件路径') + fileName = filePath.split('/')[-1] + request.add_query_param('Size', os.path.getsize(filePath)) + request.add_query_param('Ext', fileName[-fileName[::-1].index('.'):]) + request.add_query_param('Md5', client.get_md5_01(filePath)) + request.add_query_param('StoreName', StoreName) - # request = client.createAction('GetPhotoStore') - # request.set_accept_format('json') - # request.add_query_param('StoreName', StoreName) - # response = json.loads(client.do_action_with_exception(request).decode()) - # print(response) - # buckets = response['PhotoStore']['Buckets'] - # - # request = client.createAction('CreateTransaction') - # filePath = 'C:/Users/10295/Desktop/灵梦.png' - # fileName = filePath.split('/')[-1] - # request.add_query_param('Size', os.path.getsize(filePath)) - # request.add_query_param('Ext', fileName[-fileName[::-1].index('.'):]) - # request.add_query_param('Md5', client.get_md5_01(filePath)) - # request.add_query_param('StoreName', StoreName) - # - # liberaryId = input('输入liberaryId\n') # request.add_query_param('LibraryId', liberaryId) # response = json.loads(client.do_action_with_exception(request)) # print(response) @@ -100,9 +115,3 @@ if __name__ == '__main__': # request.add_query_param('LibraryId', liberaryId) # response = json.loads(client.do_action_with_exception(request)) # print(response) - - request=client.createAction('FetchLibraries') - request.add_query_param('StoreName', StoreName) - request.add_query_param('LibraryId', '') - response = json.loads(client.do_action_with_exception(request)) - print(response) \ No newline at end of file From 6dcb3ee1f2f17256668cf65504fa16de508f57fc Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Sat, 28 Apr 2018 02:05:39 +0800 Subject: [PATCH 14/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BA=91=E7=9B=B8=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/photo/AliyunPhoto.py | 136 +++++++++++++++--------- PixivSearch/baidu/downLoadBduImg.py | 44 ++++---- 2 files changed, 110 insertions(+), 70 deletions(-) diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index 6d8a534..69fb51e 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -1,6 +1,8 @@ +import datetime import hashlib import json import os +import time import aliyunsdkcore import oss2 as oss2 @@ -19,6 +21,8 @@ class UploadPhoto(AcsClient): aliyunsdkcore.request.set_default_protocol_type("https") self.domain = 'cloudphoto.cn-shanghai.aliyuncs.com' self.version = '2017-07-11' + self.LibraryId = None + self.StoreName = None def get_md5_01(self, file_path): md5 = None @@ -54,64 +58,94 @@ class UploadPhoto(AcsClient): print('{index}:{name}'.format(index=index + 1, name=option[key])) return choose[(int)(input('输入{desc}\n'.format(desc=desc))) - 1] + def getSL(self): + request = self.createAction('ListPhotoStores') + response = self.do_action_with_exception(request) + print('PhotoStores:') + photoStores = response['PhotoStores'] + self.StoreName = self.showOption(photoStores, 'Name', 'StoreName') + + # request = self.createAction('GetPhotoStore') + # request.add_query_param('StoreName', StoreName) + # response = self.do_action_with_exception(request) + # print(response) + + request = self.createAction('FetchLibraries') + request.add_query_param('StoreName', self.StoreName) + response = self.do_action_with_exception(request) + Libraries = response['Libraries'] + self.LibraryId = self.showOption(Libraries, 'LibraryId', 'LibraryId') + + # request = self.createAction('GetLibrary') + # request.add_body_params('StoreName', StoreName) + # request.add_body_params('LibraryId', LibraryId) + # response = self.do_action_with_exception(request) + # print(response) + + def uploadPhoto(self): + if self.StoreName is None or self.LibraryId is None: + return '未选择StoreName、LibraryId' + + request = self.createAction('CreateTransaction') + filePath = input('输入上传文件路径') + fileName = filePath.split('/')[-1] + request.add_query_param('Size', os.path.getsize(filePath)) + request.add_query_param('Ext', fileName[-fileName[::-1].index('.'):]) + request.add_query_param('Md5', self.get_md5_01(filePath)) + request.add_query_param('StoreName', self.StoreName) + + request.add_query_param('LibraryId', self.LibraryId) + response = self.do_action_with_exception(request) + print(response) + Upload = response['Transaction']['Upload'] + FileId = Upload['FileId'] + SessionId = Upload['SessionId'] + Bucket = Upload['Bucket'] + OssEndpoint = Upload['OssEndpoint'] + ObjectKey = Upload['ObjectKey'] + + auth = oss2.Auth(self.get_access_key(), self.get_access_secret()) + bucket = oss2.Bucket(auth, OssEndpoint, Bucket) + with open(filePath, 'rb') as fileobj: + result = bucket.put_object(ObjectKey, fileobj) + print('文件上传状态{status}'.format(status=result.status)) + + request = self.createAction('CreatePhoto') + request.add_query_param('FileId', FileId) + request.add_query_param('PhotoTitle', fileName) + request.add_query_param('SessionId', SessionId) + request.add_query_param('StoreName', self.StoreName) + request.add_query_param('UploadType', 'manual') + request.add_query_param('LibraryId', self.LibraryId) + response = self.do_action_with_exception(request) + print(response) + if __name__ == '__main__': client = UploadPhoto() - - request = client.createAction('ListPhotoStores') - response = client.do_action_with_exception(request) - print('PhotoStores:') - photoStores = response['PhotoStores'] - StoreName = client.showOption(photoStores, 'Name', 'StoreName') - - request = client.createAction('GetPhotoStore') - request.add_query_param('StoreName', StoreName) + client.getSL() + request = client.createAction('ListPhotos') + request.add_query_param('StoreName', client.StoreName) + request.add_query_param('LibraryId', client.LibraryId) + request.add_query_param('State', 'all') response = client.do_action_with_exception(request) print(response) - request = client.createAction('FetchLibraries') - request.add_query_param('StoreName', StoreName) - response = client.do_action_with_exception(request) - Libraries = response['Libraries'] - LibraryId = client.showOption(Libraries, 'LibraryId', 'LibraryId') + IdStr = client.showOption(response['Photos'], 'IdStr', '照片Id') - request = client.createAction('GetLibrary') - request.add_body_params('StoreName', StoreName) - request.add_body_params('LibraryId', LibraryId) + request = client.createAction('EditPhotos') + request.add_query_param('PhotoId.1', IdStr) + request.add_query_param('StoreName', client.StoreName) + request.add_query_param('LibraryId', client.LibraryId) + request.add_query_param('ShareExpireTime', int(round(time.time() * 1000)) + 60 * 60) response = client.do_action_with_exception(request) print(response) - request = client.createAction('CreateTransaction') - filePath = input('输入上传文件路径') - fileName = filePath.split('/')[-1] - request.add_query_param('Size', os.path.getsize(filePath)) - request.add_query_param('Ext', fileName[-fileName[::-1].index('.'):]) - request.add_query_param('Md5', client.get_md5_01(filePath)) - request.add_query_param('StoreName', StoreName) - - # request.add_query_param('LibraryId', liberaryId) - # response = json.loads(client.do_action_with_exception(request)) - # print(response) - # - # Upload=response['Transaction']['Upload'] - # FileId = Upload['FileId'] - # SessionId = Upload['SessionId'] - # Bucket=Upload['Bucket'] - # OssEndpoint=Upload['OssEndpoint'] - # ObjectKey=Upload['ObjectKey'] - # - # auth = oss2.Auth(client.get_access_key(), client.get_access_secret()) - # bucket = oss2.Bucket(auth, OssEndpoint,Bucket) - # with open(filePath, 'rb') as fileobj: - # result=bucket.put_object(ObjectKey,fileobj) - # print('文件上传状态{status}'.format(status=result.status)) - # - # request = client.createAction('CreatePhoto') - # request.add_query_param('FileId', FileId) - # request.add_query_param('PhotoTitle', fileName) - # request.add_query_param('SessionId', SessionId) - # request.add_query_param('StoreName', StoreName) - # request.add_query_param('UploadType', 'manual') - # request.add_query_param('LibraryId', liberaryId) - # response = json.loads(client.do_action_with_exception(request)) - # print(response) + request = client.createAction('GetPublicAccessUrls') + request.add_query_param('DomainType', 'OSS') + request.add_query_param('PhotoId.1', IdStr) + request.add_query_param('StoreName', client.StoreName) + request.add_query_param('ZoomType', 'style/1') + request.add_query_param('LibraryId', client.LibraryId) + response = client.do_action_with_exception(request) + print(response) \ No newline at end of file diff --git a/PixivSearch/baidu/downLoadBduImg.py b/PixivSearch/baidu/downLoadBduImg.py index 5085c44..03b1edc 100644 --- a/PixivSearch/baidu/downLoadBduImg.py +++ b/PixivSearch/baidu/downLoadBduImg.py @@ -91,22 +91,28 @@ def downImage(params): if __name__ == '__main__': - str = '' - while True: - str = input('输入要下载图片的关键字，输入 exit 退出程序\n') - if not str == 'exit': - while True: - savePath = input('输入图片存放目录：例如 E:/图片,注意目录之间使用正斜杠隔开"/"\n') - if re.fullmatch(r"[a-zA-z]:(/[\u4e00-\u9fa5_a-zA-Z0-9]+)+", savePath) is None: - print(r'图片目录{savePath}不合法请重新输入'.format(savePath=savePath)) - else: - break - getBaiduImage(str) - print(r'使用{threadNum}线程成功下载{count}张图片到目录{path}下，耗时：{second}'.format(threadNum=threadNum, - count=len(os.listdir(savePath)), - path=savePath, - second=datetime.now() - startTime)) - flag = False - else: - print('exit') - break + # str = '' + # while True: + # str = input('输入要下载图片的关键字，输入 exit 退出程序\n') + # if not str == 'exit': + # while True: + # savePath = input('输入图片存放目录：例如 E:/图片,注意目录之间使用正斜杠隔开"/"\n') + # if re.fullmatch(r"[a-zA-z]:(/[\u4e00-\u9fa5_a-zA-Z0-9]+)+", savePath) is None: + # print(r'图片目录{savePath}不合法请重新输入'.format(savePath=savePath)) + # else: + # break + # getBaiduImage(str) + # print(r'使用{threadNum}线程成功下载{count}张图片到目录{path}下，耗时：{second}'.format(threadNum=threadNum, + # count=len(os.listdir(savePath)), + # path=savePath, + # second=datetime.now() - startTime)) + # flag = False + # else: + # print('exit') + # break + req = requests.post('https://gifmaker.develophelper.com/gif/make', {'tplid': 1, + 'content': '好啊##$@?$?@$##就算你是一流工程师##$@?$?@$##就算你出报告再完美##$@?$?@$##我叫你改报告你就要改##$@?$?@$##毕竟我是客户##$@?$?@$##客户了不起啊##$@?$?@$##sorry 客户真的了不起##$@?$?@$##以后叫他天天改报告##$@?$?@$##天天改 天天改'}) + if req.status_code==200: + response = json.loads(req.content.decode()) + if 'd' in response: + print(response['d']) From 53a51d101be07e2073212fa5880ac32be1f6b697 Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Sat, 28 Apr 2018 18:37:57 +0800 Subject: [PATCH 15/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E7=9B=B8?= =?UTF-8?q?=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/{tsdm.py => pixiv/__init__.py} | 0 PixivSearch/{ => pixiv}/pixiv.py | 0 PixivSearch/thrift/TestQry/Client.py | 25 +++ PixivSearch/thrift/TestQry/Server.py | 29 +++ PixivSearch/thrift/TestQry/TestQry-remote | 116 ++++++++++ PixivSearch/thrift/TestQry/TestQry.py | 247 +++++++++++++++++++++ PixivSearch/thrift/TestQry/__init__.py | 1 + PixivSearch/thrift/TestQry/constants.py | 14 ++ PixivSearch/thrift/TestQry/ttypes.py | 91 ++++++++ PixivSearch/thrift/__init__.py | 0 10 files changed, 523 insertions(+) rename PixivSearch/{tsdm.py => pixiv/__init__.py} (100%) rename PixivSearch/{ => pixiv}/pixiv.py (100%) create mode 100644 PixivSearch/thrift/TestQry/Client.py create mode 100644 PixivSearch/thrift/TestQry/Server.py create mode 100755 PixivSearch/thrift/TestQry/TestQry-remote create mode 100644 PixivSearch/thrift/TestQry/TestQry.py create mode 100644 PixivSearch/thrift/TestQry/__init__.py create mode 100644 PixivSearch/thrift/TestQry/constants.py create mode 100644 PixivSearch/thrift/TestQry/ttypes.py create mode 100644 PixivSearch/thrift/__init__.py diff --git a/PixivSearch/tsdm.py b/PixivSearch/pixiv/__init__.py similarity index 100% rename from PixivSearch/tsdm.py rename to PixivSearch/pixiv/__init__.py diff --git a/PixivSearch/pixiv.py b/PixivSearch/pixiv/pixiv.py similarity index 100% rename from PixivSearch/pixiv.py rename to PixivSearch/pixiv/pixiv.py diff --git a/PixivSearch/thrift/TestQry/Client.py b/PixivSearch/thrift/TestQry/Client.py new file mode 100644 index 0000000..2839dc7 --- /dev/null +++ b/PixivSearch/thrift/TestQry/Client.py @@ -0,0 +1,25 @@ +import sys + +from thrift.protocol import TBinaryProtocol +from thrift.transport import TSocket, TTransport + +sys.path.append("/home/hua/PycharmProjects/PixivSearch") +from PixivSearch.thrift.TestQry.TestQry import Client + +if __name__ == '__main__': + # Make socket + socket = TSocket.TSocket('127.0.0.1', 2233) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TFramedTransport(socket) + + if not transport.isOpen(): + transport.open() + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + + # Create a client to use the protocol encoder + client = Client(protocol) + + print(client.qryTest(0)) diff --git a/PixivSearch/thrift/TestQry/Server.py b/PixivSearch/thrift/TestQry/Server.py new file mode 100644 index 0000000..de53097 --- /dev/null +++ b/PixivSearch/thrift/TestQry/Server.py @@ -0,0 +1,29 @@ +from thrift.protocol import TBinaryProtocol +from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory +from thrift.server.TNonblockingServer import TNonblockingServer +from thrift.transport import TSocket +from thrift.transport.TTransport import TFramedTransport + +from PixivSearch.thrift.TestQry import TestQry +from PixivSearch.thrift.TestQry.ttypes import QryResult + + +class QueryImpl(TestQry.Iface): + + def qryTest(self, qryCode): + result = QryResult() + if qryCode == 1: + result.code = 1 + result.msg = 'success' + else: + result.code = 0 + result.msg = 'fail' + return result + + +if __name__ == '__main__': + socket = TSocket.TSocket(port=2233) + processor = TestQry.Processor(QueryImpl()) + server = TNonblockingServer(processor, socket, TBinaryProtocolFactory(), TFramedTransport()) + + server.serve() diff --git a/PixivSearch/thrift/TestQry/TestQry-remote b/PixivSearch/thrift/TestQry/TestQry-remote new file mode 100755 index 0000000..3a5edd1 --- /dev/null +++ b/PixivSearch/thrift/TestQry/TestQry-remote @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +import sys +import pprint +from urllib.parse import urlparse +sys.path.append("/home/hua/PycharmProjects/PixivSearch") +from PixivSearch.thrift import TestQry + +from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient +from thrift.protocol.TBinaryProtocol import TBinaryProtocol + + + +if len(sys.argv) <= 1 or sys.argv[1] == '--help': + print('') + print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') + print('') + print('Functions:') + print(' QryResult qryTest(i32 qryCode)') + print('') + sys.exit(0) + +pp = pprint.PrettyPrinter(indent=2) +host = 'localhost' +port = 9090 +uri = '' +framed = False +ssl = False +validate = True +ca_certs = None +keyfile = None +certfile = None +http = False +argi = 1 + +if sys.argv[argi] == '-h': + parts = sys.argv[argi + 1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + argi += 2 + +if sys.argv[argi] == '-u': + url = urlparse(sys.argv[argi + 1]) + parts = url[1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 80 + uri = url[2] + if url[4]: + uri += '?%s' % url[4] + http = True + argi += 2 + +if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': + framed = True + argi += 1 + +if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': + ssl = True + argi += 1 + +if sys.argv[argi] == '-novalidate': + validate = False + argi += 1 + +if sys.argv[argi] == '-ca_certs': + ca_certs = sys.argv[argi+1] + argi += 2 + +if sys.argv[argi] == '-keyfile': + keyfile = sys.argv[argi+1] + argi += 2 + +if sys.argv[argi] == '-certfile': + certfile = sys.argv[argi+1] + argi += 2 + +cmd = sys.argv[argi] +args = sys.argv[argi + 1:] + +if http: + transport = THttpClient.THttpClient(host, port, uri) +else: + if ssl: + socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) + else: + socket = TSocket.TSocket(host, port) + if framed: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) +protocol = TBinaryProtocol(transport) +client = TestQry.Client(protocol) +transport.open() + +if cmd == 'qryTest': + if len(args) != 1: + print('qryTest requires 1 args') + sys.exit(1) + pp.pprint(client.qryTest(eval(args[0]),)) + +else: + print('Unrecognized method %s' % cmd) + sys.exit(1) + +transport.close() diff --git a/PixivSearch/thrift/TestQry/TestQry.py b/PixivSearch/thrift/TestQry/TestQry.py new file mode 100644 index 0000000..4114cd0 --- /dev/null +++ b/PixivSearch/thrift/TestQry/TestQry.py @@ -0,0 +1,247 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# +import logging + +from thrift.TRecursive import fix_spec +from thrift.Thrift import TProcessor +from thrift.Thrift import TType, TMessageType, TApplicationException +from thrift.transport import TTransport + +from PixivSearch.thrift.TestQry.ttypes import QryResult + +all_structs = [] + +port=2233 + +class Iface(object): + def qryTest(self, qryCode): + """ + 测试查询接口，当qryCode值为1时返回"成功"的响应信息，qryCode值为其他值时返回"失败"的响应信息 + @param qryCode测试参数 + + Parameters: + - qryCode + """ + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def qryTest(self, qryCode): + """ + 测试查询接口，当qryCode值为1时返回"成功"的响应信息，qryCode值为其他值时返回"失败"的响应信息 + @param qryCode测试参数 + + Parameters: + - qryCode + """ + self.send_qryTest(qryCode) + return self.recv_qryTest() + + def send_qryTest(self, qryCode): + self._oprot.writeMessageBegin('qryTest', TMessageType.CALL, self._seqid) + args = qryTest_args() + args.qryCode = qryCode + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_qryTest(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = qryTest_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "qryTest failed: unknown result") + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["qryTest"] = Processor.process_qryTest + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_qryTest(self, seqid, iprot, oprot): + args = qryTest_args() + args.read(iprot) + iprot.readMessageEnd() + result = qryTest_result() + try: + result.success = self._handler.qryTest(args.qryCode) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("qryTest", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + +# HELPER FUNCTIONS AND STRUCTURES + + +class qryTest_args(object): + """ + Attributes: + - qryCode + """ + + + def __init__(self, qryCode=None,): + self.qryCode = qryCode + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.qryCode = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('qryTest_args') + if self.qryCode is not None: + oprot.writeFieldBegin('qryCode', TType.I32, 1) + oprot.writeI32(self.qryCode) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(qryTest_args) +qryTest_args.thrift_spec = ( + None, # 0 + (1, TType.I32, 'qryCode', None, None, ), # 1 +) + + +class qryTest_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = QryResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('qryTest_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(qryTest_result) +qryTest_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [QryResult, None], None, ), # 0 +) +fix_spec(all_structs) +del all_structs + + + + diff --git a/PixivSearch/thrift/TestQry/__init__.py b/PixivSearch/thrift/TestQry/__init__.py new file mode 100644 index 0000000..22182f6 --- /dev/null +++ b/PixivSearch/thrift/TestQry/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'TestQry'] diff --git a/PixivSearch/thrift/TestQry/constants.py b/PixivSearch/thrift/TestQry/constants.py new file mode 100644 index 0000000..0c217ce --- /dev/null +++ b/PixivSearch/thrift/TestQry/constants.py @@ -0,0 +1,14 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +from .ttypes import * diff --git a/PixivSearch/thrift/TestQry/ttypes.py b/PixivSearch/thrift/TestQry/ttypes.py new file mode 100644 index 0000000..523a81b --- /dev/null +++ b/PixivSearch/thrift/TestQry/ttypes.py @@ -0,0 +1,91 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys + +from thrift.transport import TTransport +all_structs = [] + + +class QryResult(object): + """ + Attributes: + - code: 返回码, 1成功，0失败 + - msg: 响应信息 + """ + + + def __init__(self, code=None, msg=None,): + self.code = code + self.msg = msg + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.code = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.msg = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('QryResult') + if self.code is not None: + oprot.writeFieldBegin('code', TType.I32, 1) + oprot.writeI32(self.code) + oprot.writeFieldEnd() + if self.msg is not None: + oprot.writeFieldBegin('msg', TType.STRING, 2) + oprot.writeString(self.msg.encode('utf-8') if sys.version_info[0] == 2 else self.msg) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(QryResult) +QryResult.thrift_spec = ( + None, # 0 + (1, TType.I32, 'code', None, None, ), # 1 + (2, TType.STRING, 'msg', 'UTF8', None, ), # 2 +) +fix_spec(all_structs) +del all_structs diff --git a/PixivSearch/thrift/__init__.py b/PixivSearch/thrift/__init__.py new file mode 100644 index 0000000..e69de29 From 667c78dc229e9fb227776bc8f7985a1ffb7cb01f Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Mon, 30 Apr 2018 23:53:55 +0800 Subject: [PATCH 16/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E3=80=81?= =?UTF-8?q?=E5=BC=B9=E5=B9=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/photo/AliyunPhoto.py | 180 ++++++++++++------------ PixivSearch/dao/Comment.py | 132 +++++++++++++++++ PixivSearch/dao/bangumi.py | 85 ++--------- PixivSearch/logging.conf | 2 +- 4 files changed, 237 insertions(+), 162 deletions(-) create mode 100644 PixivSearch/dao/Comment.py diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index 69fb51e..615ffad 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -1,4 +1,3 @@ -import datetime import hashlib import json import os @@ -6,23 +5,27 @@ import time import aliyunsdkcore import oss2 as oss2 -from aliyunsdkcore.client import AcsClient, DEFAULT_SDK_CONNECTION_TIMEOUT_IN_SECONDS +from aliyunsdkcloudphoto.request.v20170711 import ListPhotoStoresRequest, FetchLibrariesRequest, \ + CreateTransactionRequest, CreatePhotoRequest, EditPhotosRequest, GetPublicAccessUrlsRequest, ListPhotosRequest, \ + DeletePhotosRequest, InactivatePhotosRequest +from aliyunsdkcore.auth.credentials import RamRoleArnCredential +from aliyunsdkcore.client import AcsClient +from aliyunsdkcore.profile import region_provider from aliyunsdkcore.request import CommonRequest +region_id = 'cn-shanghai' +ak = 'LTAIWzPnmkJs2qpL' +secret = 'LIIq3HumctXPp0WT8c06yDiFbKKiVe' +region_provider.add_endpoint('Oss', region_id, 'oss-cn-shanghai.aliyuncs.com') +region_provider.add_endpoint('CloudPhoto', region_id, 'cloudphoto.cn-shanghai.aliyuncs.com') +aliyunsdkcore.request.set_default_protocol_type("https") -class UploadPhoto(AcsClient): - def __init__(self, ak='LTAIeS8aBuPBZxV2', secret='hyPeTaDQBQs6jetYcqY0BUdpacXTH3', region_id="cn-hongkong", - auto_retry=True, max_retry_time=3, - user_agent=None, port=80, timeout=DEFAULT_SDK_CONNECTION_TIMEOUT_IN_SECONDS, public_key_id=None, - private_key=None, session_period=3600, credential=None, debug=False): - super().__init__(ak, secret, region_id, auto_retry, max_retry_time, user_agent, port, timeout, public_key_id, - private_key, session_period, credential, debug) - aliyunsdkcore.request.set_default_protocol_type("https") - self.domain = 'cloudphoto.cn-shanghai.aliyuncs.com' - self.version = '2017-07-11' - self.LibraryId = None - self.StoreName = None +class MyClient(AcsClient): + + def __init__(self, arn, sessionName): + super().__init__(region_id=region_id, credential=RamRoleArnCredential(ak, secret, arn, sessionName)) + self.StoreName=None def get_md5_01(self, file_path): md5 = None @@ -35,19 +38,6 @@ class UploadPhoto(AcsClient): md5 = str(hash_code).lower() return md5 - def assumeRole(self): - request = CommonRequest(domain='sts.aliyuncs.com', version='2015-04-01', - action_name='AssumeRole') - RoleSessionName = 'pqh' - request.add_query_param('RoleArn', 'acs:ram::1098806312754985:role/aliyuncloudphotodefaultrole') - request.add_query_param('RoleSessionName', RoleSessionName) - response = json.loads(self.do_action_with_exception(request).decode()) - print(response['Credentials']['SecurityToken']) - - def createAction(self, action): - return CommonRequest(domain=self.domain, version=self.version, - action_name=action) - def do_action_with_exception(self, acs_request): return json.loads(super().do_action_with_exception(acs_request).decode()) @@ -58,43 +48,30 @@ class UploadPhoto(AcsClient): print('{index}:{name}'.format(index=index + 1, name=option[key])) return choose[(int)(input('输入{desc}\n'.format(desc=desc))) - 1] - def getSL(self): - request = self.createAction('ListPhotoStores') + def listPhotoStores(self): + request = ListPhotoStoresRequest.ListPhotoStoresRequest() response = self.do_action_with_exception(request) print('PhotoStores:') photoStores = response['PhotoStores'] self.StoreName = self.showOption(photoStores, 'Name', 'StoreName') - # request = self.createAction('GetPhotoStore') - # request.add_query_param('StoreName', StoreName) - # response = self.do_action_with_exception(request) - # print(response) - - request = self.createAction('FetchLibraries') - request.add_query_param('StoreName', self.StoreName) + def listLibraries(self): + request = FetchLibrariesRequest.FetchLibrariesRequest() + request.set_StoreName(self.StoreName) response = self.do_action_with_exception(request) - Libraries = response['Libraries'] - self.LibraryId = self.showOption(Libraries, 'LibraryId', 'LibraryId') - - # request = self.createAction('GetLibrary') - # request.add_body_params('StoreName', StoreName) - # request.add_body_params('LibraryId', LibraryId) - # response = self.do_action_with_exception(request) - # print(response) + self.Libraries = response['Libraries'] def uploadPhoto(self): - if self.StoreName is None or self.LibraryId is None: - return '未选择StoreName、LibraryId' - - request = self.createAction('CreateTransaction') - filePath = input('输入上传文件路径') + if self.StoreName is None: + self.listPhotoStores() + request = CreateTransactionRequest.CreateTransactionRequest() + filePath = input('输入上传文件路径\n').replace('\\', '/') + filePath = 'C:/Users/47418/Desktop/照片/IMG_20170218_212837.jpg' fileName = filePath.split('/')[-1] - request.add_query_param('Size', os.path.getsize(filePath)) - request.add_query_param('Ext', fileName[-fileName[::-1].index('.'):]) - request.add_query_param('Md5', self.get_md5_01(filePath)) - request.add_query_param('StoreName', self.StoreName) - - request.add_query_param('LibraryId', self.LibraryId) + request.set_Size(os.path.getsize(filePath)) + request.set_Ext(fileName[-fileName[::-1].index('.'):]) + request.set_Md5(self.get_md5_01(filePath)) + request.set_StoreName(self.StoreName) response = self.do_action_with_exception(request) print(response) Upload = response['Transaction']['Upload'] @@ -104,48 +81,71 @@ class UploadPhoto(AcsClient): OssEndpoint = Upload['OssEndpoint'] ObjectKey = Upload['ObjectKey'] - auth = oss2.Auth(self.get_access_key(), self.get_access_secret()) + auth = oss2.StsAuth(self._signer._session_credential[0], self._signer._session_credential[1], + self._signer._session_credential[2]) bucket = oss2.Bucket(auth, OssEndpoint, Bucket) with open(filePath, 'rb') as fileobj: result = bucket.put_object(ObjectKey, fileobj) print('文件上传状态{status}'.format(status=result.status)) - request = self.createAction('CreatePhoto') - request.add_query_param('FileId', FileId) - request.add_query_param('PhotoTitle', fileName) - request.add_query_param('SessionId', SessionId) - request.add_query_param('StoreName', self.StoreName) - request.add_query_param('UploadType', 'manual') - request.add_query_param('LibraryId', self.LibraryId) + request = CreatePhotoRequest.CreatePhotoRequest() + request.set_FileId(FileId) + request.set_PhotoTitle(fileName) + request.set_SessionId(SessionId) + request.set_StoreName(self.StoreName) + request.set_UploadType('manual') + response = self.do_action_with_exception(request) + print(response) + + def listPhotos(self): + if self.StoreName == None: + self.listPhotoStores() + request = ListPhotosRequest.ListPhotosRequest() + request.set_StoreName(self.StoreName) + request.set_State('all') response = self.do_action_with_exception(request) print(response) + return response['Photos'] + + def getPhoto(self): + return self.showOption(self.listPhotos()['Photos'], 'IdStr', '照片Id') + + def sharePhoto(self): + IdStr = self.listPhotos() + request = EditPhotosRequest.EditPhotosRequest() + request.set_PhotoIds([IdStr]) + request.set_StoreName(self.StoreName) + request.set_ShareExpireTime((int(round(time.time())) + 60 * 60) * 1000) + response = self.do_action_with_exception(request) + print(response) + + request = GetPublicAccessUrlsRequest.GetPublicAccessUrlsRequest() + request.set_DomainType('OSS') + request.set_PhotoIds([IdStr]) + request.set_StoreName(self.StoreName) + request.set_ZoomType('style/2') + response = self.do_action_with_exception(request) + print(response) + +def client(arn, sessionName): + ram_role_arn_credential = RamRoleArnCredential('LTAIWzPnmkJs2qpL', 'LIIq3HumctXPp0WT8c06yDiFbKKiVe', + arn, sessionName) + return AcsClient(region_id='cn-shanghai', credential=ram_role_arn_credential) if __name__ == '__main__': - client = UploadPhoto() - client.getSL() - request = client.createAction('ListPhotos') - request.add_query_param('StoreName', client.StoreName) - request.add_query_param('LibraryId', client.LibraryId) - request.add_query_param('State', 'all') - response = client.do_action_with_exception(request) - print(response) - - IdStr = client.showOption(response['Photos'], 'IdStr', '照片Id') - - request = client.createAction('EditPhotos') - request.add_query_param('PhotoId.1', IdStr) - request.add_query_param('StoreName', client.StoreName) - request.add_query_param('LibraryId', client.LibraryId) - request.add_query_param('ShareExpireTime', int(round(time.time() * 1000)) + 60 * 60) - response = client.do_action_with_exception(request) - print(response) - - request = client.createAction('GetPublicAccessUrls') - request.add_query_param('DomainType', 'OSS') - request.add_query_param('PhotoId.1', IdStr) - request.add_query_param('StoreName', client.StoreName) - request.add_query_param('ZoomType', 'style/1') - request.add_query_param('LibraryId', client.LibraryId) - response = client.do_action_with_exception(request) - print(response) \ No newline at end of file + + myClient = MyClient('acs:ram::1098806312754985:role/aliyunosstokengeneratorrole', 'pqh001') + myClient.listPhotoStores() + request = FetchLibrariesRequest.FetchLibrariesRequest() + request.set_StoreName(myClient.StoreName) + myClient.listLibraries() + storeName = myClient.StoreName + for Library in myClient.Libraries: + myClient = MyClient('acs:ram::1098806312754985:role/aliyunosstokengeneratorrole', Library['LibraryId']) + for id in myClient.listPhotos(): + request=InactivatePhotosRequest.InactivatePhotosRequest() + request.set_StoreName(storeName) + request.set_PhotoIds([id['IdStr']]) + response=myClient.do_action_with_exception(request) + print(response) diff --git a/PixivSearch/dao/Comment.py b/PixivSearch/dao/Comment.py new file mode 100644 index 0000000..ad893f4 --- /dev/null +++ b/PixivSearch/dao/Comment.py @@ -0,0 +1,132 @@ +import json +import threading +from concurrent import futures + +import requests +import xlrd +from lxml import etree + +from PixivSearch.dao.bangumi import episodeIdToCid + + +class Comment: + lock = threading.Lock() # 多线程全局资源锁 + + def __init__(self, keywords_=None) -> None: + super().__init__() + self.obj = {'data': {}, 'flag': False} + self.keywords = keywords_ + + # 获取番剧合集弹幕排行榜 + def getCommentSort(self, cids): + + urls = [] + for cid in cids: + urls.extend(getCidUrls(cid)) + with futures.ThreadPoolExecutor(32) as executor: + executor.map(self.count, urls) + for index, data in enumerate( + sorted(self.obj["data"].items(), key=lambda d: d[1], reverse=True)[ + :50]): + print('{index}:{data}'.format(index=index+1, data=data)) + + # 获取番剧合集弹幕排行榜 + def count(self, url, desc=None): + bytes = requests.get(url).content + comment_selector = etree.HTML(bytes) + if not desc is None: + print(desc) + print("url=%s" % url) + for comment in comment_selector.xpath('//i//d/text()'): + if comment in self.obj["data"]: + with self.lock: + self.obj["data"][comment] = self.obj["data"][comment] + 1 + else: + with self.lock: + self.obj["data"][comment] = 1 + if not self.obj["flag"]: + for keyword in self.keywords: + if keyword in comment: + self.obj["flag"] = True + + +# 根据cid获取历史弹幕地址 +def getCidUrls(cid): + urls = [] + url = "https://comment.bilibili.com/rolldate,%d" % cid + req = requests.get(url) + if len(req.text) > 0: + for i in json.loads(req.text): + urls.append("https://comment.bilibili.com/dmroll,%s,%d" % (i['timestamp'], cid)) + else: + urls.append("https://comment.bilibili.com/%d.xml" % cid) + return urls + + +# 下载历史弹幕 +def parseXmlByHistory(path, cid,size=None): + dlist = set() + flag=parseXml(getCidUrls(cid), dlist, size) + if size is None or not size is None and flag: + f = open(path, 'wb') + f.write(b'') + for i in dlist: + f.write(('\r\n' + i).encode()) + f.write(b'\r\n') + f.close() + + +def parseXml(urls, dlist, size=None): + if isinstance(urls, str): + urls = [urls] + if not size is None: + size = float(size.strip('%')) / 100.0 + for url in urls: + bytes = requests.get(url).content + comment_selector = etree.HTML(bytes) + list = comment_selector.xpath('//i//d/text()') + maxlimit = int(comment_selector.xpath('//i//maxlimit/text()')[0]) + + if len(list) > 0: + print('弹幕数：{list},最大弹幕数：{maxlimit},弹幕池填充：{p}'.format(list=len(list), maxlimit=maxlimit, + p='%.2f%%' % (len(list) / maxlimit * 100))) + for element in comment_selector.xpath('//i//d'): + if len(element.xpath("text()")) > 0: + fstr = '{content}'.format(p=str(element.xpath("@p")[0]), + content=str(element.xpath("text()")[0])) + dlist.add(fstr) + + currentSize = len(dlist) / maxlimit + print('填充率：{l}'.format(l='%.2f%%' % (currentSize * 100))) + if not size is None and currentSize >= size: + return True + return False + +if __name__ == '__main__': + # parseXmlByHistory('10815558.xml', 10815558) + # Comment('').getCommentSort(episodeIdToCid(172095)) + cids=[11664778, +11662541, +11661412, +11664304, +11666093, + +] + + for cid in cids: + parseXmlByHistory('{cid}.xml'.format(cid=cid),cid) + + # path = 'D:/QQ/1029559041/FileRecv/tmp001.xlsx' + # ExcelFile = xlrd.open_workbook(path) + # + # + # for sheetName in ExcelFile.sheet_names(): + # sheet = ExcelFile.sheet_by_name(sheetName) + # for row in sheet.get_rows(): + # try: + # row[5].value.index('日剧') + # cid = int(row[2].value) + # print('aid={aid},cid={cid},title={title}'.format(aid=int(row[1].value),cid=cid,title=row[5].value)) + # parseXmlByHistory('{cid}.xml'.format(cid=cid),cid,'50%') + # except BaseException as e: + # repr(e) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 7fc523b..6b02cce 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -92,7 +92,7 @@ def listen(): _thread.start_new_thread(listen, ()) - +#遍历所有专题视频收藏数信息 def getIds(): seasonIdList = [] page = 1 @@ -140,22 +140,12 @@ def getIds(): logger.error(repr(e)) continue - -# def testA(): -# req = requests.post('https://api.bilibili.com/x/report/web/heartbeat', -# data={"aid": 29416,"cid":49052,"csrf": "c0d296db7e33085f9f4730cfee66660b"}, -# cookies=_cookies) -# print(req.status_code) - -_cookies = {'DedeUserID': '4372744', 'DedeUserID__ckMd5': 'e8179b74444cae8e', - 'SESSDATA': '919b17d2%2C1524917631%2C3eede719'} - - -def getCid(aid, type=True): +#根据aid获取cid +def getCid(aid, type=None): while True and aid > 0: url = "https://api.bilibili.com/x/web-interface/archive/stat?aid=%d" % aid print(url) - req = requests.get(url, cookies=_cookies) + req = requests.get(url) code = json.loads(req.text)["code"] if code == 0: req = requests.get("https://www.bilibili.com/video/av%d" % aid) @@ -166,17 +156,20 @@ def getCid(aid, type=True): cid = json_obj['videoData']['pages'][0]['cid'] print('cid=%s' % cid) return cid - if type: - aid = aid - 1 + if type is None: + break else: - aid = aid + 1 - + if type: + aid = aid - 1 + else: + aid = aid + 1 +#根据aid获取cid def getCids(aid): s = {"min": getCid(aid, True), "max": getCid(aid, False)} return s - +#获取专题所有cid def episodeIdToCid(episode_id): cids = [] url = "https://www.bilibili.com/bangumi/play/ep%d" % episode_id @@ -189,59 +182,9 @@ def episodeIdToCid(episode_id): return cids -def parseXml(url): - print("url=%s" % url) - comment_selector = etree.HTML(requests.get(url).content) - comment_content = comment_selector.xpath('//i') - for comment_each in comment_content: - comments = comment_each.xpath('//d/text()') - if comments: - for comment in comments: - if comment in obj["data"]: - with lock: - obj["data"][comment] = obj["data"][comment] + 1 - else: - with lock: - obj["data"][comment] = 1 - if not obj["flag"]: - for keyword in keywords: - if keyword in comment: - obj["flag"] = True - - -lock = threading.Lock() # 多线程全局资源锁 - - -def loadData(cids): - params = [] - for cid in cids: - url = "https://comment.bilibili.com/rolldate,%d" % cid - req = requests.get(url) - urls = ["https://comment.bilibili.com/%d.xml" % cid] - if len(req.text) > 0: - for i in json.loads(req.text): - urls.append("https://comment.bilibili.com/dmroll,%s,%d" % (i['timestamp'], cid)) - for url in urls: - params.append(url) - with futures.ThreadPoolExecutor(32) as executor: - executor.map(parseXml, params) - return obj - - -def getCommentSort(cids, keywords_): - global keywords, obj - keywords = keywords_ - obj = {"data": {}, "flag": False} - return loadData(cids) - if __name__ == '__main__': - # print(getCids(29416)) - # obj = loadData( - # [49052, 49053, 51525, 51526, 53407, 54180, 55295, 55296, 57255, 57256, 59288, 59289, 61559, 61560, 64034, 64035, - # 67024, 67025, 69284, 73333, 73334, 74024, 74025], ['穹']) - f = getCommentSort(episodeIdToCid(172095), []) + print(getCids(29416)) # obj = loadData([34807341], []) - for i in sorted(f["data"].items(), key=lambda d: d[1], reverse=True)[:50]: - print(i) + diff --git a/PixivSearch/logging.conf b/PixivSearch/logging.conf index dc3325b..82c59e0 100644 --- a/PixivSearch/logging.conf +++ b/PixivSearch/logging.conf @@ -18,7 +18,7 @@ handlers=fileHandler [handler_consoleHandler] class=StreamHandler -level=DEBUG +level=INFO formatter=fmt args=(sys.stdout,) From 972e42fa36b41d9e219a1b7810e7f7f77f47e75d Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Wed, 2 May 2018 18:48:44 +0800 Subject: [PATCH 17/22] thift --- PixivSearch/aliyun/photo/AliyunPhoto.py | 1 - PixivSearch/thrift/TestQry/Client.py | 2 +- PixivSearch/thrift/TestQry/Server.py | 7 ++----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index 615ffad..a81f1dc 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -66,7 +66,6 @@ class MyClient(AcsClient): self.listPhotoStores() request = CreateTransactionRequest.CreateTransactionRequest() filePath = input('输入上传文件路径\n').replace('\\', '/') - filePath = 'C:/Users/47418/Desktop/照片/IMG_20170218_212837.jpg' fileName = filePath.split('/')[-1] request.set_Size(os.path.getsize(filePath)) request.set_Ext(fileName[-fileName[::-1].index('.'):]) diff --git a/PixivSearch/thrift/TestQry/Client.py b/PixivSearch/thrift/TestQry/Client.py index 2839dc7..22237b5 100644 --- a/PixivSearch/thrift/TestQry/Client.py +++ b/PixivSearch/thrift/TestQry/Client.py @@ -22,4 +22,4 @@ if __name__ == '__main__': # Create a client to use the protocol encoder client = Client(protocol) - print(client.qryTest(0)) + print(client.qryTest(1)) diff --git a/PixivSearch/thrift/TestQry/Server.py b/PixivSearch/thrift/TestQry/Server.py index de53097..4e78d52 100644 --- a/PixivSearch/thrift/TestQry/Server.py +++ b/PixivSearch/thrift/TestQry/Server.py @@ -1,8 +1,5 @@ -from thrift.protocol import TBinaryProtocol -from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory from thrift.server.TNonblockingServer import TNonblockingServer from thrift.transport import TSocket -from thrift.transport.TTransport import TFramedTransport from PixivSearch.thrift.TestQry import TestQry from PixivSearch.thrift.TestQry.ttypes import QryResult @@ -22,8 +19,8 @@ class QueryImpl(TestQry.Iface): if __name__ == '__main__': - socket = TSocket.TSocket(port=2233) + socket = TSocket.TServerSocket(port=2233) processor = TestQry.Processor(QueryImpl()) - server = TNonblockingServer(processor, socket, TBinaryProtocolFactory(), TFramedTransport()) + server = TNonblockingServer(processor, socket) server.serve() From ba24e22147e9c2d13447c6509974fec4a11ffae7 Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Fri, 4 May 2018 18:44:29 +0800 Subject: [PATCH 18/22] thift --- PixivSearch/templates/index.html | 6 +++--- PixivSearch/thrift/TestQry/Client.py | 4 ++-- PixivSearch/thrift/TestQry/Server.py | 4 ++++ PixivSearch/thrift/TestQry/TestQry-remote | 7 +++++-- PixivSearch/thrift/TestQry/TestQry.py | 2 -- root.ini | 2 ++ root.py | 11 +++++++++++ 7 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 root.ini create mode 100644 root.py diff --git a/PixivSearch/templates/index.html b/PixivSearch/templates/index.html index 9d2d15b..24979ab 100644 --- a/PixivSearch/templates/index.html +++ b/PixivSearch/templates/index.html @@ -31,13 +31,13 @@ - - + +{# #} +{# #} {% if download %} 图包提取(文件大小:{{ size}}) -{# #} {% endif %} diff --git a/PixivSearch/thrift/TestQry/Client.py b/PixivSearch/thrift/TestQry/Client.py index 22237b5..89fea10 100644 --- a/PixivSearch/thrift/TestQry/Client.py +++ b/PixivSearch/thrift/TestQry/Client.py @@ -3,12 +3,12 @@ import sys from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -sys.path.append("/home/hua/PycharmProjects/PixivSearch") +sys.path.append('/root/PixivSearch') from PixivSearch.thrift.TestQry.TestQry import Client if __name__ == '__main__': # Make socket - socket = TSocket.TSocket('127.0.0.1', 2233) + socket = TSocket.TSocket('mikuhime.xyz', 2233) # Buffering is critical. Raw sockets are very slow transport = TTransport.TFramedTransport(socket) diff --git a/PixivSearch/thrift/TestQry/Server.py b/PixivSearch/thrift/TestQry/Server.py index 4e78d52..a31e9e5 100644 --- a/PixivSearch/thrift/TestQry/Server.py +++ b/PixivSearch/thrift/TestQry/Server.py @@ -1,6 +1,9 @@ +import sys + from thrift.server.TNonblockingServer import TNonblockingServer from thrift.transport import TSocket +sys.path.append('/root/PixivSearch') from PixivSearch.thrift.TestQry import TestQry from PixivSearch.thrift.TestQry.ttypes import QryResult @@ -12,6 +15,7 @@ class QueryImpl(TestQry.Iface): if qryCode == 1: result.code = 1 result.msg = 'success' + else: result.code = 0 result.msg = 'fail' diff --git a/PixivSearch/thrift/TestQry/TestQry-remote b/PixivSearch/thrift/TestQry/TestQry-remote index 3a5edd1..c12dd5a 100755 --- a/PixivSearch/thrift/TestQry/TestQry-remote +++ b/PixivSearch/thrift/TestQry/TestQry-remote @@ -10,8 +10,11 @@ import sys import pprint from urllib.parse import urlparse -sys.path.append("/home/hua/PycharmProjects/PixivSearch") -from PixivSearch.thrift import TestQry + +from PixivSearch.thrift.TestQry import TestQry + +sys.path.append('/root/PixivSearch') + from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient from thrift.protocol.TBinaryProtocol import TBinaryProtocol diff --git a/PixivSearch/thrift/TestQry/TestQry.py b/PixivSearch/thrift/TestQry/TestQry.py index 4114cd0..228ef8d 100644 --- a/PixivSearch/thrift/TestQry/TestQry.py +++ b/PixivSearch/thrift/TestQry/TestQry.py @@ -16,8 +16,6 @@ from PixivSearch.thrift.TestQry.ttypes import QryResult all_structs = [] -port=2233 - class Iface(object): def qryTest(self, qryCode): """ diff --git a/root.ini b/root.ini new file mode 100644 index 0000000..828fda0 --- /dev/null +++ b/root.ini @@ -0,0 +1,2 @@ +[config] +bastPath=/root/PixivSearch \ No newline at end of file diff --git a/root.py b/root.py new file mode 100644 index 0000000..8994557 --- /dev/null +++ b/root.py @@ -0,0 +1,11 @@ +import configparser + +cf = configparser.ConfigParser() +cf.read('root.ini') + +def getConfigValue(name): + value = cf.get("config", name) + return value + +if __name__ == '__main__': + print(getConfigValue('bastPath')) \ No newline at end of file From 06a5a453f4c234efdbd99f673c294134768bb441 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Mon, 7 May 2018 00:50:43 +0800 Subject: [PATCH 19/22] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BA=91=E7=9B=B8=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/aliyun/photo/AliyunPhoto.py | 22 ++++++---------------- PixivSearch/dao/Comment.py | 22 +++++++++------------- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/PixivSearch/aliyun/photo/AliyunPhoto.py b/PixivSearch/aliyun/photo/AliyunPhoto.py index a81f1dc..80954dc 100644 --- a/PixivSearch/aliyun/photo/AliyunPhoto.py +++ b/PixivSearch/aliyun/photo/AliyunPhoto.py @@ -7,7 +7,7 @@ import aliyunsdkcore import oss2 as oss2 from aliyunsdkcloudphoto.request.v20170711 import ListPhotoStoresRequest, FetchLibrariesRequest, \ CreateTransactionRequest, CreatePhotoRequest, EditPhotosRequest, GetPublicAccessUrlsRequest, ListPhotosRequest, \ - DeletePhotosRequest, InactivatePhotosRequest + DeletePhotosRequest, InactivatePhotosRequest, GetLibraryRequest from aliyunsdkcore.auth.credentials import RamRoleArnCredential from aliyunsdkcore.client import AcsClient from aliyunsdkcore.profile import region_provider @@ -25,7 +25,7 @@ class MyClient(AcsClient): def __init__(self, arn, sessionName): super().__init__(region_id=region_id, credential=RamRoleArnCredential(ak, secret, arn, sessionName)) - self.StoreName=None + self.StoreName = None def get_md5_01(self, file_path): md5 = None @@ -107,10 +107,10 @@ class MyClient(AcsClient): return response['Photos'] def getPhoto(self): - return self.showOption(self.listPhotos()['Photos'], 'IdStr', '照片Id') + return self.showOption(self.listPhotos(), 'IdStr', '照片Id') def sharePhoto(self): - IdStr = self.listPhotos() + IdStr = self.getPhoto() request = EditPhotosRequest.EditPhotosRequest() request.set_PhotoIds([IdStr]) request.set_StoreName(self.StoreName) @@ -126,6 +126,7 @@ class MyClient(AcsClient): response = self.do_action_with_exception(request) print(response) + def client(arn, sessionName): ram_role_arn_credential = RamRoleArnCredential('LTAIWzPnmkJs2qpL', 'LIIq3HumctXPp0WT8c06yDiFbKKiVe', arn, sessionName) @@ -136,15 +137,4 @@ if __name__ == '__main__': myClient = MyClient('acs:ram::1098806312754985:role/aliyunosstokengeneratorrole', 'pqh001') myClient.listPhotoStores() - request = FetchLibrariesRequest.FetchLibrariesRequest() - request.set_StoreName(myClient.StoreName) - myClient.listLibraries() - storeName = myClient.StoreName - for Library in myClient.Libraries: - myClient = MyClient('acs:ram::1098806312754985:role/aliyunosstokengeneratorrole', Library['LibraryId']) - for id in myClient.listPhotos(): - request=InactivatePhotosRequest.InactivatePhotosRequest() - request.set_StoreName(storeName) - request.set_PhotoIds([id['IdStr']]) - response=myClient.do_action_with_exception(request) - print(response) + myClient.sharePhoto() diff --git a/PixivSearch/dao/Comment.py b/PixivSearch/dao/Comment.py index ad893f4..294f423 100644 --- a/PixivSearch/dao/Comment.py +++ b/PixivSearch/dao/Comment.py @@ -3,11 +3,8 @@ import threading from concurrent import futures import requests -import xlrd from lxml import etree -from PixivSearch.dao.bangumi import episodeIdToCid - class Comment: lock = threading.Lock() # 多线程全局资源锁 @@ -75,6 +72,9 @@ def parseXmlByHistory(path, cid,size=None): f.write(b'\r\n') f.close() +def xml(url): + bytes = requests.get(url).content + return etree.HTML(bytes) def parseXml(urls, dlist, size=None): if isinstance(urls, str): @@ -82,8 +82,7 @@ def parseXml(urls, dlist, size=None): if not size is None: size = float(size.strip('%')) / 100.0 for url in urls: - bytes = requests.get(url).content - comment_selector = etree.HTML(bytes) + comment_selector = xml(url) list = comment_selector.xpath('//i//d/text()') maxlimit = int(comment_selector.xpath('//i//maxlimit/text()')[0]) @@ -105,16 +104,13 @@ def parseXml(urls, dlist, size=None): if __name__ == '__main__': # parseXmlByHistory('10815558.xml', 10815558) # Comment('').getCommentSort(episodeIdToCid(172095)) - cids=[11664778, -11662541, -11661412, -11664304, -11666093, - -] + cids=[653701,653702,653703,653704,653705,653738,653739,653740,653741,653742,653743,655269,655270,655271,655272,655273,655274,655275,655276,655277,655278,655279,1153426,1160531,1160532,1160533,1160534,1166828,3462691,3526913,4566294,4566295,4566296,4566297,4566298,4566299,4997067,4997068,4997069,4997070,4997071,4997072,4997073,4997074,4997075,4997076,4997077,4997078,4997079,4997080,4997081,4997082,4997083,4997084,4997085,4997086,4997087,4997088,4997089,5679996,7091113,7091114,7091115,7091116,7091117,7091118,7091119,7091120,7102322,7102323,7102324,7105422,7105423,7105424,7105425,7105426,7105427,7105428,7105429,7105430,7105431,7105432,7111387,7862786,7862787,7862788,7862789,7862790,7862791,7862792,7862793,7862794,7862795,7862796,8097054,8462197,8462198,8463198,8463856,8464066,8465726,8466012,8466263,8466715,8467897,8468074,8469339,8470303,8470792,8471514,8471910,8472168,8472542,8473404,8474331,8476130,8974866,9006609,9914921,9914922,9930188,9930189,9930190,9935817,9935818,9935819,9946753,10240252,10240253,10240254,10240255,10240256,10240257,10240258,10240259,10240260,10240261,10240262,12029055,12107619,13109117,13109118,13109119,13109120,13109121,13109122,13109123,13109124,13109125,13109126,13109127,13109128,13109129,13109130,13109131,13109132,13109133,13109134,13109135,13109136,13109137,13109138,13109139,13109140,13109141,13109142,13109143,13109144,13109145,13109146,13109147,13109148,13109149,13109150,13109151,13109152,13109153,13109154,13109155,13280053,13386513,13386514,13386515,13386516,13386517,13386518,13386519,13386520,13386521,13386522,13386523,13494289,14369143,14842278] + cids=[6675884,6675885,6675886,6722497,6722498,6722499,6722500,6761162,6761163,6761164,6761165,7008191,7008192,7008193,7008194,7008195,7008196,7008197,7008198,7008199,7008200,7008201,7018471,7018472,7018473,7018474,7018475,7018476,7018477,7018478,7018479,7018480,7018481,7636499,7636500,7636501,7636502,7636503,7636504,7636505,7636506,7636507,7636508,7636509,9019798,9019799,9019800,9019801,9019802,9019803,9019804,9019805,9019806,9019807,9019808,9019809,9832709] for cid in cids: - parseXmlByHistory('{cid}.xml'.format(cid=cid),cid) + comment_selector=xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid)) + print('cid:{cid},弹幕数：{length}'.format(cid=cid,length=len(comment_selector.xpath('//i//d/text()')))) + # parseXmlByHistory('{cid}.xml'.format(cid=cid),cid) # path = 'D:/QQ/1029559041/FileRecv/tmp001.xlsx' # ExcelFile = xlrd.open_workbook(path) From f0e5adb10574afb99cbf82eb920bc79bd448b577 Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Mon, 4 Jun 2018 01:28:02 +0800 Subject: [PATCH 20/22] =?UTF-8?q?thrift=E5=BC=B9=E5=B9=95=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/Comment.py | 53 +- PixivSearch/dao/bangumi.py | 3 +- PixivSearch/pixiv/pixiv.py | 24 +- PixivSearch/settings.py | 6 +- .../{TestQry => QueryComment}/Client.py | 12 +- .../QueryComment-remote} | 38 +- .../thrift/QueryComment/QueryComment.py | 660 ++++++++++++++++++ PixivSearch/thrift/QueryComment/Server.py | 48 ++ PixivSearch/thrift/QueryComment/__init__.py | 1 + .../{TestQry => QueryComment}/constants.py | 0 PixivSearch/thrift/QueryComment/ttypes.py | 18 + PixivSearch/thrift/TestQry/Server.py | 30 - PixivSearch/thrift/TestQry/TestQry.py | 245 ------- PixivSearch/thrift/TestQry/__init__.py | 1 - PixivSearch/thrift/TestQry/ttypes.py | 91 --- PixivSearch/util/Util.py | 10 + PixivSearch/util/__init__.py | 0 PixivSearch/view.py | 2 +- 18 files changed, 803 insertions(+), 439 deletions(-) rename PixivSearch/thrift/{TestQry => QueryComment}/Client.py (63%) rename PixivSearch/thrift/{TestQry/TestQry-remote => QueryComment/QueryComment-remote} (74%) mode change 100755 => 100644 create mode 100644 PixivSearch/thrift/QueryComment/QueryComment.py create mode 100644 PixivSearch/thrift/QueryComment/Server.py create mode 100644 PixivSearch/thrift/QueryComment/__init__.py rename PixivSearch/thrift/{TestQry => QueryComment}/constants.py (100%) create mode 100644 PixivSearch/thrift/QueryComment/ttypes.py delete mode 100644 PixivSearch/thrift/TestQry/Server.py delete mode 100644 PixivSearch/thrift/TestQry/TestQry.py delete mode 100644 PixivSearch/thrift/TestQry/__init__.py delete mode 100644 PixivSearch/thrift/TestQry/ttypes.py create mode 100644 PixivSearch/util/Util.py create mode 100644 PixivSearch/util/__init__.py diff --git a/PixivSearch/dao/Comment.py b/PixivSearch/dao/Comment.py index 294f423..2a9f5c7 100644 --- a/PixivSearch/dao/Comment.py +++ b/PixivSearch/dao/Comment.py @@ -1,4 +1,6 @@ import json +import os +import shutil import threading from concurrent import futures @@ -25,7 +27,7 @@ class Comment: for index, data in enumerate( sorted(self.obj["data"].items(), key=lambda d: d[1], reverse=True)[ :50]): - print('{index}:{data}'.format(index=index+1, data=data)) + print('{index}:{data}'.format(index=index + 1, data=data)) # 获取番剧合集弹幕排行榜 def count(self, url, desc=None): @@ -60,22 +62,31 @@ def getCidUrls(cid): return urls -# 下载历史弹幕 -def parseXmlByHistory(path, cid,size=None): +# 下载弹幕 +def downloadXml(path, cid, size=None, histroy=True): dlist = set() - flag=parseXml(getCidUrls(cid), dlist, size) - if size is None or not size is None and flag: - f = open(path, 'wb') + flag = None + if histroy: + flag = parseXml(getCidUrls(cid), dlist, size) + else: + parseXml("https://comment.bilibili.com/%d.xml" % cid, dlist, size) + if size is None or (histroy and not size is None and flag): + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path) + f = open('{path}/{cid}.xml'.format(path=path, cid=cid), 'wb') f.write(b'') for i in dlist: f.write(('\r\n' + i).encode()) f.write(b'\r\n') f.close() + def xml(url): bytes = requests.get(url).content return etree.HTML(bytes) + def parseXml(urls, dlist, size=None): if isinstance(urls, str): urls = [urls] @@ -92,7 +103,7 @@ def parseXml(urls, dlist, size=None): for element in comment_selector.xpath('//i//d'): if len(element.xpath("text()")) > 0: fstr = '{content}'.format(p=str(element.xpath("@p")[0]), - content=str(element.xpath("text()")[0])) + content=str(element.xpath("text()")[0])) dlist.add(fstr) currentSize = len(dlist) / maxlimit @@ -101,28 +112,8 @@ def parseXml(urls, dlist, size=None): return True return False + if __name__ == '__main__': - # parseXmlByHistory('10815558.xml', 10815558) - # Comment('').getCommentSort(episodeIdToCid(172095)) - cids=[653701,653702,653703,653704,653705,653738,653739,653740,653741,653742,653743,655269,655270,655271,655272,655273,655274,655275,655276,655277,655278,655279,1153426,1160531,1160532,1160533,1160534,1166828,3462691,3526913,4566294,4566295,4566296,4566297,4566298,4566299,4997067,4997068,4997069,4997070,4997071,4997072,4997073,4997074,4997075,4997076,4997077,4997078,4997079,4997080,4997081,4997082,4997083,4997084,4997085,4997086,4997087,4997088,4997089,5679996,7091113,7091114,7091115,7091116,7091117,7091118,7091119,7091120,7102322,7102323,7102324,7105422,7105423,7105424,7105425,7105426,7105427,7105428,7105429,7105430,7105431,7105432,7111387,7862786,7862787,7862788,7862789,7862790,7862791,7862792,7862793,7862794,7862795,7862796,8097054,8462197,8462198,8463198,8463856,8464066,8465726,8466012,8466263,8466715,8467897,8468074,8469339,8470303,8470792,8471514,8471910,8472168,8472542,8473404,8474331,8476130,8974866,9006609,9914921,9914922,9930188,9930189,9930190,9935817,9935818,9935819,9946753,10240252,10240253,10240254,10240255,10240256,10240257,10240258,10240259,10240260,10240261,10240262,12029055,12107619,13109117,13109118,13109119,13109120,13109121,13109122,13109123,13109124,13109125,13109126,13109127,13109128,13109129,13109130,13109131,13109132,13109133,13109134,13109135,13109136,13109137,13109138,13109139,13109140,13109141,13109142,13109143,13109144,13109145,13109146,13109147,13109148,13109149,13109150,13109151,13109152,13109153,13109154,13109155,13280053,13386513,13386514,13386515,13386516,13386517,13386518,13386519,13386520,13386521,13386522,13386523,13494289,14369143,14842278] - - cids=[6675884,6675885,6675886,6722497,6722498,6722499,6722500,6761162,6761163,6761164,6761165,7008191,7008192,7008193,7008194,7008195,7008196,7008197,7008198,7008199,7008200,7008201,7018471,7018472,7018473,7018474,7018475,7018476,7018477,7018478,7018479,7018480,7018481,7636499,7636500,7636501,7636502,7636503,7636504,7636505,7636506,7636507,7636508,7636509,9019798,9019799,9019800,9019801,9019802,9019803,9019804,9019805,9019806,9019807,9019808,9019809,9832709] - for cid in cids: - comment_selector=xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid)) - print('cid:{cid},弹幕数：{length}'.format(cid=cid,length=len(comment_selector.xpath('//i//d/text()')))) - # parseXmlByHistory('{cid}.xml'.format(cid=cid),cid) - - # path = 'D:/QQ/1029559041/FileRecv/tmp001.xlsx' - # ExcelFile = xlrd.open_workbook(path) - # - # - # for sheetName in ExcelFile.sheet_names(): - # sheet = ExcelFile.sheet_by_name(sheetName) - # for row in sheet.get_rows(): - # try: - # row[5].value.index('日剧') - # cid = int(row[2].value) - # print('aid={aid},cid={cid},title={title}'.format(aid=int(row[1].value),cid=cid,title=row[5].value)) - # parseXmlByHistory('{cid}.xml'.format(cid=cid),cid,'50%') - # except BaseException as e: - # repr(e) + cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505] + + downloadXml('F:/ABC',12026697,histroy=False) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 6b02cce..4f048f6 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -184,7 +184,8 @@ def episodeIdToCid(episode_id): if __name__ == '__main__': - print(getCids(29416)) + # print(getCids(29416)) + req=requests.post('https://api.bilibili.com/x/v2/history/shadow/set','') # obj = loadData([34807341], []) diff --git a/PixivSearch/pixiv/pixiv.py b/PixivSearch/pixiv/pixiv.py index a3312b5..17c278f 100644 --- a/PixivSearch/pixiv/pixiv.py +++ b/PixivSearch/pixiv/pixiv.py @@ -1,17 +1,18 @@ #!/usr/bin/env python # coding:utf-8 -from concurrent import futures -import threading import json -import requests -from bs4 import BeautifulSoup +import os import sys +import threading +from concurrent import futures from datetime import datetime -import os -import zipfile + +import requests +from bs4 import BeautifulSoup from PixivSearch.model import config from PixivSearch.settings import logger +from PixivSearch.util import Util headers = { 'X-Requested-With': 'XMLHttpRequest', @@ -74,15 +75,6 @@ def get_Img(params): params[0], params[1]['bookmarkCount'], params[1]['illustTitle'], ','.join(params[1]['tags']), '', params[1]['imgUrl'])) - -def zip(inputFile, outFile): - f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED) - for dirpath, dirnames, filenames in os.walk(inputFile): - for filename in filenames: - f.write(os.path.join(dirpath, filename), filename) - f.close() - - fsize = '' @@ -119,7 +111,7 @@ def get_nodes(param): zipPath = 'PixivSearch/static/download/' + param[1] + '.zip' logger.info('图片打包到：%s' % (zipPath)) - zip(imgPath, zipPath) + Util.zip(imgPath, zipPath) fsize = str(round(os.path.getsize(zipPath) / float(1024 * 1024), 2)) + 'MB' logger.info('图包大小：%s' % (fsize)) diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index f584b76..9be76e0 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -79,9 +79,9 @@ DATABASES = { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'bangumi', 'USER': 'root', - 'PASSWORD': 'Luffy9412!', - # 'HOST': '127.0.0.1', - 'HOST': 'mikuhime.xyz', + 'PASSWORD': '', + 'HOST': '127.0.0.1', + # 'HOST': 'sukura.top', 'PORT': '3306', } } diff --git a/PixivSearch/thrift/TestQry/Client.py b/PixivSearch/thrift/QueryComment/Client.py similarity index 63% rename from PixivSearch/thrift/TestQry/Client.py rename to PixivSearch/thrift/QueryComment/Client.py index 89fea10..9abc8e7 100644 --- a/PixivSearch/thrift/TestQry/Client.py +++ b/PixivSearch/thrift/QueryComment/Client.py @@ -1,14 +1,11 @@ -import sys - from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -sys.path.append('/root/PixivSearch') -from PixivSearch.thrift.TestQry.TestQry import Client +from PixivSearch.thrift.QueryComment.QueryComment import Client if __name__ == '__main__': - # Make socket - socket = TSocket.TSocket('mikuhime.xyz', 2233) + + socket = TSocket.TSocket('sukura.top', 2233) # Buffering is critical. Raw sockets are very slow transport = TTransport.TFramedTransport(socket) @@ -22,4 +19,5 @@ if __name__ == '__main__': # Create a client to use the protocol encoder client = Client(protocol) - print(client.qryTest(1)) + cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505] + print(client.commentSumMap(cids)) \ No newline at end of file diff --git a/PixivSearch/thrift/TestQry/TestQry-remote b/PixivSearch/thrift/QueryComment/QueryComment-remote old mode 100755 new mode 100644 similarity index 74% rename from PixivSearch/thrift/TestQry/TestQry-remote rename to PixivSearch/thrift/QueryComment/QueryComment-remote index c12dd5a..a76303e --- a/PixivSearch/thrift/TestQry/TestQry-remote +++ b/PixivSearch/thrift/QueryComment/QueryComment-remote @@ -9,24 +9,24 @@ import sys import pprint -from urllib.parse import urlparse - -from PixivSearch.thrift.TestQry import TestQry - -sys.path.append('/root/PixivSearch') - - +if sys.version_info[0] > 2: + from urllib.parse import urlparse +else: + from urlparse import urlparse from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient from thrift.protocol.TBinaryProtocol import TBinaryProtocol - +from QueryComment import QueryComment +from QueryComment.ttypes import * if len(sys.argv) <= 1 or sys.argv[1] == '--help': print('') print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') print('') print('Functions:') - print(' QryResult qryTest(i32 qryCode)') + print(' i32 commentSum(i32 cid)') + print(' commentSumList( cids)') + print(' string download( cids, string fileName)') print('') sys.exit(0) @@ -103,14 +103,26 @@ else: else: transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol(transport) -client = TestQry.Client(protocol) +client = QueryComment.Client(protocol) transport.open() -if cmd == 'qryTest': +if cmd == 'commentSum': if len(args) != 1: - print('qryTest requires 1 args') + print('commentSum requires 1 args') + sys.exit(1) + pp.pprint(client.commentSum(eval(args[0]),)) + +elif cmd == 'commentSumList': + if len(args) != 1: + print('commentSumList requires 1 args') + sys.exit(1) + pp.pprint(client.commentSumList(eval(args[0]),)) + +elif cmd == 'download': + if len(args) != 2: + print('download requires 2 args') sys.exit(1) - pp.pprint(client.qryTest(eval(args[0]),)) + pp.pprint(client.download(eval(args[0]), args[1],)) else: print('Unrecognized method %s' % cmd) diff --git a/PixivSearch/thrift/QueryComment/QueryComment.py b/PixivSearch/thrift/QueryComment/QueryComment.py new file mode 100644 index 0000000..9b4bffb --- /dev/null +++ b/PixivSearch/thrift/QueryComment/QueryComment.py @@ -0,0 +1,660 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +import logging +from .ttypes import * +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +all_structs = [] + + +class Iface(object): + def commentSum(self, cid): + """ + 查询单个cid弹幕数 + + + Parameters: + - cid + """ + pass + + def commentSumList(self, cids): + """ + * 批量查询多个cid弹幕数 + * + + Parameters: + - cids + """ + pass + + def download(self, cids, fileName): + """ + * 批量下载弹幕 + * + + Parameters: + - cids + - fileName + """ + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def commentSum(self, cid): + """ + 查询单个cid弹幕数 + + + Parameters: + - cid + """ + self.send_commentSum(cid) + return self.recv_commentSum() + + def send_commentSum(self, cid): + self._oprot.writeMessageBegin('commentSum', TMessageType.CALL, self._seqid) + args = commentSum_args() + args.cid = cid + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_commentSum(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = commentSum_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSum failed: unknown result") + + def commentSumList(self, cids): + """ + * 批量查询多个cid弹幕数 + * + + Parameters: + - cids + """ + self.send_commentSumList(cids) + return self.recv_commentSumList() + + def send_commentSumList(self, cids): + self._oprot.writeMessageBegin('commentSumList', TMessageType.CALL, self._seqid) + args = commentSumList_args() + args.cids = cids + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_commentSumList(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = commentSumList_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSumList failed: unknown result") + + def download(self, cids, fileName): + """ + * 批量下载弹幕 + * + + Parameters: + - cids + - fileName + """ + self.send_download(cids, fileName) + return self.recv_download() + + def send_download(self, cids, fileName): + self._oprot.writeMessageBegin('download', TMessageType.CALL, self._seqid) + args = download_args() + args.cids = cids + args.fileName = fileName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_download(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = download_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "download failed: unknown result") + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["commentSum"] = Processor.process_commentSum + self._processMap["commentSumList"] = Processor.process_commentSumList + self._processMap["download"] = Processor.process_download + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_commentSum(self, seqid, iprot, oprot): + args = commentSum_args() + args.read(iprot) + iprot.readMessageEnd() + result = commentSum_result() + try: + result.success = self._handler.commentSum(args.cid) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("commentSum", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_commentSumList(self, seqid, iprot, oprot): + args = commentSumList_args() + args.read(iprot) + iprot.readMessageEnd() + result = commentSumList_result() + try: + result.success = self._handler.commentSumList(args.cids) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("commentSumList", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_download(self, seqid, iprot, oprot): + args = download_args() + args.read(iprot) + iprot.readMessageEnd() + result = download_result() + try: + result.success = self._handler.download(args.cids, args.fileName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("download", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + +# HELPER FUNCTIONS AND STRUCTURES + + +class commentSum_args(object): + """ + Attributes: + - cid + """ + + + def __init__(self, cid=None,): + self.cid = cid + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cid = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSum_args') + if self.cid is not None: + oprot.writeFieldBegin('cid', TType.I32, 1) + oprot.writeI32(self.cid) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSum_args) +commentSum_args.thrift_spec = ( + None, # 0 + (1, TType.I32, 'cid', None, None, ), # 1 +) + + +class commentSum_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSum_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSum_result) +commentSum_result.thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 +) + + +class commentSumList_args(object): + """ + Attributes: + - cids + """ + + + def __init__(self, cids=None,): + self.cids = cids + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.cids = [] + (_etype3, _size0) = iprot.readListBegin() + for _i4 in range(_size0): + _elem5 = iprot.readI32() + self.cids.append(_elem5) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSumList_args') + if self.cids is not None: + oprot.writeFieldBegin('cids', TType.LIST, 1) + oprot.writeListBegin(TType.I32, len(self.cids)) + for iter6 in self.cids: + oprot.writeI32(iter6) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSumList_args) +commentSumList_args.thrift_spec = ( + None, # 0 + (1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1 +) + + +class commentSumList_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype10, _size7) = iprot.readListBegin() + for _i11 in range(_size7): + _elem12 = iprot.readI32() + self.success.append(_elem12) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSumList_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.LIST, 0) + oprot.writeListBegin(TType.I32, len(self.success)) + for iter13 in self.success: + oprot.writeI32(iter13) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSumList_result) +commentSumList_result.thrift_spec = ( + (0, TType.LIST, 'success', (TType.I32, None, False), None, ), # 0 +) + + +class download_args(object): + """ + Attributes: + - cids + - fileName + """ + + + def __init__(self, cids=None, fileName=None,): + self.cids = cids + self.fileName = fileName + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.cids = [] + (_etype17, _size14) = iprot.readListBegin() + for _i18 in range(_size14): + _elem19 = iprot.readI32() + self.cids.append(_elem19) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.fileName = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('download_args') + if self.cids is not None: + oprot.writeFieldBegin('cids', TType.LIST, 1) + oprot.writeListBegin(TType.I32, len(self.cids)) + for iter20 in self.cids: + oprot.writeI32(iter20) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.fileName is not None: + oprot.writeFieldBegin('fileName', TType.STRING, 2) + oprot.writeString(self.fileName.encode('utf-8') if sys.version_info[0] == 2 else self.fileName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(download_args) +download_args.thrift_spec = ( + None, # 0 + (1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1 + (2, TType.STRING, 'fileName', 'UTF8', None, ), # 2 +) + + +class download_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('download_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRING, 0) + oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(download_result) +download_result.thrift_spec = ( + (0, TType.STRING, 'success', 'UTF8', None, ), # 0 +) +fix_spec(all_structs) +del all_structs + diff --git a/PixivSearch/thrift/QueryComment/Server.py b/PixivSearch/thrift/QueryComment/Server.py new file mode 100644 index 0000000..956f024 --- /dev/null +++ b/PixivSearch/thrift/QueryComment/Server.py @@ -0,0 +1,48 @@ +import os +import sys + +from thrift.server.TNonblockingServer import TNonblockingServer +from thrift.transport import TSocket + +sys.path.append('/root/PixivSearch') +from PixivSearch.util import Util +from PixivSearch.dao.Comment import xml, downloadXml +from PixivSearch.thrift.QueryComment import QueryComment +from PixivSearch.thrift.QueryComment.QueryComment import Iface + +commentPath = '/root/PixivSearch/PixivSearch/thrift/tmpFile/comment' + + +class Server(Iface): + def commentSumList(self, cids): + result = [] + for cid in cids: + comment_selector = xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid)) + length = len(comment_selector.xpath('//i//d/text()')) + print('cid:{cid},弹幕数：{length}'.format(cid=cid, length=length)) + result.append(length) + return result + + def commentSum(self, cid): + return self.commentSumList([cid])[0]; + + def download(self, cids, fileName): + path = '{commentPath}/{fileName}'.format(commentPath=commentPath, fileName=fileName) + for cid in cids: + downloadXml(path, cid, + histroy=False) + zipFile = '{path}/{fileName}.zip'.format(path=path,fileName=fileName) + print(zipFile) + Util.zip(path, zipFile) + if os.path.isfile(zipFile): + print('压缩包成功生成到{zipFile}'.format(zipFile=zipFile)) + return zipFile + else: + return None + + +if __name__ == '__main__': + socket = TSocket.TServerSocket(port=2233) + processor = QueryComment.Processor(Server()) + server = TNonblockingServer(processor, socket) + server.serve() diff --git a/PixivSearch/thrift/QueryComment/__init__.py b/PixivSearch/thrift/QueryComment/__init__.py new file mode 100644 index 0000000..52b572e --- /dev/null +++ b/PixivSearch/thrift/QueryComment/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'QueryComment'] diff --git a/PixivSearch/thrift/TestQry/constants.py b/PixivSearch/thrift/QueryComment/constants.py similarity index 100% rename from PixivSearch/thrift/TestQry/constants.py rename to PixivSearch/thrift/QueryComment/constants.py diff --git a/PixivSearch/thrift/QueryComment/ttypes.py b/PixivSearch/thrift/QueryComment/ttypes.py new file mode 100644 index 0000000..9fb0f57 --- /dev/null +++ b/PixivSearch/thrift/QueryComment/ttypes.py @@ -0,0 +1,18 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys + +from thrift.transport import TTransport +all_structs = [] +fix_spec(all_structs) +del all_structs diff --git a/PixivSearch/thrift/TestQry/Server.py b/PixivSearch/thrift/TestQry/Server.py deleted file mode 100644 index a31e9e5..0000000 --- a/PixivSearch/thrift/TestQry/Server.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys - -from thrift.server.TNonblockingServer import TNonblockingServer -from thrift.transport import TSocket - -sys.path.append('/root/PixivSearch') -from PixivSearch.thrift.TestQry import TestQry -from PixivSearch.thrift.TestQry.ttypes import QryResult - - -class QueryImpl(TestQry.Iface): - - def qryTest(self, qryCode): - result = QryResult() - if qryCode == 1: - result.code = 1 - result.msg = 'success' - - else: - result.code = 0 - result.msg = 'fail' - return result - - -if __name__ == '__main__': - socket = TSocket.TServerSocket(port=2233) - processor = TestQry.Processor(QueryImpl()) - server = TNonblockingServer(processor, socket) - - server.serve() diff --git a/PixivSearch/thrift/TestQry/TestQry.py b/PixivSearch/thrift/TestQry/TestQry.py deleted file mode 100644 index 228ef8d..0000000 --- a/PixivSearch/thrift/TestQry/TestQry.py +++ /dev/null @@ -1,245 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# -import logging - -from thrift.TRecursive import fix_spec -from thrift.Thrift import TProcessor -from thrift.Thrift import TType, TMessageType, TApplicationException -from thrift.transport import TTransport - -from PixivSearch.thrift.TestQry.ttypes import QryResult - -all_structs = [] - -class Iface(object): - def qryTest(self, qryCode): - """ - 测试查询接口，当qryCode值为1时返回"成功"的响应信息，qryCode值为其他值时返回"失败"的响应信息 - @param qryCode测试参数 - - Parameters: - - qryCode - """ - pass - - -class Client(Iface): - def __init__(self, iprot, oprot=None): - self._iprot = self._oprot = iprot - if oprot is not None: - self._oprot = oprot - self._seqid = 0 - - def qryTest(self, qryCode): - """ - 测试查询接口，当qryCode值为1时返回"成功"的响应信息，qryCode值为其他值时返回"失败"的响应信息 - @param qryCode测试参数 - - Parameters: - - qryCode - """ - self.send_qryTest(qryCode) - return self.recv_qryTest() - - def send_qryTest(self, qryCode): - self._oprot.writeMessageBegin('qryTest', TMessageType.CALL, self._seqid) - args = qryTest_args() - args.qryCode = qryCode - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_qryTest(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = qryTest_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "qryTest failed: unknown result") - - -class Processor(Iface, TProcessor): - def __init__(self, handler): - self._handler = handler - self._processMap = {} - self._processMap["qryTest"] = Processor.process_qryTest - - def process(self, iprot, oprot): - (name, type, seqid) = iprot.readMessageBegin() - if name not in self._processMap: - iprot.skip(TType.STRUCT) - iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) - oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) - x.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - return - else: - self._processMap[name](self, seqid, iprot, oprot) - return True - - def process_qryTest(self, seqid, iprot, oprot): - args = qryTest_args() - args.read(iprot) - iprot.readMessageEnd() - result = qryTest_result() - try: - result.success = self._handler.qryTest(args.qryCode) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("qryTest", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - -# HELPER FUNCTIONS AND STRUCTURES - - -class qryTest_args(object): - """ - Attributes: - - qryCode - """ - - - def __init__(self, qryCode=None,): - self.qryCode = qryCode - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.qryCode = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('qryTest_args') - if self.qryCode is not None: - oprot.writeFieldBegin('qryCode', TType.I32, 1) - oprot.writeI32(self.qryCode) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(qryTest_args) -qryTest_args.thrift_spec = ( - None, # 0 - (1, TType.I32, 'qryCode', None, None, ), # 1 -) - - -class qryTest_result(object): - """ - Attributes: - - success - """ - - - def __init__(self, success=None,): - self.success = success - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = QryResult() - self.success.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('qryTest_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(qryTest_result) -qryTest_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [QryResult, None], None, ), # 0 -) -fix_spec(all_structs) -del all_structs - - - - diff --git a/PixivSearch/thrift/TestQry/__init__.py b/PixivSearch/thrift/TestQry/__init__.py deleted file mode 100644 index 22182f6..0000000 --- a/PixivSearch/thrift/TestQry/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['ttypes', 'constants', 'TestQry'] diff --git a/PixivSearch/thrift/TestQry/ttypes.py b/PixivSearch/thrift/TestQry/ttypes.py deleted file mode 100644 index 523a81b..0000000 --- a/PixivSearch/thrift/TestQry/ttypes.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys - -from thrift.transport import TTransport -all_structs = [] - - -class QryResult(object): - """ - Attributes: - - code: 返回码, 1成功，0失败 - - msg: 响应信息 - """ - - - def __init__(self, code=None, msg=None,): - self.code = code - self.msg = msg - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.code = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRING: - self.msg = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('QryResult') - if self.code is not None: - oprot.writeFieldBegin('code', TType.I32, 1) - oprot.writeI32(self.code) - oprot.writeFieldEnd() - if self.msg is not None: - oprot.writeFieldBegin('msg', TType.STRING, 2) - oprot.writeString(self.msg.encode('utf-8') if sys.version_info[0] == 2 else self.msg) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(QryResult) -QryResult.thrift_spec = ( - None, # 0 - (1, TType.I32, 'code', None, None, ), # 1 - (2, TType.STRING, 'msg', 'UTF8', None, ), # 2 -) -fix_spec(all_structs) -del all_structs diff --git a/PixivSearch/util/Util.py b/PixivSearch/util/Util.py new file mode 100644 index 0000000..d69b89a --- /dev/null +++ b/PixivSearch/util/Util.py @@ -0,0 +1,10 @@ +import os +import zipfile + + +def zip(inputFile, outFile): + f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED) + for dirpath, dirnames, filenames in os.walk(inputFile): + for filename in filenames: + f.write(os.path.join(dirpath, filename), filename) + f.close() \ No newline at end of file diff --git a/PixivSearch/util/__init__.py b/PixivSearch/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/view.py b/PixivSearch/view.py index d01f309..9153a04 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -7,13 +7,13 @@ from django.http import Http404, StreamingHttpResponse, HttpResponse from django.shortcuts import render from PixivSearch.dao.bangumi import get_, stop_, getIds +from PixivSearch.pixiv.pixiv import get_nodes from PixivSearch.settings import logger os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") django.setup() # 添加的代码 from PixivSearch.model import config -from PixivSearch.pixiv import get_nodes def search(request): From ea41314e0b9d016e5c24c78d06f3e407ba1f65f8 Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Fri, 15 Jun 2018 18:28:21 +0800 Subject: [PATCH 21/22] thift --- PixivSearch/thrift/QueryComment/Server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PixivSearch/thrift/QueryComment/Server.py b/PixivSearch/thrift/QueryComment/Server.py index 956f024..d3d6d14 100644 --- a/PixivSearch/thrift/QueryComment/Server.py +++ b/PixivSearch/thrift/QueryComment/Server.py @@ -11,7 +11,7 @@ from PixivSearch.thrift.QueryComment import QueryComment from PixivSearch.thrift.QueryComment.QueryComment import Iface commentPath = '/root/PixivSearch/PixivSearch/thrift/tmpFile/comment' - +#commentPath='/home/hua/下载/comment' class Server(Iface): def commentSumList(self, cids): @@ -31,7 +31,7 @@ class Server(Iface): for cid in cids: downloadXml(path, cid, histroy=False) - zipFile = '{path}/{fileName}.zip'.format(path=path,fileName=fileName) + zipFile = '{commentPath}/{fileName}.zip'.format(commentPath=commentPath,fileName=fileName) print(zipFile) Util.zip(path, zipFile) if os.path.isfile(zipFile): From 5993db080c74b6d1d2e123de6c3822d9b84dc480 Mon Sep 17 00:00:00 2001 From: WuXianChaoPin <1029559041@qq.com> Date: Tue, 19 Jun 2018 18:34:57 +0800 Subject: [PATCH 22/22] thift --- PixivSearch/thrift/task/Server.py | 22 ++ PixivSearch/thrift/task/TSDM-remote | 124 +++++++++ PixivSearch/thrift/task/TSDM.py | 366 +++++++++++++++++++++++++++ PixivSearch/thrift/task/__init__.py | 1 + PixivSearch/thrift/task/constants.py | 14 + PixivSearch/thrift/task/ttypes.py | 18 ++ 6 files changed, 545 insertions(+) create mode 100644 PixivSearch/thrift/task/Server.py create mode 100755 PixivSearch/thrift/task/TSDM-remote create mode 100644 PixivSearch/thrift/task/TSDM.py create mode 100644 PixivSearch/thrift/task/__init__.py create mode 100644 PixivSearch/thrift/task/constants.py create mode 100644 PixivSearch/thrift/task/ttypes.py diff --git a/PixivSearch/thrift/task/Server.py b/PixivSearch/thrift/task/Server.py new file mode 100644 index 0000000..c804b0a --- /dev/null +++ b/PixivSearch/thrift/task/Server.py @@ -0,0 +1,22 @@ +import sys + +from thrift.server.TNonblockingServer import TNonblockingServer +from thrift.transport import TSocket + +sys.path.append('/root/PixivSearch') +from PixivSearch.thrift.task import TSDM +from PixivSearch.thrift.task.TSDM import Iface + + +class Server(Iface): + def qiandao(self): + return False + + def word(self): + return True + +if __name__ == '__main__': + socket = TSocket.TServerSocket(port=2233) + processor = TSDM.Processor(Server()) + server = TNonblockingServer(processor, socket) + server.serve() \ No newline at end of file diff --git a/PixivSearch/thrift/task/TSDM-remote b/PixivSearch/thrift/task/TSDM-remote new file mode 100755 index 0000000..c06ca34 --- /dev/null +++ b/PixivSearch/thrift/task/TSDM-remote @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +import sys +import pprint +if sys.version_info[0] > 2: + from urllib.parse import urlparse +else: + from urlparse import urlparse +from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient +from thrift.protocol.TBinaryProtocol import TBinaryProtocol + +from core.thrift.task import TSDM +from core.thrift.task.ttypes import * + +if len(sys.argv) <= 1 or sys.argv[1] == '--help': + print('') + print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') + print('') + print('Functions:') + print(' bool qiandao()') + print(' bool word()') + print('') + sys.exit(0) + +pp = pprint.PrettyPrinter(indent=2) +host = 'localhost' +port = 9090 +uri = '' +framed = False +ssl = False +validate = True +ca_certs = None +keyfile = None +certfile = None +http = False +argi = 1 + +if sys.argv[argi] == '-h': + parts = sys.argv[argi + 1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + argi += 2 + +if sys.argv[argi] == '-u': + url = urlparse(sys.argv[argi + 1]) + parts = url[1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 80 + uri = url[2] + if url[4]: + uri += '?%s' % url[4] + http = True + argi += 2 + +if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': + framed = True + argi += 1 + +if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': + ssl = True + argi += 1 + +if sys.argv[argi] == '-novalidate': + validate = False + argi += 1 + +if sys.argv[argi] == '-ca_certs': + ca_certs = sys.argv[argi+1] + argi += 2 + +if sys.argv[argi] == '-keyfile': + keyfile = sys.argv[argi+1] + argi += 2 + +if sys.argv[argi] == '-certfile': + certfile = sys.argv[argi+1] + argi += 2 + +cmd = sys.argv[argi] +args = sys.argv[argi + 1:] + +if http: + transport = THttpClient.THttpClient(host, port, uri) +else: + if ssl: + socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) + else: + socket = TSocket.TSocket(host, port) + if framed: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) +protocol = TBinaryProtocol(transport) +client = TSDM.Client(protocol) +transport.open() + +if cmd == 'qiandao': + if len(args) != 0: + print('qiandao requires 0 args') + sys.exit(1) + pp.pprint(client.qiandao()) + +elif cmd == 'word': + if len(args) != 0: + print('word requires 0 args') + sys.exit(1) + pp.pprint(client.word()) + +else: + print('Unrecognized method %s' % cmd) + sys.exit(1) + +transport.close() diff --git a/PixivSearch/thrift/task/TSDM.py b/PixivSearch/thrift/task/TSDM.py new file mode 100644 index 0000000..7768372 --- /dev/null +++ b/PixivSearch/thrift/task/TSDM.py @@ -0,0 +1,366 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +import logging +from .ttypes import * +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +all_structs = [] + + +class Iface(object): + def qiandao(self): + pass + + def word(self): + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def qiandao(self): + self.send_qiandao() + return self.recv_qiandao() + + def send_qiandao(self): + self._oprot.writeMessageBegin('qiandao', TMessageType.CALL, self._seqid) + args = qiandao_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_qiandao(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = qiandao_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "qiandao failed: unknown result") + + def word(self): + self.send_word() + return self.recv_word() + + def send_word(self): + self._oprot.writeMessageBegin('word', TMessageType.CALL, self._seqid) + args = word_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_word(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = word_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "word failed: unknown result") + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["qiandao"] = Processor.process_qiandao + self._processMap["word"] = Processor.process_word + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_qiandao(self, seqid, iprot, oprot): + args = qiandao_args() + args.read(iprot) + iprot.readMessageEnd() + result = qiandao_result() + try: + result.success = self._handler.qiandao() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("qiandao", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_word(self, seqid, iprot, oprot): + args = word_args() + args.read(iprot) + iprot.readMessageEnd() + result = word_result() + try: + result.success = self._handler.word() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("word", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + +# HELPER FUNCTIONS AND STRUCTURES + + +class qiandao_args(object): + + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('qiandao_args') + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(qiandao_args) +qiandao_args.thrift_spec = ( +) + + +class qiandao_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('qiandao_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(qiandao_result) +qiandao_result.thrift_spec = ( + (0, TType.BOOL, 'success', None, None, ), # 0 +) + + +class word_args(object): + + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('word_args') + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(word_args) +word_args.thrift_spec = ( +) + + +class word_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('word_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(word_result) +word_result.thrift_spec = ( + (0, TType.BOOL, 'success', None, None, ), # 0 +) +fix_spec(all_structs) +del all_structs + diff --git a/PixivSearch/thrift/task/__init__.py b/PixivSearch/thrift/task/__init__.py new file mode 100644 index 0000000..c42116e --- /dev/null +++ b/PixivSearch/thrift/task/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'TSDM'] diff --git a/PixivSearch/thrift/task/constants.py b/PixivSearch/thrift/task/constants.py new file mode 100644 index 0000000..0c217ce --- /dev/null +++ b/PixivSearch/thrift/task/constants.py @@ -0,0 +1,14 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +from .ttypes import * diff --git a/PixivSearch/thrift/task/ttypes.py b/PixivSearch/thrift/task/ttypes.py new file mode 100644 index 0000000..9fb0f57 --- /dev/null +++ b/PixivSearch/thrift/task/ttypes.py @@ -0,0 +1,18 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys + +from thrift.transport import TTransport +all_structs = [] +fix_spec(all_structs) +del all_structs