#!/usr/bin/env python
#coding:utf-8
from concurrent import futures
import threading
import json
import requests
from bs4 import BeautifulSoup
import sys
from datetime import datetime
import os
import zipfile
import logging.config

headers = {
    'X-Requested-With': 'XMLHttpRequest',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/56.0.2924.87 Safari/537.36'
}

lock = threading.Lock()     # 多线程全局资源锁
total = 1
logging.config.fileConfig('PixivSearch/logging.conf')
logger = logging.getLogger('file')

def get_cookies():
    _cookies = {}
    array = "p_ab_id=5; p_ab_id_2=9; login_ever=yes; device_token=c8c37fdf24b917b4e7fb191fe11c5ca5; search_tools_toggle=1; _ga=GA1.2.887334537.1498628532; PHPSESSID=25745470_75a76e86ff3145b53e21b440183b4822; a_type=0; is_sensei_service_user=1; module_orders_mypage=%5B%7B%22name%22%3A%22recommended_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22everyone_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22following_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22mypixiv_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22fanbox%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22featured_tags%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22contests%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22sensei_courses%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22spotlight%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22booth_follow_items%22%2C%22visible%22%3Atrue%7D%5D; __utmt=1; __utma=235335808.887334537.1498628532.1498720739.1498819600.5; __utmb=235335808.1.10.1498819600; __utmc=235335808; __utmz=235335808.1498713152.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=235335808.|2=login%20ever=yes=1^3=plan=normal=1^5=gender=male=1^6=user_id=25745470=1^9=p_ab_id=5=1^10=p_ab_id_2=9=1^11=lang=zh_tw=1".split(';')
    for row in array:
        k, v = row.strip().split('=', 1)
        _cookies[k] = v
    return _cookies




def crawl(url):
    global total
    req = requests.get(url, headers=headers, cookies=get_cookies()).text
    tag=BeautifulSoup(req, 'lxml').select('#js-mount-point-search-result-list')[0].attrs['data-items']
    imageNodes=json.loads(tag)
    for imageNode in imageNodes:
        with lock:
            nodes.append(imageNode)

def get_urls(search, page):
    fmt = 'https://www.pixiv.net/search.php?word={}&order=date_d&p={}'
    return [fmt.format(search, p) for p in range(1, page)]

def get_Img(params):
    params[1]['imgUrl']='https://i.pximg.net/img-original/img/'+params[1]['url'][-46:-15]

    headers['referer']='https://www.pixiv.net/member_illust.php?mode=medium&illust_id='+params[1]['illustId']

    suffix=".jpg"
    logger.info('开始下载图片：%s%s' % (params[1]['imgUrl'], suffix))

    s=requests.get(params[1]['imgUrl']+suffix, headers=headers, cookies=get_cookies())
    if(s.status_code==404):
        suffix='.png'
        s=requests.get(params[1]['imgUrl']+suffix,headers=headers, cookies=get_cookies())
    if(s.status_code==404):
        logger.error('无法下载图片：%s' % (params[1]['illustTitle']))
        return

    logger.info('下载图片："%s"到%s'%(params[1]['illustTitle'],os.getcwd().replace('\\','/')+'/'+imgPath+params[1]['illustId']+suffix))
    f = open(imgPath+params[1]['illustId']+suffix, 'wb') #写入多媒体文件要 b 这个参数
    f.write(s.content)  #多媒体文件要是用conctent
    f.close()

    params[1]['localName']=params[1]['illustId']+suffix
    logger.info('排行第%d名，收藏数%d，标题：%s，标签：%s，(%s)前投稿，链接：%s'%(params[0],params[1]['bookmarkCount'],params[1]['illustTitle'], ','.join(params[1]['tags']),'',params[1]['imgUrl']))

def zip(inputFile,outFile):
    f = zipfile.ZipFile(outFile,'w',zipfile.ZIP_DEFLATED)
    for dirpath, dirnames, filenames in os.walk(inputFile):
        for filename in filenames:
            f.write(os.path.join(dirpath,filename),filename)
    f.close()

fsize=''
def get_nodes(param):
    global nodes,fsize,imgPath
    nodes=[]
    start = datetime.now()
    urls = get_urls(param[1], int(param[2])+1)
    logger.info('开始从P站获取图片数据')
    with futures.ThreadPoolExecutor(32) as executor:
        executor.map(crawl, urls)

    # for url in urls:
    #     crawl(url)

    length=len(nodes)
    logger.info('获取到%d张图片'%(length))
    logger.info('对图片收藏数进行排序')
    nodes=sorted(nodes, key=lambda v: v.get('bookmarkCount'), reverse=True)[:int(param[3])]    # 按star数降序排序
    if(param[4]!=None and param[4]=='img'):
        imgPath='PixivSearch/static/images/'
        for file in os.listdir(imgPath):
            os.remove(imgPath+file)
        nodes_tup=[]
        start_d=datetime.now()
        for index,img in enumerate(nodes):
            nodes_tup.append((index+1,img))
            # get_Img((index+1,img))

        with futures.ThreadPoolExecutor(32) as executor:
                executor.map(get_Img, nodes_tup)
        print('下载图片花费时间：%s' % (datetime.now() - start_d))
        logger.info('%s张图片下载完毕'%(len(os.listdir(imgPath))))

        zipPath='PixivSearch/static/download/' + param[1] + '.zip'
        logger.info('图片打包到：%s' % (zipPath))
        zip(imgPath,zipPath)
        fsize = str(round(os.path.getsize(zipPath)/float(1024*1024),2))+'MB'
        logger.info('图包大小：%s'%(fsize))

    tip='从%d张图片中筛选出收藏数前%s的图片,处理耗时：%s'%(length,param[3],datetime.now()-start)
    logger.info(tip)
    return [nodes,tip,fsize]



if __name__ == "__main__":
    if (len(sys.argv))==5 and sys.argv[2].isdigit() and sys.argv[3].isdigit():
        try:
            get_nodes(sys.argv)
        except BaseException as e:
            repr(e)

    else :
        logger.error('参数不合法')