#!/user/bin/python
# -*- coding: UTF-8 -*-

import urllib
import urllib2
import lxml
import re
import MySQLdb
import time
from bs4 import BeautifulSoup

import httplib
httplib.HTTPConnection._http_vsn = 10
httplib.HTTPConnection._http_vsn_str = 'HTTP/1.1'

user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36'  
hdr = { 'User-Agent' : user_agent }

db = MySQLdb.connect(host="localhost", port=3306, user="root", passwd="123456", db="xiaoshuo", charset="utf8")
str_sql = '''INSERT INTO `xiaoshuo`.`book1` (`bookName`, `author`, `url`, `classifyName`, `brief`, `updateTime`,
            `status`) VALUES'''

def getBookInfoBaseOnUrl(url):

    global str_sql

    request = urllib2.Request(url, headers=hdr)
    response = urllib2.urlopen(request)
    html_data = response.read().decode('gbk')

    soup = BeautifulSoup(html_data,'lxml')
    mylist = soup.select('head')

    for item in mylist: 
        bookName = item.find(property="og:novel:book_name").get("content")
        print "书名:", bookName

        author = item.find(property="og:novel:author").get("content")
        print "作者:", author

        url = item.find(property="og:novel:read_url").get("content")
        print "链接:", url

        classifyName = item.find(property="og:novel:category").get("content")
        print "类型:", classifyName

        description = item.find(property="og:description").get("content")
        print "brief:", description

        updateTime = item.find(property="og:novel:update_time").get("content")
        print "更新时间:", updateTime

        status = item.find(property="og:novel:status").get("content")
        print "status:", status

        str_sql += '("' + bookName + '", "' + author + '", "' + url + '", "' + classifyName + '", "' + description + '", "' + updateTime + '", "' + status + '"),'
        
        print "-----------------------------------------------------------------------------------------"

def get_book( ):

    global str_sql
    cursor = db.cursor()
    #count = 0

    soup = BeautifulSoup(open('biquge.html'),'lxml')
    mylist = soup.find_all('div', class_ ='content')
    for item in mylist:
        #print item
        xiaoshuo_list = item.find_all('li')
        for item in xiaoshuo_list:
            #coutn2 = 0
            url = item.find('a').get('href')
            print "书的连接:" , url
            getBookInfoBaseOnUrl(url)

            #coutn2 = coutn2 + 1
            #if coutn2 == 2:
                #break
        #count = count + 1
        #if count == 2:
            #break

    for item in mylist:
        xiaoshuo_list = item.find_all('dl')
        for item in xiaoshuo_list:
            url = item.find('a').get('href')
            print "书的连接:", url
            getBookInfoBaseOnUrl(url)

    str_sql = str_sql.encode("utf-8")
    str_sql = str_sql[0:len(str_sql)-1]
    print "tmp_sql:", str_sql
    cursor.execute(str_sql)
    db.commit()
    cursor.close()

if __name__ == "__main__":
    print ("<<<-----Start Get Book INFO And Save Db------>>")
    get_book()

    db.close()
    print str_sql