读取几个目录下及其各自的子目录中所有的txt文件，并从中筛选出domain，然后按照每个domain出现的次数进行一个排序,最后将这个排序结果写入到一个文件中。

#coding=utf8
# 统计data_naver和data_google两个目录下出现的每个domain的次数，按照从多到少进行排序

import os
import re

# 将给定目录下所有的txt文件整合到一个list中
def merge_txt(path_list):
    file_list = []
    for path in path_list:
        for (dirpath, dirnames, filenames) in os.walk(path):
            # print filenames
            for filename in filenames:
                filename = os.path.join(dirpath, filename)
                # print filename
                file_list.append(filename)
    print "所有文件的总数是：",len(file_list)
    return file_list

# 将文件列表中每个文件里面的url合并到一个list
def merge_to_domain_list(file_list):
    all_domain_list = []
    domain_pat = '"https*://(.*?)/'
    # 对于每个文件，读取每行，并对每行进行处理
    for file in file_list:
        file_object = open(file)
        # 对每行用正则匹配出url
        for line in file_object:
            # print line
            domain_list = re.findall(domain_pat,line)
            all_domain_list = all_domain_list + domain_list
        file_object.close()
    print all_domain_list
    print "url的总数是：",len(all_domain_list)
    return all_domain_list


# 对list按照出现次数排序，得出一个dict,key是url，value是count（次数）
def sort_list_by_count(domain_list):
    domain_dict = {}
    for domain in domain_list:
        domain_count = domain_dict.get(domain)
        if domain_count:
            domain_count = int(domain_count)
            domain_count += 1
            domain_dict[domain] = domain_count
        else:
            domain_dict[domain] = 1
    # print domain_dict
    items = domain_dict.items()
    backitems = [[v[1], v[0]] for v in items]
    # print backitems
    # backitems.sort()
    backitems = sorted(backitems)
    # backitems = reversed(backitems)
    backitems.reverse()
    print [[v[1], v[0]] for v in backitems]
    list_sorted_by_count = [[v[1], v[0]] for v in backitems]
    return list_sorted_by_count

# 将一个list写入文件
def write_list_to_file(sorted_list,full_path):
    sorted_list_file = open(full_path,'a')
    for l in sorted_list:
        sorted_list_file.write(str(l) + "\n")
    sorted_list_file.close()


if __name__ == '__main__':
    dir = ['./data_google','./data_naver']
    file_list = merge_txt(dir)
    domain_list = merge_to_domain_list(file_list)
    # print type(domain_list)
    list_sorted_by_count = sort_list_by_count(domain_list)
    full_path = './data/sorted.txt'
    write_list_to_file(list_sorted_by_count, full_path)