求教关于Python并行计算共用参数的问题

Civiano 2016-02-23 10:27:00

我使用map函数并行计算，对数据进行筛选

#!/usr/bin/python

# encoding: utf-8

import MySQLdb,MySQLdb.cursors

import string



import os



from multiprocessing import Pool

from multiprocessing.dummy import Pool as ThreadPool





##



# 打开数据库连接

##conn = MySQLdb.connect("localhost","root","lab121","weibodata")

conn = MySQLdb.connect(host='localhost', user='root',\

                       passwd='lab121',db="weibodata",\

                       use_unicode=0, charset='gb2312',\

                       cursorclass=MySQLdb.cursors.SSCursor)

# 使用cursor()方法获取操作游标

cursor = conn.cursor()



cursor.execute('select index_id from userswith2000deg')

data = cursor.fetchall()  





users = []



for row in data:

    users.append(row[0])

print "get %s users" % len(users)



##conn.close()



folder = 'tests'

results = []



def get_paths(folder):

    return (os.path.join(folder, f) 

            for f in os.listdir(folder) 

            if 'txt' in f)



def judgement(filename):

    print filename



 

    f = open(filename,'r')

    for line in f:

        line = line.split()

        a = string.atoi(line[0])

        b = string.atoi(line[1])

        c = string.atoi(line[2])

        if a in users:

            if b in users:

                results.append((a,b,c))

    f.close()



if __name__ == '__main__':

    folder = os.path.abspath(

        'tests')

    files = get_paths(folder)



    pool = ThreadPool(3)

    pool.map(judgement, files)

    pool.close()

    pool.join()

    print results

有三个文件要处理，E:\weibo_datasets\weibo_retweet_diffusion\codes\tests\graph_170w_1month.txt.0001E:\weibo_datasets\weibo_retweet_diffusion\codes\tests\graph_170w_1month.txt.0002E:\weibo_datasets\weibo_retweet_diffusion\codes\tests\graph_170w_1month.txt.0003，
但是结果显示只处理了第一个文件。

...全文