首页 > 代码库 > python写的多线程下载工具

python写的多线程下载工具

其实只是想练习一下threading的用法。

写完后发现其实下载速度也没增加多少,略显尴尬,汗

# -*- coding: cp936 -*-import urllib2import threadingimport os,os.pathimport sysimport time,datetimeurl = ‘‘path = os.getcwd()file_name = ‘‘file_fp=‘‘net_filesize = 0size_download = 0threadlock = threading.Lock()Thread_Num = 10Block_Size = 16384def get_filename_from_url(url):    return url[url.rfind(‘/‘)+1:]def get_net_file_info(url):    r = urllib2.urlopen(url)    fs = int(r.info()[‘Content-Length‘])    if r.info().has_key(‘Content-Disposition‘):        fn = r.info()[‘Content-Disposition‘].split(‘filename=‘)[1]        fn = fn.replace(‘"‘, ‘‘).replace("‘", "")    else:        fn = get_filename_from_url(url)    r.close()    return fs,fndef file_write_init():    global file_fp,net_filesize,file_name    net_filesize,file_name = get_net_file_info(url)    full_path= os.path.join(path,file_name)    if os.path.isfile(full_path) == True:return False    file_fp = file(full_path,"wb")    file_fp.truncate(net_filesize)    return True    def file_write(pos,dat):    global size_download    threadlock.acquire()    size_download += len(dat)    #print ‘size_download:%d‘%(size_download )    file_fp.seek(pos)    file_fp.write(dat)    threadlock.release()    def file_write_finish():    file_fp.close()class mpdown(threading.Thread):    def __init__(self,range_start,range_end):        threading.Thread.__init__(self)        self.range_start = range_start        self.range_end = range_end    def run(self):        req = urllib2.Request(url)        req.add_header(‘Range‘,‘bytes=%d-%d‘%(self.range_start,self.range_end))        r = urllib2.urlopen(req)        #print r.info()        pos_base = self.range_start        pos = 0        size_to_get = self.range_end - self.range_start +1        while pos<size_to_get:            dat = r.read(Block_Size)            l = len(dat)            if pos+len(dat)>size_to_get:                dat = dat[:size_to_get-pos]            file_write(pos+pos_base,dat)            pos += len(dat)def download_one_thread():    passdef download():    #check_range_acceptable()    #if file_length<thread_num  use single thread    #download_one_thread()    #block size of each thread    time_start = datetime.datetime.now()    if file_write_init() == False:        return     print ‘file name:%s‘%(file_name)    print ‘net_filesize:%dbyte‘%(net_filesize)    print ‘multi process downloading...‘    threads = []    thread_size = net_filesize / Thread_Num    left_size = net_filesize % Thread_Num    pos = 0    for i in xrange(0,Thread_Num):        range_start = pos        pos += thread_size-1        if i==0:pos += left_size        range_end = pos        pos += 1        t = mpdown(range_start,range_end)        threads.append(t)        #print ‘range=%d-%d‘%(range_start,range_end)    for t in threads:        t.start()    for t in threads:        t.join()    file_write_finish()    time_end = datetime.datetime.now()    s = (time_end-time_start).seconds    if s<1:s=1    print ‘time passed:%dh %dm %ds‘%(s/3600,s/60,s%60)    print ‘speed:%dkb/s‘%(net_filesize/1000.0/s)    def main():    global url    args = len(sys.argv)    if args>1:        for i in range(1,args):            url = sys.argv[i]            download()    else:        url = raw_input("input download url:")        download()def test():    req = urllib2.Request(url)    req.add_header(‘Range‘,‘bytes=%d-%d‘%(1,500))    print req    r = urllib2.urlopen(req)    print r.info()if __name__ == ‘__main__‘:    main()

  

python写的多线程下载工具