我這就拋磚引玉了,大家多指教。
參考:
http://pycurl.sourceforge.net/doc/pycurl.html
如果想多線程下載,請關注下面url
http://groups.google.com/group/python-cn/msg/cdbe671b814fd3af
http://www.pythonclub.org/doku/index.php/python-network-application/pycurl
(個人粗略認為)原理:
把服務器上的文件分n“等份”,再在準備下載地址按等分一一填充到文件中。
當 下載 “等份數(shù)據(jù)” 時如果未能正常結(jié)束,從頭下載次等份數(shù)據(jù)。正常結(jié)束后線程下載下一等分。
好比:
服務器文件(x): 1111111
待填充文件(y): 0000000
運行1:(2分鐘)
文件(x):0101001
接著運行2:(1分鐘,斷點續(xù)傳)
文件(x):1111001
最后:文件(x):1111111
需要技術(shù):
1.按切割下載
2.本地創(chuàng)造與服務器文件一樣大小空文件。
2.等份寫入
技術(shù)1代碼(按切割下載).
c.setopt(pycurl.RANGE, '%d-%d' % (500, 1000))
import pycurl
c = pycurl.Curl()
c.setopt(pycurl.URL, "http://www.python.org/")
c.setopt(pycurl.HTTPHEADER, ["Accept:"])
import StringIO
b = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.RANGE, '%d-%d' % (500, 1000))
c.perform()
print b.getvalue()
c = pycurl.Curl()
c.setopt(pycurl.URL, "http://www.python.org/")
c.setopt(pycurl.HTTPHEADER, ["Accept:"])
import StringIO
b = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.RANGE, '%d-%d' % (500, 1000))
c.perform()
print b.getvalue()
技術(shù)2(本地創(chuàng)造與服務器文件一樣大小空文件)
def allocate_space(self):
'''
預分配文件空間(通用?)
'''
afile = file(self.url_info['file'], 'wb')
afile.truncate(self.url_info['size'])#文件大小限定
afile.close()
'''
預分配文件空間(通用?)
'''
afile = file(self.url_info['file'], 'wb')
afile.truncate(self.url_info['size'])#文件大小限定
afile.close()
技術(shù)3(等份寫入 file.seek):
#----關鍵----
def write_cb(self, data):
if self.piece:
self.result.seek(self.piece[1] + self.piece_downloaded, 0)
self.result.write(data)
self.result.flush()
size = len(data)
self.link_downloaded += size
self.piece_downloaded += size
self.total_downloaded += size
if self.is_stop: return -1
def write_cb(self, data):
if self.piece:
self.result.seek(self.piece[1] + self.piece_downloaded, 0)
self.result.write(data)
self.result.flush()
size = len(data)
self.link_downloaded += size
self.piece_downloaded += size
self.total_downloaded += size
if self.is_stop: return -1
pycurl 學習筆記:
$ftp讀取
import pycurl
tfile='ftp://name:pass@218.200.***.***/tongbu/test.txt'
c = pycurl.Curl()
c.setopt(pycurl.URL, tfile)
import StringIO
b = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
#c.setopt(pycurl.RANGE, '%d-%d' % (2, 6))
c.perform()
print b.getvalue()
tfile='ftp://name:pass@218.200.***.***/tongbu/test.txt'
c = pycurl.Curl()
c.setopt(pycurl.URL, tfile)
import StringIO
b = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
#c.setopt(pycurl.RANGE, '%d-%d' % (2, 6))
c.perform()
print b.getvalue()
下載寫入:
bean_pycurl=pycurl.Curl()
...
bean_pycurl.setopt(pycurl.WRITEFUNCTION, write_cb)
...
def write_cb(data):
file.write(data)
..
...
bean_pycurl.setopt(pycurl.WRITEFUNCTION, write_cb)
...
def write_cb(data):
file.write(data)
..
整理 www.aygfsteel.com/Good-Game