|
#!/usr/bin/env python
import openanything
USER_HTTP_AGENT = 'python http downloader'
class http_downloader():
def __init__(self,url):
self.result = {}
self.result['url'] = url
self.result['etag'] = None
self.result['lastmodified'] = None
def fetch(self):
f = openanything.openAnything(self.result['url'],self.result['etag'],self.result['lastmodified'],USER_HTTP_AGENT)
self.result['data'] = f.read()
if hasattr(f, 'headers'):
# save ETag, if the server sent one
self.result['etag'] = f.headers.get('ETag')
# save Last-Modified header, if the server sent one
self.result['lastmodified'] = f.headers.get('Last-Modified')
if f.headers.get('content-encoding') == 'gzip':
# data came back gzip-compressed, decompress it
self.result['data'] = gzip.GzipFile(fileobj=StringIO(result['data'])).read()
if hasattr(f, 'url'):
self.result['url'] = f.url
#if f.read success,f has not attr 'status ', so set default here
self.result['status'] = 200
if hasattr(f, 'status'):
self.result['status'] = f.status
f.close()
def set_url(self,url):
if url != self.result['url']:
self.result['etag'] = None
self.result['lastmodified'] = None
self.result['url'] = url
def debug_print(self):
print self.result['url']
print self.result['status']
print self.result['etag']
print self.result['lastmodified']
#print self.result['data']
|
