Here is the quick script I wrote to download a gz file using python requests module:
#!/usr/bin/env python
import requests
import gzip
import logging
import sys
import StringIO
import zlib
# setup logging
logging.basicConfig(stream = sys.stdout, level = logging.ERROR)
log = logging.getLogger('threat-feeds-logger')
#proxy configuration
proxy_host='10.1.1.11'
proxy_port=3128
proxy_user = 'xxxx'
proxy_password = 'xxxx'
feed_url = 'https://foo.org/foo.gz'
proxy_dict = {
'http':'http://%s:%s@%s:%s' % (proxy_user, proxy_password, proxy_host, proxy_port),
'https':'http://%s:%s@%s:%s' % (proxy_user, proxy_password, proxy_host, proxy_port)
}
try:
response = requests.get(feed_url,proxies = proxy_dict)
except Exception as e:
log.error("Error while getting data from url - %s" %feed_url)
if response.status_code == 200:
buf_data = StringIO.StringIO(response.content)
f = gzip.GzipFile(fileobj=buf_data)
for row in f.readlines():
print row
#!/usr/bin/env python
import requests
import gzip
import logging
import sys
import StringIO
import zlib
# setup logging
logging.basicConfig(stream = sys.stdout, level = logging.ERROR)
log = logging.getLogger('threat-feeds-logger')
#proxy configuration
proxy_host='10.1.1.11'
proxy_port=3128
proxy_user = 'xxxx'
proxy_password = 'xxxx'
feed_url = 'https://foo.org/foo.gz'
proxy_dict = {
'http':'http://%s:%s@%s:%s' % (proxy_user, proxy_password, proxy_host, proxy_port),
'https':'http://%s:%s@%s:%s' % (proxy_user, proxy_password, proxy_host, proxy_port)
}
try:
response = requests.get(feed_url,proxies = proxy_dict)
except Exception as e:
log.error("Error while getting data from url - %s" %feed_url)
if response.status_code == 200:
buf_data = StringIO.StringIO(response.content)
f = gzip.GzipFile(fileobj=buf_data)
for row in f.readlines():
print row
No comments:
Post a Comment