saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

use requests streaming for uploads/downloads to file (return_bin unchanged)

Browse source 
allows downloading files larger than amount of memory (non-stream reads into memory before writing to disk or uploading)
This commit is contained in:
Ethan Moore 2016-05-27 23:04:01 +00:00 • committed by rallytime
parent 4a9b23f03f
commit 45143a599b
1 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
salt/utils/s3.py
View file

@ -28,7 +28,8 @@ log = logging.getLogger(__name__)
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path='', return_bin=False, action=None, local_file=None,
verify_ssl=True, location=None, full_headers=False, role_arn=None):
verify_ssl=True, location=None, full_headers=False, role_arn=None,
chunk_size=16384):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
@ -95,9 +96,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
payload_hash = None
if method == 'PUT':
if local_file:
payload_hash = salt.file.get_hash(local_file, form='sha256')
#with salt.utils.fopen(local_file, 'r') as ifile:
# data = ifile.read()
payload_hash = salt.utils.get_hash(local_file, form='sha256')
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
@ -191,7 +190,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'wb') as out:
for chunk in result.iter_content(chunk_size=2048):
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return 'Saved to local file: {0}'.format(local_file)