Wednesday, 7 November 2018

how to upload chunks of a string longer than 2147483647 bytes?

I am trying to upload a file around ~5GB size as below but, it throws the error string longer than 2147483647 bytes. It sounds like there is a limit of 2 GB to upload. Is there a way to upload data in chunks? Can anyone provide guidance?

logger.debug(attachment_path)
currdir = os.path.abspath(os.getcwd())
os.chdir(os.path.dirname(attachment_path))
headers = self._headers
headers['Content-Type'] = content_type
headers['X-Override-File'] = 'true'
if not os.path.exists(attachment_path):
    raise Exception, "File path was invalid, no file found at the path %s" % attachment_path
filesize = os.path.getsize(attachment_path) 
fileToUpload = open(attachment_path, 'rb').read()
logger.info(filesize)
logger.debug(headers)
r = requests.put(self._baseurl + 'problems/' + problemID + "/" + attachment_type + "/" + urllib.quote(os.path.basename(attachment_path)), 
                 headers=headers,data=fileToUpload,timeout=300)

ERROR:

string longer than 2147483647 bytes

UPDATE:

def read_in_chunks(file_object,chunk_size=30720*30720):
    """Lazy function (generator) to read a file piece by piece.
    Default chunk size: 1k."""
    while True:
        data = file_object.read(chunk_size)
        if not data:
            break
        yield data
        f = open(attachment_path)

for piece in read_in_chunks(f):
      r = requests.put(self._baseurl + 'problems/' + problemID + "/" + attachment_type + "/" + urllib.quote(os.path.basename(attachment_path)), 
                        headers=headers,data=piece,timeout=300)



from how to upload chunks of a string longer than 2147483647 bytes?

No comments:

Post a Comment