import hashlib import logging import os import requests from bootstrapvz.common.bytes import Bytes class OracleStorageAPIClient(object): def __init__(self, username, password, identity_domain, container): self.username = username self.password = password self.identity_domain = identity_domain self.container = container self.base_url = 'https://' + identity_domain + '.storage.oraclecloud.com' self.log = logging.getLogger(__name__) # Avoid 'requests' INFO/DEBUG log messages logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) def _fail(self, error): raise RuntimeError('Oracle Storage Cloud API - ' + error) @property def auth_token(self): headers = { 'X-Storage-User': 'Storage-{id_domain}:{user}'.format( id_domain=self.identity_domain, user=self.username, ), 'X-Storage-Pass': self.password, } url = self.base_url + '/auth/v1.0' response = requests.get(url, headers=headers) if response.status_code == 200: return response.headers.get('x-auth-token') else: self._fail(response.text) @property def chunk_size(self): file_size = os.path.getsize(self.file_path) if file_size > int(Bytes('300MiB')): chunk_size = int(Bytes('100MiB')) else: chunk_size = int(Bytes('50MiB')) return chunk_size def compare_files(self): uploaded_file_md5 = hashlib.md5() downloaded_file_md5 = hashlib.md5() files = [self.file_path, self.target_file_path] hashes = [uploaded_file_md5, downloaded_file_md5] for f, h in zip(files, hashes): with open(f, 'rb') as current_file: while True: data = current_file.read(int(Bytes('8MiB'))) if not data: break h.update(data) if uploaded_file_md5.hexdigest() != downloaded_file_md5.hexdigest(): self.log.error('File hashes mismatch') else: self.log.debug('Both files have the same hash') def create_manifest(self): headers = { 'X-Auth-Token': self.auth_token, 'X-Object-Manifest': '{container}/{object_name}-'.format( container=self.container, object_name=self.file_name, ), 'Content-Length': '0', } url = self.object_url self.log.debug('Creating remote manifest to join chunks') response = requests.put(url, headers=headers) if response.status_code != 201: self._fail(response.text) def download_file(self): headers = { 'X-Auth-Token': self.auth_token, } url = self.object_url response = requests.get(url, headers=headers, stream=True) if response.status_code != 200: self._fail(response.text) with open(self.target_file_path, 'wb') as f: for chunk in response.iter_content(chunk_size=int(Bytes('8MiB'))): if chunk: f.write(chunk) @property def file_name(self): return os.path.basename(self.file_path) @property def object_url(self): url = '{base}/v1/Storage-{id_domain}/{container}/{object_name}'.format( base=self.base_url, id_domain=self.identity_domain, container=self.container, object_name=self.file_name, ) return url def upload_file(self): f = open(self.file_path, 'rb') n = 1 while True: chunk = f.read(self.chunk_size) if not chunk: break chunk_name = '{name}-{number}'.format( name=self.file_name, number='{0:04d}'.format(n), ) headers = { 'X-Auth-Token': self.auth_token, } url = '{base}/v1/Storage-{id_domain}/{container}/{object_chunk_name}'.format( base=self.base_url, id_domain=self.identity_domain, container=self.container, object_chunk_name=chunk_name, ) self.log.debug('Uploading chunk ' + chunk_name) response = requests.put(url, data=chunk, headers=headers) if response.status_code != 201: self._fail(response.text) n += 1 self.create_manifest()