Made Minio default AWS driver

os_benchmark/benchmarks/download.py

@@ -35,9 +35,14 @@ def download_objet(url):
                 self.errors.append(err)
 
         def download_objets(urls):
+            futures = []
             with ThreadPoolExecutor(max_workers=self.params['parallel_objects']) as executor:
                 for url in urls:
-                    executor.submit(download_objet, url=url)
+                    future = executor.submit(download_objet, url=url)
+                    futures.append(future)
+
+            for futures in futures:
+                future.result()
 
         self.sleep(self.params['warmup_sleep'])
         self.total_time = self.timeit(download_objets, urls=self.urls)[0]

os_benchmark/drivers/aws.py

@@ -1,6 +1,6 @@
 """
 .. note::
-  This driver requires `boto3`_.
+  This driver requires `minio`_.
 
 Configuration
 ~~~~~~~~~~~~~
@@ -10,22 +10,30 @@
   ---
   aws:
     driver: aws
-    aws_access_key_id: <your_ak>
-    aws_secret_access_key: <your_sk>
-    endpoint_url: https://s3.<region_id>.amazonaws.com
-    region_name: <region_id>
+    access_key: <your_ak>
+    secret_key: <your_sk>
+    endpoint: s3.<region_id>.amazonaws.com
+    region: <region_id>
 
 .. _boto3: https://github.com/boto/boto3
 """
+from minio.xml import Element, SubElement, getbytes
 from os_benchmark.drivers import minio_sdk
 
 
 class Driver(minio_sdk.Driver):
     """AWS S3 Driver"""
     id = 'aws'
-    default_acl = None
     default_object_acl = None
 
     def get_url(self, bucket_id, name, **kwargs):
-        url = '%s/%s/%s' % (self.kwargs['endpoint_url'], bucket_id, name)
+        url = 'https://%s/%s/%s' % (self.kwargs['endpoint'], bucket_id, name)
         return url
+
+    def _make_create_bucket_params(self, params):
+        if self.kwargs['region'] == 'us-east-1':
+            params['body'] = None
+        elif self.kwargs.get('region'):
+            element = Element("CreateBucketConfiguration")
+            SubElement(element, "LocationConstraint", self.kwargs['region'])
+            params['body'] = getbytes(element)

os_benchmark/drivers/aws_boto.py

@@ -0,0 +1,12 @@
+from os_benchmark.drivers import s3
+
+
+class Driver(s3.Driver):
+    """AWS S3 Boto Driver"""
+    id = 'aws_s3'
+    old_acl = False
+    default_object_acl = 'public-read'
+
+    def get_url(self, bucket_id, name, **kwargs):
+        url = '%s/%s/%s' % (self.kwargs['endpoint_url'], bucket_id, name)
+        return url

os_benchmark/drivers/base.py

@@ -416,8 +416,9 @@ def download(self, url, block_size=65536, headers=None, **kwargs):
         try:
             with self.session.get(url, stream=True, headers=headers) as response:
                 if response.status_code != 200:
+                    self.logger.warning('GET %s: %s', url, response.status_code)
                     msg = '%s %s' % (url, response.content)
-                    raise errors.base.InvalidHttpCode(msg, response.status_code)
+                    raise errors.InvalidHttpCode(msg, response.status_code)
                 for chunk in response.iter_content(chunk_size=block_size):
                     pass
         except requests.exceptions.ConnectionError as err:

os_benchmark/drivers/errors.py

@@ -1,5 +1,6 @@
 """Driver errors module."""
 from os_benchmark import errors as base
+from os_benchmark.errors import InvalidHttpCode
 
 
 class DriverError(base.OsbError):

os_benchmark/drivers/minio_sdk.py

@@ -72,6 +72,9 @@ def list_buckets(self, **kwargs):
         buckets = [{'id': b.name} for b in buckets]
         return buckets
 
+    def _make_create_bucket_params(self, params):
+        pass
+
     def create_bucket(self, name, acl=None, bucket_lock=None, **kwargs):
         location = self.kwargs['region']
         headers = {}
@@ -83,16 +86,12 @@ def create_bucket(self, name, acl=None, bucket_lock=None, **kwargs):
         if bucket_lock:
             headers["x-amz-bucket-object-lock-enabled"] = "true"
 
-        body = None
-        if self.kwargs.get('extra') and self.kwargs['extra'].get('location_constraint'):
-            element = Element("CreateBucketConfiguration")
-            SubElement(element, "LocationConstraint", location)
-            body = getbytes(element)
         params = {
             'bucket_name': name,
-            'body': body,
+            'body': None,
             'headers': headers,
         }
+        self._make_create_bucket_params(params)
         self.logger.debug("Create bucket params: %s", params)
         try:
             self.client._url_open("PUT", location, **params)
@@ -167,16 +166,23 @@ def get_presigned_url(self, bucket_id, name, method='GET', **kwargs):
         )
         return url
 
-    def put_bucket_policy(self, bucket_id, **kwargs):
+    def put_bucket_policy(self, bucket_id, object_id='*', **kwargs):
         policy = json.dumps({
+            "Version": "2012-10-17",
             "Statement": [{
-                "Action": ["s3:GetObject"],
                 "Effect": "Allow",
-                "Principal": {"AWS": ["*"]},
-                "Resource": [f"arn:aws:s3:::{bucket_id}/*"],
-                "Sid":"UCDefaultPublicPolicy"
+                "Action": [
+                    "s3:GetObject",
+                    "s3:ListBucket",
+                ],
+                "Principal": {"AWS": "*"},
+                "Resource": [f"arn:aws:s3:::{bucket_id}"],
+            }, {
+                "Effect": "Allow",
+                "Principal": {"AWS": "*"},
+                "Action": ["s3:GetObject"],
+                "Resource": [f"arn:aws:s3:::{bucket_id}/{object_id}"],
             }],
-            "Version": "2012-10-17"
         })
         self.client.set_bucket_policy(bucket_id, policy)
 

os_benchmark/drivers/s3.py

@@ -14,7 +14,7 @@
     driver: s3
     aws_access_key_id: <your_ak>
     aws_secret_access_key: <your_sk>
-    region: eu-west-1
+    region_name: eu-west-1
 
 All parameters except ``driver`` will be passed to ``boto3.resource``.
 """
@@ -70,8 +70,10 @@ def _handle_request(self, *args, **kwargs):
 
 class Driver(base.RequestsMixin, base.BaseDriver):
     id = 's3'
-    default_acl = 'public-read'
-    default_object_acl = 'public-read'
+    default_acl = None
+    default_object_acl = None
+    old_acl = True
+
     default_kwargs = {}
     default_config = {}
     _default_config = {
@@ -120,8 +122,10 @@ def list_buckets(self, **kwargs):
     def _get_create_request_params(self, name, acl, **kwargs):
         params = {
             'Bucket': name,
-            'ACL': acl,
         }
+        if self.old_acl:
+            params['ACL'] = acl
+
         if 'region_name' in self.kwargs:
             params['CreateBucketConfiguration'] = {
                 'LocationConstraint': self.kwargs['region_name']
@@ -148,6 +152,32 @@ def create_bucket(self, name, acl=None, bucket_lock=None, **kwargs):
             if code == 'BucketAlreadyExists':
                 raise errors.DriverBucketAlreadyExistError(msg)
             raise
+
+        if not self.old_acl:
+            self.logger.debug("Allow public access block")
+            self.s3.meta.client.put_public_access_block(
+                Bucket=name,
+                PublicAccessBlockConfiguration={
+                    'BlockPublicAcls': False,
+                    'IgnorePublicAcls': False,
+                    'BlockPublicPolicy': False,
+                    'RestrictPublicBuckets': False,
+                }
+            )
+            policy = json.dumps({
+                "Version": "2012-10-17",
+                "Statement": [{
+                    "Effect": "Allow",
+                    "Principal": {"AWS": "*"},
+                    "Action": ["s3:GetObject"],
+                    "Resource": [f"arn:aws:s3:::{name}/*"],
+                }],
+            })
+            self.s3.meta.client.put_bucket_policy(
+                Bucket=name,
+                Policy=policy,
+            )
+
         return {'id': bucket.name}
 
     @handle_request
@@ -284,8 +314,10 @@ def upload(self, bucket_id, name, content, acl=None,
                multipart_threshold=None, multipart_chunksize=None,
                max_concurrency=None, storage_class=None,
                **kwargs):
-        acl = acl or self.default_object_acl
-        extra = {'ACL': acl}
+        extra = {}
+        if self.old_acl:
+            acl = acl or self.default_object_acl
+            extra['ACL'] = acl
         if storage_class:
             extra['StorageClass'] = storage_class
         multipart_threshold = multipart_threshold or base.MULTIPART_THRESHOLD
@@ -297,14 +329,16 @@ def upload(self, bucket_id, name, content, acl=None,
             max_concurrency=max_concurrency,
             multipart_chunksize=multipart_chunksize,
         )
+        params = {
+            'Fileobj': content,
+            'Bucket': bucket_id,
+            'Key': name,
+            'ExtraArgs': extra,
+            'Config': transfer_config,
+        }
+        self.logger.debug("Upload obj params: %s", params)
         try:
-            self.s3.meta.client.upload_fileobj(
-                Fileobj=content,
-                Bucket=bucket_id,
-                Key=name,
-                ExtraArgs=extra,
-                Config=transfer_config,
-            )
+            self.s3.meta.client.upload_fileobj(**params)
         except botocore.exceptions.ClientError as err:
             code = err.response['Error']['Code']
             msg = err.response['Error']['Message']
@@ -313,6 +347,7 @@ def upload(self, bucket_id, name, content, acl=None,
             elif code == 'AccessDenied':
                 raise errors.DriverBucketUnfoundError(msg)
             raise
+
         return {'name': name}
 
     @handle_request

setup.cfg

@@ -59,6 +59,7 @@ swift =
 
 alibaba = oss2
 aws = minio
+aws_boto = boto3
 azure = azure-storage-blob
 backblaze = b2sdk
 backblaze_s3 = boto3