Merge pull request #38 from dt3310321/s3

S3

Author: dt3310321

.travis.yml

@@ -10,7 +10,7 @@ install:
 - pip install dicttoxml
 script:
 - pep8 --max-line-length=180 qcloud_cos/.
-- nosetests -s -v
+- nosetests -s -v ut/
 deploy:
   provider: pypi
   distributions: sdist bdist_wheel

qcloud_cos/cos_client.py

@@ -8,12 +8,14 @@
 import xml.dom.minidom
 import xml.etree.ElementTree
 from urllib import quote
+from urllib import unquote
 from xml2dict import Xml2Dict
 from dicttoxml import dicttoxml
 from cos_exception import CosClientError
 from cos_exception import CosServiceError
 
 SINGLE_UPLOAD_LENGTH = 5*1024*1024*1024  # 单次上传文件最大为5G
+LOGGING_UIN = 'id="qcs::cam::uin/100001001014:uin/100001001014"'
 # kwargs中params到http headers的映射
 maplist = {
             'ContentLength': 'Content-Length',
@@ -45,7 +47,12 @@
             'CopySourceIfNoneMatch': 'x-cos-copy-source-If-None-Match',
             'CopySourceIfModifiedSince': 'x-cos-copy-source-If-Modified-Since',
             'CopySourceIfUnmodifiedSince': 'x-cos-copy-source-If-Unmodified-Since',
-            'VersionId': 'x-cos-version-id',
+            'VersionId': 'versionId',
+            'ServerSideEncryption': 'x-cos-server-side-encryption',
+            'SSECustomerAlgorithm': 'x-cos-server-side-encryption-customer-algorithm',
+            'SSECustomerKey': 'x-cos-server-side-encryption-customer-key',
+            'SSECustomerKeyMD5': 'x-cos-server-side-encryption-customer-key-MD5',
+            'SSEKMSKeyId': 'x-cos-server-side-encryption-cos-kms-key-id'
            }
 
 
@@ -62,6 +69,21 @@ def get_md5(data):
     return MD5
 
 
+def get_content_md5(body):
+    body_type = type(body)
+    if body_type == str:
+        return get_md5(body)
+    elif body_type == file:
+        if hasattr(body, 'tell') and hasattr(body, 'seek') and hasattr(body, 'read'):
+            file_position = body.tell()  # 记录文件当前位置
+            md5_str = get_md5(body.read())
+            body.seek(file_position)  # 恢复初始的文件位置
+            return md5_str
+        else:
+            raise CosClientError('can not get md5 digest for file without necessary attrs, including tell, seek and read')
+    return None
+
+
 def dict_to_xml(data):
     """V5使用xml格式，将输入的dict转换为xml"""
     doc = xml.dom.minidom.Document()
@@ -98,6 +120,8 @@ def xml_to_dict(data, origin_str="", replace_str=""):
     xmldict = Xml2Dict(root)
     xmlstr = str(xmldict)
     xmlstr = xmlstr.replace("{http://www.qcloud.com/document/product/436/7751}", "")
+    xmlstr = xmlstr.replace("{https://cloud.tencent.com/document/product/436}", "")
+    xmlstr = xmlstr.replace("{http://doc.s3.amazonaws.com/2006-03-01}", "")
     xmlstr = xmlstr.replace("{http://www.w3.org/2001/XMLSchema-instance}", "")
     if origin_str:
         xmlstr = xmlstr.replace(origin_str, replace_str)
@@ -193,6 +217,7 @@ def format_path(path):
 def get_copy_source_info(CopySource):
     """获取拷贝源的所有信息"""
     appid = ""
+    versionid = ""
     if 'Appid' in CopySource.keys():
         appid = CopySource['Appid']
     if 'Bucket' in CopySource.keys():
@@ -209,13 +234,17 @@ def get_copy_source_info(CopySource):
         path = CopySource['Key']
     else:
         raise CosClientError('CopySource Need Parameter Key')
-    return bucket, path, region
+    if 'VersionId' in CopySource.keys():
+        versionid = CopySource['VersionId']
+    return bucket, path, region, versionid
 
 
 def gen_copy_source_url(CopySource):
     """拼接拷贝源url"""
-    bucket, path, region = get_copy_source_info(CopySource)
+    bucket, path, region, versionid = get_copy_source_info(CopySource)
     path = format_path(path)
+    if versionid != '':
+        path = path + '?versionId=' + versionid
     url = "{bucket}.{region}.myqcloud.com/{path}".format(
             bucket=bucket,
             region=region,
@@ -245,3 +274,27 @@ def deal_with_empty_file_stream(data):
         except io.UnsupportedOperation:
             return ""
     return data
+
+
+def format_dict(data, key_lst):
+    """转换返回dict中的可重复字段为list"""
+    for key in key_lst:
+        # 将dict转为list，保持一致
+        if key in data.keys() and isinstance(data[key], dict):
+            lst = []
+            lst.append(data[key])
+            data[key] = lst
+    return data
+
+
+def decode_result(data, key_lst, multi_key_list):
+    """decode结果中的字段"""
+    for key in key_lst:
+        if key in data.keys() and data[key]:
+            data[key] = unquote(data[key])
+    for multi_key in multi_key_list:
+        if multi_key[0] in data.keys():
+            for item in data[multi_key[0]]:
+                if multi_key[1] in item.keys() and item[multi_key[1]]:
+                    item[multi_key[1]] = unquote(item[multi_key[1]])
+    return data

qcloud_cos/cos_comm.py

@@ -43,9 +43,9 @@ def get_result(self):
 
 class SimpleThreadPool:
 
-    def __init__(self, num_threads=5):
+    def __init__(self, num_threads=5, num_queue=0):
         self._num_threads = num_threads
-        self._queue = Queue()
+        self._queue = Queue(num_queue)
         self._lock = Lock()
         self._active = False
         self._workers = list()

qcloud_cos/cos_threadpool.py

@@ -4,12 +4,16 @@
 from qcloud_cos import CosServiceError
 from qcloud_cos import CosClientError
 
+import logging
+
 # 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7
 
 # pip安装指南:pip install -U cos-python-sdk-v5
 
 # cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224
 
+logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
+
 # 设置用户属性, 包括secret_id, secret_key, region
 # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成
 secret_id = 'AKID15IsskiBQACGbAo6WhgcQbVls7HmuG00'     # 替换为用户的secret_id

qcloud_cos/demo.py

@@ -16,7 +16,7 @@ def long_description():
 
 setup(
     name='cos-python-sdk-v5',
-    version='1.3.2',
+    version='1.4.1',
     url='https://www.qcloud.com/',
     license='MIT',
     author='tiedu, lewzylu, channingliu',

setup.py

@@ -5,9 +5,9 @@
 import hashlib
 import os
 import requests
-from cos_client import CosS3Client
-from cos_client import CosConfig
-from cos_exception import CosServiceError
+from qcloud_cos import CosS3Client
+from qcloud_cos import CosConfig
+from qcloud_cos import CosServiceError
 
 SECRET_ID = os.environ["SECRET_ID"]
 SECRET_KEY = os.environ["SECRET_KEY"]
@@ -297,7 +297,7 @@ def test_delete_multiple_objects():
     )
     assert response2
     objects = {
-        "Quite": "true",
+        "Quiet": "true",
         "Object": [
             {
                 "Key": file_name1
@@ -311,7 +311,6 @@ def test_delete_multiple_objects():
         Bucket=test_bucket,
         Delete=objects
     )
-    assert response
 
 
 def test_create_head_delete_bucket():
@@ -353,7 +352,9 @@ def test_list_objects():
     """列出bucket下的objects"""
     response = client.list_objects(
         Bucket=test_bucket,
-        MaxKeys=100
+        MaxKeys=100,
+        Prefix='中文',
+        Delimiter='/'
     )
     assert response
 
@@ -589,7 +590,7 @@ def test_upload_empty_file():
 
 def test_copy_10G_file_in_same_region():
     """同园区的拷贝,应该直接用copy_object接口,可以直接秒传"""
-    copy_source = {'Bucket': 'test01-1252448703', 'Key': '/10G.txt', 'Region': 'ap-beijing-1'}
+    copy_source = {'Bucket': 'test01-1252448703', 'Key': '10G.txt', 'Region': 'ap-beijing-1'}
     response = client.copy(
         Bucket='test04-1252448703',
         Key='10G.txt',
@@ -610,8 +611,70 @@ def test_use_get_auth():
     assert response.status_code == 200
 
 
+def test_upload_with_server_side_encryption():
+    """上传带上加密头部,下载时验证有该头部"""
+    response = client.put_object(
+        Bucket=test_bucket,
+        Key=test_object,
+        Body='123',
+        ServerSideEncryption='AES256'
+    )
+    assert response['x-cos-server-side-encryption'] == 'AES256'
+
+
+def test_put_get_bucket_logging():
+    """测试bucket的logging服务"""
+    logging_bucket = 'logging-beijing-1252448703'
+    logging_config = {
+        'LoggingEnabled': {
+            'TargetBucket': logging_bucket,
+            'TargetPrefix': 'test'
+        }
+    }
+    beijing_conf = CosConfig(
+        Region="ap-beijing",
+        Secret_id=SECRET_ID,
+        Secret_key=SECRET_KEY
+    )
+    logging_client = CosS3Client(beijing_conf)
+    response = logging_client.put_bucket_logging(
+        Bucket=logging_bucket,
+        BucketLoggingStatus=logging_config
+    )
+    time.sleep(4)
+    response = logging_client.get_bucket_logging(
+        Bucket=logging_bucket
+    )
+    print response
+    assert response['LoggingEnabled']['TargetBucket'] == logging_bucket
+    assert response['LoggingEnabled']['TargetPrefix'] == 'test'
+
+
+def test_put_object_enable_md5():
+    """上传文件,SDK计算content-md5头部"""
+    file_size = 10
+    file_name = 'test_object_sdk_caculate_md5.file'
+    gen_file(file_name, 10)
+    with open(file_name, 'rb') as f:
+        etag = get_raw_md5(f.read())
+    with open(file_name, 'rb') as fp:
+        put_response = client.put_object(
+            Bucket=test_bucket,
+            Body=fp,
+            Key=file_name,
+            EnableMD5=True,
+            CacheControl='no-cache',
+            ContentDisposition='download.txt'
+        )
+        assert etag == put_response['Etag']
+    if os.path.exists(file_name):
+        os.remove(file_name)
+
+
 if __name__ == "__main__":
     setUp()
+    test_put_object_enable_md5()
+    test_upload_with_server_side_encryption()
     test_upload_empty_file()
     test_put_get_delete_object_10MB()
     test_put_get_versioning()
@@ -620,5 +683,7 @@ def test_use_get_auth():
     test_upload_file_multithreading()
     test_copy_file_automatically()
     test_copy_10G_file_in_same_region()
+    test_list_objects()
     test_use_get_auth()
+    test_put_get_bucket_logging()
     tearDown()