s3 api接口的調用

 

最近公司使用s3作文件存儲服務器，所以在程序中須要調用s3的api，目前程序中使用了python和java版本的s3的api，簡單作下記錄，方便之後使用。

1、s3 api使用python版

1.安裝boto3的sdk

pip install boto3==1.4.7

2.建立s3的api客戶端

import boto3
from botocore.client import Config
import os
import sys
import threading

#建立s3連接，若是s3服務器是第四代，則須要設置signature_version='s3v4'
s3_client = boto3.client('s3', endpoint_url='s3服務器地址',
                         aws_access_key_id='s3服務器的access_key_id',
                         aws_secret_access_key='s3服務器的secret_access_key',
                         region_name='s3服務器的時區，這裏能夠填寫cn-north-1',
                         config=Config(signature_version='s3'))

3.獲取s3中bucket列表

bucket_list = s3_client.list_buckets()

print bucket_list

4.建立bucket

bucket = s3_client.create_bucket(Bucket='bucket的名稱')

print bucket

5.獲取bucket信息
bucket_info = s3_client.head_bucket(Bucket='bucket的名稱')

print bucket_info

6.刪除bucket
#bucket_delete = s3_client.delete_bucket(Bucket='bucket的名稱')

7.上傳文件到s3服務器

# s3_client.upload_file("上傳的源文件地址", "bucket名稱", "上傳文件在s3上對應的key名稱", ExtraArgs={'ACL': 'public-read'})

或者

s3_client.upload_file("上傳的源文件地址", "bucket名稱", "上傳文件在s3上對應的key名稱",

　　　　　　ExtraArgs={'ACL': 'public-read'},

　　　　　　Callback=UploadProgressPercentage("上傳的源文件地址"))
Callback屬性對應的類方法以下，該類方法在控制檯中打印了上傳文件的進度

class UploadProgressPercentage(object):
    def __init__(self, filename):
        self._filename = filename
        self._size = float(os.path.getsize(filename))
        self._seen_so_far = 0
        self._lock = threading.Lock()
    def __call__(self, bytes_amount):
        # To simplify we'll assume this is hooked up
        # to a single filename.
        with self._lock:
            self._seen_so_far += bytes_amount
            percentage = (self._seen_so_far / self._size) * 100
            sys.stdout.write(
                "\r%s  %s / %s  (%.2f%%)" % (
                    self._filename, self._seen_so_far, self._size,
                    percentage))
            sys.stdout.flush()

8.獲取bucket下文件列表

object_list = s3_client.list_objects(Bucket='bucket名稱')
print object_list

9.查看bucket下的某個文件信息
object_info = s3_client.get_object(Bucket='bucket名稱', Key='文件對應的key名稱')
print object_info
10.刪除文件
object_delete = s3_client.delete_object(Bucket='bucket名稱', Key='文件對應的key名稱')

11.下載文件

s3_client.download_file("bucket名稱", "文件對應的key名稱", "文件下載到的地址")

或者

s3client.download_file("bucket名稱", "文件對應的key名稱", "文件下載到的地址",Callback=DownloadProgressPercentage("文件下載到的地址"))

Callback屬性對應的類方法以下，該類方法在控制檯中打印了下載文件的進度

class _DownloadProgressPercentage(object):
    def __init__(self, filename):
        self._filename = filename
        self._seen_so_far = 0
        self._lock = threading.Lock()
    def __call__(self, bytes_amount):
        # To simplify we'll assume this is hooked up
        # to a single filename.
        with self._lock:
            self._seen_so_far += bytes_amount
            sys.stdout.write(
                "\r%s --> %s bytes transferred" % (
                    self._filename, self._seen_so_far))
            sys.stdout.flush()

 

參考文檔

boto3 api官方文檔：http://boto3.readthedocs.io/en/latest/guide/quickstart.html

1.s3的client的api

2.s3的client的api上傳下載示例

2、3 api使用java版

1.在pom.xml中增長依賴包

<dependency>
　　<groupId>com.amazonaws</groupId>
　　<artifactId>aws-java-sdk</artifactId>
　　<version>1.7.3</version>
</dependency>

2.java中使用s3的api的demo

package org.jenkinsci.plugins.s3_step;import java.io.BufferedInputStream;import java.io.BufferedOutputStream;import java.io.ByteArrayInputStream;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.net.URL;import java.security.SecureRandom;import java.util.ArrayList;import java.util.List;import com.amazonaws.ClientConfiguration;import com.amazonaws.auth.AWSCredentials;import com.amazonaws.auth.BasicAWSCredentials;import com.amazonaws.regions.Region;import com.amazonaws.regions.Regions;import com.amazonaws.services.s3.AmazonS3;import com.amazonaws.services.s3.AmazonS3Client;import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;import com.amazonaws.services.s3.model.Bucket;import com.amazonaws.services.s3.model.BucketPolicy;import com.amazonaws.services.s3.model.CannedAccessControlList;import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;import com.amazonaws.services.s3.model.CopyObjectResult;import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;import com.amazonaws.services.s3.model.GetBucketPolicyRequest;import com.amazonaws.services.s3.model.GetObjectRequest;import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;import com.amazonaws.services.s3.model.ListObjectsRequest;import com.amazonaws.services.s3.model.ListPartsRequest;import com.amazonaws.services.s3.model.MultipartUpload;import com.amazonaws.services.s3.model.MultipartUploadListing;import com.amazonaws.services.s3.model.ObjectListing;import com.amazonaws.services.s3.model.ObjectMetadata;import com.amazonaws.services.s3.model.PartETag;import com.amazonaws.services.s3.model.PartListing;import com.amazonaws.services.s3.model.PartSummary;import com.amazonaws.services.s3.model.PutObjectRequest;import com.amazonaws.services.s3.model.PutObjectResult;import com.amazonaws.services.s3.model.S3Object;import com.amazonaws.services.s3.model.S3ObjectInputStream;import com.amazonaws.services.s3.model.S3ObjectSummary;import com.amazonaws.services.s3.model.UploadPartRequest;import com.amazonaws.services.s3.model.UploadPartResult;public class Demo {    AWSCredentials credentials = null;    public AmazonS3 getS3Client(String access_key, String secret_key,                                String endpoint) {        ClientConfiguration conf = null;        if (conf == null) {            conf = new ClientConfiguration();            credentials = new BasicAWSCredentials(access_key, secret_key);        }        AmazonS3 s3 = new AmazonS3Client(credentials, conf);        s3.setRegion(Region.getRegion(Regions.CN_NORTH_1));        if (endpoint != null)            s3.setEndpoint(endpoint);        return s3;    }    // ============================ bucket ================================    public void testCreateBucket(AmazonS3 s3Client, String bucketName) {        System.out.println("======================建立 Bucket==========================");        Bucket bucket = s3Client.createBucket(bucketName);        System.out.println(bucket);        System.out.println("建立 Bucket : " + bucket.getName());        System.out.println("======================建立 Bucket==========================");    }    public void testDeleteBucket(AmazonS3 s3Client, String bucketName) {        System.out.println("======================刪除 Bucket==========================");        System.out.println("刪除 Bucket : " + bucketName);        s3Client.deleteBucket(bucketName);        System.out.println("======================刪除 Bucket==========================");    }    public void testDoesBucketExist(AmazonS3 s3Client, String bucketName) {        System.out.println("======================Bucket 是否存在==========================");        boolean bucketExist = s3Client.doesBucketExist(bucketName);        System.out.println(String.format("%s : %s", bucketName, bucketExist));        System.out.println("======================Bucket 是否存在==========================");    }    public void testListBuckets(AmazonS3 s3Client) {        System.out.println("======================枚舉 Buckets==========================");        List<Bucket> buckets = s3Client.listBuckets();        for (Bucket bucket : buckets) {            System.out.println(String.format("%s - %s - %s", bucket.getName(), bucket.getOwner(), bucket.getCreationDate()));        }        System.out.println("======================枚舉 Buckets==========================");    }    public void testListObjects(AmazonS3 s3Client, String bucketName) {        System.out.println("======================枚舉 Objects==========================");        List<String> objectKeys = new ArrayList<String>();        String prefix = "/s3demo/test/";//        for (int i = 0; i < 5; i++) {//            objectKeys.add(prefix + i);//        }//        for (String objectKey : objectKeys) {//            testPutObject(s3Client, bucketName, objectKey);//        }        ObjectListing objects = s3Client.listObjects(bucketName);        List<S3ObjectSummary> s3Objects = objects.getObjectSummaries();        for (S3ObjectSummary s3os : s3Objects) {            System.out.println(String.format("%s - %s - %s", s3os.getKey(), s3os.getETag(), s3os.getSize()));        }        // prefix        ListObjectsRequest listObjReq = new ListObjectsRequest();        listObjReq.setBucketName(bucketName);        listObjReq.setDelimiter("/");        listObjReq.setPrefix(prefix);        ObjectListing preobjects = s3Client.listObjects(listObjReq);        List<String> preobjs = preobjects.getCommonPrefixes();        for (String preobj : preobjs) {            System.out.println(preobj + "\n");        }        System.out.println("======================枚舉 Objects==========================");    }    // ============================ bucket ================================    // ============================ object ================================    public void testPutObject(AmazonS3 s3Client, String bucketName, String objectKey,String file_source_address) {      System.out.println("======================上傳 object==========================");        s3Client.putObject(new PutObjectRequest(bucketName, objectKey, new File(file_source_address)).withCannedAcl(CannedAccessControlList.PublicRead));        System.out.println("======================上傳 object==========================");    }    public void testGetObject(AmazonS3 s3Client, String bucketName,String objectKey) throws Exception {        System.out.println("======================下載 object==========================");        try {            S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName,objectKey));            //下載文件到指定地方            InputStream reader = new BufferedInputStream(s3Object.getObjectContent());            File file = new File("localFilename");                FileOutputStream fos = new FileOutputStream(file);            OutputStream writer = new BufferedOutputStream(fos);            int read = -1;            while ( ( read = reader.read() ) != -1 ) {                        writer.write(read);            }            writer.flush();            writer.close();            reader.close();            //獲取一個request，生成一個下載的url地址，能夠在瀏覽器上下載            GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey);              //生成公用的url                URL url = s3Client.generatePresignedUrl(urlRequest);                System.out.println("=========URL=================" + url + "============URL=============");                if (url == null) {                    throw new Exception("can't get s3 file url!");                }                System.out.println("URL=" + url.toString());            } catch (Exception e) {            e.printStackTrace();        }                System.out.println("======================下載 object==========================");    }    public void testDeleteObject(AmazonS3 s3Client, String bucketName,String objectKey) {        System.out.println("======================刪除 object==========================");        s3Client.deleteObject(bucketName, objectKey);        System.out.println("======================刪除 object==========================");    }    public void testCopyObject(AmazonS3 s3Client, String srcBucketName, String srcObjectKey, String dstBucketName, String dstObjectKey) {        System.out.println("======================複製 object==========================");        CopyObjectResult cpres = s3Client.copyObject(srcBucketName, srcObjectKey, dstBucketName, dstObjectKey);        String text = String.format("test copy objcet %s from %s to %s.%s", srcObjectKey, srcBucketName, dstBucketName, dstObjectKey);        System.out.println(text);        System.out.println("object etag : " + cpres.getETag());        System.out.println("======================複製 objects==========================");    }    public void testMultiUpload(AmazonS3 s3Client, String bucketName, String objectKey) {        System.out.println("======================分塊上傳==========================");        System.out.println("初始化分塊上傳");        InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey);        InitiateMultipartUploadResult initMultiUpRes = s3Client.initiateMultipartUpload(initMultiUp);        String uploadId = initMultiUpRes.getUploadId();        System.out.println("分塊上傳 MultiUpload Id : " + uploadId);        System.out.println("上傳塊");        List<PartETag> partETags = new ArrayList<PartETag>();        for (int i = 1; i <= 3; i++) {            UploadPartRequest uploadPartReq = new UploadPartRequest();            uploadPartReq.setBucketName(bucketName);            uploadPartReq.setKey(objectKey);            uploadPartReq.setUploadId(uploadId);            uploadPartReq.setPartNumber(i);            byte[] part = new byte[1024 * 1024 * 6]; // 分塊上傳一個塊必須大於等於5M            if (i == 3) {                uploadPartReq.setLastPart(true);                part = new byte[1024 * 1024 * 3]; // 最後一個塊能夠小於5M            }            SecureRandom random = new SecureRandom();            random.nextBytes(part);            System.out.println("塊大小 = " + part.length);            uploadPartReq.setPartSize(part.length);            InputStream inputStream = new ByteArrayInputStream(part);            uploadPartReq.setInputStream(inputStream);            UploadPartResult uploadPartRes = s3Client.uploadPart(uploadPartReq);            PartETag partETag = uploadPartRes.getPartETag();            partETags.add(partETag);            System.out.println(String.format("上傳塊 %s - %s", partETag.getPartNumber(), partETag.getETag()));        }        CompleteMultipartUploadRequest comMultiReq = new CompleteMultipartUploadRequest(                bucketName, objectKey, uploadId, partETags);        CompleteMultipartUploadResult comMultires = s3Client.completeMultipartUpload(comMultiReq);        System.out.println("完成上傳 etag : " + comMultires.getETag());        System.out.println("測試終止上傳");        String objectKey1 = "test_abort_multi_upload";        InitiateMultipartUploadRequest initMultiUp1 = new InitiateMultipartUploadRequest(                bucketName, objectKey1);        InitiateMultipartUploadResult initMultiUpRes1 = s3Client                .initiateMultipartUpload(initMultiUp1);        String uploadId1 = initMultiUpRes1.getUploadId();        System.out.println("終止上傳 upload id  : " + uploadId1);        AbortMultipartUploadRequest abortMultiUp = new AbortMultipartUploadRequest(bucketName, objectKey1, uploadId1);        s3Client.abortMultipartUpload(abortMultiUp);        System.out.println("======================分塊上傳==========================");    }    public void testListUploadIds(AmazonS3 s3Client, String bucketName) {        System.out.println("======================枚舉分塊上傳uploadId==========================");        String keyPrefix = "/demo/listupids";        for (int i = 1; i < 5; i++) {            String objectKey = keyPrefix + i;            InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey);            s3Client.initiateMultipartUpload(initMultiUp);        }        ListMultipartUploadsRequest listMultiUpReq = new ListMultipartUploadsRequest(bucketName);        MultipartUploadListing multiUploads = s3Client.listMultipartUploads(listMultiUpReq);        List<MultipartUpload> uploads = multiUploads.getMultipartUploads();        for (MultipartUpload up : uploads) {            System.out.println(String.format("uploadId : %s - key : %s", up.getUploadId(), up.getKey()));        }        System.out.println("======================枚舉分塊上傳uploadId==========================");    }    public void testListMultiParts(AmazonS3 s3Client, String bucketName, String objectkey, String uploadId) {        System.out.println("======================枚舉分塊上傳part==========================");        ListPartsRequest listPartsReq = new ListPartsRequest(bucketName, objectkey, uploadId);        PartListing parts = s3Client.listParts(listPartsReq);        List<PartSummary> ps = parts.getParts();        for (PartSummary p : ps) {            System.out.println(String.format("partNum : %s - partEtag : %s", p.getPartNumber(), p.getETag()));        }        System.out.println("======================枚舉分塊上傳part==========================");    }        public void testIsFileExists(AmazonS3 s3Client, String s3Path,String keyName){        System.out.println("======================判斷文件是否已經存在==========================");        try{            boolean flag = false;            S3Object s3Object = s3Client.getObject(s3Path, keyName);            if(s3Object!=null){                flag = true;            }            if(flag){                System.out.println("File exists，filename:"+keyName);            }        }catch(Exception e){            e.printStackTrace();        }    }    // ============================ object ================================    public static void main(String[] args) throws IOException {        String access_key = "s3的access key";        String secret_key = "s3的secret key";        String endpoint = "s3的地址";        Demo d = new Demo();        AmazonS3 s3Client = d.getS3Client(access_key, secret_key, endpoint);        String bucketName = "demo";        String bucketName1 = "demo1";        //建立bucket//        d.testCreateBucket(s3Client, bucketName);        //查詢bucket列表//        d.testListBuckets(s3Client);        //判斷bucket是否存在//        d.testDoesBucketExist(s3Client, bucketName);        //刪除bucket//        d.testDeleteBucket(s3Client, bucketName2);        //上傳文件//        String objectKey = "var/lib/pom.xml";//        String objectKey1 = "var/lib/pom.xml";//        String file_source_address = "E:/demo/test1/pom.xml";//        d.testPutObject(s3Client, bucketName, objectKey,file_source_address);        //下載文件//        d.testGetObject(s3Client, bucketName, objectKey);//        //複製文件//        d.testCopyObject(s3Client, bucketName, objectKey, bucketName1, objectKey1);        //bucket下面文件列表//        d.testListObjects(s3Client, bucketName);        //判斷文件是否存在//        d.testIsFileExists(s3Client, bucketName, objectKey);    }}