Skip to content

Commit

Permalink
Merge pull request #42 from kytrun/dev
Browse files Browse the repository at this point in the history
feat: 谷歌云存储
  • Loading branch information
1171736840 authored Nov 9, 2022
2 parents bfa9745 + 2e7cbb6 commit c24910d
Show file tree
Hide file tree
Showing 10 changed files with 261 additions and 2 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ AWS S3、金山云 KS3、美团云 MSS、京东云 OSS、天翼云 OOS、移动

查看 [所有支持的存储平台](https://spring-file-storage.xuyanwu.cn/#/存储平台)

后续即将支持 谷歌云存储、Samba、NFS
后续即将支持 Samba、NFS

> 通过 WebDAV 连接到 Alist 后,可以使用百度网盘、天翼云盘、阿里云盘、迅雷网盘等常见存储服务,[查看 Alist 支持的存储平台](https://alist-doc.nn.ci/docs/webdav)
Expand Down
2 changes: 1 addition & 1 deletion docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ AWS S3、金山云 KS3、美团云 MSS、京东云 OSS、天翼云 OOS、移动

查看 [所有支持的存储平台](存储平台)

后续即将支持 谷歌云存储、Samba、NFS
后续即将支持 Samba、NFS

> 通过 WebDAV 连接到 Alist 后,可以使用百度网盘、天翼云盘、阿里云盘、迅雷网盘等常见存储服务,[查看 Alist 支持的存储平台](https://alist-doc.nn.ci/docs/webdav)
Expand Down
1 change: 1 addition & 0 deletions docs/存储平台.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
| 平安云 OBS | × || [查看](https://yun.pingan.com/ssr/help/storage/obs/OBS_SDK_.Java_SDK_) |
| 首云 OSS | × || [查看](http://www.capitalonline.net.cn/zh-cn/service/distribution/oss-new/#product-adv) |
| IBM COS | × || [查看](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-compatibility-api) |
| 谷歌云存储 || × | - |
| 其它兼容 S3 协议的平台 | × || - |

如果想通 AWS S3 SDK 使用对应的存储平台,直接将配置写在 ASW S3 中。
Expand Down
22 changes: 22 additions & 0 deletions docs/快速入门.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,20 @@
<version>5.10</version>
</dependency>

<!-- 谷歌云 Google Cloud Storage-->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>2.14.0</version>
</dependency>

<!--因 guava 存在较多冲突版本导致谷歌云存储无法使用,故引入独立版本-->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>

</dependencies>
```

Expand Down Expand Up @@ -223,6 +237,14 @@ spring:
domain: ?? # 访问域名,注意“/”结尾,例如:https://file.abc.com/
base-path: webdav/ # 基础路径
storage-path: / # 存储路径,可以配合 Nginx 实现访问,注意“/”结尾,默认“/”
google-cloud: # 谷歌云存储
- platform: google-1 # 存储平台标识
enable-storage: true # 启用存储
project-id: ?? # 项目 id
bucket-name: ??
credentials-location: /deploy/example-key.json # 授权 key json 文件路径
domain: ?? # 访问域名,注意“/”结尾,例如:https://storage.googleapis.com/test-bucket/
base-path: hy/ # 基础路径
```
注意配置每个平台前面都有个`-`号,通过以下方式可以配置多个
Expand Down
14 changes: 14 additions & 0 deletions spring-file-storage-test/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,20 @@
<version>3.22.3.1</version>
</dependency>

<!-- 谷歌云 Google Cloud Platform Storage-->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>2.14.0</version>
</dependency>

<!--因存在较多冲突版本导致谷歌云无法使用,故引入独立版本-->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>

<!--糊涂工具类核心-->
<dependency>
<groupId>cn.hutool</groupId>
Expand Down
8 changes: 8 additions & 0 deletions spring-file-storage-test/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,3 +129,11 @@ spring:
domain: ?? # 访问域名,注意“/”结尾,例如:https://file.abc.com/
base-path: webdav/ # 基础路径
storage-path: / # 存储路径,上传的文件都会存储在这个路径下面,默认“/”,注意“/”结尾
google-cloud: # 谷歌云存储
- platform: google-1 # 存储平台标识
enable-storage: true # 启用存储
project-id: ?? # 项目 id
bucket-name: ??
credentials-location: /deploy/example-key.json # 授权 key json 文件路径
domain: ?? # 访问域名,注意“/”结尾,例如:https://storage.googleapis.com/test-bucket/
base-path: hy/ # 基础路径
9 changes: 9 additions & 0 deletions spring-file-storage/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,15 @@
<optional>true</optional>
</dependency>

<!-- 谷歌云 Google Cloud Platform Storage-->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>2.14.0</version>
<scope>provided</scope>
<optional>true</optional>
</dependency>

<!--糊涂工具类核心-->
<dependency>
<groupId>cn.hutool</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,23 @@ public List<WebDavFileStorage> webDavFileStorageList() {
return storage;
}).filter(Objects::nonNull).collect(Collectors.toList());
}

@Bean
@ConditionalOnClass(name = "com.google.cloud.storage.Storage")
public List<GoogleCloudStorage> googleCloudStorageList() {
return properties.getGoogleCloud().stream().map(googleCloud -> {
if (!googleCloud.getEnableStorage()) return null;
log.info("加载存储平台:{}", googleCloud.getPlatform());
GoogleCloudStorage storage = new GoogleCloudStorage();
storage.setPlatform(googleCloud.getPlatform());
storage.setProjectId(googleCloud.getProjectId());
storage.setBucketName(googleCloud.getBucketName());
storage.setCredentialsLocation(googleCloud.getCredentialsLocation());
storage.setDomain(googleCloud.getDomain());
storage.setBasePath(googleCloud.getBasePath());
return storage;
}).filter(Objects::nonNull).collect(Collectors.toList());
}

/**
* 当没有找到 FileRecorder 时使用默认的 FileRecorder
Expand Down Expand Up @@ -395,6 +412,9 @@ public void initDetect() {
if (CollUtil.isNotEmpty(properties.getAwsS3()) && doesNotExistClass("com.github.sardine.Sardine")) {
log.warn(template," WebDAV ");
}
if (CollUtil.isNotEmpty(properties.getGoogleCloud()) && doesNotExistClass("com.google.cloud.storage.Storage")) {
log.warn(template, " 谷歌云存储 ");
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ public class FileStorageProperties {
*/
private List<WebDAV> WebDav = new ArrayList<>();

/**
* 谷歌云存储
*/
private List<GoogleCloud> googleCloud = new ArrayList<>();

/**
* 本地存储
*/
Expand Down Expand Up @@ -523,4 +528,27 @@ public static class WebDAV {
*/
private String storagePath = "/";
}

@Data
public static class GoogleCloud {
private String projectId;
private String credentialsLocation;
private String bucketName;
/**
* 访问域名
*/
private String domain = "";
/**
* 启用存储
*/
private Boolean enableStorage = false;
/**
* 存储平台
*/
private String platform = "";
/**
* 基础路径
*/
private String basePath = "";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package cn.xuyanwu.spring.file.storage.platform;

import cn.hutool.core.util.StrUtil;
import cn.xuyanwu.spring.file.storage.FileInfo;
import cn.xuyanwu.spring.file.storage.UploadPretreatment;
import cn.xuyanwu.spring.file.storage.exception.FileStorageRuntimeException;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.ReadChannel;
import com.google.cloud.storage.*;
import lombok.Getter;
import lombok.Setter;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.Channels;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;

/**
* @author Kytrun
* @version 1.0
* @date 2022/11/4 9:56
*/
@Getter
@Setter
public class GoogleCloudStorage implements FileStorage {
private String projectId;
private String bucketName;
private String credentialsLocation;
/* 基础路径 */
private String basePath;
/* 存储平台 */
private String platform;
/* 访问域名 */
private String domain;

private Storage client;

/**
* 单例模式运行,不需要每次使用完再销毁了
*/
public Storage getClient() {
if (client == null) {
ServiceAccountCredentials credentialsFromStream;
try {
InputStream stream = Files.newInputStream(Paths.get(credentialsLocation));
credentialsFromStream = ServiceAccountCredentials.fromStream(stream);
} catch (IOException e) {
throw new FileStorageRuntimeException("Google Cloud Platform 授权 key 文件获取失败!credentialsLocation:" + credentialsLocation);
}
List<String> scopes = Collections.singletonList("https://www.googleapis.com/auth/cloud-platform");
ServiceAccountCredentials credentials = credentialsFromStream.toBuilder().setScopes(scopes).build();
StorageOptions storageOptions = StorageOptions.newBuilder().setProjectId(projectId).setCredentials(credentials).build();
client = storageOptions.getService();
}
return client;
}


@Override
public void close() {
if (client != null) {
try {
client.close();
} catch (Exception e) {
throw new FileStorageRuntimeException(e);
}
client = null;
}
}

@Override
public boolean save(FileInfo fileInfo, UploadPretreatment pre) {
String newFileKey = basePath + fileInfo.getPath() + fileInfo.getFilename();
fileInfo.setBasePath(basePath);
fileInfo.setUrl(domain + newFileKey);
Storage client = getClient();

BlobId blobId = BlobId.of(bucketName, newFileKey);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(fileInfo.getContentType()).build();
try (InputStream in = pre.getFileWrapper().getInputStream()) {
// 上传原文件
client.createFrom(blobInfo, in);
//上传缩略图
byte[] thumbnailBytes = pre.getThumbnailBytes();
if (thumbnailBytes != null) {
String newThFileKey = basePath + fileInfo.getPath() + fileInfo.getThFilename();
fileInfo.setThUrl(domain + newThFileKey);
BlobId thBlobId = BlobId.of(bucketName, newThFileKey);
BlobInfo thBlobInfo = BlobInfo.newBuilder(thBlobId).setContentType(fileInfo.getThContentType()).build();
client.createFrom(thBlobInfo, new ByteArrayInputStream(thumbnailBytes));
}
return true;
} catch (IOException e) {
checkAndDelete(newFileKey);
throw new FileStorageRuntimeException("文件上传失败!platform:" + platform + ",filename:" + fileInfo.getOriginalFilename(), e);
}
}


/**
* 检查并删除对象
* <a href="https://github.com/googleapis/java-storage/blob/main/samples/snippets/src/main/java/com/example/storage/object/DeleteObject.java">Source Example</a>
*
* @param fileKey 对象 key
*/
private void checkAndDelete(String fileKey) {
Storage client = getClient();
Blob blob = client.get(bucketName, fileKey);
if (blob != null) {
Storage.BlobSourceOption precondition = Storage.BlobSourceOption.generationMatch(blob.getGeneration());
client.delete(bucketName, fileKey, precondition);
}
}

@Override
public boolean delete(FileInfo fileInfo) {
//删除缩略图
if (fileInfo.getThFilename() != null) {
checkAndDelete(fileInfo.getBasePath() + fileInfo.getPath() + fileInfo.getThFilename());
}
checkAndDelete(fileInfo.getBasePath() + fileInfo.getPath() + fileInfo.getFilename());
return true;
}

@Override
public boolean exists(FileInfo fileInfo) {
Storage client = getClient();
BlobId blobId = BlobId.of(bucketName, fileInfo.getBasePath() + fileInfo.getPath() + fileInfo.getFilename());
return client.get(blobId) != null;
}

@Override
public void download(FileInfo fileInfo, Consumer<InputStream> consumer) {
Storage client = getClient();
BlobId blobId = BlobId.of(bucketName, fileInfo.getBasePath() + fileInfo.getPath() + fileInfo.getFilename());
ReadChannel readChannel = client.reader(blobId);
InputStream in = Channels.newInputStream(readChannel);
consumer.accept(in);
}

@Override
public void downloadTh(FileInfo fileInfo, Consumer<InputStream> consumer) {
if (StrUtil.isBlank(fileInfo.getThFilename())) {
throw new FileStorageRuntimeException("缩略图文件下载失败,文件不存在!fileInfo:" + fileInfo);
}
Storage client = getClient();
BlobId thBlobId = BlobId.of(bucketName, fileInfo.getBasePath() + fileInfo.getPath() + fileInfo.getThFilename());
ReadChannel readChannel = client.reader(thBlobId);
InputStream in = Channels.newInputStream(readChannel);
consumer.accept(in);
}
}

0 comments on commit c24910d

Please sign in to comment.