Refactor storage system (#319)

- Abstract interface StorageServiceClient for generalized storage implementation use; - Refactor all usage of BlobStorageClient.java to StorageServiceClient.java; - Add StorageManageService for StorageServiceClient init control; - Enable fastjson auto type support for all entity classes under common/entity/common package, move classes that would be used in Websocket communication to common/entity/common; - Modify CDNUrl usage in StorageFileInfo, from CDN endpoint to absolute CDN URL; - Gradle plugin compatibility; - Tested in plugin and frontend; - Fix UT for modified classes; - Refactor entity class BlobFileInfo to StorageFileInfo;
microsoft · Mar 2, 2023 · b49072b · b49072b
1 parent faf2459
commit b49072b
Show file tree

Hide file tree

Showing 49 changed files with 884 additions and 518 deletions.
diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/config/AppConfiguration.java b/agent/src/main/java/com/microsoft/hydralab/agent/config/AppConfiguration.java
@@ -19,7 +19,7 @@
 import com.microsoft.hydralab.common.monitor.MetricPushGateway;
 import com.microsoft.hydralab.common.util.ADBOperateUtil;
 import com.microsoft.hydralab.common.util.Const;
-import com.microsoft.hydralab.common.util.blob.BlobStorageClient;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.prometheus.client.CollectorRegistry;
 import io.prometheus.client.exporter.PushGateway;
@@ -33,6 +33,7 @@
 import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusPushGatewayManager;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -97,10 +98,8 @@ public AgentWebSocketClient agentWebSocketClient(AgentWebSocketClientService age
     }
 
     @Bean
-    public DeviceManager initDeviceManager(BlobStorageClient deviceLabBlobClient, ADBOperateUtil adbOperateUtil,
-                                           AppiumServerManager appiumServerManager,
+    public DeviceManager initDeviceManager(StorageServiceClientProxy storageServiceClientProxy, ADBOperateUtil adbOperateUtil, AppiumServerManager appiumServerManager,
                                            DeviceStatusListenerManager deviceStatusListenerManager) {
-
         AgentType agentType = AgentType.formAgentType(agentTypeValue);
         DeviceManager deviceManager = agentType.getManager();
         if (deviceManager instanceof AndroidDeviceManager) {
@@ -135,7 +134,7 @@ public DeviceManager initDeviceManager(BlobStorageClient deviceLabBlobClient, AD
             }
         }
         deviceManager.setDeviceLogBaseDir(deviceLogBaseDir);
-        deviceManager.setBlobStorageClient(deviceLabBlobClient);
+        deviceManager.setStorageServiceClientProxy(storageServiceClientProxy);
 
         deviceManager.setScreenshotDir(getScreenshotDir());
         deviceManager.setDeviceFolderUrlPrefix(AppOptions.DEVICE_STORAGE_MAPPING_REL_PATH);
@@ -172,11 +171,6 @@ public FastJsonHttpMessageConverter fastJsonHttpMessageConverter() {
         return fastConverter;
     }
 
-    @Bean
-    public BlobStorageClient blobStorageClient() {
-        return new BlobStorageClient();
-    }
-
     @Bean
     public SmartTestUtil smartTestUtil() {
         return new SmartTestUtil(appOptions.getLocation());
@@ -225,4 +219,9 @@ public PrometheusPushGatewayManager monitorPrometheusPushGatewayManager(PushGate
         return new PrometheusPushGatewayManager(pushGateway, registry,
                 pushRate, job, groupingKey, shutdownOperation);
     }
+
+    @Bean
+    public StorageServiceClientProxy storageServiceClientProxy(ApplicationContext applicationContext) {
+        return new StorageServiceClientProxy(applicationContext);
+    }
 }
diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/service/AgentManageService.java b/agent/src/main/java/com/microsoft/hydralab/agent/service/AgentManageService.java
@@ -10,8 +10,8 @@
 import com.microsoft.hydralab.common.management.impl.IOSDeviceManager;
 import com.microsoft.hydralab.common.util.CommandOutputReceiver;
 import com.microsoft.hydralab.common.util.Const;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import com.microsoft.hydralab.common.util.ThreadPoolUtil;
-import com.microsoft.hydralab.common.util.blob.BlobStorageClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
@@ -32,14 +32,13 @@ public class AgentManageService {
     @Resource
     private AppOptions appOptions;
     @Resource
-    BlobStorageClient blobStorageClient;
+    StorageServiceClientProxy storageServiceClientProxy;
 
     public void updateAgentPackage(AgentUpdateTask updateTask, String path) {
         Runnable run = () -> {
             sendMessageToCenter(true, "Download package file.", "", path);
             File downloadToFile = new File(appOptions.getLocation(), updateTask.getPackageInfo().getFileName());
-            blobStorageClient.downloadFileFromBlob(downloadToFile, updateTask.getPackageInfo().getBlobContainer(),
-                    updateTask.getPackageInfo().getBlobPath());
+            storageServiceClientProxy.download(downloadToFile, updateTask.getPackageInfo());
             sendMessageToCenter(true, "Download Package Success!", "", path);
 
             restartAgent(updateTask.getPackageInfo().getFileName(), path);

diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/service/AgentWebSocketClientService.java b/agent/src/main/java/com/microsoft/hydralab/agent/service/AgentWebSocketClientService.java
@@ -18,8 +18,8 @@
 import com.microsoft.hydralab.common.monitor.MetricPushGateway;
 import com.microsoft.hydralab.common.util.Const;
 import com.microsoft.hydralab.common.util.GlobalConstant;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import com.microsoft.hydralab.common.util.ThreadUtils;
-import com.microsoft.hydralab.common.util.blob.BlobStorageClient;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Tags;
 import io.prometheus.client.exporter.BasicAuthHttpConnectionFactory;
@@ -54,14 +54,15 @@ public class AgentWebSocketClientService implements TestTaskRunCallback {
     @Resource
     AgentManageService agentManageService;
     @Resource
-    BlobStorageClient blobStorageClient;
+    MeterRegistry meterRegistry;
+    AgentUser agentUser;
+    @Resource
+    private StorageServiceClientProxy storageServiceClientProxy;
+    private boolean isStorageClientInit = false;
     @Resource
     private AppOptions appOptions;
     @Resource
     private AgentWebSocketClient agentWebSocketClient;
-    @Resource
-    MeterRegistry meterRegistry;
-    AgentUser agentUser;
     @Value("${agent.version}")
     private String versionName;
     @Value("${agent.versionCode}")
@@ -169,16 +170,14 @@ public void onMessage(Message message) {
 
     private void heartbeatResponse(Message message) {
         AgentMetadata agentMetadata = (AgentMetadata) message.getBody();
-        blobStorageClient.setSASData(agentMetadata.getBlobSAS());
-        syncAgentStatus(agentMetadata.getAgentUser());
-        if (isPrometheusEnabled && !pushGateway.isBasicAuthSet.get()) {
-            pushGateway.setConnectionFactory(
-                    new BasicAuthHttpConnectionFactory(agentMetadata.getPushgatewayUsername(),
-                            agentMetadata.getPushgatewayPassword()));
-            ThreadUtils.safeSleep(1000);
-            pushGateway.isBasicAuthSet.set(true);
-            log.info("Pushgateway has set basic auth now, data can be pushed correctly.");
+
+        if (!isStorageClientInit) {
+            storageServiceClientProxy.initAgentStorageClient(agentMetadata.getStorageType());
+            isStorageClientInit = true;
         }
+        storageServiceClientProxy.updateAccessToken(agentMetadata.getAccessToken());
+        syncAgentStatus(agentMetadata.getAgentUser());
+        prometheusPushgatewayInit(agentMetadata);
     }
 
     private void syncAgentStatus(AgentUser passedAgent) {
@@ -187,6 +186,15 @@ private void syncAgentStatus(AgentUser passedAgent) {
         agentUser.setBatteryStrategy(passedAgent.getBatteryStrategy());
     }
 
+    private void prometheusPushgatewayInit(AgentMetadata agentMetadata) {
+        if (isPrometheusEnabled && !pushGateway.isBasicAuthSet.get()) {
+            pushGateway.setConnectionFactory(new BasicAuthHttpConnectionFactory(agentMetadata.getPushgatewayUsername(), agentMetadata.getPushgatewayPassword()));
+            ThreadUtils.safeSleep(1000);
+            pushGateway.isBasicAuthSet.set(true);
+            log.info("Pushgateway has set basic auth now, data can be pushed correctly.");
+        }
+    }
+
     private void provideAuthInfo(Message message) {
         Message responseAuth = new Message();
         responseAuth.setSessionId(message.getSessionId());

diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/service/TestTaskEngineService.java b/agent/src/main/java/com/microsoft/hydralab/agent/service/TestTaskEngineService.java
@@ -226,13 +226,12 @@ private StorageFileInfo saveFileToBlob(File file, File folder, Logger logger) {
         StorageFileInfo storageFileInfo = new StorageFileInfo(file,
                 "test/result/" + folder.getParentFile().getName() + "/" + folder.getName(),
                 StorageFileInfo.FileType.COMMON_FILE);
-        return attachmentService.addFileInfo(storageFileInfo, file, EntityType.TEST_RESULT,
-                logger);
+        return attachmentService.saveFileInStorageAndDB(storageFileInfo, file, EntityType.TEST_RESULT, logger);
     }
 
     private void processAndSaveDeviceTestResultBlobUrl(TestRun result) {
         Assert.isTrue(result.getAttachments().size() > 0, "deviceTestResultBlobUrl should not null");
-        String deviceTestResultBlobUrl = result.getAttachments().get(0).getBlobUrl();
+        String deviceTestResultBlobUrl = result.getAttachments().get(0).getCDNUrl();
         String fileName = result.getAttachments().get(0).getFileName();
         log.info("deviceTestResultBlobUrl is {}", deviceTestResultBlobUrl);
 

diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/util/FileLoadUtil.java b/agent/src/main/java/com/microsoft/hydralab/agent/util/FileLoadUtil.java
@@ -8,7 +8,7 @@
 import com.microsoft.hydralab.common.entity.common.TestTask;
 import com.microsoft.hydralab.common.util.CommandOutputReceiver;
 import com.microsoft.hydralab.common.util.FileUtil;
-import com.microsoft.hydralab.common.util.blob.BlobStorageClient;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
@@ -28,7 +28,7 @@ public class FileLoadUtil {
     @Resource
     private AppOptions appOptions;
     @Resource
-    BlobStorageClient blobStorageClient;
+    StorageServiceClientProxy storageServiceClientProxy;
 
     public void clearAttachments(TestTask testTask) {
         List<StorageFileInfo> attachments = testTask.getTestFileSet().getAttachments();
@@ -58,17 +58,17 @@ public void loadAttachments(TestTask testTask) {
                     loadCommonFile(attachment);
                     break;
                 case StorageFileInfo.FileType.APP_FILE:
-                    File appFile = downloadFromBlob(attachment);
+                    File appFile = downloadFile(attachment);
                     Assert.isTrue(appFile != null && appFile.exists(), "Download app file failed!");
                     testTask.setAppFile(appFile);
                     break;
                 case StorageFileInfo.FileType.TEST_APP_FILE:
-                    File testAppFile = downloadFromBlob(attachment);
+                    File testAppFile = downloadFile(attachment);
                     Assert.isTrue(testAppFile != null && testAppFile.exists(), "Download test app file failed!");
                     testTask.setTestAppFile(testAppFile);
                     break;
                 case StorageFileInfo.FileType.T2C_JSON_FILE:
-                    File testJsonFile = downloadFromBlob(attachment);
+                    File testJsonFile = downloadFile(attachment);
                     Assert.isTrue(testJsonFile != null && testJsonFile.exists(), "Download test json file failed!");
                     testTask.addTestJsonFile(testJsonFile);
                     break;
@@ -81,7 +81,7 @@ public void loadAttachments(TestTask testTask) {
     public void installWinApp(StorageFileInfo attachment) {
         try {
             Runtime runtime = Runtime.getRuntime();
-            File attachmentFile = downloadFromBlob(attachment, appOptions.getTestPackageLocation(), attachment.getBlobPath());
+            File attachmentFile = downloadFile(attachment, appOptions.getTestPackageLocation(), attachment.getBlobPath());
             String installCommand = "& { Add-AppxPackage -ForceApplicationShutdown -forceupdatefromanyversion -Path '" +
                     attachmentFile.getAbsolutePath() + "' }";
             String[] command = new String[]{"Powershell.exe", "-Command", installCommand};
@@ -105,7 +105,7 @@ public void loadCommonFile(StorageFileInfo attachment) {
             File loadFolder = new File(appOptions.getLocation() + "/" + attachment.getLoadDir());
             Assert.isTrue(!loadFolder.exists(), "Load file error : folder has been existed!");
             log.info("Load common file start filename:{} path:{}", attachment.getFileName(), loadFolder.getAbsolutePath());
-            File attachmentFile = downloadFromBlob(attachment, appOptions.getLocation(), attachment.getLoadDir() + "/" + attachment.getFileName());
+            File attachmentFile = downloadFile(attachment, appOptions.getLocation(), attachment.getLoadDir() + "/" + attachment.getFileName());
             if (StorageFileInfo.LoadType.UNZIP.equalsIgnoreCase(attachment.getLoadType())) {
                 FileUtil.unzipFile(attachmentFile.getAbsolutePath(), loadFolder.getAbsolutePath());
             }
@@ -115,17 +115,17 @@ public void loadCommonFile(StorageFileInfo attachment) {
         }
     }
 
-    private File downloadFromBlob(StorageFileInfo attachment, String location, String targetFilePath) throws IOException {
+    private File downloadFile(StorageFileInfo attachment, String location, String targetFilePath) throws IOException {
         File file = new File(location, targetFilePath);
         log.debug("download file from {} to {}", attachment.getBlobUrl(), file.getAbsolutePath());
-        blobStorageClient.downloadFileFromBlob(file, attachment.getBlobContainer(), attachment.getBlobPath());
+        storageServiceClientProxy.download(file, attachment);
         return file;
     }
 
-    private File downloadFromBlob(StorageFileInfo attachment) {
+    private File downloadFile(StorageFileInfo attachment) {
         File file = null;
         try {
-            file = downloadFromBlob(attachment, appOptions.getTestPackageLocation(), attachment.getBlobPath());
+            file = downloadFile(attachment, appOptions.getTestPackageLocation(), attachment.getBlobPath());
             log.info("Download file success");
         } catch (IOException e) {
             log.error("Download file failed", e);

diff --git a/agent/src/test/java/com/microsoft/hydralab/agent/test/BaseTest.java b/agent/src/test/java/com/microsoft/hydralab/agent/test/BaseTest.java
@@ -5,7 +5,7 @@
 
 import com.microsoft.hydralab.agent.service.AgentWebSocketClientService;
 import com.microsoft.hydralab.agent.socket.AgentWebSocketClient;
-import com.microsoft.hydralab.common.util.blob.BlobStorageClient;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.slf4j.Logger;
@@ -30,7 +30,7 @@
 public class BaseTest {
     protected Logger baseLogger = LoggerFactory.getLogger(BaseTest.class);
     @MockBean
-    BlobStorageClient blobStorageClient;
+    StorageServiceClientProxy storageServiceClientProxy;
     @MockBean
     AgentWebSocketClient agentWebSocketClient;
     @MockBean

diff --git a/build.gradle b/build.gradle
@@ -101,7 +101,7 @@ import com.microsoft.hydralab.compile.UMLImageGenerator
 
 task generateUMLImage(group: 'documentation') {
     doFirst {
-        def scanningDirList = ['agent/doc/UML', 'gradle_plugin/doc/UML']
+        def scanningDirList = ['agent/doc/UML', 'common/doc/UML', 'gradle_plugin/doc/UML']
         def outputDir = new File(projectDir, 'docs/images/UML')
 
         def generator = new UMLImageGenerator()

diff --git a/center/application-sample.yml b/center/application-sample.yml
@@ -22,6 +22,8 @@ spring:
 app:
   # Mail Address Format
   default-user: ${DEFAULT_USER}
-  blob:
-    connection: ${BLOB_CONNECTION_STR}
-    CDNUrl: ${CDN_URL}
+  storage:
+    type: ${STORAGE_TYPE:AZURE} # current available options: LOCAL, AZURE
+    blob:
+      connection: ${BLOB_CONNECTION_STR}
+      CDNUrl: ${CDN_URL}
diff --git a/center/src/main/java/com/microsoft/hydralab/center/config/AppConfig.java b/center/src/main/java/com/microsoft/hydralab/center/config/AppConfig.java
@@ -7,9 +7,9 @@
 import com.alibaba.fastjson.serializer.SerializerFeature;
 import com.alibaba.fastjson.support.config.FastJsonConfig;
 import com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter;
-import com.microsoft.hydralab.common.entity.center.BlobProperty;
 import com.microsoft.hydralab.common.monitor.MetricPushGateway;
-import com.microsoft.hydralab.common.util.blob.BlobStorageClient;
+import com.microsoft.hydralab.common.util.Const;
+import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import io.prometheus.client.CollectorRegistry;
 import io.prometheus.client.exporter.BasicAuthHttpConnectionFactory;
 import io.prometheus.client.exporter.PushGateway;
@@ -18,6 +18,7 @@
 import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusPushGatewayManager;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
@@ -37,6 +38,8 @@ public class AppConfig {
     private String pushgatewayUsername;
     @Value("${management.metrics.export.prometheus.pushgateway.password}")
     private String pushgatewayPassword;
+    @Value("${app.storage.type}")
+    private String storageType;
 
     @Bean
     @ConditionalOnClass({JSON.class})
@@ -55,8 +58,11 @@ public FastJsonHttpMessageConverter fastJsonHttpMessageConverter() {
     }
 
     @Bean
-    public BlobStorageClient blobStorageClient(BlobProperty blobProperty) {
-        return new BlobStorageClient(blobProperty);
+    public StorageServiceClientProxy storageServiceClientProxy(ApplicationContext applicationContext) {
+        StorageServiceClientProxy storageServiceClientProxy = new StorageServiceClientProxy(applicationContext);
+        storageServiceClientProxy.initCenterStorageClient(storageType);
+
+        return storageServiceClientProxy;
     }
 
     @Bean