feat: in tree plugin (dragonflyoss#1276)

* feat: in tree plugin

Signed-off-by: Jim Ma <[email protected]>

cdn/supervisor/cdn/cache_detector_test.go

@@ -35,7 +35,7 @@ import (
 	storageMock "d7y.io/dragonfly/v2/cdn/supervisor/cdn/storage/mock"
 	"d7y.io/dragonfly/v2/cdn/supervisor/task"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 	sourceMock "d7y.io/dragonfly/v2/pkg/source/mock"
 	"d7y.io/dragonfly/v2/pkg/util/digestutils"
 	"d7y.io/dragonfly/v2/pkg/util/rangeutils"

cdn/supervisor/cdn/manager_test.go

@@ -36,7 +36,7 @@ import (
 	"d7y.io/dragonfly/v2/pkg/idgen"
 	"d7y.io/dragonfly/v2/pkg/rpc/base"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 	sourceMock "d7y.io/dragonfly/v2/pkg/source/mock"
 	"d7y.io/dragonfly/v2/pkg/util/net/urlutils"
 	"d7y.io/dragonfly/v2/pkg/util/rangeutils"

cdn/supervisor/progress/manager_test.go

@@ -26,7 +26,7 @@ import (
 
 	"d7y.io/dragonfly/v2/cdn/supervisor/task"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 	sourcemock "d7y.io/dragonfly/v2/pkg/source/mock"
 	"d7y.io/dragonfly/v2/pkg/util/rangeutils"
 )

cdn/supervisor/task/manager_test.go

@@ -29,7 +29,7 @@ import (
 	"d7y.io/dragonfly/v2/internal/util"
 	"d7y.io/dragonfly/v2/pkg/rpc/base"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 	sourcemock "d7y.io/dragonfly/v2/pkg/source/mock"
 )
 

client/daemon/peer/peertask_manager_test.go

@@ -59,7 +59,7 @@ import (
 	schedulerclient "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client"
 	mock_scheduler "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client/mocks"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 	sourceMock "d7y.io/dragonfly/v2/pkg/source/mock"
 	"d7y.io/dragonfly/v2/pkg/util/digestutils"
 )

client/daemon/peer/peertask_stream_backsource_partial_test.go

@@ -53,7 +53,7 @@ import (
 	schedulerclient "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client"
 	mock_scheduler "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client/mocks"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 	sourceMock "d7y.io/dragonfly/v2/pkg/source/mock"
 	"d7y.io/dragonfly/v2/pkg/util/digestutils"
 )

client/daemon/peer/piece_downloader_test.go

@@ -40,7 +40,7 @@ import (
 	logger "d7y.io/dragonfly/v2/internal/dflog"
 	"d7y.io/dragonfly/v2/pkg/rpc/base"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 )
 
 func TestPieceDownloader_DownloadPiece(t *testing.T) {

client/daemon/peer/piece_manager_test.go

@@ -43,7 +43,7 @@ import (
 	_ "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/server"
 	"d7y.io/dragonfly/v2/pkg/rpc/scheduler"
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/source/httpprotocol"
+	"d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol"
 )
 
 func TestPieceManager_DownloadSource(t *testing.T) {

cmd/cdn/main.go

@@ -19,8 +19,7 @@ package main
 import (
 	_ "d7y.io/dragonfly/v2/cdn/supervisor/cdn/storage/disk"   //nolint:gci    // Register disk storage manager
 	_ "d7y.io/dragonfly/v2/cdn/supervisor/cdn/storage/hybrid" // Register hybrid storage manager
-	_ "d7y.io/dragonfly/v2/pkg/source/httpprotocol"           // Register http client
-	_ "d7y.io/dragonfly/v2/pkg/source/ossprotocol"            // Register oss client
+	_ "d7y.io/dragonfly/v2/pkg/source/loader"                 // Register all source clients
 
 	"d7y.io/dragonfly/v2/cmd/cdn/cmd" //nolint:gci
 )

cmd/dependency/plugin_cmd.go

@@ -25,6 +25,7 @@ import (
 
 	"d7y.io/dragonfly/v2/internal/dfplugin"
 	"d7y.io/dragonfly/v2/pkg/dfpath"
+	"d7y.io/dragonfly/v2/pkg/source"
 )
 
 var PluginCmd = &cobra.Command{
@@ -35,11 +36,19 @@ var PluginCmd = &cobra.Command{
 	DisableAutoGenTag: true,
 	SilenceUsage:      true,
 	Run: func(cmd *cobra.Command, args []string) {
-		ListAvailablePlugins()
+		ListAvailableInTreePlugins()
+		ListAvailableOutOfTreePlugins()
 	},
 }
 
-func ListAvailablePlugins() {
+func ListAvailableInTreePlugins() {
+	clients := source.ListClients()
+	for _, scheme := range clients {
+		fmt.Printf("source plugin: %s, location: in-tree\n", scheme)
+	}
+}
+
+func ListAvailableOutOfTreePlugins() {
 	d, err := dfpath.New()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to get plugin path: %q\n", err)
@@ -56,6 +65,10 @@ func ListAvailablePlugins() {
 		fmt.Fprintf(os.Stderr, "read plugin dir %s error: %s\n", d.PluginDir(), err)
 		return
 	}
+	if len(files) == 0 {
+		fmt.Fprintf(os.Stderr, "no out of tree plugin found\n")
+		return
+	}
 	for _, file := range files {
 		var attr []byte
 		fileName := file.Name()
@@ -86,6 +99,6 @@ func ListAvailablePlugins() {
 			continue
 		}
 
-		fmt.Printf("%s plugin %s, location: %s, attribute: %s\n", typ, name, fileName, string(attr))
+		fmt.Printf("%s plugin: %s, location: %s, attribute: %s\n", typ, name, fileName, string(attr))
 	}
 }

cmd/dfget/main.go

@@ -18,10 +18,8 @@ package main
 
 import (
 	"d7y.io/dragonfly/v2/cmd/dfget/cmd"
-	// Register http client
-	_ "d7y.io/dragonfly/v2/pkg/source/httpprotocol"
-	// Register oss client
-	_ "d7y.io/dragonfly/v2/pkg/source/ossprotocol"
+	// register all source clients
+	_ "d7y.io/dragonfly/v2/pkg/source/loader"
 )
 
 func main() {

pkg/source/clients/example/dfs.go

@@ -0,0 +1,61 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package example
+
+import (
+	"bytes"
+	"io"
+
+	"d7y.io/dragonfly/v2/pkg/source"
+)
+
+const scheme = "dfs"
+
+var data = "hello world"
+
+type client struct {
+}
+
+func init() {
+	if err := source.Register(scheme, New(), nil); err != nil {
+		panic(err)
+	}
+}
+
+func New() source.ResourceClient {
+	return &client{}
+}
+
+func (c *client) GetContentLength(request *source.Request) (int64, error) {
+	return int64(len(data)), nil
+}
+
+func (c *client) IsSupportRange(request *source.Request) (bool, error) {
+	return false, nil
+}
+
+func (c *client) IsExpired(request *source.Request, info *source.ExpireInfo) (bool, error) {
+	panic("implement me")
+}
+
+func (c *client) Download(request *source.Request) (*source.Response, error) {
+	return source.NewResponse(io.NopCloser(bytes.NewBufferString(data))), nil
+}
+
+func (c *client) GetLastModified(request *source.Request) (int64, error) {
+	panic("implement me")
+}

pkg/source/loader/dfs.go.example

@@ -0,0 +1,21 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package loader
+
+import (
+	_ "d7y.io/dragonfly/v2/pkg/source/clients/example" // Register dfs client
+)

pkg/source/loader/http.go

@@ -0,0 +1,21 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package loader
+
+import (
+	_ "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" // Register http client
+)

pkg/source/loader/oss.go

@@ -0,0 +1,21 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package loader
+
+import (
+	_ "d7y.io/dragonfly/v2/pkg/source/clients/ossprotocol" // Register oss client
+)

pkg/source/source_client.go

@@ -120,14 +120,17 @@ type ResourceLister interface {
 }
 
 type ClientManager interface {
-	// Register a source client with scheme
+	// Register registers a source client with scheme
 	Register(scheme string, resourceClient ResourceClient, adapter requestAdapter, hook ...Hook) error
 
-	// UnRegister a source client from manager
+	// UnRegister revoke a source client from manager
 	UnRegister(scheme string)
 
-	// GetClient a source client by scheme
+	// GetClient gets a source client by scheme
 	GetClient(scheme string, options ...Option) (ResourceClient, bool)
+
+	// ListClients lists all supported client scheme
+	ListClients() []string
 }
 
 // clientManager implements the interface ClientManager
@@ -186,6 +189,16 @@ func (m *clientManager) UnRegister(scheme string) {
 	delete(m.clients, scheme)
 }
 
+func (m *clientManager) ListClients() []string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var clients []string
+	for c := range m.clients {
+		clients = append(clients, c)
+	}
+	return clients
+}
+
 func (m *clientManager) GetClient(scheme string, options ...Option) (ResourceClient, bool) {
 	logger.Debugf("current clients: %#v", m.clients)
 	m.mu.RLock()
@@ -226,6 +239,10 @@ func UnRegister(scheme string) {
 	_defaultManager.UnRegister(scheme)
 }
 
+func ListClients() []string {
+	return _defaultManager.ListClients()
+}
+
 type requestAdapter func(request *Request) *Request
 
 // Hook TODO hook