diff --git a/cdn/supervisor/cdn/cache_detector_test.go b/cdn/supervisor/cdn/cache_detector_test.go index 12bfc628a..58b671668 100644 --- a/cdn/supervisor/cdn/cache_detector_test.go +++ b/cdn/supervisor/cdn/cache_detector_test.go @@ -35,7 +35,7 @@ import ( storageMock "d7y.io/dragonfly/v2/cdn/supervisor/cdn/storage/mock" "d7y.io/dragonfly/v2/cdn/supervisor/task" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" sourceMock "d7y.io/dragonfly/v2/pkg/source/mock" "d7y.io/dragonfly/v2/pkg/util/digestutils" "d7y.io/dragonfly/v2/pkg/util/rangeutils" diff --git a/cdn/supervisor/cdn/manager_test.go b/cdn/supervisor/cdn/manager_test.go index 6384bb8db..367c65d40 100644 --- a/cdn/supervisor/cdn/manager_test.go +++ b/cdn/supervisor/cdn/manager_test.go @@ -36,7 +36,7 @@ import ( "d7y.io/dragonfly/v2/pkg/idgen" "d7y.io/dragonfly/v2/pkg/rpc/base" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" sourceMock "d7y.io/dragonfly/v2/pkg/source/mock" "d7y.io/dragonfly/v2/pkg/util/net/urlutils" "d7y.io/dragonfly/v2/pkg/util/rangeutils" diff --git a/cdn/supervisor/progress/manager_test.go b/cdn/supervisor/progress/manager_test.go index 77a0df646..fa6ba4172 100644 --- a/cdn/supervisor/progress/manager_test.go +++ b/cdn/supervisor/progress/manager_test.go @@ -26,7 +26,7 @@ import ( "d7y.io/dragonfly/v2/cdn/supervisor/task" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" sourcemock "d7y.io/dragonfly/v2/pkg/source/mock" "d7y.io/dragonfly/v2/pkg/util/rangeutils" ) diff --git a/cdn/supervisor/task/manager_test.go b/cdn/supervisor/task/manager_test.go index 572a2d694..ffdf3c345 100644 --- a/cdn/supervisor/task/manager_test.go +++ b/cdn/supervisor/task/manager_test.go @@ -29,7 +29,7 @@ import ( "d7y.io/dragonfly/v2/internal/util" "d7y.io/dragonfly/v2/pkg/rpc/base" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" sourcemock "d7y.io/dragonfly/v2/pkg/source/mock" ) diff --git a/client/daemon/peer/peertask_manager_test.go b/client/daemon/peer/peertask_manager_test.go index a8b86c993..142f41627 100644 --- a/client/daemon/peer/peertask_manager_test.go +++ b/client/daemon/peer/peertask_manager_test.go @@ -59,7 +59,7 @@ import ( schedulerclient "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client" mock_scheduler "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client/mocks" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" sourceMock "d7y.io/dragonfly/v2/pkg/source/mock" "d7y.io/dragonfly/v2/pkg/util/digestutils" ) diff --git a/client/daemon/peer/peertask_stream_backsource_partial_test.go b/client/daemon/peer/peertask_stream_backsource_partial_test.go index 7242dd49c..a81159fdb 100644 --- a/client/daemon/peer/peertask_stream_backsource_partial_test.go +++ b/client/daemon/peer/peertask_stream_backsource_partial_test.go @@ -53,7 +53,7 @@ import ( schedulerclient "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client" mock_scheduler "d7y.io/dragonfly/v2/pkg/rpc/scheduler/client/mocks" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" sourceMock "d7y.io/dragonfly/v2/pkg/source/mock" "d7y.io/dragonfly/v2/pkg/util/digestutils" ) diff --git a/client/daemon/peer/piece_downloader_test.go b/client/daemon/peer/piece_downloader_test.go index 4a71c80f0..63917dadf 100644 --- a/client/daemon/peer/piece_downloader_test.go +++ b/client/daemon/peer/piece_downloader_test.go @@ -40,7 +40,7 @@ import ( logger "d7y.io/dragonfly/v2/internal/dflog" "d7y.io/dragonfly/v2/pkg/rpc/base" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" ) func TestPieceDownloader_DownloadPiece(t *testing.T) { diff --git a/client/daemon/peer/piece_manager_test.go b/client/daemon/peer/piece_manager_test.go index 391136ae6..b8d9f0f78 100644 --- a/client/daemon/peer/piece_manager_test.go +++ b/client/daemon/peer/piece_manager_test.go @@ -43,7 +43,7 @@ import ( _ "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/server" "d7y.io/dragonfly/v2/pkg/rpc/scheduler" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/source/httpprotocol" + "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" ) func TestPieceManager_DownloadSource(t *testing.T) { diff --git a/cmd/cdn/main.go b/cmd/cdn/main.go index b9086ccd2..deee0feed 100644 --- a/cmd/cdn/main.go +++ b/cmd/cdn/main.go @@ -19,8 +19,7 @@ package main import ( _ "d7y.io/dragonfly/v2/cdn/supervisor/cdn/storage/disk" //nolint:gci // Register disk storage manager _ "d7y.io/dragonfly/v2/cdn/supervisor/cdn/storage/hybrid" // Register hybrid storage manager - _ "d7y.io/dragonfly/v2/pkg/source/httpprotocol" // Register http client - _ "d7y.io/dragonfly/v2/pkg/source/ossprotocol" // Register oss client + _ "d7y.io/dragonfly/v2/pkg/source/loader" // Register all source clients "d7y.io/dragonfly/v2/cmd/cdn/cmd" //nolint:gci ) diff --git a/cmd/dependency/plugin_cmd.go b/cmd/dependency/plugin_cmd.go index b0c5469ab..3beb732bb 100644 --- a/cmd/dependency/plugin_cmd.go +++ b/cmd/dependency/plugin_cmd.go @@ -25,6 +25,7 @@ import ( "d7y.io/dragonfly/v2/internal/dfplugin" "d7y.io/dragonfly/v2/pkg/dfpath" + "d7y.io/dragonfly/v2/pkg/source" ) var PluginCmd = &cobra.Command{ @@ -35,11 +36,19 @@ var PluginCmd = &cobra.Command{ DisableAutoGenTag: true, SilenceUsage: true, Run: func(cmd *cobra.Command, args []string) { - ListAvailablePlugins() + ListAvailableInTreePlugins() + ListAvailableOutOfTreePlugins() }, } -func ListAvailablePlugins() { +func ListAvailableInTreePlugins() { + clients := source.ListClients() + for _, scheme := range clients { + fmt.Printf("source plugin: %s, location: in-tree\n", scheme) + } +} + +func ListAvailableOutOfTreePlugins() { d, err := dfpath.New() if err != nil { fmt.Fprintf(os.Stderr, "failed to get plugin path: %q\n", err) @@ -56,6 +65,10 @@ func ListAvailablePlugins() { fmt.Fprintf(os.Stderr, "read plugin dir %s error: %s\n", d.PluginDir(), err) return } + if len(files) == 0 { + fmt.Fprintf(os.Stderr, "no out of tree plugin found\n") + return + } for _, file := range files { var attr []byte fileName := file.Name() @@ -86,6 +99,6 @@ func ListAvailablePlugins() { continue } - fmt.Printf("%s plugin %s, location: %s, attribute: %s\n", typ, name, fileName, string(attr)) + fmt.Printf("%s plugin: %s, location: %s, attribute: %s\n", typ, name, fileName, string(attr)) } } diff --git a/cmd/dfget/main.go b/cmd/dfget/main.go index 06915819f..432448625 100644 --- a/cmd/dfget/main.go +++ b/cmd/dfget/main.go @@ -18,10 +18,8 @@ package main import ( "d7y.io/dragonfly/v2/cmd/dfget/cmd" - // Register http client - _ "d7y.io/dragonfly/v2/pkg/source/httpprotocol" - // Register oss client - _ "d7y.io/dragonfly/v2/pkg/source/ossprotocol" + // register all source clients + _ "d7y.io/dragonfly/v2/pkg/source/loader" ) func main() { diff --git a/pkg/source/clients/example/dfs.go b/pkg/source/clients/example/dfs.go new file mode 100644 index 000000000..64f5e1422 --- /dev/null +++ b/pkg/source/clients/example/dfs.go @@ -0,0 +1,61 @@ +/* + * Copyright 2022 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package example + +import ( + "bytes" + "io" + + "d7y.io/dragonfly/v2/pkg/source" +) + +const scheme = "dfs" + +var data = "hello world" + +type client struct { +} + +func init() { + if err := source.Register(scheme, New(), nil); err != nil { + panic(err) + } +} + +func New() source.ResourceClient { + return &client{} +} + +func (c *client) GetContentLength(request *source.Request) (int64, error) { + return int64(len(data)), nil +} + +func (c *client) IsSupportRange(request *source.Request) (bool, error) { + return false, nil +} + +func (c *client) IsExpired(request *source.Request, info *source.ExpireInfo) (bool, error) { + panic("implement me") +} + +func (c *client) Download(request *source.Request) (*source.Response, error) { + return source.NewResponse(io.NopCloser(bytes.NewBufferString(data))), nil +} + +func (c *client) GetLastModified(request *source.Request) (int64, error) { + panic("implement me") +} diff --git a/pkg/source/hdfsprotocol/hdfs_source_client.go b/pkg/source/clients/hdfsprotocol/hdfs_source_client.go similarity index 100% rename from pkg/source/hdfsprotocol/hdfs_source_client.go rename to pkg/source/clients/hdfsprotocol/hdfs_source_client.go diff --git a/pkg/source/hdfsprotocol/hdfs_source_client_test.go b/pkg/source/clients/hdfsprotocol/hdfs_source_client_test.go similarity index 100% rename from pkg/source/hdfsprotocol/hdfs_source_client_test.go rename to pkg/source/clients/hdfsprotocol/hdfs_source_client_test.go diff --git a/pkg/source/httpprotocol/http_source_client.go b/pkg/source/clients/httpprotocol/http_source_client.go similarity index 100% rename from pkg/source/httpprotocol/http_source_client.go rename to pkg/source/clients/httpprotocol/http_source_client.go diff --git a/pkg/source/httpprotocol/http_source_client_test.go b/pkg/source/clients/httpprotocol/http_source_client_test.go similarity index 100% rename from pkg/source/httpprotocol/http_source_client_test.go rename to pkg/source/clients/httpprotocol/http_source_client_test.go diff --git a/pkg/source/ossprotocol/oss_source_client.go b/pkg/source/clients/ossprotocol/oss_source_client.go similarity index 100% rename from pkg/source/ossprotocol/oss_source_client.go rename to pkg/source/clients/ossprotocol/oss_source_client.go diff --git a/pkg/source/loader/dfs.go.example b/pkg/source/loader/dfs.go.example new file mode 100644 index 000000000..49b092f2f --- /dev/null +++ b/pkg/source/loader/dfs.go.example @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + _ "d7y.io/dragonfly/v2/pkg/source/clients/example" // Register dfs client +) diff --git a/pkg/source/loader/http.go b/pkg/source/loader/http.go new file mode 100644 index 000000000..a74a3bfe1 --- /dev/null +++ b/pkg/source/loader/http.go @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + _ "d7y.io/dragonfly/v2/pkg/source/clients/httpprotocol" // Register http client +) diff --git a/pkg/source/loader/oss.go b/pkg/source/loader/oss.go new file mode 100644 index 000000000..39b15ddb2 --- /dev/null +++ b/pkg/source/loader/oss.go @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + _ "d7y.io/dragonfly/v2/pkg/source/clients/ossprotocol" // Register oss client +) diff --git a/pkg/source/source_client.go b/pkg/source/source_client.go index 49d6e8bd4..349c17975 100644 --- a/pkg/source/source_client.go +++ b/pkg/source/source_client.go @@ -120,14 +120,17 @@ type ResourceLister interface { } type ClientManager interface { - // Register a source client with scheme + // Register registers a source client with scheme Register(scheme string, resourceClient ResourceClient, adapter requestAdapter, hook ...Hook) error - // UnRegister a source client from manager + // UnRegister revoke a source client from manager UnRegister(scheme string) - // GetClient a source client by scheme + // GetClient gets a source client by scheme GetClient(scheme string, options ...Option) (ResourceClient, bool) + + // ListClients lists all supported client scheme + ListClients() []string } // clientManager implements the interface ClientManager @@ -186,6 +189,16 @@ func (m *clientManager) UnRegister(scheme string) { delete(m.clients, scheme) } +func (m *clientManager) ListClients() []string { + m.mu.Lock() + defer m.mu.Unlock() + var clients []string + for c := range m.clients { + clients = append(clients, c) + } + return clients +} + func (m *clientManager) GetClient(scheme string, options ...Option) (ResourceClient, bool) { logger.Debugf("current clients: %#v", m.clients) m.mu.RLock() @@ -226,6 +239,10 @@ func UnRegister(scheme string) { _defaultManager.UnRegister(scheme) } +func ListClients() []string { + return _defaultManager.ListClients() +} + type requestAdapter func(request *Request) *Request // Hook TODO hook