Skip to content

Commit

Permalink
Added an endpoint allowing file metadata to be requested by ID.
Browse files Browse the repository at this point in the history
  • Loading branch information
jeff-cohere committed Aug 2, 2024
1 parent 8c4a60c commit 8396b1c
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 25 deletions.
52 changes: 52 additions & 0 deletions services/prototype.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,57 @@ func (service *prototype) searchDatabaseWithSpecificParams(ctx context.Context,
return searchDatabase(ctx, &searchInput, body.Specific)
}

type FileMetadataOutput struct {
Body FileMetadataResponse `doc:"Metadata for files with the given IDs"`
}

// fetches file metadata given a list of file identifiers
func (service *prototype) fetchFileMetadata(ctx context.Context,
input *struct {
Authorization string `header:"authorization" doc:"Authorization header with encoded access token"`
Database string `json:"database" query:"database" example:"jdp" doc:"The ID of the database for which file metadata is fetched"`
Ids string `json:"ids" query:"ids" example:"JDP:6101cc0f2b1f2eeea564c978" doc:"A comma-separated list of file IDs"`
Offset int `json:"offset" query:"offset" example:"100" doc:"Metadata records begin at the given offset"`
Limit int `json:"limit" query:"limit" example:"50" doc:"Limits the number of metadata records returned"`
}) (*FileMetadataOutput, error) {

orcid, err := authorize(input.Authorization)
if err != nil {
return nil, err
}

// is the database valid?
_, ok := config.Databases[input.Database]
if !ok {
return nil, fmt.Errorf("Database %s not found", input.Database)
}

// have we been given any IDs?
if strings.TrimSpace(input.Ids) == "" {
return nil, huma.Error400BadRequest("No file IDs were provided!")
}
ids := strings.Split(input.Ids, ",")

slog.Info(fmt.Sprintf("Fetching file metadata for %d files in database %s...",
len(ids), input.Database))
db, err := databases.NewDatabase(orcid, input.Database)
if err != nil {
return nil, err
}

results, err := db.Resources(ids)
if err != nil {
slog.Error(err.Error())
return nil, err
}
return &FileMetadataOutput{
Body: FileMetadataResponse{
Database: input.Database,
Resources: results,
},
}, nil
}

type TransferOutput struct {
Body TransferResponse `doc:"A UUID for the requested transfer"`
Status int
Expand Down Expand Up @@ -527,6 +578,7 @@ func NewDTSPrototype() (TransferService, error) {
huma.Get(api, "/api/v1/databases/{db}/search-parameters", service.getDatabaseSearchParameters)
huma.Get(api, "/api/v1/files", service.searchDatabase)
huma.Post(api, "/api/v1/files", service.searchDatabaseWithSpecificParams)
huma.Get(api, "/api/v1/files/by-id", service.fetchFileMetadata)
huma.Post(api, "/api/v1/transfers", service.createTransfer)
huma.Get(api, "/api/v1/transfers/{id}", service.getTransferStatus)
huma.Delete(api, "/api/v1/transfers/{id}", service.deleteTransfer)
Expand Down
29 changes: 29 additions & 0 deletions services/prototype_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,35 @@ func TestSearchDatabase(t *testing.T) {
assert.Equal("file1", results.Resources[0].Name)
}

// fetches file metadata from the JDP for some specific files
func TestFetchJdpMetadata(t *testing.T) {
assert := assert.New(t)

// try omitting file IDs
resp, err := get(baseUrl + apiPrefix + "files/by-id?database=jdp")
assert.Nil(err)
assert.Equal(http.StatusBadRequest, resp.StatusCode)

// now let's fetch 3 records
resp, err = get(baseUrl + apiPrefix +
"files/by-id?database=jdp&ids=JDP:6101cc0f2b1f2eeea564c978,JDP:613a7baa72d3a08c9a54b32d,JDP:61412246cc4ff44f36c8913d")
assert.Nil(err)

respBody, err := io.ReadAll(resp.Body)
assert.Nil(err)
assert.Equal(http.StatusOK, resp.StatusCode)
defer resp.Body.Close()

var results SearchResultsResponse
err = json.Unmarshal(respBody, &results)
assert.Nil(err)
assert.Equal("jdp", results.Database)
assert.Equal(3, len(results.Resources))
assert.Equal("JDP:6101cc0f2b1f2eeea564c978", results.Resources[0].Id)
assert.Equal("JDP:613a7baa72d3a08c9a54b32d", results.Resources[1].Id)
assert.Equal("JDP:61412246cc4ff44f36c8913d", results.Resources[2].Id)
}

// creates a transfer from source -> destination1
func TestCreateTransfer(t *testing.T) {
assert := assert.New(t)
Expand Down
35 changes: 10 additions & 25 deletions services/transfer_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,12 @@ package services

import (
"context"
"encoding/json"
"net/http"

"github.com/google/uuid"

"github.com/kbase/dts/frictionless"
)

// This package-specific helper function writes a JSON payload to an
// http.ResponseWriter.
func writeJson(w http.ResponseWriter, data []byte, code int) {
w.WriteHeader(code)
if len(data) > 0 {
w.Header().Set("Content-Type", "application/json")
w.Write(data)
}
}

// this type encodes a JSON object for responding to root queries
type ServiceInfoResponse struct {
Name string `json:"name" example:"DTS" doc:"The name of the service API"`
Expand All @@ -37,17 +25,6 @@ type ErrorResponse struct {
Error string `json:"message"`
}

// This package-specific helper function writes an error to an
// http.ResponseWriter, giving it the proper status code, and encoding an
// ErrorResponse in the response body.
func writeError(w http.ResponseWriter, message string, code int) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
e := ErrorResponse{Code: code, Error: message}
data, _ := json.Marshal(e)
w.Write(data)
}

// a response for a database-related query (GET)
type DatabaseResponse struct {
Id string `json:"id" example:"jdp" `
Expand All @@ -56,13 +33,21 @@ type DatabaseResponse struct {
URL string `json:"url" example:"https://data.jgi.doe.gov"`
}

// a response for a query (GET)
// a response for a file search query (GET)
type SearchResultsResponse struct {
// name of organization database
Database string `json:"database" example:"jdp" doc:"the database searched"`
// ElasticSearch query string
Query string `json:"query" example:"prochlorococcus" doc:"the given query string"`
// Resources matching the query
// resources matching the query
Resources []frictionless.DataResource `json:"resources" doc:"an array of Frictionless DataResources"`
}

// a response for a file metadata query (GET)
type FileMetadataResponse struct {
// name of organization database
Database string `json:"database" example:"jdp" doc:"the database searched"`
// resources corresponding to given file IDs
Resources []frictionless.DataResource `json:"resources" doc:"an array of Frictionless DataResources"`
}

Expand Down

0 comments on commit 8396b1c

Please sign in to comment.