From a9d91cc85dc8c92bacaea980ed4503239c812e0a Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sat, 27 Nov 2021 19:56:29 +0000 Subject: [PATCH 1/7] Add a reload fs --- reload/fs.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 reload/fs.go diff --git a/reload/fs.go b/reload/fs.go new file mode 100644 index 000000000..25d33e50b --- /dev/null +++ b/reload/fs.go @@ -0,0 +1,29 @@ +package reload + +import ( + "io/fs" + "os" +) + +type FS struct { + embed fs.FS + dir fs.FS +} + +func NewFS(embed fs.ReadDirFS, dir string) FS { + return FS{ + embed: embed, + dir: os.DirFS(dir), + } +} + +func (f FS) Open(name string) (fs.File, error) { + if name == "embed.go" { + return nil, fs.ErrNotExist + } + cfgFile := "./.buffalo.dev.yml" + if _, err := os.Stat(cfgFile); err != nil { + return f.embed.Open(name) + } + return f.dir.Open(name) +} From a9db1bd548fcc27e2db8b0663f7195bf3a68f530 Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sat, 27 Nov 2021 20:43:41 +0000 Subject: [PATCH 2/7] Improved hiding of embed.go --- reload/fs.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/reload/fs.go b/reload/fs.go index 25d33e50b..283a808f7 100644 --- a/reload/fs.go +++ b/reload/fs.go @@ -1,6 +1,7 @@ package reload import ( + "fmt" "io/fs" "os" ) @@ -21,9 +22,39 @@ func (f FS) Open(name string) (fs.File, error) { if name == "embed.go" { return nil, fs.ErrNotExist } + file, err := f.getFile(name) + if name != "." { + return file, err + } + return rootFile{file}, err +} + +func (f FS) getFile(name string) (fs.File, error) { cfgFile := "./.buffalo.dev.yml" if _, err := os.Stat(cfgFile); err != nil { return f.embed.Open(name) } return f.dir.Open(name) } + +type rootFile struct { + fs.File +} + +func (f rootFile) ReadDir(n int) ([]fs.DirEntry, error) { + dir, ok := f.File.(fs.ReadDirFile) + if !ok { + return nil, fmt.Errorf("failed at hiding embed.go file") + } + entries, err := dir.ReadDir(n) + if err != nil { + return entries, err + } + + for i, entry := range entries { + if entry.Name() == "embed.go" { + entries = append(entries[:i], entries[i+1:]...) + } + } + return entries, nil +} From eedf7675fedba0da40757fbbe28efbb1bc93fb7e Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sun, 28 Nov 2021 11:57:50 +0000 Subject: [PATCH 3/7] Review changes --- fs.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ reload/fs.go | 60 ------------------------------------ 2 files changed, 87 insertions(+), 60 deletions(-) create mode 100644 fs.go delete mode 100644 reload/fs.go diff --git a/fs.go b/fs.go new file mode 100644 index 000000000..cc7dcfa0a --- /dev/null +++ b/fs.go @@ -0,0 +1,87 @@ +package buffalo + +import ( + "fmt" + "io" + "io/fs" + "os" +) + +// FS wraps a directory and an embed FS that are expected to have the same contents. +// it prioritizes the directory FS and falls back to the embedded FS if the file cannot +// be found on disk. This is useful during development or when deploying with +// assets not embedded in the binary. +// +// Additionally FS hiddes any file named embed.go from the FS. +type FS struct { + embed fs.FS + dir fs.FS +} + +// NewFS returns a new FS that wraps the given directory and embedded FS. +// the embed.FS is expected to embed the same files as the directory FS. +func NewFS(embed fs.ReadDirFS, dir string) FS { + return FS{ + embed: embed, + dir: os.DirFS(dir), + } +} + +// Open implements the FS interface. +func (f FS) Open(name string) (fs.File, error) { + if name == "embed.go" { + return nil, fs.ErrNotExist + } + file, err := f.getFile(name) + if name == "." { + return rootFile{file}, err + } + return file, err +} + +func (f FS) getFile(name string) (fs.File, error) { + file, err := f.dir.Open(name) + if err == nil { + return file, nil + } + + return f.embed.Open(name) +} + +// rootFile wraps the "." directory for hidding the embed.go file. +type rootFile struct { + fs.File +} + +// ReadDir implements the fs.ReadDirFile interface. +func (f rootFile) ReadDir(n int) (entries []fs.DirEntry, err error) { + dir, ok := f.File.(fs.ReadDirFile) + if !ok { + return nil, fmt.Errorf("%T is not a directory", f.File) + } + + if n <= 0 { + entries, err = dir.ReadDir(n) + entries = hideEmbedFile(entries) + } else { + entries, err = dir.ReadDir(n + 1) + entries = hideEmbedFile(entries) + if len(entries) > n { + entries = entries[:n-1] + } + } + + if len(entries) == 0 { + return entries, io.EOF + } + return entries, err +} + +func hideEmbedFile(entries []fs.DirEntry) []fs.DirEntry { + for i, entry := range entries { + if entry.Name() == "embed.go" { + entries = append(entries[:i], entries[i+1:]...) + } + } + return entries +} diff --git a/reload/fs.go b/reload/fs.go deleted file mode 100644 index 283a808f7..000000000 --- a/reload/fs.go +++ /dev/null @@ -1,60 +0,0 @@ -package reload - -import ( - "fmt" - "io/fs" - "os" -) - -type FS struct { - embed fs.FS - dir fs.FS -} - -func NewFS(embed fs.ReadDirFS, dir string) FS { - return FS{ - embed: embed, - dir: os.DirFS(dir), - } -} - -func (f FS) Open(name string) (fs.File, error) { - if name == "embed.go" { - return nil, fs.ErrNotExist - } - file, err := f.getFile(name) - if name != "." { - return file, err - } - return rootFile{file}, err -} - -func (f FS) getFile(name string) (fs.File, error) { - cfgFile := "./.buffalo.dev.yml" - if _, err := os.Stat(cfgFile); err != nil { - return f.embed.Open(name) - } - return f.dir.Open(name) -} - -type rootFile struct { - fs.File -} - -func (f rootFile) ReadDir(n int) ([]fs.DirEntry, error) { - dir, ok := f.File.(fs.ReadDirFile) - if !ok { - return nil, fmt.Errorf("failed at hiding embed.go file") - } - entries, err := dir.ReadDir(n) - if err != nil { - return entries, err - } - - for i, entry := range entries { - if entry.Name() == "embed.go" { - entries = append(entries[:i], entries[i+1:]...) - } - } - return entries, nil -} From 5ec4f4583c41065fdb20a17d95cc4d20aab9ed72 Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sun, 28 Nov 2021 18:17:18 +0000 Subject: [PATCH 4/7] Add a few tests for buffalo.FS --- fs_test.go | 66 +++++++++++++++++++++++++++++ internal/testdata/disk/embed.go | 0 internal/testdata/disk/file.txt | 1 + internal/testdata/embedded/embed.go | 12 ++++++ internal/testdata/embedded/file.txt | 1 + internal/testdata/panic.txt | 1 + 6 files changed, 81 insertions(+) create mode 100644 fs_test.go create mode 100644 internal/testdata/disk/embed.go create mode 100644 internal/testdata/disk/file.txt create mode 100644 internal/testdata/embedded/embed.go create mode 100644 internal/testdata/embedded/file.txt create mode 100644 internal/testdata/panic.txt diff --git a/fs_test.go b/fs_test.go new file mode 100644 index 000000000..fd79f4a1b --- /dev/null +++ b/fs_test.go @@ -0,0 +1,66 @@ +package buffalo + +import ( + "io" + "io/fs" + "testing" + + "github.com/gobuffalo/buffalo/internal/testdata/embedded" + "github.com/stretchr/testify/require" +) + +func Test_FS_Disallows_Parent_Folders(t *testing.T) { + r := require.New(t) + + fsys := NewFS(embedded.FS(), "internal/testdata/disk") + r.NotNil(fsys) + + f, err := fsys.Open("../panic.txt") + r.ErrorIs(err, fs.ErrNotExist) + r.Nil(f) + + f, err = fsys.Open("try/../to/../trick/../panic.txt") + r.ErrorIs(err, fs.ErrNotExist) + r.Nil(f) +} + +func Test_FS_Hides_embed_go(t *testing.T) { + r := require.New(t) + + fsys := NewFS(embedded.FS(), "internal/testdata/disk") + r.NotNil(fsys) + + f, err := fsys.Open("embed.go") + r.ErrorIs(err, fs.ErrNotExist) + r.Nil(f) +} + +func Test_FS_Prioritizes_Disk(t *testing.T) { + r := require.New(t) + + fs := NewFS(embedded.FS(), "internal/testdata/disk") + r.NotNil(fs) + + f, err := fs.Open("file.txt") + r.NoError(err) + + b, err := io.ReadAll(f) + r.NoError(err) + + r.Equal("This file is on disk.", string(b)) +} + +func Test_FS_Uses_Embed_If_No_Disk(t *testing.T) { + r := require.New(t) + + fs := NewFS(embedded.FS(), "internal/testdata/empty") + r.NotNil(fs) + + f, err := fs.Open("file.txt") + r.NoError(err) + + b, err := io.ReadAll(f) + r.NoError(err) + + r.Equal("This file is embedded.", string(b)) +} diff --git a/internal/testdata/disk/embed.go b/internal/testdata/disk/embed.go new file mode 100644 index 000000000..e69de29bb diff --git a/internal/testdata/disk/file.txt b/internal/testdata/disk/file.txt new file mode 100644 index 000000000..e286859c0 --- /dev/null +++ b/internal/testdata/disk/file.txt @@ -0,0 +1 @@ +This file is on disk. \ No newline at end of file diff --git a/internal/testdata/embedded/embed.go b/internal/testdata/embedded/embed.go new file mode 100644 index 000000000..c6f4a841f --- /dev/null +++ b/internal/testdata/embedded/embed.go @@ -0,0 +1,12 @@ +package embedded + +import ( + "embed" +) + +//go:embed * +var files embed.FS + +func FS() embed.FS { + return files +} diff --git a/internal/testdata/embedded/file.txt b/internal/testdata/embedded/file.txt new file mode 100644 index 000000000..0ba5d8426 --- /dev/null +++ b/internal/testdata/embedded/file.txt @@ -0,0 +1 @@ +This file is embedded. \ No newline at end of file diff --git a/internal/testdata/panic.txt b/internal/testdata/panic.txt new file mode 100644 index 000000000..764ccb2f2 --- /dev/null +++ b/internal/testdata/panic.txt @@ -0,0 +1 @@ +This file must not be accessible from buffalo.FS. \ No newline at end of file From 71da1ca3141e9379d1e1effc7098c94cd89ea217 Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sun, 28 Nov 2021 18:52:06 +0000 Subject: [PATCH 5/7] Add tests for ReadDir and fixed bug --- fs.go | 27 ++++++------------- fs_test.go | 46 +++++++++++++++++++++++++++----- internal/testdata/disk/file2.txt | 0 3 files changed, 48 insertions(+), 25 deletions(-) create mode 100644 internal/testdata/disk/file2.txt diff --git a/fs.go b/fs.go index cc7dcfa0a..e7436c3c5 100644 --- a/fs.go +++ b/fs.go @@ -2,7 +2,6 @@ package buffalo import ( "fmt" - "io" "io/fs" "os" ) @@ -60,28 +59,18 @@ func (f rootFile) ReadDir(n int) (entries []fs.DirEntry, err error) { return nil, fmt.Errorf("%T is not a directory", f.File) } - if n <= 0 { - entries, err = dir.ReadDir(n) - entries = hideEmbedFile(entries) - } else { - entries, err = dir.ReadDir(n + 1) - entries = hideEmbedFile(entries) - if len(entries) > n { - entries = entries[:n-1] - } - } - - if len(entries) == 0 { - return entries, io.EOF - } + entries, err = dir.ReadDir(n) + entries = hideEmbedFile(entries) return entries, err } func hideEmbedFile(entries []fs.DirEntry) []fs.DirEntry { - for i, entry := range entries { - if entry.Name() == "embed.go" { - entries = append(entries[:i], entries[i+1:]...) + result := make([]fs.DirEntry, 0, len(entries)) + + for _, entry := range entries { + if entry.Name() != "embed.go" { + result = append(result, entry) } } - return entries + return result } diff --git a/fs_test.go b/fs_test.go index fd79f4a1b..c0b9cdcd6 100644 --- a/fs_test.go +++ b/fs_test.go @@ -38,10 +38,10 @@ func Test_FS_Hides_embed_go(t *testing.T) { func Test_FS_Prioritizes_Disk(t *testing.T) { r := require.New(t) - fs := NewFS(embedded.FS(), "internal/testdata/disk") - r.NotNil(fs) + fsys := NewFS(embedded.FS(), "internal/testdata/disk") + r.NotNil(fsys) - f, err := fs.Open("file.txt") + f, err := fsys.Open("file.txt") r.NoError(err) b, err := io.ReadAll(f) @@ -53,10 +53,10 @@ func Test_FS_Prioritizes_Disk(t *testing.T) { func Test_FS_Uses_Embed_If_No_Disk(t *testing.T) { r := require.New(t) - fs := NewFS(embedded.FS(), "internal/testdata/empty") - r.NotNil(fs) + fsys := NewFS(embedded.FS(), "internal/testdata/empty") + r.NotNil(fsys) - f, err := fs.Open("file.txt") + f, err := fsys.Open("file.txt") r.NoError(err) b, err := io.ReadAll(f) @@ -64,3 +64,37 @@ func Test_FS_Uses_Embed_If_No_Disk(t *testing.T) { r.Equal("This file is embedded.", string(b)) } + +func Test_FS_ReadDirFile(t *testing.T) { + r := require.New(t) + + fsys := NewFS(embedded.FS(), "internal/testdata/disk") + r.NotNil(fsys) + + f, err := fsys.Open(".") + r.NoError(err) + + dir, ok := f.(fs.ReadDirFile) + r.True(ok, "folder does not implement fs.ReadDirFile interface") + + // First read should return 1 file + entries, err := dir.ReadDir(1) + r.NoError(err) + + // The actual len will be 0 because the first file read is the embed.go file + // this is counter-intuitive, but it's how the fs.ReadDirFile interface is specified; + // if err == nil, just continue to call ReadDir until io.EOF is returned. + r.LessOrEqual(len(entries), 1, "a call to ReadDir must at most return n entries") + + // First read should return at most 2 files + entries, err = dir.ReadDir(2) + r.NoError(err) + + // The actual len will be 2 (file.txt & file2.txt) + r.LessOrEqual(len(entries), 2, "a call to ReadDir must at most return n entries") + + // trying to read next 2 files (none left) + entries, err = dir.ReadDir(2) + r.ErrorIs(err, io.EOF) + r.Empty(entries) +} diff --git a/internal/testdata/disk/file2.txt b/internal/testdata/disk/file2.txt new file mode 100644 index 000000000..e69de29bb From cf903588c184bc730228fe126c5989b437da1680 Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sun, 28 Nov 2021 18:54:34 +0000 Subject: [PATCH 6/7] Fix typo --- fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs_test.go b/fs_test.go index c0b9cdcd6..92eb11146 100644 --- a/fs_test.go +++ b/fs_test.go @@ -77,7 +77,7 @@ func Test_FS_ReadDirFile(t *testing.T) { dir, ok := f.(fs.ReadDirFile) r.True(ok, "folder does not implement fs.ReadDirFile interface") - // First read should return 1 file + // First read should return at most 1 file entries, err := dir.ReadDir(1) r.NoError(err) From 5927867a8fcf6c009c9eeefbd32ee5fb70c6a57e Mon Sep 17 00:00:00 2001 From: Matthias Fasching Date: Sun, 28 Nov 2021 18:55:35 +0000 Subject: [PATCH 7/7] Fix another typo --- fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs_test.go b/fs_test.go index 92eb11146..11d142766 100644 --- a/fs_test.go +++ b/fs_test.go @@ -86,7 +86,7 @@ func Test_FS_ReadDirFile(t *testing.T) { // if err == nil, just continue to call ReadDir until io.EOF is returned. r.LessOrEqual(len(entries), 1, "a call to ReadDir must at most return n entries") - // First read should return at most 2 files + // Second read should return at most 2 files entries, err = dir.ReadDir(2) r.NoError(err)