Skip to content

Commit

Permalink
Merge branch 'some2'
Browse files Browse the repository at this point in the history
  • Loading branch information
ajiyoshi-vg authored and ajiyoshi-vg committed Sep 26, 2024
2 parents 5b1d153 + 66e62b4 commit cf91571
Show file tree
Hide file tree
Showing 11 changed files with 133 additions and 34 deletions.
23 changes: 23 additions & 0 deletions bytes/walker.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package bytes

import (
"io"

"github.com/ajiyoshi-vg/external/scan"
"github.com/ajiyoshi-vg/hairetsu/doublearray/item"
)

Expand All @@ -21,3 +24,23 @@ func FromSlice(xs [][]byte, f Factory) (Dict, error) {
}
return dict, nil
}

func FromReadSeeker(r io.ReadSeeker, f Factory) (Dict, error) {
b := NewBuilder()
for line := range scan.ByteLines(r) {
b.Add(line)
}
dict := b.Build()

if _, err := r.Seek(0, io.SeekStart); err != nil {
return nil, err
}

var i uint32
for line := range scan.ByteLines(r) {
w := dict.Word(line)
f.Put(item.New(w, i))
i++
}
return dict, nil
}
16 changes: 16 additions & 0 deletions bytetrie.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import (
"iter"
"slices"

"github.com/ajiyoshi-vg/external/scan"
da "github.com/ajiyoshi-vg/hairetsu/doublearray"
"github.com/ajiyoshi-vg/hairetsu/doublearray/item"
"github.com/ajiyoshi-vg/hairetsu/node"
"github.com/ajiyoshi-vg/hairetsu/word"
)

type ByteTrie struct {
Expand Down Expand Up @@ -52,3 +54,17 @@ func (b *ByteTrieBuilder) StreamBuild(seq iter.Seq[[]byte]) (*ByteTrie, error) {
}
return NewByteTrie(x), nil
}

func (b *ByteTrieBuilder) BuildFromLines(r io.Reader) (*ByteTrie, error) {
f := b.builder.Factory()
var i uint32
for x := range scan.ByteLines(r) {
f.Put(item.New(word.FromBytes(x), i))
i++
}
ret, err := f.Done()
if err != nil {
return nil, err
}
return NewByteTrie(ret), nil
}
11 changes: 5 additions & 6 deletions cmd/dump/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ import (
"github.com/ajiyoshi-vg/external/scan"
"github.com/ajiyoshi-vg/hairetsu"
"github.com/ajiyoshi-vg/hairetsu/doublearray"
"github.com/ajiyoshi-vg/hairetsu/progress"

Check failure on line 16 in cmd/dump/main.go

View workflow job for this annotation

GitHub Actions / build

no required module provides package github.com/ajiyoshi-vg/hairetsu/progress; to add it:
"github.com/ikawaha/dartsclone"
dartsprog "github.com/ikawaha/dartsclone/progressbar"
"github.com/schollz/progressbar"
)

type option struct {
Expand Down Expand Up @@ -70,7 +70,7 @@ func run() error {

func options() []doublearray.Option {
ret := []doublearray.Option{
doublearray.OptionProgress(progressbar.New(0)),
doublearray.OptionProgress(&progress.ProgressBar{}),
}
if opt.verbose {
return append(ret, doublearray.Verbose)
Expand All @@ -79,23 +79,22 @@ func options() []doublearray.Option {
}

func dumpByte(file io.Reader) error {
seq := scan.ByteLines(file)
trie, err := hairetsu.NewByteTrieBuilder(options()...).StreamBuild(seq)
trie, err := hairetsu.NewByteTrieBuilder(options()...).BuildFromLines(file)
if err != nil {
return err
}
return writeTo(trie, opt.out)
}

func dumpRune(file io.Reader) error {
func dumpRune(file io.ReadSeeker) error {
trie, err := hairetsu.NewRuneTrieBuilder(options()...).BuildFromLines(file)
if err != nil {
return err
}
return writeTo(trie, opt.out)
}

func dumpDict(file io.Reader) error {
func dumpDict(file io.ReadSeeker) error {
trie, err := hairetsu.NewDictTrieBuilder(options()...).BuildFromLines(file)
if err != nil {
return err
Expand Down
19 changes: 13 additions & 6 deletions dicttrie.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package hairetsu
import (
"bytes"
"io"
"slices"

"github.com/ajiyoshi-vg/external/scan"
dict "github.com/ajiyoshi-vg/hairetsu/bytes"
"github.com/ajiyoshi-vg/hairetsu/doublearray"
da "github.com/ajiyoshi-vg/hairetsu/doublearray"
Expand Down Expand Up @@ -87,13 +85,22 @@ func (b *DictTrieBuilder) BuildFromSlice(xs [][]byte) (*DictTrie, error) {
if err != nil {
return nil, err
}
trie, err := f.Done()
return buildDictTrie(f, dict)
}

func (b *DictTrieBuilder) BuildFromLines(r io.ReadSeeker) (*DictTrie, error) {
f := b.builder.Factory()
dict, err := dict.FromReadSeeker(r, f)
if err != nil {
return nil, err
}
return NewDictTrie(trie, dict), nil
return buildDictTrie(f, dict)
}

func (b *DictTrieBuilder) BuildFromLines(r io.Reader) (*DictTrie, error) {
return b.BuildFromSlice(slices.Collect(scan.ByteLines(r)))
func buildDictTrie(f *da.Factory, dict dict.Dict) (*DictTrie, error) {
trie, err := f.Done()
if err != nil {
return nil, err
}
return NewDictTrie(trie, dict), nil
}
5 changes: 3 additions & 2 deletions dicttrie_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,13 @@ func TestDictTrieBuild(t *testing.T) {

for _, c := range cases {
t.Run(c.title, func(t *testing.T) {
data := bytes.NewBufferString(c.data)
data := bytes.NewReader([]byte(c.data))
origin, err := NewDictTrieBuilder().BuildFromLines(data)
assert.NoError(t, err)

buf := &bytes.Buffer{}
origin.WriteTo(buf)
_, err = origin.WriteTo(buf)
assert.NoError(t, err)

restored := &DictTrie{}
_, err = restored.ReadFrom(buf)
Expand Down
4 changes: 2 additions & 2 deletions doublearray/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type Builder struct {

type Progress interface {
SetMax(int)
Add(int) error
Add(int)
}

func NewBuilder(opt ...Option) *Builder {
Expand Down Expand Up @@ -237,7 +237,7 @@ func (b *Builder) insert(da *DoubleArray, prefix word.Word, branch []word.Code,
}
func (b *Builder) addProgress(n int) {
if b.progress != nil {
_ = b.progress.Add(n)
b.progress.Add(n)
}
}
func (b *Builder) progressLogf(format string, args ...interface{}) {
Expand Down
11 changes: 8 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,24 @@ go 1.23.0
require (
github.com/ajiyoshi-vg/external v0.0.0-20240925080105-8951936a439c
github.com/google/uuid v1.6.0
github.com/gosuri/uiprogress v0.0.1
github.com/ikawaha/dartsclone v1.0.0
github.com/pkg/errors v0.9.1
github.com/schollz/progressbar v1.0.0
github.com/schollz/progressbar/v3 v3.16.0
github.com/stretchr/testify v1.9.0
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286 // indirect
github.com/gosuri/uilive v0.0.4 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/schollz/progressbar/v2 v2.7.1 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/term v0.24.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
28 changes: 21 additions & 7 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,36 +1,50 @@
github.com/ajiyoshi-vg/external v0.0.0-20240925044203-7f6ccf58d02e h1:hKrcVN5c5mYbmGcjgMQIwOfURSIaO048ViU3emUWaUA=
github.com/ajiyoshi-vg/external v0.0.0-20240925044203-7f6ccf58d02e/go.mod h1:Sm/g0L0F1NiwClqtFT/A+kGJQrCFGp36Q6eBR4fZvWs=
github.com/ajiyoshi-vg/external v0.0.0-20240925080105-8951936a439c h1:Qc0Qe1Pe1Eumah1KpgDIg98zaSs9WeJeny8NkbepGLM=
github.com/ajiyoshi-vg/external v0.0.0-20240925080105-8951936a439c/go.mod h1:Sm/g0L0F1NiwClqtFT/A+kGJQrCFGp36Q6eBR4fZvWs=
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/euclidr/darts v0.0.0-20180401113647-e0859b23b68e h1:SJ6NVXT96EzZgSX2V5NBVYGsHyxHXrDhhtrWOKe4n/k=
github.com/euclidr/darts v0.0.0-20180401113647-e0859b23b68e/go.mod h1:uZOcB9m9tCkJHZFC3diz9McvyuzFL3wWG+Wn1IiNXjI=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
github.com/ikawaha/da v0.0.0-20141126165404-5556b9e515ca h1:Ye8wDuFVpINaiKj26eAOwmqBAWDEEVdhgEtyUCfUMw8=
github.com/ikawaha/da v0.0.0-20141126165404-5556b9e515ca/go.mod h1:JZfcL9+WaPkxDS7OBk09o8VkkV7YYI9yLQG610HxZds=
github.com/ikawaha/dartsclone v1.0.0 h1:Z6R0BASHSNSxlcIY6UE1+aR6azYW6j3ge6dbmT+JWA0=
github.com/ikawaha/dartsclone v1.0.0/go.mod h1:6+lg0WgpO3ZJaqHu/qOSNIjxbeD1qpTpimD8YYnBnJY=
github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286 h1:KHyL+3mQOF9sPfs26lsefckcFNDcIZtiACQiECzIUkw=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM=
github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/schollz/progressbar/v2 v2.7.1 h1:wEojKO9r+zgGEEgJSsu7QcVAM0NeEP64Ad4lHK12A5A=
github.com/schollz/progressbar/v2 v2.7.1/go.mod h1:l6tn6yU6ZdQoF8lwX/VoAUQ3FjhCbrcZDnl9xeWZzYw=
github.com/schollz/progressbar/v3 v3.16.0 h1:+MbBim/cE9DqDb8UXRfLJ6RZdyDkXG1BDy/sWc5s0Mc=
github.com/schollz/progressbar/v3 v3.16.0/go.mod h1:lLiKjKJ9/yzc9Q8jk+sVLfxWxgXKsktvUf6TO+4Y2nw=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
26 changes: 26 additions & 0 deletions runes/walker.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package runes

import (
"io"

"github.com/ajiyoshi-vg/external/scan"
"github.com/ajiyoshi-vg/hairetsu/doublearray/item"
)

Expand All @@ -24,3 +27,26 @@ func FromSlice(xs []string, f Factory) (Dict, error) {
}
return dict, nil
}

func FromReader(r io.ReadSeeker, f Factory) (Dict, error) {
b := NewBuilder()
for line := range scan.Lines(r) {
b.Add(line)
}
dict := b.Build()

if _, err := r.Seek(0, io.SeekStart); err != nil {
return nil, err
}

var i uint32
for line := range scan.Lines(r) {
w, err := dict.Word(line)
if err != nil {
return nil, err
}
f.Put(item.New(w, i))
i++
}
return dict, nil
}
19 changes: 13 additions & 6 deletions runetrie.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@ import (
"bytes"
"encoding/binary"
"io"
"slices"

"github.com/ajiyoshi-vg/external/scan"
da "github.com/ajiyoshi-vg/hairetsu/doublearray"
"github.com/ajiyoshi-vg/hairetsu/node"
"github.com/ajiyoshi-vg/hairetsu/runes"
Expand Down Expand Up @@ -102,13 +100,22 @@ func (b *RuneTrieBuilder) BuildFromSlice(xs []string) (*RuneTrie, error) {
if err != nil {
return nil, err
}
trie, err := f.Done()
return buildRuneTrie(f, dict)
}

func (b *RuneTrieBuilder) BuildFromLines(r io.ReadSeeker) (*RuneTrie, error) {
f := b.builder.Factory()
dict, err := runes.FromReader(r, f)
if err != nil {
return nil, err
}
return NewRuneTrie(trie, dict), nil
return buildRuneTrie(f, dict)
}

func (b *RuneTrieBuilder) BuildFromLines(r io.Reader) (*RuneTrie, error) {
return b.BuildFromSlice(slices.Collect(scan.Lines(r)))
func buildRuneTrie(f *da.Factory, dict runes.Dict) (*RuneTrie, error) {
trie, err := f.Done()
if err != nil {
return nil, err
}
return NewRuneTrie(trie, dict), nil
}
5 changes: 3 additions & 2 deletions runetrie_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,13 @@ func TestRuneTrieBuild(t *testing.T) {

for _, c := range cases {
t.Run(c.title, func(t *testing.T) {
data := bytes.NewBufferString(c.data)
data := bytes.NewReader([]byte(c.data))
origin, err := NewRuneTrieBuilder().BuildFromLines(data)
assert.NoError(t, err)

buf := &bytes.Buffer{}
origin.WriteTo(buf)
_, err = origin.WriteTo(buf)
assert.NoError(t, err)

restored := &RuneTrie{}
_, err = restored.ReadFrom(buf)
Expand Down

0 comments on commit cf91571

Please sign in to comment.