From 012edda3a34eea73f549618c71941a166c3591f1 Mon Sep 17 00:00:00 2001 From: Federico Muttis Date: Sat, 13 Jul 2024 17:51:22 +0200 Subject: [PATCH 1/3] When there is no value on an input, favor HTML's placeholder before using Katana's if exists --- pkg/utils/formfill.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/utils/formfill.go b/pkg/utils/formfill.go index 1ace9a35..2deeb742 100644 --- a/pkg/utils/formfill.go +++ b/pkg/utils/formfill.go @@ -47,7 +47,7 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st data := mapsutil.NewOrderedMap[string, string]() // Fill checkboxes and radioboxes first or default values first - for _, input := range inputs { + for i, input := range inputs { switch input.Type { case "radio": // Use a single radio name per value @@ -62,6 +62,11 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st // infer the values based on input types. if input.Value != "" { data.Set(input.Name, input.Value) + } else { + if value, ok := input.Attributes.Get("placeholder"); ok { + inputs[i].Value = value + data.Set(input.Name, value) + } } } } From 1a9da471ab1229201ebaa3bdca4701fe9227e602 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 20 Aug 2024 14:08:13 +0200 Subject: [PATCH 2/3] lint --- pkg/engine/passive/passive.go | 2 +- pkg/utils/formfill.go | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/engine/passive/passive.go b/pkg/engine/passive/passive.go index 0a6188f7..6457f658 100644 --- a/pkg/engine/passive/passive.go +++ b/pkg/engine/passive/passive.go @@ -49,7 +49,7 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { gologger.Fatal().Msg("No sources selected for this search") } - gologger.Debug().Msgf(fmt.Sprintf("Selected source(s) for this crawl: %s", strings.Join(maps.Keys(sources), ", "))) + gologger.Debug().Msgf("Selected source(s) for this crawl: %s", strings.Join(maps.Keys(sources), ", ")) httpClient := httpclient.NewHttpClient(options.Options.Timeout) return &Crawler{Shared: shared, sources: maps.Values(sources), httpClient: httpClient}, nil diff --git a/pkg/utils/formfill.go b/pkg/utils/formfill.go index 2deeb742..c6a660a9 100644 --- a/pkg/utils/formfill.go +++ b/pkg/utils/formfill.go @@ -62,11 +62,9 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st // infer the values based on input types. if input.Value != "" { data.Set(input.Name, input.Value) - } else { - if value, ok := input.Attributes.Get("placeholder"); ok { - inputs[i].Value = value - data.Set(input.Name, value) - } + } else if value, ok := input.Attributes.Get("placeholder"); ok { + inputs[i].Value = value + data.Set(input.Name, value) } } } From b60ab881bb9ba196768b62ccd0bd8dcd6a6c79d4 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 20 Aug 2024 14:12:47 +0200 Subject: [PATCH 3/3] conflict --- pkg/engine/passive/passive.go | 145 ---------------------------------- 1 file changed, 145 deletions(-) delete mode 100644 pkg/engine/passive/passive.go diff --git a/pkg/engine/passive/passive.go b/pkg/engine/passive/passive.go deleted file mode 100644 index 6457f658..00000000 --- a/pkg/engine/passive/passive.go +++ /dev/null @@ -1,145 +0,0 @@ -package passive - -import ( - "context" - "fmt" - "net/http" - "strings" - "sync" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/katana/pkg/engine/common" - "github.com/projectdiscovery/katana/pkg/engine/passive/httpclient" - "github.com/projectdiscovery/katana/pkg/engine/passive/source" - "github.com/projectdiscovery/katana/pkg/navigation" - "github.com/projectdiscovery/katana/pkg/types" - "github.com/projectdiscovery/katana/pkg/utils" - errorutil "github.com/projectdiscovery/utils/errors" - urlutil "github.com/projectdiscovery/utils/url" - "golang.org/x/exp/maps" -) - -// Crawler is a passive crawler instance -type Crawler struct { - *common.Shared - sources []source.Source - httpClient *httpclient.HttpClient -} - -// New returns a new passive crawler instance -func New(options *types.CrawlerOptions) (*Crawler, error) { - shared, err := common.NewShared(options) - if err != nil { - return nil, errorutil.NewWithErr(err).WithTag("passive") - } - - sources := make(map[string]source.Source, len(Sources)) - if len(options.Options.PassiveSource) > 0 { - for _, source := range options.Options.PassiveSource { - if s, ok := Sources[source]; ok { - sources[source] = s - } - } - } else { - sources = Sources - } - - if len(sources) == 0 { - gologger.Fatal().Msg("No sources selected for this search") - } - - gologger.Debug().Msgf("Selected source(s) for this crawl: %s", strings.Join(maps.Keys(sources), ", ")) - - httpClient := httpclient.NewHttpClient(options.Options.Timeout) - return &Crawler{Shared: shared, sources: maps.Values(sources), httpClient: httpClient}, nil -} - -// Close closes the crawler process -func (c *Crawler) Close() error { - return nil -} - -// Crawl crawls a URL with the specified options -func (c *Crawler) Crawl(rootURL string) error { - gologger.Info().Msgf("Enumerating passive endpoints for %s", rootURL) - - rootUrlParsed, _ := urlutil.ParseURL(rootURL, true) - results := make(chan source.Result) - var timeTaken time.Duration - go func() { - defer func(startTime time.Time) { - timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - ctx := context.Background() - wg := &sync.WaitGroup{} - for _, s := range c.sources { - wg.Add(1) - go func(source source.Source) { - for result := range source.Run(ctx, c.Shared, rootURL) { - results <- result - } - wg.Done() - }(s) - } - wg.Wait() - }() - - seenURLs := make(map[string]struct{}) - sourceStats := make(map[string]int) - for result := range results { - if _, found := seenURLs[result.Value]; found { - continue - } - - if !utils.IsURL(result.Value) { - gologger.Debug().Msgf("`%v` not a url. skipping", result.Value) - continue - } - - if ok, err := c.Options.ValidateScope(result.Value, rootUrlParsed.Hostname()); err != nil || !ok { - gologger.Debug().Msgf("`%v` not in scope. skipping", result.Value) - continue - } - - if !c.Options.ExtensionsValidator.ValidatePath(result.Value) { - gologger.Debug().Msgf("`%v` not allowed extension. skipping", result.Value) - continue - } - - seenURLs[result.Value] = struct{}{} - sourceStats[result.Source]++ - - passiveURL, _ := urlutil.Parse(result.Value) - req := &navigation.Request{ - Method: http.MethodGet, - URL: result.Value, - } - resp := &navigation.Response{ - StatusCode: http.StatusOK, - RootHostname: passiveURL.Hostname(), - Resp: &http.Response{ - StatusCode: http.StatusOK, - Request: &http.Request{ - Method: http.MethodGet, - URL: passiveURL.URL, - }, - }, - } - passiveReference := &navigation.PassiveReference{ - Source: result.Source, - Reference: result.Reference, - } - c.Output(req, resp, passiveReference, nil) - } - - var stats []string - for source, count := range sourceStats { - stats = append(stats, fmt.Sprintf("%s: %d", source, count)) - } - - gologger.Info().Msgf("Found %d endpoints for %s in %s (%s)", len(seenURLs), rootURL, timeTaken.String(), strings.Join(stats, ", ")) - return nil -}