这是indexloc提供的服务,不要输入任何密码
Skip to content

Add -unique flag for filtering duplicate response sizes #822

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ A fast web fuzzer written in Go.

- [Download](https://github.com/ffuf/ffuf/releases/latest) a prebuilt binary from [releases page](https://github.com/ffuf/ffuf/releases/latest), unpack and run!

_or_
- To use the enhanced version with unique response filtering feature, download from [cyinnove's fork](https://github.com/cyinnove/ffuf/releases/tag/v1.0.0)

_or_
- If you are on macOS with [homebrew](https://brew.sh), ffuf can be installed with: `brew install ffuf`

Expand All @@ -43,6 +46,16 @@ Michael Skelton ([@codingo](https://github.com/codingo)).

You can also practise your ffuf scans against a live host with different lessons and use cases either locally by using the docker container https://github.com/adamtlangley/ffufme or against the live hosted version at http://ffuf.me created by Adam Langley [@adamtlangley](https://twitter.com/adamtlangley).

### Filtering Duplicate Response Sizes

Using the `-unique` flag, you can filter out responses with duplicate sizes, showing only the first occurrence of each unique size. This is useful for reducing noise and identifying distinct responses:

```bash
ffuf -w wordlist.txt -u https://example.org/FUZZ -unique
```

This will only show responses with unique content lengths, helping you focus on potentially interesting endpoints.

### Typical directory discovery

[![asciicast](https://asciinema.org/a/211350.png)](https://asciinema.org/a/211350)
Expand Down Expand Up @@ -195,6 +208,7 @@ GENERAL OPTIONS:
-search Search for a FFUFHASH payload from ffuf history
-sf Stop when > 95% of responses return 403 Forbidden (default: false)
-t Number of concurrent threads. (default: 40)
-unique Only show the first occurrence of responses with same size (default: false)
-v Verbose output, printing full URL and redirect location (if any) with the results. (default: false)

MATCHER OPTIONS:
Expand Down
3 changes: 3 additions & 0 deletions b29325839bbc8a11ff17152201234d16
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

---- ↑ Request ---- Response ↓ ----

2 changes: 1 addition & 1 deletion help.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func Usage() {
Description: "",
Flags: make([]UsageFlag, 0),
Hidden: false,
ExpectedFlags: []string{"ac", "acc", "ack", "ach", "acs", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "scraperfile", "scrapers", "search", "s", "sa", "se", "sf", "t", "v", "V"},
ExpectedFlags: []string{"ac", "acc", "ack", "ach", "acs", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "scraperfile", "scrapers", "search", "s", "sa", "se", "sf", "t", "unique", "v", "V"},
}
u_compat := UsageSection{
Name: "COMPATIBILITY OPTIONS",
Expand Down
10 changes: 10 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.BoolVar(&opts.General.StopOnAll, "sa", opts.General.StopOnAll, "Stop on all error cases. Implies -sf and -se.")
flag.BoolVar(&opts.General.StopOnErrors, "se", opts.General.StopOnErrors, "Stop on spurious errors")
flag.BoolVar(&opts.General.Verbose, "v", opts.General.Verbose, "Verbose output, printing full URL and redirect location (if any) with the results.")
flag.BoolVar(&opts.General.UniqueSizes, "unique", opts.General.UniqueSizes, "Only show unique response sizes in output")
flag.BoolVar(&opts.HTTP.FollowRedirects, "r", opts.HTTP.FollowRedirects, "Follow redirects")
flag.BoolVar(&opts.HTTP.IgnoreBody, "ignore-body", opts.HTTP.IgnoreBody, "Do not fetch the response content.")
flag.BoolVar(&opts.HTTP.Raw, "raw", opts.HTTP.Raw, "Do not encode URI")
Expand Down Expand Up @@ -299,6 +300,15 @@ func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) {
func SetupFilters(parseOpts *ffuf.ConfigOptions, conf *ffuf.Config) error {
errs := ffuf.NewMultierror()
conf.MatcherManager = filter.NewMatcherManager()

// If -unique flag is set, add the unique size filter
if parseOpts.General.UniqueSizes {
err := conf.MatcherManager.AddFilter("uniquesize", "", false)
if err != nil {
return fmt.Errorf("could not setup unique size filter: %s", err)
}
}

// If any other matcher is set, ignore -mc default value
matcherSet := false
statusSet := false
Expand Down
1 change: 1 addition & 0 deletions pkg/ffuf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type Config struct {
Threads int `json:"threads"`
Timeout int `json:"timeout"`
Url string `json:"url"`
UniqueSizes bool `json:"unique_sizes"`
Verbose bool `json:"verbose"`
Wordlists []string `json:"wordlists"`
Http2 bool `json:"http2"`
Expand Down
2 changes: 1 addition & 1 deletion pkg/ffuf/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (

var (
//VERSION holds the current version number
VERSION = "2.1.0"
VERSION = "2.1.1"
//VERSION_APPENDIX holds additional version definition
VERSION_APPENDIX = "-dev"
CONFIGDIR = filepath.Join(xdg.ConfigHome, "ffuf")
Expand Down
2 changes: 2 additions & 0 deletions pkg/ffuf/optionsparser.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ type GeneralOptions struct {
StopOnAll bool `json:"stop_on_all"`
StopOnErrors bool `json:"stop_on_errors"`
Threads int `json:"threads"`
UniqueSizes bool `json:"unique_sizes"`
Verbose bool `json:"verbose"`
}

Expand Down Expand Up @@ -143,6 +144,7 @@ func NewConfigOptions() *ConfigOptions {
c.General.StopOnAll = false
c.General.StopOnErrors = false
c.General.Threads = 40
c.General.UniqueSizes = false
c.General.Verbose = false
c.HTTP.Data = ""
c.HTTP.FollowRedirects = false
Expand Down
3 changes: 3 additions & 0 deletions pkg/filter/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) {
if name == "size" {
return NewSizeFilter(value)
}
if name == "uniquesize" {
return NewUniqueSizeFilter(), nil
}
if name == "word" {
return NewWordFilter(value)
}
Expand Down
49 changes: 49 additions & 0 deletions pkg/filter/uniquesize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package filter

import (
"sync"

"github.com/ffuf/ffuf/v2/pkg/ffuf"
)

type UniqueSizeFilter struct {
seenSizes map[int64]string // maps size to first URL with that size
mutex sync.Mutex
}

func NewUniqueSizeFilter() ffuf.FilterProvider {
return &UniqueSizeFilter{
seenSizes: make(map[int64]string),
}
}

func (f *UniqueSizeFilter) Filter(response *ffuf.Response) (bool, error) {
f.mutex.Lock()
defer f.mutex.Unlock()

size := response.ContentLength

if firstURL, seen := f.seenSizes[size]; !seen {
// First time seeing this size
f.seenSizes[size] = response.Request.Url
return false, nil
} else if firstURL == response.Request.Url {
// This is the first URL we saw with this size, keep it
return false, nil
}

// Not the first URL with this size, filter it out
return true, nil
}

func (f *UniqueSizeFilter) Repr() string {
return "Unique response sizes only"
}

func (f *UniqueSizeFilter) ReprVerbose() string {
return "Unique response sizes only"
}

func (f *UniqueSizeFilter) MarshalJSON() ([]byte, error) {
return []byte(`{"type":"uniquesize"}`), nil
}
41 changes: 22 additions & 19 deletions pkg/output/stdout.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,12 @@ func (s *Stdoutput) Reset() {

// Cycle moves the CurrentResults to Results and resets the results slice
func (s *Stdoutput) Cycle() {
s.Results = append(s.Results, s.CurrentResults...)
s.Reset()
if s.config.UniqueSizes {
s.Results = append(s.Results, FilterUniqueResults(s.CurrentResults)...)
} else {
s.Results = append(s.Results, s.CurrentResults...)
}
s.CurrentResults = make([]ffuf.Result, 0)
}

// GetResults returns the result slice
Expand Down Expand Up @@ -202,7 +206,7 @@ func (s *Stdoutput) Error(errstring string) {
fmt.Fprintf(os.Stderr, "%s", errstring)
} else {
if !s.config.Colors {
fmt.Fprintf(os.Stderr, "%s[ERR] %s\n", TERMINAL_CLEAR_LINE, errstring)
fmt.Fprintf(os.Stderr, "%s[ERR] %ss\n", TERMINAL_CLEAR_LINE, errstring)
} else {
fmt.Fprintf(os.Stderr, "%s[%sERR%s] %s\n", TERMINAL_CLEAR_LINE, ANSI_RED, ANSI_CLEAR, errstring)
}
Expand Down Expand Up @@ -314,33 +318,32 @@ func (s *Stdoutput) Finalize() error {
}

func (s *Stdoutput) Result(resp ffuf.Response) {
// Do we want to write request and response to a file
if len(s.config.OutputDirectory) > 0 {
resp.ResultFile = s.writeResultToFile(resp)
}

inputs := make(map[string][]byte, len(resp.Request.Input))
for k, v := range resp.Request.Input {
inputs[k] = v
}
sResult := ffuf.Result{
Input: inputs,
res := ffuf.Result{
Input: resp.Request.Input,
Position: resp.Request.Position,
StatusCode: resp.StatusCode,
ContentLength: resp.ContentLength,
ContentWords: resp.ContentWords,
ContentLines: resp.ContentLines,
ContentType: resp.ContentType,
RedirectLocation: resp.GetRedirectLocation(false),
ScraperData: resp.ScraperData,
Url: resp.Request.Url,
Duration: resp.Time,
ResultFile: resp.ResultFile,
ResultFile: s.writeResultToFile(resp),
Host: resp.Request.Host,
}
s.CurrentResults = append(s.CurrentResults, sResult)
// Output the result
s.PrintResult(sResult)

if s.config.UniqueSizes {
// For unique sizes, we'll print immediately if it's a new size
newResult := FilterUniqueResults(append(s.CurrentResults, res))
if len(newResult) > len(s.CurrentResults) {
// This is a new unique size, print it
s.PrintResult(res)
}
} else {
s.PrintResult(res)
}
s.CurrentResults = append(s.CurrentResults, res)
}

func (s *Stdoutput) writeResultToFile(resp ffuf.Response) string {
Expand Down
20 changes: 20 additions & 0 deletions pkg/output/uniqueresults.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package output

import (
"github.com/ffuf/ffuf/v2/pkg/ffuf"
)

// FilterUniqueResults filters out results with duplicate sizes, keeping only the first occurrence
func FilterUniqueResults(results []ffuf.Result) []ffuf.Result {
seenSizes := make(map[int64]bool)
uniqueResults := make([]ffuf.Result, 0)

for _, result := range results {
if !seenSizes[result.ContentLength] {
seenSizes[result.ContentLength] = true
uniqueResults = append(uniqueResults, result)
}
}

return uniqueResults
}
Loading