go.mod: bump all deps

Bump all transitive and direct dependencies.

Signed-off-by: Casey Callendrello <c1@caseyc.net>
This commit is contained in:
Casey Callendrello
2023-04-04 16:30:47 +02:00
parent 63235a2531
commit bc5f3defe7
338 changed files with 33887 additions and 2915 deletions

View File

@ -1,42 +0,0 @@
## pwalk: parallel implementation of filepath.Walk
This is a wrapper for [filepath.Walk](https://pkg.go.dev/path/filepath?tab=doc#Walk)
which may speed it up by calling multiple callback functions (WalkFunc) in parallel,
utilizing goroutines.
By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks.
This can be changed by using WalkN function which has the additional
parameter, specifying the number of goroutines (concurrency).
### Caveats
Please note the following limitations of this code:
* Unlike filepath.Walk, the order of calls is non-deterministic;
* Only primitive error handling is supported:
* filepath.SkipDir is not supported;
* no errors are ever passed to WalkFunc;
* once any error is returned from any WalkFunc instance, no more new calls
to WalkFunc are made, and the error is returned to the caller of Walk;
* if more than one walkFunc instance will return an error, only one
of such errors will be propagated and returned by Walk, others
will be silently discarded.
### Documentation
For the official documentation, see
https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalk?tab=doc
### Benchmarks
For a WalkFunc that consists solely of the return statement, this
implementation is about 10% slower than the standard library's
filepath.Walk.
Otherwise (if a WalkFunc is doing something) this is usually faster,
except when the WalkN(..., 1) is used.

View File

@ -0,0 +1,54 @@
## pwalkdir: parallel implementation of filepath.WalkDir
This is a wrapper for [filepath.WalkDir](https://pkg.go.dev/path/filepath#WalkDir)
which may speed it up by calling multiple callback functions (WalkDirFunc)
in parallel, utilizing goroutines.
By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks.
This can be changed by using WalkN function which has the additional
parameter, specifying the number of goroutines (concurrency).
### pwalk vs pwalkdir
This package is very similar to
[pwalk](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir),
but utilizes `filepath.WalkDir` (added to Go 1.16), which does not call stat(2)
on every entry and is therefore faster (up to 3x, depending on usage scenario).
Users who are OK with requiring Go 1.16+ should switch to this
implementation.
### Caveats
Please note the following limitations of this code:
* Unlike filepath.WalkDir, the order of calls is non-deterministic;
* Only primitive error handling is supported:
* fs.SkipDir is not supported;
* no errors are ever passed to WalkDirFunc;
* once any error is returned from any walkDirFunc instance, no more calls
to WalkDirFunc are made, and the error is returned to the caller of WalkDir;
* if more than one WalkDirFunc instance will return an error, only one
of such errors will be propagated to and returned by WalkDir, others
will be silently discarded.
### Documentation
For the official documentation, see
https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir
### Benchmarks
For a WalkDirFunc that consists solely of the return statement, this
implementation is about 15% slower than the standard library's
filepath.WalkDir.
Otherwise (if a WalkDirFunc is actually doing something) this is usually
faster, except when the WalkDirN(..., 1) is used. Run `go test -bench .`
to see how different operations can benefit from it, as well as how the
level of paralellism affects the speed.

View File

@ -1,17 +1,17 @@
package pwalk
//go:build go1.16
// +build go1.16
package pwalkdir
import (
"os"
"fmt"
"io/fs"
"path/filepath"
"runtime"
"sync"
"github.com/pkg/errors"
)
type WalkFunc = filepath.WalkFunc
// Walk is a wrapper for filepath.Walk which can call multiple walkFn
// Walk is a wrapper for filepath.WalkDir which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// twice the runtime.NumCPU() walkFn will be called at any one time.
// If you want to change the maximum, use WalkN instead.
@ -20,7 +20,7 @@ type WalkFunc = filepath.WalkFunc
//
// Note that this implementation only supports primitive error handling:
//
// - no errors are ever passed to WalkFn;
// - no errors are ever passed to walkFn;
//
// - once a walkFn returns any error, all further processing stops
// and the error is returned to the caller of Walk;
@ -30,43 +30,51 @@ type WalkFunc = filepath.WalkFunc
// - if more than one walkFn instance will return an error, only one
// of such errors will be propagated and returned by Walk, others
// will be silently discarded.
func Walk(root string, walkFn WalkFunc) error {
func Walk(root string, walkFn fs.WalkDirFunc) error {
return WalkN(root, walkFn, runtime.NumCPU()*2)
}
// WalkN is a wrapper for filepath.Walk which can call multiple walkFn
// WalkN is a wrapper for filepath.WalkDir which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// num walkFn will be called at any one time.
//
// Please see Walk documentation for caveats of using this function.
func WalkN(root string, walkFn WalkFunc, num int) error {
func WalkN(root string, walkFn fs.WalkDirFunc, num int) error {
// make sure limit is sensible
if num < 1 {
return errors.Errorf("walk(%q): num must be > 0", root)
return fmt.Errorf("walk(%q): num must be > 0", root)
}
files := make(chan *walkArgs, 2*num)
errCh := make(chan error, 1) // get the first error, ignore others
errCh := make(chan error, 1) // Get the first error, ignore others.
// Start walking a tree asap
// Start walking a tree asap.
var (
err error
wg sync.WaitGroup
rootLen = len(root)
rootEntry *walkArgs
)
wg.Add(1)
go func() {
err = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
err = filepath.WalkDir(root, func(p string, entry fs.DirEntry, err error) error {
if err != nil {
close(files)
return err
}
// add a file to the queue unless a callback sent an error
if len(p) == rootLen {
// Root entry is processed separately below.
rootEntry = &walkArgs{path: p, entry: entry}
return nil
}
// Add a file to the queue unless a callback sent an error.
select {
case e := <-errCh:
close(files)
return e
default:
files <- &walkArgs{path: p, info: &info}
files <- &walkArgs{path: p, entry: entry}
return nil
}
})
@ -80,7 +88,7 @@ func WalkN(root string, walkFn WalkFunc, num int) error {
for i := 0; i < num; i++ {
go func() {
for file := range files {
if e := walkFn(file.path, *file.info, nil); e != nil {
if e := walkFn(file.path, file.entry, nil); e != nil {
select {
case errCh <- e: // sent ok
default: // buffer full
@ -93,12 +101,16 @@ func WalkN(root string, walkFn WalkFunc, num int) error {
wg.Wait()
if err == nil {
err = walkFn(rootEntry.path, rootEntry.entry, nil)
}
return err
}
// walkArgs holds the arguments that were passed to the Walk or WalkLimit
// walkArgs holds the arguments that were passed to the Walk or WalkN
// functions.
type walkArgs struct {
path string
info *os.FileInfo
entry fs.DirEntry
path string
}