diff options
18 files changed, 513 insertions, 108 deletions
@@ -21,7 +21,7 @@ require ( codeberg.org/gruf/go-structr v0.1.1 codeberg.org/superseriousbusiness/exif-terminator v0.7.0 github.com/DmitriyVTitov/size v1.5.0 - github.com/KimMachineGun/automemlimit v0.4.0 + github.com/KimMachineGun/automemlimit v0.5.0 github.com/abema/go-mp4 v1.1.1 github.com/buckket/go-blurhash v1.1.0 github.com/coreos/go-oidc/v3 v3.9.0 @@ -143,6 +143,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -79,8 +79,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g= github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0= -github.com/KimMachineGun/automemlimit v0.4.0 h1:qOjSDbAUENEL6fiKmRKuAVhPaLijpoEHFDTE+I+prp0= -github.com/KimMachineGun/automemlimit v0.4.0/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= +github.com/KimMachineGun/automemlimit v0.5.0 h1:BeOe+BbJc8L5chL3OwzVYjVzyvPALdd5wxVVOWuUZmQ= +github.com/KimMachineGun/automemlimit v0.5.0/go.mod h1:di3GCKiu9Y+1fs92erCbUvKzPkNyViN3mA0vti/ykEQ= github.com/abema/go-mp4 v1.1.1 h1:OfzkdMO6SWTBR1ltNSVwlTHatrAK9I3iYLQfkdEMMuc= github.com/abema/go-mp4 v1.1.1/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= @@ -416,6 +416,8 @@ github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNia github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw= github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= diff --git a/vendor/github.com/KimMachineGun/automemlimit/README.md b/vendor/github.com/KimMachineGun/automemlimit/README.md index 192bc0b6f..c0e66d2a7 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/README.md +++ b/vendor/github.com/KimMachineGun/automemlimit/README.md @@ -8,6 +8,14 @@ Automatically set `GOMEMLIMIT` to match Linux [cgroups(7)](https://man7.org/linu See more details about `GOMEMLIMIT` [here](https://tip.golang.org/doc/gc-guide#Memory_limit). +## Notice + +Version `v0.5.0` introduces a fallback to system memory limits as an experimental feature when cgroup limits are unavailable. Activate this by setting `AUTOMEMLIMIT_EXPERIMENT=system`. +You can also use system memory limits via `memlimit.FromSystem` provider directly. + +This feature is under evaluation and might become a default or be removed based on user feedback. +If you have any feedback about this feature, please open an issue. + ## Installation ```shell @@ -34,9 +42,17 @@ import "github.com/KimMachineGun/automemlimit/memlimit" func init() { memlimit.SetGoMemLimitWithOpts( memlimit.WithRatio(0.9), - memlimit.WithEnv(), memlimit.WithProvider(memlimit.FromCgroup), ) + memlimit.SetGoMemLimitWithOpts( + memlimit.WithRatio(0.9), + memlimit.WithProvider( + memlimit.ApplyFallback( + memlimit.FromCgroup, + memlimit.FromSystem, + ), + ), + ) memlimit.SetGoMemLimitWithEnv() memlimit.SetGoMemLimit(0.9) memlimit.SetGoMemLimitWithProvider(memlimit.Limit(1024*1024), 0.9) diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go index cfe6d0f60..979bd3937 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go @@ -1,91 +1,12 @@ -//go:build linux -// +build linux - package memlimit import ( - "path/filepath" - - "github.com/containerd/cgroups/v3" - "github.com/containerd/cgroups/v3/cgroup1" - "github.com/containerd/cgroups/v3/cgroup2" + "errors" ) -const ( - cgroupMountPoint = "/sys/fs/cgroup" +var ( + // ErrNoCgroup is returned when the process is not in cgroup. + ErrNoCgroup = errors.New("process is not in cgroup") + // ErrCgroupsNotSupported is returned when the system does not support cgroups. + ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system") ) - -// FromCgroup returns the memory limit based on the cgroups version on this system. -func FromCgroup() (uint64, error) { - switch cgroups.Mode() { - case cgroups.Legacy: - return FromCgroupV1() - case cgroups.Hybrid: - return FromCgroupHybrid() - case cgroups.Unified: - return FromCgroupV2() - } - return 0, ErrNoCgroup -} - -// FromCgroupV1 returns the memory limit from the cgroup v1. -func FromCgroupV1() (uint64, error) { - cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy( - cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory), - )) - if err != nil { - return 0, err - } - - metrics, err := cg.Stat(cgroup1.IgnoreNotExist) - if err != nil { - return 0, err - } - - if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 { - return limit, nil - } - - return 0, ErrNoLimit -} - -// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2. -// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1. -func FromCgroupHybrid() (uint64, error) { - limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified")) - if err == nil { - return limit, nil - } else if err != ErrNoLimit { - return 0, err - } - - return FromCgroupV1() -} - -// FromCgroupV2 returns the memory limit from the cgroup v2. -func FromCgroupV2() (uint64, error) { - return fromCgroupV2(cgroupMountPoint) -} - -func fromCgroupV2(mountPoint string) (uint64, error) { - path, err := cgroup2.NestedGroupPath("") - if err != nil { - return 0, err - } - - m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint)) - if err != nil { - return 0, err - } - - stats, err := m.Stat() - if err != nil { - return 0, err - } - - if limit := stats.GetMemory().GetUsageLimit(); limit != 0 { - return limit, nil - } - - return 0, ErrNoLimit -} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go new file mode 100644 index 000000000..98a954873 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go @@ -0,0 +1,98 @@ +//go:build linux +// +build linux + +package memlimit + +import ( + "math" + "os" + "path/filepath" + + "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup1" + "github.com/containerd/cgroups/v3/cgroup2" +) + +const ( + cgroupMountPoint = "/sys/fs/cgroup" +) + +// FromCgroup returns the memory limit based on the cgroups version on this system. +func FromCgroup() (uint64, error) { + switch cgroups.Mode() { + case cgroups.Legacy: + return FromCgroupV1() + case cgroups.Hybrid: + return FromCgroupHybrid() + case cgroups.Unified: + return FromCgroupV2() + } + return 0, ErrNoCgroup +} + +// FromCgroupV1 returns the memory limit from the cgroup v1. +func FromCgroupV1() (uint64, error) { + cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy( + cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory), + )) + if err != nil { + return 0, err + } + + metrics, err := cg.Stat(cgroup1.IgnoreNotExist) + if err != nil { + return 0, err + } + + if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 && limit != getCgroupV1NoLimit() { + return limit, nil + } + + return 0, ErrNoLimit +} + +func getCgroupV1NoLimit() uint64 { + ps := uint64(os.Getpagesize()) + return math.MaxInt64 / ps * ps +} + +// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2. +// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1. +func FromCgroupHybrid() (uint64, error) { + limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified")) + if err == nil { + return limit, nil + } else if err != ErrNoLimit { + return 0, err + } + + return FromCgroupV1() +} + +// FromCgroupV2 returns the memory limit from the cgroup v2. +func FromCgroupV2() (uint64, error) { + return fromCgroupV2(cgroupMountPoint) +} + +func fromCgroupV2(mountPoint string) (uint64, error) { + path, err := cgroup2.NestedGroupPath("") + if err != nil { + return 0, err + } + + m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint)) + if err != nil { + return 0, err + } + + stats, err := m.Stat() + if err != nil { + return 0, err + } + + if limit := stats.GetMemory().GetUsageLimit(); limit != 0 && limit != math.MaxUint64 { + return limit, nil + } + + return 0, ErrNoLimit +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go new file mode 100644 index 000000000..dee95f520 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go @@ -0,0 +1,14 @@ +package memlimit + +import ( + "github.com/pbnjay/memory" +) + +// FromSystem returns the total memory of the system. +func FromSystem() (uint64, error) { + limit := memory.TotalMemory() + if limit == 0 { + return 0, ErrNoLimit + } + return limit, nil +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go new file mode 100644 index 000000000..2a7c320ed --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go @@ -0,0 +1,59 @@ +package memlimit + +import ( + "fmt" + "os" + "reflect" + "strings" +) + +const ( + envAUTOMEMLIMIT_EXPERIMENT = "AUTOMEMLIMIT_EXPERIMENT" +) + +// Experiments is a set of experiment flags. +// It is used to enable experimental features. +// +// You can set the flags by setting the environment variable AUTOMEMLIMIT_EXPERIMENT. +// The value of the environment variable is a comma-separated list of experiment names. +// +// The following experiment names are known: +// +// - none: disable all experiments +// - system: enable fallback to system memory limit +type Experiments struct { + // System enables fallback to system memory limit. + System bool +} + +func parseExperiments() (Experiments, error) { + var exp Experiments + + // Create a map of known experiment names. + names := make(map[string]func(bool)) + rv := reflect.ValueOf(&exp).Elem() + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + field := rv.Field(i) + names[strings.ToLower(rt.Field(i).Name)] = field.SetBool + } + + // Parse names. + for _, f := range strings.Split(os.Getenv(envAUTOMEMLIMIT_EXPERIMENT), ",") { + if f == "" { + continue + } + if f == "none" { + exp = Experiments{} + continue + } + val := true + set, ok := names[f] + if !ok { + return Experiments{}, fmt.Errorf("unknown AUTOMEMLIMIT_EXPERIMENT %s", f) + } + set(val) + } + + return exp, nil +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go index dbb4d1c72..0d7a9853c 100644 --- a/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go @@ -22,16 +22,11 @@ const ( var ( // ErrNoLimit is returned when the memory limit is not set. ErrNoLimit = errors.New("memory is not limited") - // ErrNoCgroup is returned when the process is not in cgroup. - ErrNoCgroup = errors.New("process is not in cgroup") - // ErrCgroupsNotSupported is returned when the system does not support cgroups. - ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system") ) type config struct { logger *log.Logger ratio float64 - env bool provider Provider } @@ -50,10 +45,10 @@ func WithRatio(ratio float64) Option { // WithEnv configures whether to use environment variables. // // Default: false +// +// Deprecated: currently this does nothing. func WithEnv() Option { - return func(cfg *config) { - cfg.env = true - } + return func(cfg *config) {} } // WithProvider configures the provider. @@ -65,17 +60,24 @@ func WithProvider(provider Provider) Option { } } -// SetGoMemLimitWithOpts sets GOMEMLIMIT with options. +// SetGoMemLimitWithOpts sets GOMEMLIMIT with options and environment variables. +// +// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT +// through AUTOMEMLIMIT envrironment variable in the half-open range (0.0,1.0]. +// +// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.) +// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing. +// +// If AUTOMEMLIMIT_EXPERIMENT is set, it enables experimental features. +// Please see the documentation of Experiments for more details. // // Options: // - WithRatio -// - WithEnv (see more SetGoMemLimitWithEnv) // - WithProvider func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) { cfg := &config{ logger: log.New(io.Discard, "", log.LstdFlags), ratio: defaultAUTOMEMLIMIT, - env: false, provider: FromCgroup, } if os.Getenv(envAUTOMEMLIMIT_DEBUG) == "true" { @@ -90,6 +92,15 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) { } }() + exps, err := parseExperiments() + if err != nil { + return 0, fmt.Errorf("failed to parse experiments: %w", err) + } + if exps.System { + cfg.logger.Println("system experiment is enabled: using system memory limit as a fallback") + cfg.provider = ApplyFallback(cfg.provider, FromSystem) + } + snapshot := debug.SetMemoryLimit(-1) defer func() { err := recover() @@ -122,6 +133,10 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) { limit, err := setGoMemLimit(ApplyRatio(cfg.provider, ratio)) if err != nil { + if errors.Is(err, ErrNoLimit) { + cfg.logger.Printf("memory is not limited, skipping: %v\n", err) + return 0, nil + } return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err) } @@ -130,14 +145,8 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) { return limit, nil } -// SetGoMemLimitWithEnv sets GOMEMLIMIT with the value from the environment variable. -// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT -// through AUTOMEMLIMIT in the half-open range (0.0,1.0]. -// -// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.) -// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing. func SetGoMemLimitWithEnv() { - _, _ = SetGoMemLimitWithOpts(WithEnv()) + _, _ = SetGoMemLimitWithOpts() } // SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio. diff --git a/vendor/github.com/pbnjay/memory/LICENSE b/vendor/github.com/pbnjay/memory/LICENSE new file mode 100644 index 000000000..63ca4a6d2 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2017, Jeremy Jay +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/pbnjay/memory/README.md b/vendor/github.com/pbnjay/memory/README.md new file mode 100644 index 000000000..e98f261a0 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/README.md @@ -0,0 +1,41 @@ +# memory + +Package `memory` provides two methods reporting total physical system memory +accessible to the kernel, and free memory available to the running application. + +This package has no external dependency besides the standard library and default operating system tools. + +Documentation: +[](https://godoc.org/github.com/pbnjay/memory) + +This is useful for dynamic code to minimize thrashing and other contention, similar to the stdlib `runtime.NumCPU` +See some history of the proposal at https://github.com/golang/go/issues/21816 + + +## Example + +```go +fmt.Printf("Total system memory: %d\n", memory.TotalMemory()) +fmt.Printf("Free memory: %d\n", memory.FreeMemory()) +``` + + +## Testing + +Tested/working on: + - macOS 10.12.6 (16G29), 10.15.7 (19H2) + - Windows 10 1511 (10586.1045) + - Linux RHEL (3.10.0-327.3.1.el7.x86_64) + - Raspberry Pi 3 (ARMv8) on Raspbian, ODROID-C1+ (ARMv7) on Ubuntu, C.H.I.P + (ARMv7). + - Amazon Linux 2 aarch64 (m6a.large, 4.14.203-156.332.amzn2.aarch64) + +Tested on virtual machines: + - Windows 7 SP1 386 + - Debian stretch 386 + - NetBSD 7.1 amd64 + 386 + - OpenBSD 6.1 amd64 + 386 + - FreeBSD 11.1 amd64 + 386 + - DragonFly BSD 4.8.1 amd64 + +If you have access to untested systems please test and report success/bugs. diff --git a/vendor/github.com/pbnjay/memory/doc.go b/vendor/github.com/pbnjay/memory/doc.go new file mode 100644 index 000000000..4e4f984c0 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/doc.go @@ -0,0 +1,24 @@ +// Package memory provides a single method reporting total system memory +// accessible to the kernel. +package memory + +// TotalMemory returns the total accessible system memory in bytes. +// +// The total accessible memory is installed physical memory size minus reserved +// areas for the kernel and hardware, if such reservations are reported by +// the operating system. +// +// If accessible memory size could not be determined, then 0 is returned. +func TotalMemory() uint64 { + return sysTotalMemory() +} + +// FreeMemory returns the total free system memory in bytes. +// +// The total free memory is installed physical memory size minus reserved +// areas for other applications running on the same system. +// +// If free memory size could not be determined, then 0 is returned. +func FreeMemory() uint64 { + return sysFreeMemory() +} diff --git a/vendor/github.com/pbnjay/memory/memory_bsd.go b/vendor/github.com/pbnjay/memory/memory_bsd.go new file mode 100644 index 000000000..49d808a9e --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_bsd.go @@ -0,0 +1,19 @@ +// +build freebsd openbsd dragonfly netbsd + +package memory + +func sysTotalMemory() uint64 { + s, err := sysctlUint64("hw.physmem") + if err != nil { + return 0 + } + return s +} + +func sysFreeMemory() uint64 { + s, err := sysctlUint64("hw.usermem") + if err != nil { + return 0 + } + return s +} diff --git a/vendor/github.com/pbnjay/memory/memory_darwin.go b/vendor/github.com/pbnjay/memory/memory_darwin.go new file mode 100644 index 000000000..a3f457699 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_darwin.go @@ -0,0 +1,49 @@ +// +build darwin + +package memory + +import ( + "os/exec" + "regexp" + "strconv" +) + +func sysTotalMemory() uint64 { + s, err := sysctlUint64("hw.memsize") + if err != nil { + return 0 + } + return s +} + +func sysFreeMemory() uint64 { + cmd := exec.Command("vm_stat") + outBytes, err := cmd.Output() + if err != nil { + return 0 + } + + rePageSize := regexp.MustCompile("page size of ([0-9]*) bytes") + reFreePages := regexp.MustCompile("Pages free: *([0-9]*)\\.") + + // default: page size of 4096 bytes + matches := rePageSize.FindSubmatchIndex(outBytes) + pageSize := uint64(4096) + if len(matches) == 4 { + pageSize, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64) + if err != nil { + return 0 + } + } + + // ex: Pages free: 1126961. + matches = reFreePages.FindSubmatchIndex(outBytes) + freePages := uint64(0) + if len(matches) == 4 { + freePages, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64) + if err != nil { + return 0 + } + } + return freePages * pageSize +} diff --git a/vendor/github.com/pbnjay/memory/memory_linux.go b/vendor/github.com/pbnjay/memory/memory_linux.go new file mode 100644 index 000000000..3d07711ce --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_linux.go @@ -0,0 +1,29 @@ +// +build linux + +package memory + +import "syscall" + +func sysTotalMemory() uint64 { + in := &syscall.Sysinfo_t{} + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + // If this is a 32-bit system, then these fields are + // uint32 instead of uint64. + // So we always convert to uint64 to match signature. + return uint64(in.Totalram) * uint64(in.Unit) +} + +func sysFreeMemory() uint64 { + in := &syscall.Sysinfo_t{} + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + // If this is a 32-bit system, then these fields are + // uint32 instead of uint64. + // So we always convert to uint64 to match signature. + return uint64(in.Freeram) * uint64(in.Unit) +} diff --git a/vendor/github.com/pbnjay/memory/memory_windows.go b/vendor/github.com/pbnjay/memory/memory_windows.go new file mode 100644 index 000000000..c8500cc6f --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_windows.go @@ -0,0 +1,60 @@ +// +build windows + +package memory + +import ( + "syscall" + "unsafe" +) + +// omitting a few fields for brevity... +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx +type memStatusEx struct { + dwLength uint32 + dwMemoryLoad uint32 + ullTotalPhys uint64 + ullAvailPhys uint64 + unused [5]uint64 +} + +func sysTotalMemory() uint64 { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return 0 + } + // GetPhysicallyInstalledSystemMemory is simpler, but broken on + // older versions of windows (and uses this under the hood anyway). + globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx") + if err != nil { + return 0 + } + msx := &memStatusEx{ + dwLength: 64, + } + r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if r == 0 { + return 0 + } + return msx.ullTotalPhys +} + +func sysFreeMemory() uint64 { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return 0 + } + // GetPhysicallyInstalledSystemMemory is simpler, but broken on + // older versions of windows (and uses this under the hood anyway). + globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx") + if err != nil { + return 0 + } + msx := &memStatusEx{ + dwLength: 64, + } + r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if r == 0 { + return 0 + } + return msx.ullAvailPhys +} diff --git a/vendor/github.com/pbnjay/memory/memsysctl.go b/vendor/github.com/pbnjay/memory/memsysctl.go new file mode 100644 index 000000000..438d9eff8 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memsysctl.go @@ -0,0 +1,21 @@ +// +build darwin freebsd openbsd dragonfly netbsd + +package memory + +import ( + "syscall" + "unsafe" +) + +func sysctlUint64(name string) (uint64, error) { + s, err := syscall.Sysctl(name) + if err != nil { + return 0, err + } + // hack because the string conversion above drops a \0 + b := []byte(s) + if len(b) < 8 { + b = append(b, 0) + } + return *(*uint64)(unsafe.Pointer(&b[0])), nil +} diff --git a/vendor/github.com/pbnjay/memory/stub.go b/vendor/github.com/pbnjay/memory/stub.go new file mode 100644 index 000000000..f29473ba0 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/stub.go @@ -0,0 +1,10 @@ +// +build !linux,!darwin,!windows,!freebsd,!dragonfly,!netbsd,!openbsd + +package memory + +func sysTotalMemory() uint64 { + return 0 +} +func sysFreeMemory() uint64 { + return 0 +} diff --git a/vendor/modules.txt b/vendor/modules.txt index dc7cdc0f7..8ad21430c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -68,7 +68,7 @@ codeberg.org/superseriousbusiness/exif-terminator # github.com/DmitriyVTitov/size v1.5.0 ## explicit; go 1.14 github.com/DmitriyVTitov/size -# github.com/KimMachineGun/automemlimit v0.4.0 +# github.com/KimMachineGun/automemlimit v0.5.0 ## explicit; go 1.19 github.com/KimMachineGun/automemlimit github.com/KimMachineGun/automemlimit/memlimit @@ -418,6 +418,9 @@ github.com/oklog/ulid # github.com/opencontainers/runtime-spec v1.0.2 ## explicit github.com/opencontainers/runtime-spec/specs-go +# github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 +## explicit; go 1.16 +github.com/pbnjay/memory # github.com/pelletier/go-toml/v2 v2.1.1 ## explicit; go 1.16 github.com/pelletier/go-toml/v2 |