Create memory module dynamically and check total memory (#79)

Author: anuraaga

internal/memory/LICENSE

@@ -0,0 +1,31 @@
+Copied from https://github.com/pbnjay/memory
+
+BSD 3-Clause License
+
+Copyright (c) 2017, Jeremy Jay
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

internal/memory/doc.go

@@ -0,0 +1,24 @@
+// Package memory provides a single method reporting total system memory
+// accessible to the kernel.
+package memory
+
+// TotalMemory returns the total accessible system memory in bytes.
+//
+// The total accessible memory is installed physical memory size minus reserved
+// areas for the kernel and hardware, if such reservations are reported by
+// the operating system.
+//
+// If accessible memory size could not be determined, then 0 is returned.
+func TotalMemory() uint64 {
+	return sysTotalMemory()
+}
+
+// FreeMemory returns the total free system memory in bytes.
+//
+// The total free memory is installed physical memory size minus reserved
+// areas for other applications running on the same system.
+//
+// If free memory size could not be determined, then 0 is returned.
+func FreeMemory() uint64 {
+	return sysFreeMemory()
+}

internal/memory/memory_bsd.go

@@ -0,0 +1,20 @@
+//go:build freebsd || openbsd || dragonfly || netbsd
+// +build freebsd openbsd dragonfly netbsd
+
+package memory
+
+func sysTotalMemory() uint64 {
+	s, err := sysctlUint64("hw.physmem")
+	if err != nil {
+		return 0
+	}
+	return s
+}
+
+func sysFreeMemory() uint64 {
+	s, err := sysctlUint64("hw.usermem")
+	if err != nil {
+		return 0
+	}
+	return s
+}

internal/memory/memory_darwin.go

@@ -0,0 +1,52 @@
+//go:build darwin
+// +build darwin
+
+package memory
+
+import (
+	"os/exec"
+	"regexp"
+	"strconv"
+)
+
+func sysTotalMemory() uint64 {
+	s, err := sysctlUint64("hw.memsize")
+	if err != nil {
+		return 0
+	}
+	return s
+}
+
+var (
+	rePageSize  = regexp.MustCompile(`page size of ([0-9]*) bytes`)
+	reFreePages = regexp.MustCompile(`Pages free: *([0-9]*)\.`)
+)
+
+func sysFreeMemory() uint64 {
+	cmd := exec.Command("vm_stat")
+	outBytes, err := cmd.Output()
+	if err != nil {
+		return 0
+	}
+
+	// default: page size of 4096 bytes
+	matches := rePageSize.FindSubmatchIndex(outBytes)
+	pageSize := uint64(4096)
+	if len(matches) == 4 {
+		pageSize, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64)
+		if err != nil {
+			return 0
+		}
+	}
+
+	// ex: Pages free:                             1126961.
+	matches = reFreePages.FindSubmatchIndex(outBytes)
+	freePages := uint64(0)
+	if len(matches) == 4 {
+		freePages, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64)
+		if err != nil {
+			return 0
+		}
+	}
+	return freePages * pageSize
+}

internal/memory/memory_linux.go

@@ -0,0 +1,30 @@
+//go:build linux
+// +build linux
+
+package memory
+
+import "syscall"
+
+func sysTotalMemory() uint64 {
+	in := &syscall.Sysinfo_t{}
+	err := syscall.Sysinfo(in)
+	if err != nil {
+		return 0
+	}
+	// If this is a 32-bit system, then these fields are
+	// uint32 instead of uint64.
+	// So we always convert to uint64 to match signature.
+	return uint64(in.Totalram) * uint64(in.Unit)
+}
+
+func sysFreeMemory() uint64 {
+	in := &syscall.Sysinfo_t{}
+	err := syscall.Sysinfo(in)
+	if err != nil {
+		return 0
+	}
+	// If this is a 32-bit system, then these fields are
+	// uint32 instead of uint64.
+	// So we always convert to uint64 to match signature.
+	return uint64(in.Freeram) * uint64(in.Unit)
+}

internal/memory/memory_windows.go

@@ -0,0 +1,61 @@
+//go:build windows
+// +build windows
+
+package memory
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// omitting a few fields for brevity...
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx
+type memStatusEx struct {
+	dwLength     uint32
+	dwMemoryLoad uint32
+	ullTotalPhys uint64
+	ullAvailPhys uint64
+	unused       [5]uint64
+}
+
+func sysTotalMemory() uint64 {
+	kernel32, err := syscall.LoadDLL("kernel32.dll")
+	if err != nil {
+		return 0
+	}
+	// GetPhysicallyInstalledSystemMemory is simpler, but broken on
+	// older versions of windows (and uses this under the hood anyway).
+	globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx")
+	if err != nil {
+		return 0
+	}
+	msx := &memStatusEx{
+		dwLength: 64,
+	}
+	r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))
+	if r == 0 {
+		return 0
+	}
+	return msx.ullTotalPhys
+}
+
+func sysFreeMemory() uint64 {
+	kernel32, err := syscall.LoadDLL("kernel32.dll")
+	if err != nil {
+		return 0
+	}
+	// GetPhysicallyInstalledSystemMemory is simpler, but broken on
+	// older versions of windows (and uses this under the hood anyway).
+	globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx")
+	if err != nil {
+		return 0
+	}
+	msx := &memStatusEx{
+		dwLength: 64,
+	}
+	r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))
+	if r == 0 {
+		return 0
+	}
+	return msx.ullAvailPhys
+}

internal/memory/memsysctl.go

@@ -0,0 +1,22 @@
+//go:build darwin || freebsd || openbsd || dragonfly || netbsd
+// +build darwin freebsd openbsd dragonfly netbsd
+
+package memory
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+func sysctlUint64(name string) (uint64, error) {
+	s, err := syscall.Sysctl(name)
+	if err != nil {
+		return 0, err
+	}
+	// hack because the string conversion above drops a \0
+	b := []byte(s)
+	if len(b) < 8 {
+		b = append(b, 0)
+	}
+	return *(*uint64)(unsafe.Pointer(&b[0])), nil
+}

internal/memory/stub.go

@@ -0,0 +1,12 @@
+//go:build !linux && !darwin && !windows && !freebsd && !dragonfly && !netbsd && !openbsd
+// +build !linux,!darwin,!windows,!freebsd,!dragonfly,!netbsd,!openbsd
+
+package memory
+
+func sysTotalMemory() uint64 {
+	return 0
+}
+
+func sysFreeMemory() uint64 {
+	return 0
+}

internal/re2_wazero.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	_ "embed"
 	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"os"
 	"runtime"
@@ -18,6 +19,8 @@ import (
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/experimental"
 	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
+
+	"github.com/wasilibs/go-re2/internal/memory"
 )
 
 var (
@@ -28,11 +31,6 @@ var (
 //go:embed wasm/libcre2.so
 var libre2 []byte
 
-// memoryWasm created by `wat2wasm --enable-threads internal/wasm/memory.wat`
-//
-//go:embed wasm/memory.wasm
-var memoryWasm []byte
-
 var (
 	wasmRT       wazero.Runtime
 	wasmCompiled wazero.CompiledModule
@@ -156,7 +154,17 @@ func init() {
 
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
 
-	if _, err := rt.InstantiateWithConfig(ctx, memoryWasm, wazero.NewModuleConfig().WithName("env")); err != nil {
+	maxPages := uint32(65536)
+	if m := memory.TotalMemory(); m != 0 {
+		pages := uint32(m / 65536) // Divide by WASM page size
+		if pages < maxPages {
+			maxPages = pages
+		}
+	}
+
+	mem := encodeMemory(maxPages)
+
+	if _, err := rt.InstantiateWithConfig(ctx, mem, wazero.NewModuleConfig().WithName("env")); err != nil {
 		panic(err)
 	}
 
@@ -643,3 +651,41 @@ func (f *lazyFunction) callWithStack(ctx context.Context, callStack []uint64) (u
 	}
 	return callStack[0], nil
 }
+
+// Memory module is prefix, max pages, suffix, resulting a module looking like
+// (module (memory (export "memory") 3 <max> shared))
+const (
+	memoryPrefixHex = "0061736d010000000506010303"
+	memorySuffixHex = "070a01066d656d6f72790200"
+)
+
+func encodeMemory(maxPages uint32) []byte {
+	memoryPrefix, _ := hex.DecodeString(memoryPrefixHex)
+	memorySuffix, _ := hex.DecodeString(memorySuffixHex)
+	pages := encodeUint64(uint64(maxPages))
+	res := append([]byte(memoryPrefix), pages...)
+	return append(res, []byte(memorySuffix)...)
+}
+
+// From https://github.com/tetratelabs/wazero/blob/main/internal/leb128/leb128.go#L75
+
+func encodeUint64(value uint64) (buf []byte) {
+	// This is effectively a do/while loop where we take 7 bits of the value and encode them until it is zero.
+	for {
+		// Take 7 remaining low-order bits from the value into b.
+		b := uint8(value & 0x7f)
+		value = value >> 7
+
+		// If there are remaining bits, the value won't be zero: Set the high-
+		// order bit to tell the reader there are more bytes in this uint.
+		if value != 0 {
+			b |= 0x80
+		}
+
+		// Append b into the buffer
+		buf = append(buf, b)
+		if b&0x80 == 0 {
+			return buf
+		}
+	}
+}