I'm trying to reduce the memory footprint of the code below, which is only about initializing the data I need to then perform a certain set of operations.
Pre-allocations are needed.
func bToMb(b uint64) uint64 {
return b / 1024 / 1024
}
func printMemUsage() {
var m runtime.MemStats
runtime.ReadMemStats(&m)
// For info on each, see: https://golang.org/pkg/runtime/#MemStats
fmt.Println("\n---")
fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
fmt.Printf("\tNumGC = %v\n", m.NumGC)
fmt.Println("---")
}
type Container struct {
key string
left []*Container
right []*Container
}
func main() {
var start = time.Now()
const count = 100_000_000
const steps = 500
const childCount = 50
var m = make(map[string]*Container, count)
fmt.Printf("map initialized: %v\n", time.Since(start))
for i := 0; i < count / (childCount * 2); i++ {
if i % (count / steps) == 0 {
fmt.Printf("i = %v\n", i)
printMemUsage()
}
for j := 0; j < childCount * 2; j++ {
var key = fmt.Sprintf("%d:%d", i, j)
m[key] = &Container{
key: key,
left: make([]*Container, 0, childCount),
right: make([]*Container, 0, childCount),
}
}
}
fmt.Println("done!")
printMemUsage()
}
One issue I have is that the initialization cannot even be performed entirely... this is what I'm getting when executing the code above:
GOROOT=C:\Go #gosetup
GOPATH=C:\Users\User\go #gosetup
C:\Go\bin\go.exe build -o C:\Users\User\AppData\Local\Temp\___180go_build_main_go.exe C:\Users\User\Repos\go-playground\main.go #gosetup
C:\Users\User\AppData\Local\Temp\___180go_build_main_go.exe #gosetup
map initialized: 383.0352ms
i = 0
---
Alloc = 3536 MiB TotalAlloc = 3536 MiB Sys = 3658 MiB NumGC = 0
---
i = 200000
---
Alloc = 20983 MiB TotalAlloc = 20999 MiB Sys = 22673 MiB NumGC = 3
---
runtime: VirtualAlloc of 8192 bytes failed with errno=1455
fatal error: out of memory
runtime stack:
runtime.throw(0xa4b84e, 0xd)
C:/Go/src/runtime/panic.go:1116 +0x79
runtime.sysUsed(0xc82440e000, 0x2000)
C:/Go/src/runtime/mem_windows.go:83 +0x22e
runtime.(*mheap).allocSpan(0xb07980, 0x1, 0x230330b2c00, 0xb22508, 0x23033e6c358)
C:/Go/src/runtime/mheap.go:1276 +0x3c7
runtime.(*mheap).alloc.func1()
C:/Go/src/runtime/mheap.go:907 +0x6b
runtime.systemstack(0x0)
C:/Go/src/runtime/asm_amd64.s:370 +0x6b
runtime.mstart()
C:/Go/src/runtime/proc.go:1116
goroutine 1 [running]:
runtime.systemstack_switch()
C:/Go/src/runtime/asm_amd64.s:330 fp=0xc3acf25c20 sp=0xc3acf25c18 pc=0x9d2da0
runtime.(*mheap).alloc(0xb07980, 0x1, 0xc0dd11012c, 0x63)
C:/Go/src/runtime/mheap.go:901 +0x88 fp=0xc3acf25c70 sp=0xc3acf25c20 pc=0x995fa8
runtime.(*mcentral).grow(0xb1ac30, 0x0)
C:/Go/src/runtime/mcentral.go:506 +0x88 fp=0xc3acf25cb8 sp=0xc3acf25c70 pc=0x986e08
runtime.(*mcentral).cacheSpan(0xb1ac30, 0x23033e6c358)
C:/Go/src/runtime/mcentral.go:177 +0x3e5 fp=0xc3acf25d30 sp=0xc3acf25cb8 pc=0x986b85
runtime.(*mcache).refill(0x22fd4b50e58, 0x2c)
C:/Go/src/runtime/mcache.go:142 +0xb5 fp=0xc3acf25d50 sp=0xc3acf25d30 pc=0x9864f5
runtime.(*mcache).nextFree(0x22fd4b50e58, 0xc00003b02c, 0x0, 0xc3acf25dc8, 0x9df265)
C:/Go/src/runtime/malloc.go:880 +0xa5 fp=0xc3acf25d88 sp=0xc3acf25d50 pc=0x97c665
runtime.mallocgc(0x1a0, 0xa2b460, 0xc8243f6801, 0xc0dd110000)
C:/Go/src/runtime/malloc.go:1061 +0x894 fp=0xc3acf25e28 sp=0xc3acf25d88 pc=0x97d0b4
runtime.makeslice(0xa2b460, 0x0, 0x32, 0x2)
C:/Go/src/runtime/slice.go:98 +0x78 fp=0xc3acf25e58 sp=0xc3acf25e28 pc=0x9b81b8
main.main()
C:/Users/User/Repos/go-playground/main.go:102 +0x255 fp=0xc3acf25f88 sp=0xc3acf25e58 pc=0xa230b5
runtime.main()
C:/Go/src/runtime/proc.go:204 +0x209 fp=0xc3acf25fe0 sp=0xc3acf25f88 pc=0x9a89c9
runtime.goexit()
C:/Go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc3acf25fe8 sp=0xc3acf25fe0 pc=0x9d4b61
Process finished with exit code 2
How can I avoid the fatal error: out of memory
?
Optimization no.1: Moving to uint32
keys (instead of string
):
type UInt32Container struct {
key uint32
left []uint32
right []uint32
}
Optimization no.2: Stripping the key of the container:
type UInt32Container struct {
left []uint32
right []uint32
}
Optimization no.3: Going for a 2D uint32 array storage:
Still none of these optimizations above are enough.