How to reduce the memory footprint of this go program?

504 views Asked by At

I'm trying to reduce the memory footprint of the code below, which is only about initializing the data I need to then perform a certain set of operations.

Pre-allocations are needed.

func bToMb(b uint64) uint64 {
    return b / 1024 / 1024
}

func printMemUsage() {
    var m runtime.MemStats
    runtime.ReadMemStats(&m)
    // For info on each, see: https://golang.org/pkg/runtime/#MemStats
    fmt.Println("\n---")
    fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
    fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
    fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
    fmt.Printf("\tNumGC = %v\n", m.NumGC)
    fmt.Println("---")
}

type Container struct {
    key string
    left  []*Container
    right []*Container
}

func main() {
    var start = time.Now()
    const count = 100_000_000
    const steps = 500
    const childCount = 50
    var m = make(map[string]*Container, count)
    fmt.Printf("map initialized: %v\n", time.Since(start))
    for i := 0; i < count / (childCount * 2); i++ {
        if i % (count / steps) == 0 {
            fmt.Printf("i = %v\n", i)
            printMemUsage()
        }
        for j := 0; j < childCount * 2; j++ {
            var key = fmt.Sprintf("%d:%d", i, j)
            m[key] = &Container{
                key: key,
                left: make([]*Container, 0, childCount),
                right: make([]*Container, 0, childCount),
            }
        }
    }
    fmt.Println("done!")
    printMemUsage()
}

One issue I have is that the initialization cannot even be performed entirely... this is what I'm getting when executing the code above:

GOROOT=C:\Go #gosetup
GOPATH=C:\Users\User\go #gosetup
C:\Go\bin\go.exe build -o C:\Users\User\AppData\Local\Temp\___180go_build_main_go.exe C:\Users\User\Repos\go-playground\main.go #gosetup
C:\Users\User\AppData\Local\Temp\___180go_build_main_go.exe #gosetup
map initialized: 383.0352ms
i = 0

---
Alloc = 3536 MiB    TotalAlloc = 3536 MiB   Sys = 3658 MiB  NumGC = 0
---
i = 200000

---
Alloc = 20983 MiB   TotalAlloc = 20999 MiB  Sys = 22673 MiB NumGC = 3
---
runtime: VirtualAlloc of 8192 bytes failed with errno=1455
fatal error: out of memory

runtime stack:
runtime.throw(0xa4b84e, 0xd)
    C:/Go/src/runtime/panic.go:1116 +0x79
runtime.sysUsed(0xc82440e000, 0x2000)
    C:/Go/src/runtime/mem_windows.go:83 +0x22e
runtime.(*mheap).allocSpan(0xb07980, 0x1, 0x230330b2c00, 0xb22508, 0x23033e6c358)
    C:/Go/src/runtime/mheap.go:1276 +0x3c7
runtime.(*mheap).alloc.func1()
    C:/Go/src/runtime/mheap.go:907 +0x6b
runtime.systemstack(0x0)
    C:/Go/src/runtime/asm_amd64.s:370 +0x6b
runtime.mstart()
    C:/Go/src/runtime/proc.go:1116

goroutine 1 [running]:
runtime.systemstack_switch()
    C:/Go/src/runtime/asm_amd64.s:330 fp=0xc3acf25c20 sp=0xc3acf25c18 pc=0x9d2da0
runtime.(*mheap).alloc(0xb07980, 0x1, 0xc0dd11012c, 0x63)
    C:/Go/src/runtime/mheap.go:901 +0x88 fp=0xc3acf25c70 sp=0xc3acf25c20 pc=0x995fa8
runtime.(*mcentral).grow(0xb1ac30, 0x0)
    C:/Go/src/runtime/mcentral.go:506 +0x88 fp=0xc3acf25cb8 sp=0xc3acf25c70 pc=0x986e08
runtime.(*mcentral).cacheSpan(0xb1ac30, 0x23033e6c358)
    C:/Go/src/runtime/mcentral.go:177 +0x3e5 fp=0xc3acf25d30 sp=0xc3acf25cb8 pc=0x986b85
runtime.(*mcache).refill(0x22fd4b50e58, 0x2c)
    C:/Go/src/runtime/mcache.go:142 +0xb5 fp=0xc3acf25d50 sp=0xc3acf25d30 pc=0x9864f5
runtime.(*mcache).nextFree(0x22fd4b50e58, 0xc00003b02c, 0x0, 0xc3acf25dc8, 0x9df265)
    C:/Go/src/runtime/malloc.go:880 +0xa5 fp=0xc3acf25d88 sp=0xc3acf25d50 pc=0x97c665
runtime.mallocgc(0x1a0, 0xa2b460, 0xc8243f6801, 0xc0dd110000)
    C:/Go/src/runtime/malloc.go:1061 +0x894 fp=0xc3acf25e28 sp=0xc3acf25d88 pc=0x97d0b4
runtime.makeslice(0xa2b460, 0x0, 0x32, 0x2)
    C:/Go/src/runtime/slice.go:98 +0x78 fp=0xc3acf25e58 sp=0xc3acf25e28 pc=0x9b81b8
main.main()
    C:/Users/User/Repos/go-playground/main.go:102 +0x255 fp=0xc3acf25f88 sp=0xc3acf25e58 pc=0xa230b5
runtime.main()
    C:/Go/src/runtime/proc.go:204 +0x209 fp=0xc3acf25fe0 sp=0xc3acf25f88 pc=0x9a89c9
runtime.goexit()
    C:/Go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc3acf25fe8 sp=0xc3acf25fe0 pc=0x9d4b61

Process finished with exit code 2

How can I avoid the fatal error: out of memory?


Optimization no.1: Moving to uint32 keys (instead of string):

type UInt32Container struct {
    key uint32
    left  []uint32
    right []uint32
}

Optimization no.2: Stripping the key of the container:

type UInt32Container struct {
    left  []uint32
    right []uint32
}

Optimization no.3: Going for a 2D uint32 array storage:


Still none of these optimizations above are enough.

0

There are 0 answers