Skip to content

Commit

Permalink
rewrite representation to a sorted binary list and embed it
Browse files Browse the repository at this point in the history
This is now a binary format which is embeded as a string, this remove the need to run code at init or when lazy initializing.
It also fixes a bug where we would incorrectly assume the dataset only reports CIDR alligned ranges, however it does not.

This result in a `1.56MiB` binary file for the dataset.
It is embedded as a string into the `rodata` part of the executable. This remove the runtime memory cost (as this is usually mmaped directly from disk) and all the init execution. It also removes some dependencies, in practice here is the difference (tested with `go test -c`):
```console
> ls -l old new
-rwxr-xr-x 1 hugo hugo 10398323 Dec 19 02:40 old*
-rwxr-xr-x 1 hugo hugo  6954814 Dec 19 02:39 new*
```

This is also significantly faster, new:
```
goos: linux
goarch: amd64
pkg: github.com/libp2p/go-libp2p-asn-util
cpu: AMD Ryzen 5 3600 6-Core Processor
BenchmarkAsnForIPv6-12    	21599720	        56.85 ns/op
```
old:
```
BenchmarkAsnForIPv6-12    	 9417902	       126.3 ns/op
```
  • Loading branch information
Jorropo committed Dec 19, 2023
1 parent 263f49e commit 310bb6c
Show file tree
Hide file tree
Showing 8 changed files with 250 additions and 93,399 deletions.
127 changes: 58 additions & 69 deletions asn.go
Original file line number Diff line number Diff line change
@@ -1,95 +1,84 @@
package asnutil

import (
_ "embed"
"encoding/binary"
"errors"
"fmt"
"net"
"sync"

"github.com/libp2p/go-cidranger"
"strconv"
)

var Store *lazyAsnStore

func init() {
Store = &lazyAsnStore{}
}
//go:embed sorted-network-list.bin
var dataset string

type networkWithAsn struct {
nn net.IPNet
asn string
}
const entrySize = 8*2 + 4 // start, end 8 bytes; asn 4 bytes

func (e *networkWithAsn) Network() net.IPNet {
return e.nn
}

type asnStore struct {
cr cidranger.Ranger
func readEntry(index uint) (start, end uint64, asn uint32) {
base := entrySize * index
b := dataset[base : base+entrySize]
start = uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
b = b[8:]
end = uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
b = b[8:]
asn = uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
return
}

// AsnForIPv6 returns the AS number for the given IPv6 address.
// If no mapping exists for the given IP, this function will
// return an empty ASN and a nil error.
func (a *asnStore) AsnForIPv6(ip net.IP) (string, error) {
if ip.To16() == nil {
return "", errors.New("ONLY IPv6 addresses supported for now")
}

ns, err := a.cr.ContainingNetworks(ip)
if err != nil {
return "", fmt.Errorf("failed to find matching networks for the given ip: %w", err)
// If no mapping exists for the given network, this function will return a zero ASN number.
func AsnForIPv6(ip net.IP) (asn uint32) {
ip = ip.To16()
if ip == nil {
return
}

if len(ns) == 0 {
return "", nil
}

// longest prefix match
n := ns[len(ns)-1].(*networkWithAsn)
return n.asn, nil
return AsnForIPv6Network(binary.BigEndian.Uint64(ip))
}

func newAsnStore() (*asnStore, error) {
cr := cidranger.NewPCTrieRanger()

for _, v := range ipv6CidrToAsnPairList {
_, nn, err := net.ParseCIDR(v.cidr)
if err != nil {
return nil, fmt.Errorf("failed to parse CIDR %s: %w", v.cidr, err)
}

if err := cr.Insert(&networkWithAsn{*nn, v.asn}); err != nil {
return nil, fmt.Errorf("failed to insert CIDR %s in Trie store: %w", v.cidr, err)
// AsnForIPv6Network returns the AS number for the given IPv6 network.
// If no mapping exists for the given network, this function will return a zero ASN number.
// network is the first 64 bits of the ip address interpreted as big endian.
func AsnForIPv6Network(network uint64) (asn uint32) {
n := uint(len(dataset)) / entrySize
var i, j uint = 0, n
for i < j {
h := i/2 + j/2
start, end, asn := readEntry(h)
if start <= network {
if network <= end {
return asn
}
i = h + 1
} else {
j = h
}
}

return &asnStore{cr}, nil
if i >= n {
return 0
}
start, end, asn := readEntry(i)
if start <= network && network <= end {
return asn
}
return 0
}

// lazyAsnStore builds the underlying trie on first call to AsnForIPv6.
// Alternatively, Init can be called to manually trigger initialization.
type lazyAsnStore struct {
store *asnStore
once sync.Once
}
// Deprecated: use [AsnForIPv6] or [AsnForIPv6Network], they do not allocate.
var Store backwardCompat

type backwardCompat struct{}

// AsnForIPv6 returns the AS number for the given IPv6 address.
// If no mapping exists for the given IP, this function will
// return an empty ASN and a nil error.
func (a *lazyAsnStore) AsnForIPv6(ip net.IP) (string, error) {
a.once.Do(a.init)
return a.store.AsnForIPv6(ip)
}

func (a *lazyAsnStore) Init() {
a.once.Do(a.init)
}
func (backwardCompat) AsnForIPv6(ip net.IP) (string, error) {
ip = ip.To16()
if ip == nil {
return "", errors.New("ONLY IPv6 addresses supported")
}

func (a *lazyAsnStore) init() {
store, err := newAsnStore()
if err != nil {
panic(err)
asn := AsnForIPv6Network(binary.BigEndian.Uint64(ip))
if asn == 0 {
return "", nil
}
a.store = store
return strconv.FormatUint(uint64(asn), 10), nil
}
20 changes: 17 additions & 3 deletions asn_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package asnutil

import (
"math/rand"
"net"
"testing"

Expand Down Expand Up @@ -32,8 +33,21 @@ func TestAsnIpv6(t *testing.T) {

for name, tc := range tcs {
require.NotEmpty(t, tc.ip, name)
n, err := Store.AsnForIPv6(tc.ip)
require.NoError(t, err)
require.Equal(t, tc.expectedASN, n, name)
t.Run(name, func(t *testing.T) {
n, err := Store.AsnForIPv6(tc.ip)
require.NoError(t, err)
require.Equal(t, tc.expectedASN, n, name)
})
}
}

var leak uint32

func BenchmarkAsnForIPv6(b *testing.B) {
r := rand.New(rand.NewSource(0))
var doNotOptimize uint32
for i := b.N; i != 0; i-- {
doNotOptimize = AsnForIPv6Network(r.Uint64())
}
leak = doNotOptimize
}
1 change: 0 additions & 1 deletion doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,3 @@
package asnutil

//go:generate go run ./generate/
//go:generate go fmt ./...
Loading

0 comments on commit 310bb6c

Please sign in to comment.